From f903602b267dd1804126a97e7db55430afdf30fe Mon Sep 17 00:00:00 2001 From: Rafael Wiltz Date: Fri, 23 Oct 2026 10:52:06 -0400 Subject: [PATCH 1/6] Add keyboard, gamepad and SpaceMouse teleop to isaaclab_teleop Add builders for keyboard, gamepad and SpaceMouse SE(2)/SE(3) Isaac Capture pipelines. The keyboard is read inside the teleop session from whichever visualizer window has focus (no input group, no keys from other apps); while a pipeline reads it, the session turns off the visualizers' keyboard camera movement. The gamepad and SpaceMouse configs start the Isaac Capture gamepad and spacemouse plugins with the session, which read /dev/input/jsN and /dev/hidrawN. Like every Isaac Capture session they open an OpenXR session; without a headset they run on the standalone CloudXR profile. IsaacTeleopDevice now starts, pauses and resets teleoperation on the B, P and R keys typed into a visualizer window and resets on a SpaceMouse pipeline's right button, the bindings the teleop scripts made themselves through a Carb keyboard. Each key press is handled once, whatever the retargeting mode. device_selection translates an isaaclab.devices keyboard, gamepad or SpaceMouse config into the equivalent pipeline, so a script can serve --teleop_device and an environment's teleop_devices entries through Isaac Capture. --- .../source/api/lab_teleop/isaaclab_teleop.rst | 27 ++ .../rwiltz-keyboard-focus-surfaces.added.rst | 12 + ...rwiltz-keyboard-focus-surfaces.changed.rst | 4 + .../rwiltz-keyboard-focus-surfaces.minor | 0 .../isaaclab_teleop/__init__.pyi | 9 + .../isaaclab_teleop/_plugins.py | 24 ++ .../isaaclab_teleop/device_selection.py | 103 +++++++ .../isaaclab_teleop/gamepad/__init__.py | 10 + .../isaaclab_teleop/gamepad/__init__.pyi | 12 + .../isaaclab_teleop/gamepad/se2_gamepad.py | 92 +++++++ .../isaaclab_teleop/gamepad/se3_gamepad.py | 113 ++++++++ .../isaaclab_teleop/isaac_teleop_device.py | 37 ++- .../isaaclab_teleop/keyboard/__init__.py | 10 + .../isaaclab_teleop/keyboard/__init__.pyi | 12 + .../isaaclab_teleop/keyboard/se2_keyboard.py | 85 ++++++ .../isaaclab_teleop/keyboard/se3_keyboard.py | 104 +++++++ .../isaaclab_teleop/session_lifecycle.py | 117 +++++++- .../isaaclab_teleop/spacemouse/__init__.py | 10 + .../isaaclab_teleop/spacemouse/__init__.pyi | 12 + .../spacemouse/se2_spacemouse.py | 99 +++++++ .../spacemouse/se3_spacemouse.py | 111 ++++++++ .../test/test_device_selection.py | 154 +++++++++++ .../test/test_gamepad_constructors.py | 65 +++++ .../test/test_hand_debug_viz.py | 3 + .../test/test_keyboard_constructors.py | 72 +++++ .../test/test_keyboard_session.py | 253 ++++++++++++++++++ .../test/test_operator_input.py | 88 ++++++ .../test/test_spacemouse_constructors.py | 72 +++++ 28 files changed, 1705 insertions(+), 5 deletions(-) create mode 100644 source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.added.rst create mode 100644 source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.changed.rst create mode 100644 source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.minor create mode 100644 source/isaaclab_teleop/isaaclab_teleop/_plugins.py create mode 100644 source/isaaclab_teleop/isaaclab_teleop/device_selection.py create mode 100644 source/isaaclab_teleop/isaaclab_teleop/gamepad/__init__.py create mode 100644 source/isaaclab_teleop/isaaclab_teleop/gamepad/__init__.pyi create mode 100644 source/isaaclab_teleop/isaaclab_teleop/gamepad/se2_gamepad.py create mode 100644 source/isaaclab_teleop/isaaclab_teleop/gamepad/se3_gamepad.py create mode 100644 source/isaaclab_teleop/isaaclab_teleop/keyboard/__init__.py create mode 100644 source/isaaclab_teleop/isaaclab_teleop/keyboard/__init__.pyi create mode 100644 source/isaaclab_teleop/isaaclab_teleop/keyboard/se2_keyboard.py create mode 100644 source/isaaclab_teleop/isaaclab_teleop/keyboard/se3_keyboard.py create mode 100644 source/isaaclab_teleop/isaaclab_teleop/spacemouse/__init__.py create mode 100644 source/isaaclab_teleop/isaaclab_teleop/spacemouse/__init__.pyi create mode 100644 source/isaaclab_teleop/isaaclab_teleop/spacemouse/se2_spacemouse.py create mode 100644 source/isaaclab_teleop/isaaclab_teleop/spacemouse/se3_spacemouse.py create mode 100644 source/isaaclab_teleop/test/test_device_selection.py create mode 100644 source/isaaclab_teleop/test/test_gamepad_constructors.py create mode 100644 source/isaaclab_teleop/test/test_keyboard_constructors.py create mode 100644 source/isaaclab_teleop/test/test_keyboard_session.py create mode 100644 source/isaaclab_teleop/test/test_operator_input.py create mode 100644 source/isaaclab_teleop/test/test_spacemouse_constructors.py diff --git a/docs/source/api/lab_teleop/isaaclab_teleop.rst b/docs/source/api/lab_teleop/isaaclab_teleop.rst index 370b9bc56cf1..48fcec1918c1 100644 --- a/docs/source/api/lab_teleop/isaaclab_teleop.rst +++ b/docs/source/api/lab_teleop/isaaclab_teleop.rst @@ -30,6 +30,12 @@ isaaclab_teleop create_isaac_teleop_device create_haptic_feedback_driver remove_camera_configs + se2_gamepad_teleop_cfg + se2_keyboard_teleop_cfg + se2_spacemouse_teleop_cfg + se3_gamepad_teleop_cfg + se3_keyboard_teleop_cfg + se3_spacemouse_teleop_cfg Configuration ------------- @@ -64,6 +70,27 @@ Device .. autofunction:: create_isaac_teleop_device +Keyboard +-------- + +.. autofunction:: se2_keyboard_teleop_cfg + +.. autofunction:: se3_keyboard_teleop_cfg + +Gamepad +------- + +.. autofunction:: se2_gamepad_teleop_cfg + +.. autofunction:: se3_gamepad_teleop_cfg + +SpaceMouse +---------- + +.. autofunction:: se2_spacemouse_teleop_cfg + +.. autofunction:: se3_spacemouse_teleop_cfg + Haptic Feedback --------------- diff --git a/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.added.rst b/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.added.rst new file mode 100644 index 000000000000..64c1bb7f43d6 --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.added.rst @@ -0,0 +1,12 @@ +* Added :func:`~isaaclab_teleop.se2_keyboard_teleop_cfg`, :func:`~isaaclab_teleop.se3_keyboard_teleop_cfg`, + :func:`~isaaclab_teleop.se2_gamepad_teleop_cfg`, :func:`~isaaclab_teleop.se3_gamepad_teleop_cfg`, + :func:`~isaaclab_teleop.se2_spacemouse_teleop_cfg` and :func:`~isaaclab_teleop.se3_spacemouse_teleop_cfg`, + which build an :class:`~isaaclab_teleop.IsaacTeleopCfg` for keyboard, gamepad or SpaceMouse + SE(2)/SE(3) teleoperation. The keyboard is read in process from whichever visualizer window has + focus (no ``input`` group, no keys from other apps). The gamepad is read from ``/dev/input/jsN`` + and the SpaceMouse from ``/dev/hidrawN`` (which needs a udev rule) by the Isaac Capture + ``gamepad`` and ``spacemouse`` plugins, which their configs start with the session. Without + ``--xr`` they run headset-free on the standalone CloudXR profile. +* Added :mod:`isaaclab_teleop.device_selection`, which translates an :mod:`isaaclab.devices` + keyboard, gamepad or SpaceMouse config into the equivalent pipeline and picks the pipeline for a + script's ``--teleop_device`` request, or ``None`` when ``isaacteleop`` is not installed. diff --git a/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.changed.rst b/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.changed.rst new file mode 100644 index 000000000000..bd02528c6e67 --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.changed.rst @@ -0,0 +1,4 @@ +* Changed :class:`~isaaclab_teleop.IsaacTeleopDevice` to start, pause and reset teleoperation on the + ``B`` / ``P`` / ``R`` keys typed into a visualizer window and on a SpaceMouse pipeline's right + button, so scripts no longer bind them. While a pipeline reads the keyboard, the visualizers' + keyboard camera movement is turned off. diff --git a/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.minor b/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi index eb5c4f82838b..fe204a802a14 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi +++ b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi @@ -31,10 +31,17 @@ __all__ = [ "create_isaac_teleop_device", "poll_control_events", "remove_camera_configs", + "se2_gamepad_teleop_cfg", + "se2_keyboard_teleop_cfg", + "se2_spacemouse_teleop_cfg", + "se3_gamepad_teleop_cfg", + "se3_keyboard_teleop_cfg", + "se3_spacemouse_teleop_cfg", ] from .camera_feed import XrCameraFeedSession from .control_events import TELEOP_CONTROL_CHANNEL_UUID, ControlEvents, SupportsControlEvents, poll_control_events +from .gamepad import se2_gamepad_teleop_cfg, se3_gamepad_teleop_cfg from .haptic_feedback import ( ControllerHapticFeedbackCfg, GloveHapticFeedbackCfg, @@ -52,7 +59,9 @@ from .isaac_teleop_cfg import ( XrCameraFeedLayoutCfg, ) from .isaac_teleop_device import IsaacTeleopDevice, create_isaac_teleop_device +from .keyboard import se2_keyboard_teleop_cfg, se3_keyboard_teleop_cfg from .session_lifecycle import cloudxr_eula_accepted +from .spacemouse import se2_spacemouse_teleop_cfg, se3_spacemouse_teleop_cfg from .system_check import SystemCheckItem, SystemCheckResult, check_system_requirements from .xr_anchor_utils import XrAnchorSynchronizer from .xr_cfg import XrAnchorRotationMode, XrCfg, remove_camera_configs diff --git a/source/isaaclab_teleop/isaaclab_teleop/_plugins.py b/source/isaaclab_teleop/isaaclab_teleop/_plugins.py new file mode 100644 index 000000000000..a36e0ae3e021 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/_plugins.py @@ -0,0 +1,24 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""The Isaac Capture plugins that read the gamepad and the SpaceMouse for their teleop configs.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from isaacteleop.teleop_session_manager import PluginConfig + + +def bundled_plugin_configs(name: str) -> list[PluginConfig]: + """The Isaac Capture plugin ``name`` (``"gamepad"`` or ``"spacemouse"``), started with the session. + + It runs from the copy bundled with Isaac Capture and pushes to the collection the pipeline's + source of the same name reads. + """ + from isaacteleop.teleop_session_manager import BUNDLED_PLUGINS_DIR, PluginConfig + + return [PluginConfig(plugin_name=name, plugin_root_id=name, search_paths=[BUNDLED_PLUGINS_DIR], required=True)] diff --git a/source/isaaclab_teleop/isaaclab_teleop/device_selection.py b/source/isaaclab_teleop/isaaclab_teleop/device_selection.py new file mode 100644 index 000000000000..03f66d446acd --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/device_selection.py @@ -0,0 +1,103 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Serve a script's teleop device request with an Isaac Capture pipeline when one can. + +An environment declares its keyboard, gamepad and SpaceMouse devices with the +:mod:`isaaclab.devices` configs (:class:`~isaaclab.devices.Se3KeyboardCfg`, ...). +:func:`select_isaac_teleop_cfg` translates such a config into the equivalent Isaac Capture +pipeline. Without the ``isaacteleop`` package, or for a device it has no pipeline for (e.g. OpenXR +hand tracking), it returns ``None`` and the caller uses the :mod:`isaaclab.devices` device. +""" + +from __future__ import annotations + +import importlib.util +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any + +from isaaclab.devices.gamepad import Se2GamepadCfg, Se3GamepadCfg +from isaaclab.devices.keyboard import Se2KeyboardCfg, Se3KeyboardCfg +from isaaclab.devices.spacemouse import Se2SpaceMouseCfg, Se3SpaceMouseCfg + +from .gamepad import se2_gamepad_teleop_cfg, se3_gamepad_teleop_cfg +from .keyboard import se2_keyboard_teleop_cfg, se3_keyboard_teleop_cfg +from .spacemouse import se2_spacemouse_teleop_cfg, se3_spacemouse_teleop_cfg + +if TYPE_CHECKING: + from isaaclab.devices import DeviceCfg + + from .isaac_teleop_cfg import IsaacTeleopCfg + +# Each device config type, the builder of its pipeline, and the config fields the builder takes. +_PIPELINES = { + Se3KeyboardCfg: (se3_keyboard_teleop_cfg, ("pos_sensitivity", "rot_sensitivity", "gripper_term")), + Se2KeyboardCfg: (se2_keyboard_teleop_cfg, ("v_x_sensitivity", "v_y_sensitivity", "omega_z_sensitivity")), + Se3GamepadCfg: (se3_gamepad_teleop_cfg, ("pos_sensitivity", "rot_sensitivity", "dead_zone", "gripper_term")), + Se2GamepadCfg: ( + se2_gamepad_teleop_cfg, + ("v_x_sensitivity", "v_y_sensitivity", "omega_z_sensitivity", "dead_zone"), + ), + Se3SpaceMouseCfg: (se3_spacemouse_teleop_cfg, ("pos_sensitivity", "rot_sensitivity", "gripper_term")), + Se2SpaceMouseCfg: (se2_spacemouse_teleop_cfg, ("v_x_sensitivity", "v_y_sensitivity", "omega_z_sensitivity")), +} + + +def _isaacteleop_available() -> bool: + return importlib.util.find_spec("isaacteleop") is not None + + +def pipeline_from_device_cfg(device_cfg: DeviceCfg, sim_device: str) -> IsaacTeleopCfg | None: + """Build the Isaac Capture pipeline equivalent to a device config. + + Args: + device_cfg: A device config, e.g. :class:`~isaaclab.devices.Se3KeyboardCfg`. + sim_device: Torch device for the pipeline's output tensors. + + Returns: + The pipeline, or ``None`` when no pipeline serves this device config type. + """ + entry = _PIPELINES.get(type(device_cfg)) + if entry is None: + return None + builder, fields = entry + return builder(**{field: getattr(device_cfg, field) for field in fields}, sim_device=sim_device) + + +def select_isaac_teleop_cfg( + env_cfg: Any, + device_name: str | None, + default_device_cfgs: Mapping[str, DeviceCfg], + sim_device: str, +) -> IsaacTeleopCfg | None: + """Pick the Isaac Capture pipeline that serves a script's device request. + + Without ``device_name``, the environment's declared ``isaac_teleop`` pipeline is used, else + the ``"keyboard"`` entry of ``default_device_cfgs``. With it, the environment's + ``teleop_devices`` entry of that name is used, else the ``default_device_cfgs`` entry. + + Args: + env_cfg: The environment config, read for ``isaac_teleop`` and ``teleop_devices``. + device_name: The requested device (e.g. ``--teleop_device``), or ``None``. + default_device_cfgs: The script's own device configs, by name, for devices the + environment does not declare. + sim_device: Torch device for the pipeline's output tensors. + + Returns: + The pipeline, or ``None`` when ``isaacteleop`` is not installed or no pipeline serves the + requested device; the caller then uses the :mod:`isaaclab.devices` device. + """ + if not _isaacteleop_available(): + return None + if device_name is None: + declared = getattr(env_cfg, "isaac_teleop", None) + if declared is not None: + return declared + device_name = "keyboard" + env_devices = getattr(getattr(env_cfg, "teleop_devices", None), "devices", None) or {} + device_cfg = env_devices.get(device_name) or default_device_cfgs.get(device_name) + if device_cfg is None: + return None + return pipeline_from_device_cfg(device_cfg, sim_device) diff --git a/source/isaaclab_teleop/isaaclab_teleop/gamepad/__init__.py b/source/isaaclab_teleop/isaaclab_teleop/gamepad/__init__.py new file mode 100644 index 000000000000..2957e08ed477 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/gamepad/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Gamepad device for SE(2) and SE(3) control.""" + +from isaaclab.utils.module import lazy_export + +lazy_export() diff --git a/source/isaaclab_teleop/isaaclab_teleop/gamepad/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/gamepad/__init__.pyi new file mode 100644 index 000000000000..cd2a8c1216bf --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/gamepad/__init__.pyi @@ -0,0 +1,12 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "se2_gamepad_teleop_cfg", + "se3_gamepad_teleop_cfg", +] + +from .se2_gamepad import se2_gamepad_teleop_cfg +from .se3_gamepad import se3_gamepad_teleop_cfg diff --git a/source/isaaclab_teleop/isaaclab_teleop/gamepad/se2_gamepad.py b/source/isaaclab_teleop/isaaclab_teleop/gamepad/se2_gamepad.py new file mode 100644 index 000000000000..6dbaae854f26 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/gamepad/se2_gamepad.py @@ -0,0 +1,92 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Gamepad-driven IsaacTeleopCfg builder for SE(2) control.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..isaac_teleop_cfg import IsaacTeleopCfg + + +def se2_gamepad_teleop_cfg( + v_x_sensitivity: float = 1.0, + v_y_sensitivity: float = 1.0, + omega_z_sensitivity: float = 1.0, + dead_zone: float = 0.15, + sim_device: str = "cpu", +) -> IsaacTeleopCfg: + r"""Build an :class:`~isaaclab_teleop.IsaacTeleopCfg` for gamepad-driven SE(2) velocity control. + + Raw stick state is retargeted via :class:`~isaacteleop.retargeters.GamepadToSe2Retargeter`. + + The gamepad is read by the Isaac Capture ``gamepad`` plugin from the first connected Linux + joystick-API device (``/dev/input/jsN``); the config starts the plugin with the session, so no + headset or separate process is needed, and udev grants the logged-in user access without the + ``input`` group. + + Key bindings: + ====================== ========================= ======================== + Command Key (+ve axis) Key (-ve axis) + ====================== ========================= ======================== + Move along x-axis left stick up left stick down + Move along y-axis left stick right left stick left + Rotate along z-axis right stick right right stick left + ====================== ========================= ======================== + + No physical button drives session start/stop/reset -- those come from the teleop session's + control channel or an auxiliary control device, matching the legacy ``Se2Gamepad``. + + Args: + v_x_sensitivity: Linear x-velocity scale per step. + v_y_sensitivity: Linear y-velocity scale per step. + omega_z_sensitivity: Angular z-velocity scale per step. + dead_zone: Stick deflection magnitude below which input is treated as zero. + sim_device: Torch device string for the pipeline's output tensors. + + Returns: + IsaacTeleopCfg driving this pipeline. + """ + from .._plugins import bundled_plugin_configs + from ..isaac_teleop_cfg import IsaacTeleopCfg + + def build_pipeline(): + from isaacteleop.retargeters import GamepadToSe2Retargeter, GamepadToSe2RetargeterConfig, TensorReorderer + from isaacteleop.retargeting_engine.deviceio_source_nodes import GamepadSource + from isaacteleop.retargeting_engine.interface import OutputCombiner + + gamepad_source = GamepadSource("gamepad") + + se2 = GamepadToSe2Retargeter( + GamepadToSe2RetargeterConfig( + v_x_sensitivity=v_x_sensitivity, + v_y_sensitivity=v_y_sensitivity, + omega_z_sensitivity=omega_z_sensitivity, + dead_zone=dead_zone, + ), + name="se2", + ) + connected_se2 = se2.connect({"gamepad": gamepad_source.output("gamepad")}) + + base_command_elements = ["v_x", "v_y", "omega_z"] + reorderer = TensorReorderer( + input_config={"base_command": base_command_elements}, + output_order=base_command_elements, + name="action_reorderer", + input_types={"base_command": "array"}, + ) + connected_reorderer = reorderer.connect({"base_command": connected_se2.output("base_command")}) + + return OutputCombiner({"action": connected_reorderer.output("output")}) + + return IsaacTeleopCfg( + pipeline_builder=build_pipeline, + sim_device=sim_device, + teleoperation_active_default=True, + app_name="IsaacLabGamepadSe2", + plugins=bundled_plugin_configs("gamepad"), + ) diff --git a/source/isaaclab_teleop/isaaclab_teleop/gamepad/se3_gamepad.py b/source/isaaclab_teleop/isaaclab_teleop/gamepad/se3_gamepad.py new file mode 100644 index 000000000000..020df585c509 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/gamepad/se3_gamepad.py @@ -0,0 +1,113 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Gamepad-driven IsaacTeleopCfg builder for SE(3) control.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..isaac_teleop_cfg import IsaacTeleopCfg + + +def se3_gamepad_teleop_cfg( + pos_sensitivity: float = 1.0, + rot_sensitivity: float = 1.6, + dead_zone: float = 0.15, + gripper_term: bool = True, + sim_device: str = "cpu", +) -> IsaacTeleopCfg: + """Build an :class:`~isaaclab_teleop.IsaacTeleopCfg` for gamepad-driven SE(3) delta-pose control. + + Raw stick/button state is retargeted via + :class:`~isaacteleop.retargeters.GamepadToSe3RelRetargeter` / + :class:`~isaacteleop.retargeters.GamepadGripperRetargeter`. + + The gamepad is read by the Isaac Capture ``gamepad`` plugin from the first connected Linux + joystick-API device (``/dev/input/jsN``); the config starts the plugin with the session, so no + headset or separate process is needed, and udev grants the logged-in user access without the + ``input`` group. + + Stick and Button bindings: + ============================ ========================= ========================= + Description Stick/Button (+ve axis) Stick/Button (-ve axis) + ============================ ========================= ========================= + Toggle gripper(open/close) West Button (X, Square) West Button (X, Square) + Move along x-axis Left Stick Up Left Stick Down + Move along y-axis Left Stick Right Left Stick Left + Move along z-axis Right Stick Up Right Stick Down + Rotate along x-axis D-Pad Left D-Pad Right + Rotate along y-axis D-Pad Down D-Pad Up + Rotate along z-axis Right Stick Right Right Stick Left + ============================ ========================= ========================= + + No physical button drives session start/stop/reset -- those come from the teleop session's + control channel or an auxiliary control device, matching the legacy ``Se3Gamepad``. + + Args: + pos_sensitivity: Position delta scale per step [m]. + rot_sensitivity: Rotation delta scale per step [rad]. + dead_zone: Stick deflection magnitude below which input is treated as zero. + gripper_term: Whether to include a gripper open/close command as a 7th action element. + sim_device: Torch device string for the pipeline's output tensors. + + Returns: + IsaacTeleopCfg driving this pipeline. + """ + from .._plugins import bundled_plugin_configs + from ..isaac_teleop_cfg import IsaacTeleopCfg + + def build_pipeline(): + from isaacteleop.retargeters import ( + GamepadGripperRetargeter, + GamepadToSe3RelRetargeter, + GamepadToSe3RelRetargeterConfig, + TensorReorderer, + ) + from isaacteleop.retargeting_engine.deviceio_source_nodes import GamepadSource + from isaacteleop.retargeting_engine.interface import OutputCombiner + + gamepad_source = GamepadSource("gamepad") + + se3 = GamepadToSe3RelRetargeter( + GamepadToSe3RelRetargeterConfig( + pos_sensitivity=pos_sensitivity, rot_sensitivity=rot_sensitivity, dead_zone=dead_zone + ), + name="se3", + ) + connected_se3 = se3.connect({"gamepad": gamepad_source.output("gamepad")}) + + ee_delta_elements = ["dx", "dy", "dz", "drx", "dry", "drz"] + input_config = {"ee_delta": ee_delta_elements} + input_types = {"ee_delta": "array"} + reorder_inputs = {"ee_delta": connected_se3.output("ee_delta")} + output_order = list(ee_delta_elements) + + if gripper_term: + gripper = GamepadGripperRetargeter(name="gripper") + connected_gripper = gripper.connect({"gamepad": gamepad_source.output("gamepad")}) + input_config["gripper_command"] = ["gripper_value"] + input_types["gripper_command"] = "scalar" + reorder_inputs["gripper_command"] = connected_gripper.output("gripper_command") + output_order.append("gripper_value") + + reorderer = TensorReorderer( + input_config=input_config, + output_order=output_order, + name="action_reorderer", + input_types=input_types, + ) + connected_reorderer = reorderer.connect(reorder_inputs) + + return OutputCombiner({"action": connected_reorderer.output("output")}) + + return IsaacTeleopCfg( + pipeline_builder=build_pipeline, + sim_device=sim_device, + teleoperation_active_default=True, + app_name="IsaacLabGamepadSe3", + plugins=bundled_plugin_configs("gamepad"), + ) diff --git a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py index 2170a29207db..aad98b878f27 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py +++ b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py @@ -75,7 +75,10 @@ class IsaacTeleopDevice: Teleop commands: The device supports callbacks for START, STOP, and RESET commands that can be triggered via the message-channel control pipeline or - registered directly via :meth:`add_callback`. + registered directly via :meth:`add_callback`. Without a headset, the + ``B`` / ``P`` / ``R`` keys typed into a visualizer window start, pause + and reset teleoperation, and a SpaceMouse pipeline's right button + resets it. Example: .. code-block:: python @@ -170,6 +173,7 @@ def __init__( ) self._prev_right_a_pressed = False + self._prev_spacemouse_right_pressed = False self._prev_control_is_active: bool | None = None # Each visualizer is created lazily on the first frame with matching @@ -361,6 +365,7 @@ def advance(self, target_T_world: np.ndarray | torch.Tensor | SupportsDLPack | N if self._enable_debug_visualization: self._update_debug_visualization() + self._handle_operator_input() self._dispatch_control_callbacks() return action @@ -461,6 +466,36 @@ def _dispatch_control_callbacks(self) -> None: self._command_handler.fire("START" if events.is_active else "STOP") self._prev_control_is_active = events.is_active + # ------------------------------------------------------------------ + # Operator keys and buttons + # ------------------------------------------------------------------ + + _START_KEY, _STOP_KEY, _RESET_KEY = 48, 25, 19 # evdev codes of B, P and R + _SPACEMOUSE_RIGHT_BUTTON = 1 + + def _handle_operator_input(self) -> None: + """Start, pause or reset on the operator's ``B`` / ``P`` / ``R`` keys and SpaceMouse right button. + + An operator reset also pauses, so teleop resumes on ``B``. The keys come from the visualizer + windows, each press once whatever the retargeting mode. + """ + for code in self._session_lifecycle.drain_pressed_keys(): + if code == self._START_KEY: + self.request_start() + elif code == self._STOP_KEY: + self.request_stop() + elif code == self._RESET_KEY: + self.reset(pause=True) + + result = self._session_lifecycle.last_step_result + buttons = None if result is None else result.get("spacemouse_buttons") + if buttons is None or buttons.is_none: + return + pressed = bool(buttons[0][self._SPACEMOUSE_RIGHT_BUTTON]) + if pressed and not self._prev_spacemouse_right_pressed: + self.reset(pause=True) + self._prev_spacemouse_right_pressed = pressed + # ------------------------------------------------------------------ # Controller button polling (glue between session and anchor manager) # ------------------------------------------------------------------ diff --git a/source/isaaclab_teleop/isaaclab_teleop/keyboard/__init__.py b/source/isaaclab_teleop/isaaclab_teleop/keyboard/__init__.py new file mode 100644 index 000000000000..d5a91b84fdd1 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/keyboard/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Keyboard device for SE(2) and SE(3) control.""" + +from isaaclab.utils.module import lazy_export + +lazy_export() diff --git a/source/isaaclab_teleop/isaaclab_teleop/keyboard/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/keyboard/__init__.pyi new file mode 100644 index 000000000000..87898b0a4ce5 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/keyboard/__init__.pyi @@ -0,0 +1,12 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "se2_keyboard_teleop_cfg", + "se3_keyboard_teleop_cfg", +] + +from .se2_keyboard import se2_keyboard_teleop_cfg +from .se3_keyboard import se3_keyboard_teleop_cfg diff --git a/source/isaaclab_teleop/isaaclab_teleop/keyboard/se2_keyboard.py b/source/isaaclab_teleop/isaaclab_teleop/keyboard/se2_keyboard.py new file mode 100644 index 000000000000..d226dcbd9ac0 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/keyboard/se2_keyboard.py @@ -0,0 +1,85 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Keyboard-driven IsaacTeleopCfg builder for SE(2) control.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..isaac_teleop_cfg import IsaacTeleopCfg + + +def se2_keyboard_teleop_cfg( + v_x_sensitivity: float = 0.8, + v_y_sensitivity: float = 0.4, + omega_z_sensitivity: float = 1.0, + sim_device: str = "cpu", +) -> IsaacTeleopCfg: + r"""Build an :class:`~isaaclab_teleop.IsaacTeleopCfg` for keyboard-driven SE(2) velocity control. + + Keys come from an in-process :class:`~isaacteleop.retargeting_engine.deviceio_source_nodes.KeyboardSource`, + fed by whichever visualizer window has focus (see :ref:`isaac-teleop-keyboard`), and are + retargeted via :class:`~isaacteleop.retargeters.KeyboardToSe2Retargeter`. The pipeline needs + no plugin process or headset. + + Key bindings: + ====================== ========================= ======================== + Command Key (+ve axis) Key (-ve axis) + ====================== ========================= ======================== + Move along x-axis Numpad 8 / Arrow Up Numpad 2 / Arrow Down + Move along y-axis Numpad 4 / Arrow Left Numpad 6 / Arrow Right + Rotate along z-axis Numpad 7 / Z Numpad 9 / X + ====================== ========================= ======================== + + ``B`` / ``P`` / ``R`` start, pause and reset teleoperation (see + :class:`~isaaclab_teleop.IsaacTeleopDevice`). + + Args: + v_x_sensitivity: Linear x-velocity scale per step. + v_y_sensitivity: Linear y-velocity scale per step. + omega_z_sensitivity: Angular z-velocity scale per step. + sim_device: Torch device string for the pipeline's output tensors. + + Returns: + IsaacTeleopCfg driving this pipeline. + """ + from ..isaac_teleop_cfg import IsaacTeleopCfg + + def build_pipeline(): + from isaacteleop.retargeters import KeyboardToSe2Retargeter, KeyboardToSe2RetargeterConfig, TensorReorderer + from isaacteleop.retargeting_engine.deviceio_source_nodes import KeyboardSource + from isaacteleop.retargeting_engine.interface import OutputCombiner + + keyboard_source = KeyboardSource("keyboard") + + se2 = KeyboardToSe2Retargeter( + KeyboardToSe2RetargeterConfig( + v_x_sensitivity=v_x_sensitivity, + v_y_sensitivity=v_y_sensitivity, + omega_z_sensitivity=omega_z_sensitivity, + ), + name="se2", + ) + connected_se2 = se2.connect({"keyboard_held": keyboard_source.output("keyboard_held")}) + + base_command_elements = ["v_x", "v_y", "omega_z"] + reorderer = TensorReorderer( + input_config={"base_command": base_command_elements}, + output_order=base_command_elements, + name="action_reorderer", + input_types={"base_command": "array"}, + ) + connected_reorderer = reorderer.connect({"base_command": connected_se2.output("base_command")}) + + return OutputCombiner({"action": connected_reorderer.output("output")}) + + return IsaacTeleopCfg( + pipeline_builder=build_pipeline, + sim_device=sim_device, + teleoperation_active_default=True, + app_name="IsaacLabKeyboardSe2", + ) diff --git a/source/isaaclab_teleop/isaaclab_teleop/keyboard/se3_keyboard.py b/source/isaaclab_teleop/isaaclab_teleop/keyboard/se3_keyboard.py new file mode 100644 index 000000000000..8d1fd6f1da2f --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/keyboard/se3_keyboard.py @@ -0,0 +1,104 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Keyboard-driven IsaacTeleopCfg builder for SE(3) control.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..isaac_teleop_cfg import IsaacTeleopCfg + + +def se3_keyboard_teleop_cfg( + pos_sensitivity: float = 0.4, + rot_sensitivity: float = 0.8, + gripper_term: bool = True, + sim_device: str = "cpu", +) -> IsaacTeleopCfg: + """Build an :class:`~isaaclab_teleop.IsaacTeleopCfg` for keyboard-driven SE(3) delta-pose control. + + Keys come from an in-process :class:`~isaacteleop.retargeting_engine.deviceio_source_nodes.KeyboardSource`, + fed by whichever visualizer window has focus (see :ref:`isaac-teleop-keyboard`), and are + retargeted via :class:`~isaacteleop.retargeters.KeyboardToSe3RelRetargeter` / + :class:`~isaacteleop.retargeters.KeyboardGripperRetargeter`. The pipeline needs no plugin + process or headset. + + Key bindings: + ============================== ================= ================= + Description Key (+ve axis) Key (-ve axis) + ============================== ================= ================= + Toggle gripper (open/close) K + Move along x-axis W S + Move along y-axis A D + Move along z-axis Q E + Rotate along x-axis Z X + Rotate along y-axis T G + Rotate along z-axis C V + ============================== ================= ================= + + ``B`` / ``P`` / ``R`` start, pause and reset teleoperation (see + :class:`~isaaclab_teleop.IsaacTeleopDevice`). + + Args: + pos_sensitivity: Position delta scale per step [m]. + rot_sensitivity: Rotation delta scale per step [rad]. + gripper_term: Whether to include a gripper open/close command as a 7th action element. + sim_device: Torch device string for the pipeline's output tensors. + + Returns: + IsaacTeleopCfg driving this pipeline. + """ + from ..isaac_teleop_cfg import IsaacTeleopCfg + + def build_pipeline(): + from isaacteleop.retargeters import ( + KeyboardGripperRetargeter, + KeyboardToSe3RelRetargeter, + KeyboardToSe3RelRetargeterConfig, + TensorReorderer, + ) + from isaacteleop.retargeting_engine.deviceio_source_nodes import KeyboardSource + from isaacteleop.retargeting_engine.interface import OutputCombiner + + keyboard_source = KeyboardSource("keyboard") + + se3 = KeyboardToSe3RelRetargeter( + KeyboardToSe3RelRetargeterConfig(pos_sensitivity=pos_sensitivity, rot_sensitivity=rot_sensitivity), + name="se3", + ) + connected_se3 = se3.connect({"keyboard_held": keyboard_source.output("keyboard_held")}) + + ee_delta_elements = ["dx", "dy", "dz", "drx", "dry", "drz"] + input_config = {"ee_delta": ee_delta_elements} + input_types = {"ee_delta": "array"} + reorder_inputs = {"ee_delta": connected_se3.output("ee_delta")} + output_order = list(ee_delta_elements) + + if gripper_term: + gripper = KeyboardGripperRetargeter(name="gripper") + connected_gripper = gripper.connect({"keyboard_pressed": keyboard_source.output("keyboard_pressed")}) + input_config["gripper_command"] = ["gripper_value"] + input_types["gripper_command"] = "scalar" + reorder_inputs["gripper_command"] = connected_gripper.output("gripper_command") + output_order.append("gripper_value") + + reorderer = TensorReorderer( + input_config=input_config, + output_order=output_order, + name="action_reorderer", + input_types=input_types, + ) + connected_reorderer = reorderer.connect(reorder_inputs) + + return OutputCombiner({"action": connected_reorderer.output("output")}) + + return IsaacTeleopCfg( + pipeline_builder=build_pipeline, + sim_device=sim_device, + teleoperation_active_default=True, + app_name="IsaacLabKeyboardSe3", + ) diff --git a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py index 75f34b9310ae..bf943de491a4 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py +++ b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py @@ -177,6 +177,33 @@ def _execution_events_to_control(ee: ExecutionEvents) -> ControlEvents: return ControlEvents(is_active=is_active, should_reset=ee.reset) +class _ControlKeyListener: + """Queues one visualizer window's key presses for :meth:`TeleopSessionLifecycle.drain_pressed_keys`. + + Keeps that window's held keys, so key repeat is not a press and a focus loss on one window + leaves the others alone. + """ + + def __init__(self, pressed: list[int]) -> None: + self._pressed = pressed + self._held: set[int] = set() + + def on_key(self, code: str, pressed: bool) -> None: + from isaacteleop.deviceio_trackers import evdev_code_from_w3c + + evdev_code = evdev_code_from_w3c(code) + if evdev_code is None: + return + if not pressed: + self._held.discard(evdev_code) + elif evdev_code not in self._held: + self._held.add(evdev_code) + self._pressed.append(evdev_code) + + def on_focus_lost(self) -> None: + self._held.clear() + + class TeleopSessionLifecycle: """Manages the Isaac Capture session lifecycle. @@ -191,6 +218,7 @@ class TeleopSessionLifecycle: 6. Building external inputs for pipeline leaf nodes (e.g. world-to-anchor transform) 7. Stepping the session and extracting the flattened action tensor 8. Managing the optional retargeting tuning UI + 9. Feeding the pipeline's keyboards from the focused visualizer windows """ WORLD_T_ANCHOR_INPUT_NAME = "world_T_anchor" @@ -330,6 +358,13 @@ def __init__( # Fallback for host-initiated resets when no control pipeline is configured self._pending_reset = False + # Keyboards: the visualizer windows feeding the pipeline's keyboards, and the operator's key + # presses, read straight from those windows so each press is consumed once whatever the + # retargeting mode. + self._keyboard_detachers: list[Callable[[], None]] = [] + self._attached_visualizers: list[Any] = [] + self._pressed_keys: list[int] = [] + # CloudXR runtime launcher (created in start if configured, stopped in stop) self._cloudxr_launcher: CloudXRLauncher | None = None @@ -559,6 +594,8 @@ def start(self) -> None: self._last_step_result = None self._pipeline = self._build_combined_pipeline(user_pipeline) + self._pressed_keys.clear() + self._attach_keyboards() # Build the optional haptic sink. It reuses the button-controller # tracker so no additional ControllersSource (and thus no duplicate @@ -607,6 +644,9 @@ def stop(self, exc_type=None, exc_val=None, exc_tb=None) -> None: exc_val: Exception value. exc_tb: Exception traceback. """ + self._detach_keyboards() + self._pressed_keys.clear() + # Close the retargeting tuning UI first if self._retargeting_ui_ctx is not None: self._retargeting_ui_ctx.__exit__(exc_type, exc_val, exc_tb) @@ -671,18 +711,83 @@ def _build_combined_pipeline(self, user_pipeline): from isaacteleop.retargeting_engine.deviceio_source_nodes import ControllersSource from isaacteleop.retargeting_engine.interface import OutputCombiner + # Forward every output the user pipeline declares, not just "action", so the + # device can read the others (e.g. the SpaceMouse buttons) from last_step_result. + pipeline_outputs: dict[str, Any] = {name: user_pipeline.output(name) for name in user_pipeline.output_types()} + # Stored on self so start() can reuse this ControllersSource's tracker for # the haptic sink, avoiding a second controller action-set attachment. self._button_controllers = ControllersSource("_button_controllers") - pipeline_outputs: dict[str, Any] = { - "action": user_pipeline.output("action"), - self._CONTROLLER_RIGHT_KEY: self._button_controllers.output(ControllersSource.RIGHT), - } + pipeline_outputs[self._CONTROLLER_RIGHT_KEY] = self._button_controllers.output(ControllersSource.RIGHT) if self._enable_debug_visualization: pipeline_outputs[self._CONTROLLER_LEFT_KEY] = self._button_controllers.output(ControllersSource.LEFT) self._chain_hand_debug_outputs(user_pipeline, pipeline_outputs) return OutputCombiner(pipeline_outputs) + # ------------------------------------------------------------------ + # Keyboards + # ------------------------------------------------------------------ + + def _attach_keyboards(self) -> None: + """Feed the pipeline's keyboards and the operator's key presses from the visualizer windows. + + Called at start and, until a window is attached, on each step, since visualizer windows can + open after the session starts. While the pipeline has a keyboard, each visualizer's keyboard + camera movement is turned off. + """ + if self._pipeline is None or self._is_replay: + return + try: + from isaaclab.sim import SimulationContext + + sim = SimulationContext.instance() + except Exception: + return + if sim is None: + return + from isaacteleop.retargeting_engine.deviceio_source_nodes import KeyboardSource, find_sources + + keyboards = find_sources(self._pipeline, KeyboardSource) + for visualizer in sim.visualizers: + # a visualizer without a window that reports keys has no source + source = visualizer.key_event_source + if source is None or any(visualizer is attached for attached in self._attached_visualizers): + continue + # attached even if this fails, so a failing window is not retried every step + self._attached_visualizers.append(visualizer) + try: + for keyboard in keyboards: + self._keyboard_detachers.append(keyboard.attach(source, capture=False).close) + if keyboards: + visualizer.set_camera_keys_enabled(False) + self._keyboard_detachers.append(lambda v=visualizer: v.set_camera_keys_enabled(True)) + listener = _ControlKeyListener(self._pressed_keys) + self._keyboard_detachers.append(source.add_key_listener(listener.on_key, listener.on_focus_lost).close) + except Exception as e: + logger.warning(f"Keyboard input from {type(visualizer).__name__} is unavailable: {e}") + continue + logger.info(f"Isaac Capture keyboard input from {type(visualizer).__name__}") + + def _detach_keyboards(self) -> None: + """Detach every visualizer window, releasing held keys and restoring keyboard camera movement.""" + detachers, self._keyboard_detachers = self._keyboard_detachers, [] + for detach in detachers: + try: + detach() + except Exception as e: + logger.debug(f"Suppressed error detaching a visualizer keyboard: {e}") + self._attached_visualizers = [] + + def drain_pressed_keys(self) -> list[int]: + """Evdev codes of the keys pressed in the visualizer windows since the last call, in order. + + Each press is returned once, whatever the retargeting execution mode; key repeat is not a + press. Presses from before the session started or stopped are dropped. + """ + pressed = list(self._pressed_keys) + self._pressed_keys.clear() + return pressed + @staticmethod def _chain_hand_debug_outputs(user_pipeline, pipeline_outputs: dict) -> None: """Auto-discover a ``HandsSource`` and chain its raw outputs. @@ -1196,6 +1301,10 @@ def step( if not self._try_start_session(): return None + # Visualizer windows can open after the session starts. + if not self._attached_visualizers: + self._attach_keyboards() + # Build external inputs (e.g. world-to-anchor transform) if the # pipeline contains ValueInput leaf nodes. external_inputs = self._build_external_inputs(anchor_world_matrix_fn, target_T_world) diff --git a/source/isaaclab_teleop/isaaclab_teleop/spacemouse/__init__.py b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/__init__.py new file mode 100644 index 000000000000..9e7710143178 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""SpaceMouse device for SE(2) and SE(3) control.""" + +from isaaclab.utils.module import lazy_export + +lazy_export() diff --git a/source/isaaclab_teleop/isaaclab_teleop/spacemouse/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/__init__.pyi new file mode 100644 index 000000000000..68cd178c261f --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/__init__.pyi @@ -0,0 +1,12 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "se2_spacemouse_teleop_cfg", + "se3_spacemouse_teleop_cfg", +] + +from .se2_spacemouse import se2_spacemouse_teleop_cfg +from .se3_spacemouse import se3_spacemouse_teleop_cfg diff --git a/source/isaaclab_teleop/isaaclab_teleop/spacemouse/se2_spacemouse.py b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/se2_spacemouse.py new file mode 100644 index 000000000000..48e0b8961c24 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/se2_spacemouse.py @@ -0,0 +1,99 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Spacemouse-driven IsaacTeleopCfg builder for SE(2) control.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..isaac_teleop_cfg import IsaacTeleopCfg + + +def se2_spacemouse_teleop_cfg( + v_x_sensitivity: float = 0.8, + v_y_sensitivity: float = 0.4, + omega_z_sensitivity: float = 1.0, + sim_device: str = "cpu", +) -> IsaacTeleopCfg: + r"""Build an :class:`~isaaclab_teleop.IsaacTeleopCfg` for spacemouse-driven SE(2) velocity control. + + Raw axis/button state is retargeted via + :class:`~isaacteleop.retargeters.SpaceMouseToSe2Retargeter`. + + The SpaceMouse is read by the Isaac Capture ``spacemouse`` plugin from its HID device + (``/dev/hidrawN``); the config starts the plugin with the session, so no headset or separate + process is needed. Reading ``/dev/hidraw*`` needs a udev rule granting the user access (see + :ref:`SpaceMouse permissions `). + + The interface finds and uses the first supported device connected to the computer. + Currently tested for the SpaceMouse Compact + (https://3dconnexion.com/de/product/spacemouse-compact/). + + The pipeline also outputs ``"spacemouse_buttons"``, from which + :class:`~isaaclab_teleop.IsaacTeleopDevice` resets teleoperation on the right button. + + Args: + v_x_sensitivity: Linear x-velocity scale per step. + v_y_sensitivity: Linear y-velocity scale per step. + omega_z_sensitivity: Angular z-velocity scale per step. + sim_device: Torch device string for the pipeline's output tensors. + + Returns: + IsaacTeleopCfg driving this pipeline. + """ + from .._plugins import bundled_plugin_configs + from ..isaac_teleop_cfg import IsaacTeleopCfg + + def build_pipeline(): + from isaacteleop.retargeters import ( + SpaceMouseToSe2Retargeter, + SpaceMouseToSe2RetargeterConfig, + TensorReorderer, + ) + from isaacteleop.retargeting_engine.deviceio_source_nodes import SpaceMouseSource + from isaacteleop.retargeting_engine.interface import OutputCombiner + + spacemouse_source = SpaceMouseSource("spacemouse") + + se2 = SpaceMouseToSe2Retargeter( + SpaceMouseToSe2RetargeterConfig( + v_x_sensitivity=v_x_sensitivity, + v_y_sensitivity=v_y_sensitivity, + omega_z_sensitivity=omega_z_sensitivity, + ), + name="se2", + ) + connected_se2 = se2.connect( + { + "spacemouse_translation": spacemouse_source.output("spacemouse_translation"), + "spacemouse_rotation": spacemouse_source.output("spacemouse_rotation"), + } + ) + + base_command_elements = ["v_x", "v_y", "omega_z"] + reorderer = TensorReorderer( + input_config={"base_command": base_command_elements}, + output_order=base_command_elements, + name="action_reorderer", + input_types={"base_command": "array"}, + ) + connected_reorderer = reorderer.connect({"base_command": connected_se2.output("base_command")}) + + return OutputCombiner( + { + "action": connected_reorderer.output("output"), + "spacemouse_buttons": spacemouse_source.output("spacemouse_buttons"), + } + ) + + return IsaacTeleopCfg( + pipeline_builder=build_pipeline, + sim_device=sim_device, + teleoperation_active_default=True, + app_name="IsaacLabSpaceMouseSe2", + plugins=bundled_plugin_configs("spacemouse"), + ) diff --git a/source/isaaclab_teleop/isaaclab_teleop/spacemouse/se3_spacemouse.py b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/se3_spacemouse.py new file mode 100644 index 000000000000..56750c3fc524 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/se3_spacemouse.py @@ -0,0 +1,111 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Spacemouse-driven IsaacTeleopCfg builder for SE(3) control.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..isaac_teleop_cfg import IsaacTeleopCfg + + +def se3_spacemouse_teleop_cfg( + pos_sensitivity: float = 0.4, + rot_sensitivity: float = 0.8, + gripper_term: bool = True, + sim_device: str = "cpu", +) -> IsaacTeleopCfg: + """Build an :class:`~isaaclab_teleop.IsaacTeleopCfg` for spacemouse-driven SE(3) delta-pose control. + + Raw axis/button state is retargeted via + :class:`~isaacteleop.retargeters.SpaceMouseToSe3RelRetargeter` / + :class:`~isaacteleop.retargeters.SpaceMouseGripperRetargeter`. + + The SpaceMouse is read by the Isaac Capture ``spacemouse`` plugin from its HID device + (``/dev/hidrawN``); the config starts the plugin with the session, so no headset or separate + process is needed. Reading ``/dev/hidraw*`` needs a udev rule granting the user access (see + :ref:`SpaceMouse permissions `). + + The interface finds and uses the first supported device connected to the computer. + Currently tested for the SpaceMouse Compact + (https://3dconnexion.com/de/product/spacemouse-compact/). + + The left button's gripper toggle is handled inside the pipeline by + ``SpaceMouseGripperRetargeter``. The pipeline also outputs ``"spacemouse_buttons"``, from which + :class:`~isaaclab_teleop.IsaacTeleopDevice` resets teleoperation on the right button. + + Args: + pos_sensitivity: Position delta scale per step [m]. + rot_sensitivity: Rotation delta scale per step [rad]. + gripper_term: Whether to include a gripper open/close command as a 7th action element. + sim_device: Torch device string for the pipeline's output tensors. + + Returns: + IsaacTeleopCfg driving this pipeline. + """ + from .._plugins import bundled_plugin_configs + from ..isaac_teleop_cfg import IsaacTeleopCfg + + def build_pipeline(): + from isaacteleop.retargeters import ( + SpaceMouseGripperRetargeter, + SpaceMouseToSe3RelRetargeter, + SpaceMouseToSe3RelRetargeterConfig, + TensorReorderer, + ) + from isaacteleop.retargeting_engine.deviceio_source_nodes import SpaceMouseSource + from isaacteleop.retargeting_engine.interface import OutputCombiner + + spacemouse_source = SpaceMouseSource("spacemouse") + + se3 = SpaceMouseToSe3RelRetargeter( + SpaceMouseToSe3RelRetargeterConfig(pos_sensitivity=pos_sensitivity, rot_sensitivity=rot_sensitivity), + name="se3", + ) + connected_se3 = se3.connect( + { + "spacemouse_translation": spacemouse_source.output("spacemouse_translation"), + "spacemouse_rotation": spacemouse_source.output("spacemouse_rotation"), + } + ) + + ee_delta_elements = ["dx", "dy", "dz", "drx", "dry", "drz"] + input_config = {"ee_delta": ee_delta_elements} + input_types = {"ee_delta": "array"} + reorder_inputs = {"ee_delta": connected_se3.output("ee_delta")} + output_order = list(ee_delta_elements) + + if gripper_term: + gripper = SpaceMouseGripperRetargeter(name="gripper") + connected_gripper = gripper.connect({"spacemouse_buttons": spacemouse_source.output("spacemouse_buttons")}) + input_config["gripper_command"] = ["gripper_value"] + input_types["gripper_command"] = "scalar" + reorder_inputs["gripper_command"] = connected_gripper.output("gripper_command") + output_order.append("gripper_value") + + reorderer = TensorReorderer( + input_config=input_config, + output_order=output_order, + name="action_reorderer", + input_types=input_types, + ) + connected_reorderer = reorderer.connect(reorder_inputs) + + return OutputCombiner( + { + "action": connected_reorderer.output("output"), + "spacemouse_buttons": spacemouse_source.output("spacemouse_buttons"), + } + ) + + return IsaacTeleopCfg( + pipeline_builder=build_pipeline, + sim_device=sim_device, + teleoperation_active_default=True, + app_name="IsaacLabSpaceMouseSe3", + plugins=bundled_plugin_configs("spacemouse"), + ) diff --git a/source/isaaclab_teleop/test/test_device_selection.py b/source/isaaclab_teleop/test/test_device_selection.py new file mode 100644 index 000000000000..8cf1dacffb57 --- /dev/null +++ b/source/isaaclab_teleop/test/test_device_selection.py @@ -0,0 +1,154 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for :mod:`isaaclab_teleop.device_selection`: device configs -> Isaac Capture pipelines.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import isaaclab_teleop.device_selection as device_selection +import pytest +from isaaclab_teleop import IsaacTeleopCfg +from isaaclab_teleop.device_selection import pipeline_from_device_cfg, select_isaac_teleop_cfg + +from isaaclab.devices import ( + DeviceCfg, + DevicesCfg, + Se2GamepadCfg, + Se2KeyboardCfg, + Se2SpaceMouseCfg, + Se3GamepadCfg, + Se3KeyboardCfg, + Se3SpaceMouseCfg, +) + +pytestmark = pytest.mark.unit + + +BUILDERS = { + Se3KeyboardCfg: "se3_keyboard_teleop_cfg", + Se2KeyboardCfg: "se2_keyboard_teleop_cfg", + Se3GamepadCfg: "se3_gamepad_teleop_cfg", + Se2GamepadCfg: "se2_gamepad_teleop_cfg", + Se3SpaceMouseCfg: "se3_spacemouse_teleop_cfg", + Se2SpaceMouseCfg: "se2_spacemouse_teleop_cfg", +} +"""Each device config type and the builder that must serve it.""" + + +@pytest.fixture +def recorded(monkeypatch): + """Record which builder ``device_selection`` calls, and with what, without changing its routing.""" + calls = [] + for cfg_type, name in BUILDERS.items(): + + def builder(name=name, **kwargs): + calls.append((name, kwargs)) + return f"pipeline:{name}" + + fields = device_selection._PIPELINES[cfg_type][1] + monkeypatch.setitem(device_selection._PIPELINES, cfg_type, (builder, fields)) + monkeypatch.setattr(device_selection, "_isaacteleop_available", lambda: True) + return calls + + +def _env_cfg(isaac_teleop=None, **devices): + return SimpleNamespace(isaac_teleop=isaac_teleop, teleop_devices=DevicesCfg(devices=devices)) + + +DEFAULTS = {"keyboard": Se3KeyboardCfg(pos_sensitivity=0.2), "spacemouse": Se3SpaceMouseCfg(pos_sensitivity=0.3)} + + +class TestPipelineFromDeviceCfg: + @pytest.mark.parametrize( + ("cfg_type", "source_name"), + [ + (Se2KeyboardCfg, "KeyboardSource"), + (Se3KeyboardCfg, "KeyboardSource"), + (Se2GamepadCfg, "GamepadSource"), + (Se3GamepadCfg, "GamepadSource"), + (Se2SpaceMouseCfg, "SpaceMouseSource"), + (Se3SpaceMouseCfg, "SpaceMouseSource"), + ], + ) + def test_every_legacy_device_reads_its_own_device(self, cfg_type, source_name): + nodes = pytest.importorskip("isaacteleop.retargeting_engine.deviceio_source_nodes") + + cfg = pipeline_from_device_cfg(cfg_type(), "cpu") + + assert isinstance(cfg, IsaacTeleopCfg) + assert len(nodes.find_sources(cfg.pipeline_builder(), getattr(nodes, source_name))) == 1 + + @pytest.mark.parametrize( + ("device_cfg", "expected"), + [ + ( + Se3KeyboardCfg(pos_sensitivity=0.7, rot_sensitivity=0.6, gripper_term=False), + {"pos_sensitivity": 0.7, "rot_sensitivity": 0.6, "gripper_term": False}, + ), + ( + Se2KeyboardCfg(v_x_sensitivity=0.1, v_y_sensitivity=0.2, omega_z_sensitivity=0.3), + {"v_x_sensitivity": 0.1, "v_y_sensitivity": 0.2, "omega_z_sensitivity": 0.3}, + ), + ( + Se3GamepadCfg(pos_sensitivity=0.7, rot_sensitivity=0.6, dead_zone=0.05, gripper_term=False), + {"pos_sensitivity": 0.7, "rot_sensitivity": 0.6, "dead_zone": 0.05, "gripper_term": False}, + ), + ( + Se2GamepadCfg(v_x_sensitivity=0.1, v_y_sensitivity=0.2, omega_z_sensitivity=0.3, dead_zone=0.05), + {"v_x_sensitivity": 0.1, "v_y_sensitivity": 0.2, "omega_z_sensitivity": 0.3, "dead_zone": 0.05}, + ), + ( + Se3SpaceMouseCfg(pos_sensitivity=0.7, rot_sensitivity=0.6, gripper_term=False), + {"pos_sensitivity": 0.7, "rot_sensitivity": 0.6, "gripper_term": False}, + ), + ( + Se2SpaceMouseCfg(v_x_sensitivity=0.1, v_y_sensitivity=0.2, omega_z_sensitivity=0.3), + {"v_x_sensitivity": 0.1, "v_y_sensitivity": 0.2, "omega_z_sensitivity": 0.3}, + ), + ], + ids=lambda value: type(value).__name__ if not isinstance(value, dict) else "", + ) + def test_config_reaches_its_builder_with_every_setting(self, recorded, device_cfg, expected): + pipeline_from_device_cfg(device_cfg, "cuda:0") + + assert recorded == [(BUILDERS[type(device_cfg)], {**expected, "sim_device": "cuda:0"})] + + def test_unsupported_config_has_no_pipeline(self): + assert pipeline_from_device_cfg(DeviceCfg(), "cpu") is None + + +class TestSelectIsaacTeleopCfg: + def test_without_isaacteleop_selects_nothing(self, recorded, monkeypatch): + monkeypatch.setattr(device_selection, "_isaacteleop_available", lambda: False) + + assert select_isaac_teleop_cfg(_env_cfg(isaac_teleop="declared"), None, DEFAULTS, "cpu") is None + + def test_declared_pipeline_wins_without_a_device_name(self, recorded): + assert select_isaac_teleop_cfg(_env_cfg(isaac_teleop="declared"), None, DEFAULTS, "cpu") == "declared" + assert recorded == [] + + def test_default_keyboard_without_a_declared_pipeline(self, recorded): + assert select_isaac_teleop_cfg(_env_cfg(), None, DEFAULTS, "cpu") == "pipeline:se3_keyboard_teleop_cfg" + assert recorded[0][1]["pos_sensitivity"] == 0.2 + + def test_environment_device_wins_over_the_default(self, recorded): + env_cfg = _env_cfg(isaac_teleop="declared", spacemouse=Se3SpaceMouseCfg(pos_sensitivity=0.9)) + + assert select_isaac_teleop_cfg(env_cfg, "spacemouse", DEFAULTS, "cpu") == "pipeline:se3_spacemouse_teleop_cfg" + assert recorded[0][1]["pos_sensitivity"] == 0.9 + + def test_default_serves_a_name_the_environment_lacks(self, recorded): + assert ( + select_isaac_teleop_cfg(_env_cfg(), "spacemouse", DEFAULTS, "cpu") == "pipeline:se3_spacemouse_teleop_cfg" + ) + assert recorded[0][1]["pos_sensitivity"] == 0.3 + + def test_unknown_or_unsupported_device_selects_nothing(self, recorded): + env_cfg = _env_cfg(handtracking=DeviceCfg()) + + assert select_isaac_teleop_cfg(env_cfg, "handtracking", DEFAULTS, "cpu") is None + assert select_isaac_teleop_cfg(env_cfg, "joystick", DEFAULTS, "cpu") is None diff --git a/source/isaaclab_teleop/test/test_gamepad_constructors.py b/source/isaaclab_teleop/test/test_gamepad_constructors.py new file mode 100644 index 000000000000..b5b901fdbb3f --- /dev/null +++ b/source/isaaclab_teleop/test/test_gamepad_constructors.py @@ -0,0 +1,65 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the gamepad :class:`~isaaclab_teleop.IsaacTeleopCfg` builders.""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("isaacteleop") + +from isaaclab_teleop.gamepad.se2_gamepad import se2_gamepad_teleop_cfg +from isaaclab_teleop.gamepad.se3_gamepad import se3_gamepad_teleop_cfg + + +def _has_gamepad_source(cfg) -> bool: + """Whether the config's pipeline reads the gamepad.""" + from isaacteleop.retargeting_engine.deviceio_source_nodes import GamepadSource, find_sources + + return len(find_sources(cfg.pipeline_builder(), GamepadSource)) == 1 + + +def _starts_the_gamepad_plugin(cfg) -> bool: + """Whether the config starts the Isaac Capture gamepad plugin that feeds its source.""" + from isaacteleop.teleop_session_manager import BUNDLED_PLUGINS_DIR + + return [(p.plugin_name, p.plugin_root_id, p.search_paths) for p in cfg.plugins] == [ + ("gamepad", "gamepad", [BUNDLED_PLUGINS_DIR]) + ] + + +class TestSe3GamepadTeleopCfg: + def test_defaults(self): + cfg = se3_gamepad_teleop_cfg() + + assert cfg.sim_device == "cpu" + assert cfg.teleoperation_active_default is True + assert cfg.app_name == "IsaacLabGamepadSe3" + assert _starts_the_gamepad_plugin(cfg) + assert _has_gamepad_source(cfg) + assert callable(cfg.pipeline_builder) + + def test_custom_sim_device(self): + cfg = se3_gamepad_teleop_cfg(sim_device="cuda:0") + + assert cfg.sim_device == "cuda:0" + + +class TestSe2GamepadTeleopCfg: + def test_defaults(self): + cfg = se2_gamepad_teleop_cfg() + + assert cfg.sim_device == "cpu" + assert cfg.teleoperation_active_default is True + assert cfg.app_name == "IsaacLabGamepadSe2" + assert _starts_the_gamepad_plugin(cfg) + assert _has_gamepad_source(cfg) + assert callable(cfg.pipeline_builder) + + def test_custom_sim_device(self): + cfg = se2_gamepad_teleop_cfg(sim_device="cuda:0") + + assert cfg.sim_device == "cuda:0" diff --git a/source/isaaclab_teleop/test/test_hand_debug_viz.py b/source/isaaclab_teleop/test/test_hand_debug_viz.py index 15e22b494ff9..c67f6d8da4cf 100644 --- a/source/isaaclab_teleop/test/test_hand_debug_viz.py +++ b/source/isaaclab_teleop/test/test_hand_debug_viz.py @@ -201,6 +201,9 @@ def get_leaf_nodes(self) -> list: def output(self, key: str) -> str: return f"selector:user_pipeline:{key}" + def output_types(self) -> dict: + return {"action": None} + class _FakeTensorGroup: """Stand-in for an ``OptionalTensorGroup`` carrying joint positions.""" diff --git a/source/isaaclab_teleop/test/test_keyboard_constructors.py b/source/isaaclab_teleop/test/test_keyboard_constructors.py new file mode 100644 index 000000000000..50dbf6fbeec4 --- /dev/null +++ b/source/isaaclab_teleop/test/test_keyboard_constructors.py @@ -0,0 +1,72 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the keyboard ``IsaacTeleopCfg`` builders.""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("isaacteleop") + +from isaaclab_teleop.keyboard.se2_keyboard import se2_keyboard_teleop_cfg +from isaaclab_teleop.keyboard.se3_keyboard import se3_keyboard_teleop_cfg + +# --------------------------------------------------------------------------- +# se2_keyboard_teleop_cfg / se3_keyboard_teleop_cfg +# --------------------------------------------------------------------------- + + +def _keyboard_sources(pipeline): + from isaacteleop.retargeting_engine.deviceio_source_nodes import KeyboardSource, find_sources + + return find_sources(pipeline, KeyboardSource) + + +class TestSe3KeyboardTeleopCfg: + def test_defaults(self): + cfg = se3_keyboard_teleop_cfg() + + assert cfg.sim_device == "cpu" + assert cfg.teleoperation_active_default is True + assert cfg.app_name == "IsaacLabKeyboardSe3" + # the keyboard runs in process: no plugin + assert cfg.plugins == [] + assert callable(cfg.pipeline_builder) + + def test_custom_sim_device(self): + cfg = se3_keyboard_teleop_cfg(sim_device="cuda:0") + + assert cfg.sim_device == "cuda:0" + + def test_pipeline_reads_one_keyboard(self): + assert len(_keyboard_sources(se3_keyboard_teleop_cfg().pipeline_builder())) == 1 + + @pytest.mark.parametrize(("gripper_term", "action_size"), [(True, 7), (False, 6)]) + def test_action_is_the_pose_delta_plus_an_optional_gripper(self, gripper_term, action_size): + """The action is ``[dx, dy, dz, drx, dry, drz]``, plus the gripper command when ``gripper_term``.""" + pipeline = se3_keyboard_teleop_cfg(gripper_term=gripper_term).pipeline_builder() + + (action,) = pipeline.output_types()["action"].types + assert action.shape == (action_size,) + + +class TestSe2KeyboardTeleopCfg: + def test_defaults(self): + cfg = se2_keyboard_teleop_cfg() + + assert cfg.sim_device == "cpu" + assert cfg.teleoperation_active_default is True + assert cfg.app_name == "IsaacLabKeyboardSe2" + assert cfg.plugins == [] + assert callable(cfg.pipeline_builder) + + def test_custom_sim_device(self): + cfg = se2_keyboard_teleop_cfg(sim_device="cuda:0") + + assert cfg.sim_device == "cuda:0" + + def test_pipeline_reads_one_keyboard(self): + assert len(_keyboard_sources(se2_keyboard_teleop_cfg().pipeline_builder())) == 1 diff --git a/source/isaaclab_teleop/test/test_keyboard_session.py b/source/isaaclab_teleop/test/test_keyboard_session.py new file mode 100644 index 000000000000..781921324ff7 --- /dev/null +++ b/source/isaaclab_teleop/test/test_keyboard_session.py @@ -0,0 +1,253 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# pyright: reportPrivateUsage=none + +"""Tests for keyboard teleop sessions fed by scripted key events in place of a visualizer window. + +The ``se3_keyboard_teleop_cfg`` pipeline runs in a real Isaac Capture session. The in-process +keyboard never touches the OpenXR session handles, so a placeholder OpenXR session stands in for a +CloudXR runtime. The session lifecycle is checked without stepping, since its combined pipeline +also polls the controllers through OpenXR. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +pytest.importorskip("isaacteleop") + +from isaaclab_teleop.keyboard import se3_keyboard_teleop_cfg +from isaaclab_teleop.session_lifecycle import TeleopSessionLifecycle +from isaacteleop.retargeting_engine.deviceio_source_nodes import KeyboardSource, find_sources + +_SETTLE_STEPS = 3 + + +def _keyboard_cfg(**kwargs): + """The SE(3) keyboard config, retargeting synchronously so each step reflects the current keys.""" + from isaacteleop.teleop_session_manager import RetargetingExecutionConfig + + cfg = se3_keyboard_teleop_cfg(**kwargs) + cfg.retargeting_execution = RetargetingExecutionConfig(mode="sync") + return cfg + + +class _PlaceholderOpenXRSession: + """Non-null handles that only OpenXR-backed tracker impls would dereference.""" + + def __init__(self, *args, **kwargs): + pass + + def get_handles(self): + from isaacteleop.oxr import OpenXRSessionHandles + + return OpenXRSessionHandles(1, 1, 1, 1) + + def get_provider_snapshot(self): + return SimpleNamespace(state="AVAILABLE", headset_state="CONNECTED", reason="NONE", result_code=None, error="") + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +@pytest.fixture +def keyboard_session(visualizer, monkeypatch): + """The keyboard config's pipeline in a real Isaac Capture session, fed by *visualizer*.""" + from isaacteleop import oxr + from isaacteleop.teleop_session_manager import TeleopSession, TeleopSessionConfig + + monkeypatch.setattr(oxr, "OpenXRSession", _PlaceholderOpenXRSession) + cfg = _keyboard_cfg(pos_sensitivity=0.5, rot_sensitivity=0.5) + pipeline = cfg.pipeline_builder() + source = visualizer.key_event_source + attachments = [keyboard.attach(source, capture=False) for keyboard in find_sources(pipeline, KeyboardSource)] + session_config = TeleopSessionConfig( + app_name="KeyboardSessionTest", pipeline=pipeline, retargeting_execution=cfg.retargeting_execution + ) + with TeleopSession(session_config) as session: + yield session + for attachment in attachments: + attachment.close() + + +def _settle(session): + result = None + for _ in range(_SETTLE_STEPS): + result = session.step() + return result["action"][0] + + +def test_held_key_drives_the_action(keyboard_session, visualizer): + visualizer.press("KeyW") + action = _settle(keyboard_session) + assert action[0] > 0.0 + + visualizer.release("KeyW") + action = _settle(keyboard_session) + assert action[0] == 0.0 + + +def test_focus_loss_releases_held_keys(keyboard_session, visualizer): + visualizer.press("KeyW") + _settle(keyboard_session) + + visualizer.blur() + action = _settle(keyboard_session) + + assert action[0] == 0.0 + + +class _Visualizer: + """A running visualizer: an Isaac Lab keyboard source with key helpers, and its camera-keys toggle.""" + + def __init__(self) -> None: + from isaaclab.visualizers import KeyEventSource + + self.key_event_source = KeyEventSource() + self.camera_keys: list[bool] = [] + + def set_camera_keys_enabled(self, enabled: bool) -> None: + self.camera_keys.append(enabled) + + @property + def listener_count(self) -> int: + return len(self.key_event_source._subscriptions) + + def press(self, code: str) -> None: + self.key_event_source._emit_key(code, True) + + def release(self, code: str) -> None: + self.key_event_source._emit_key(code, False) + + def tap(self, code: str) -> None: + self.press(code) + self.release(code) + + def blur(self) -> None: + self.key_event_source._emit_focus_lost() + + +def _start(lifecycle, *visualizers): + """Start a lifecycle reading *visualizers*, without launching CloudXR, a system check or OpenXR.""" + import isaacteleop.teleop_session_manager as tsm + + sim = SimpleNamespace(visualizers=[*visualizers, SimpleNamespace(key_event_source=None)]) + with ( + patch("isaaclab.sim.SimulationContext.instance", return_value=sim), + patch.object(lifecycle, "_ensure_cloudxr_runtime") as ensure_cloudxr, + patch.object(lifecycle, "_run_system_check"), + patch.object(tsm, "TeleopSession", MagicMock()), + ): + lifecycle.start() + return ensure_cloudxr + + +def _started_lifecycle(cfg, *visualizers, **kwargs): + lifecycle = TeleopSessionLifecycle(cfg, use_kit_xr_bridge=False, **kwargs) + return lifecycle, _start(lifecycle, *visualizers) + + +@pytest.fixture +def visualizer(): + return _Visualizer() + + +def test_keyboard_session_takes_the_standard_path(visualizer): + """A keyboard session launches the CloudXR profile and uses the control channel, like any other.""" + lifecycle, ensure_cloudxr = _started_lifecycle( + _keyboard_cfg(), visualizer, cloudxr_env_file="/nonexistent/cloudxr.env" + ) + try: + ensure_cloudxr.assert_called_once() + assert lifecycle._message_processor is not None + outputs = lifecycle._pipeline.output_types() + assert TeleopSessionLifecycle._CONTROLLER_RIGHT_KEY in outputs + finally: + lifecycle.stop() + + +def test_visualizer_feeds_the_keyboard_and_the_control_keys(visualizer): + """The pipeline's keyboard and the control keys read the window; its camera keys are off until stop.""" + lifecycle, _ = _started_lifecycle(_keyboard_cfg(), visualizer) + assert visualizer.listener_count == 2 + assert visualizer.camera_keys == [False] + + lifecycle.stop() + assert visualizer.listener_count == 0 + assert visualizer.camera_keys == [False, True] + + +KEY_R, KEY_B = 19, 48 + + +def test_each_control_key_press_is_drained_once(visualizer): + """Presses reach the app once each, with no session step in between: repeats of a held key are dropped.""" + lifecycle, _ = _started_lifecycle(_keyboard_cfg(), visualizer) + try: + visualizer.press("KeyR") + visualizer.press("KeyR") # still held: a repeat, not a press + visualizer.release("KeyR") + visualizer.tap("KeyB") + assert lifecycle.drain_pressed_keys() == [KEY_R, KEY_B] + assert lifecycle.drain_pressed_keys() == [] + finally: + lifecycle.stop() + + +def test_control_keys_work_without_a_pipeline_keyboard(visualizer): + """A pipeline with no keyboard still gets B/P/R, and the window keeps its camera keys.""" + from isaaclab_teleop import se3_spacemouse_teleop_cfg + + lifecycle, _ = _started_lifecycle(se3_spacemouse_teleop_cfg(), visualizer) + try: + assert visualizer.camera_keys == [] + visualizer.tap("KeyB") + assert lifecycle.drain_pressed_keys() == [KEY_B] + finally: + lifecycle.stop() + + +def test_presses_do_not_carry_into_the_next_session(visualizer): + lifecycle, _ = _started_lifecycle(_keyboard_cfg(), visualizer) + visualizer.tap("KeyR") + lifecycle.stop() + _start(lifecycle, visualizer) + try: + visualizer.press("KeyR") # the stale tap is gone; a new press counts + assert lifecycle.drain_pressed_keys() == [KEY_R] + finally: + lifecycle.stop() + + +def test_another_window_losing_focus_does_not_repeat_a_held_key(): + """Each window tracks its own held keys, so one window's blur leaves another's repeat suppression intact.""" + first, second = _Visualizer(), _Visualizer() + lifecycle, _ = _started_lifecycle(_keyboard_cfg(), first, second) + try: + first.press("KeyR") + second.blur() + first.press("KeyR") # an autorepeat of the key the first window still holds + assert lifecycle.drain_pressed_keys() == [KEY_R] + finally: + lifecycle.stop() + + +def test_a_failing_window_does_not_stop_the_session(visualizer): + """A window whose keyboard cannot be attached is skipped with a warning; the others still work.""" + broken = _Visualizer() + broken.set_camera_keys_enabled = MagicMock(side_effect=RuntimeError("no window")) + lifecycle, _ = _started_lifecycle(_keyboard_cfg(), broken, visualizer) + try: + visualizer.tap("KeyB") + assert lifecycle.drain_pressed_keys() == [KEY_B] + finally: + lifecycle.stop() diff --git a/source/isaaclab_teleop/test/test_operator_input.py b/source/isaaclab_teleop/test/test_operator_input.py new file mode 100644 index 000000000000..3b0fa9132031 --- /dev/null +++ b/source/isaaclab_teleop/test/test_operator_input.py @@ -0,0 +1,88 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# pyright: reportPrivateUsage=none + +"""Tests for the operator's ``B`` / ``P`` / ``R`` keys and SpaceMouse right button on +:class:`~isaaclab_teleop.IsaacTeleopDevice`, against a mocked session lifecycle.""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import numpy as np +import pytest +from isaaclab_teleop.isaac_teleop_device import IsaacTeleopDevice + +KEY_R, KEY_P, KEY_B, KEY_N = 19, 25, 48, 49 + + +class _Buttons: + """Stand-in for the ``spacemouse_buttons`` entry of a pipeline step result.""" + + def __init__(self, *pressed: int) -> None: + self.is_none = False + self._bitmap = np.zeros(8, dtype=np.uint8) + self._bitmap[list(pressed)] = 1 + + def __getitem__(self, index): + return self._bitmap + + +def _device(pressed_keys=(), step_result=None) -> IsaacTeleopDevice: + device = object.__new__(IsaacTeleopDevice) + device._session_lifecycle = MagicMock() + device._session_lifecycle.drain_pressed_keys.return_value = list(pressed_keys) + device._session_lifecycle.last_step_result = step_result + device._prev_spacemouse_right_pressed = False + device.request_start = MagicMock() + device.request_stop = MagicMock() + device.reset = MagicMock() + return device + + +@pytest.mark.parametrize(("code", "action"), [(KEY_B, "request_start"), (KEY_P, "request_stop"), (KEY_R, "reset")]) +def test_control_keys(code, action): + device = _device(pressed_keys=[code]) + device._handle_operator_input() + + for name in ("request_start", "request_stop", "reset"): + assert getattr(device, name).call_count == (name == action) + if action == "reset": + device.reset.assert_called_once_with(pause=True) + + +def test_other_keys_do_nothing(): + device = _device(pressed_keys=[KEY_N]) + device._handle_operator_input() + + device.request_start.assert_not_called() + device.reset.assert_not_called() + + +def test_spacemouse_right_button_resets_once_per_press(): + device = _device(step_result={"spacemouse_buttons": _Buttons(1)}) + device._handle_operator_input() + device._handle_operator_input() # still held + device.reset.assert_called_once_with(pause=True) + + device._session_lifecycle.last_step_result = {"spacemouse_buttons": _Buttons()} + device._handle_operator_input() + device._session_lifecycle.last_step_result = {"spacemouse_buttons": _Buttons(1)} + device._handle_operator_input() + assert device.reset.call_count == 2 + + +def test_spacemouse_left_button_does_not_reset(): + device = _device(step_result={"spacemouse_buttons": _Buttons(0)}) + device._handle_operator_input() + device.reset.assert_not_called() + + +@pytest.mark.parametrize("step_result", [None, {"action": object()}]) +def test_without_spacemouse_buttons_nothing_resets(step_result): + device = _device(step_result=step_result) + device._handle_operator_input() + device.reset.assert_not_called() diff --git a/source/isaaclab_teleop/test/test_spacemouse_constructors.py b/source/isaaclab_teleop/test/test_spacemouse_constructors.py new file mode 100644 index 000000000000..724de0f86e1d --- /dev/null +++ b/source/isaaclab_teleop/test/test_spacemouse_constructors.py @@ -0,0 +1,72 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# pyright: reportPrivateUsage=none + +"""Tests for the SpaceMouse ``IsaacTeleopCfg`` builders.""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("isaacteleop") + +from isaaclab_teleop.spacemouse.se2_spacemouse import se2_spacemouse_teleop_cfg +from isaaclab_teleop.spacemouse.se3_spacemouse import se3_spacemouse_teleop_cfg + + +def _has_spacemouse_source(cfg) -> bool: + """Whether the config's pipeline reads the SpaceMouse.""" + from isaacteleop.retargeting_engine.deviceio_source_nodes import SpaceMouseSource, find_sources + + return len(find_sources(cfg.pipeline_builder(), SpaceMouseSource)) == 1 + + +def _starts_the_spacemouse_plugin(cfg) -> bool: + """Whether the config starts the Isaac Capture spacemouse plugin that feeds its source.""" + from isaacteleop.teleop_session_manager import BUNDLED_PLUGINS_DIR + + return [(p.plugin_name, p.plugin_root_id, p.search_paths) for p in cfg.plugins] == [ + ("spacemouse", "spacemouse", [BUNDLED_PLUGINS_DIR]) + ] + + +# --------------------------------------------------------------------------- +# se2_spacemouse_teleop_cfg / se3_spacemouse_teleop_cfg +# --------------------------------------------------------------------------- + + +class TestSe3SpaceMouseTeleopCfg: + def test_defaults(self): + cfg = se3_spacemouse_teleop_cfg() + + assert cfg.sim_device == "cpu" + assert cfg.teleoperation_active_default is True + assert cfg.app_name == "IsaacLabSpaceMouseSe3" + assert _starts_the_spacemouse_plugin(cfg) + assert _has_spacemouse_source(cfg) + assert callable(cfg.pipeline_builder) + + def test_custom_sim_device(self): + cfg = se3_spacemouse_teleop_cfg(sim_device="cuda:0") + + assert cfg.sim_device == "cuda:0" + + +class TestSe2SpaceMouseTeleopCfg: + def test_defaults(self): + cfg = se2_spacemouse_teleop_cfg() + + assert cfg.sim_device == "cpu" + assert cfg.teleoperation_active_default is True + assert cfg.app_name == "IsaacLabSpaceMouseSe2" + assert _starts_the_spacemouse_plugin(cfg) + assert _has_spacemouse_source(cfg) + assert callable(cfg.pipeline_builder) + + def test_custom_sim_device(self): + cfg = se2_spacemouse_teleop_cfg(sim_device="cuda:0") + + assert cfg.sim_device == "cuda:0" From dd51e88462e6164edfe4a2ed479c6d0cc9ee2591 Mon Sep 17 00:00:00 2001 From: Rafael Wiltz Date: Fri, 23 Oct 2026 10:52:06 -0400 Subject: [PATCH 2/6] Run the teleop and recording scripts' devices through Isaac Capture teleop_se3_agent.py and record_demos.py now serve keyboard, gamepad and SpaceMouse devices through Isaac Capture when the isaacteleop package is installed, and through the isaaclab.devices devices otherwise (e.g. on aarch64 or Windows). --teleop_device keeps its meaning: an entry of the environment's teleop_devices, or a built-in device. Environments need no changes. The scripts no longer create a Carb keyboard for the B, P and R control keys, which the device now handles. --- .../imitation-learning/teleop_imitation.rst | 14 ++-- docs/source/features/isaac_teleop.rst | 81 +++++++++++++++---- .../teleoperation/teleop_se3_agent.py | 58 +++++++------ scripts/tools/record_demos.py | 61 +++++++------- 4 files changed, 134 insertions(+), 80 deletions(-) diff --git a/docs/source/features/imitation-learning/teleop_imitation.rst b/docs/source/features/imitation-learning/teleop_imitation.rst index 42f6096c43ae..233c97737abb 100644 --- a/docs/source/features/imitation-learning/teleop_imitation.rst +++ b/docs/source/features/imitation-learning/teleop_imitation.rst @@ -88,7 +88,8 @@ Teleoperation Isaac Lab supports teleoperation of robots through a variety of input devices including keyboard, SpaceMouse, and XR headsets. -Run the following command to spin up the environment and teleoperate the robot using the keyboard. Familiarize yourself +Run the following command to spin up the environment and teleoperate the robot using the keyboard: click the +visualizer window and type (see :ref:`isaac-teleop-keyboard`). Familiarize yourself with the controls and perform the stacking task. The order of the stacked cubes should be blue (bottom), red (middle), green (top). Once you feel sufficiently comfortable with the controls, you may shutdown the environment by quitting the script with Ctrl+C. @@ -145,12 +146,15 @@ the key bindings are: Move arm along z-axis: Push or pull the SpaceMouse Rotate arm: Twist the SpaceMouse +.. _isaac-teleop-spacemouse-permissions: + .. tip:: - If the SpaceMouse is not detected, you most likely need additional user permissions. The ``hidapi`` - wheel installed by Isaac Lab bundles a backend that talks to the device over ``libusb``, so it needs - read and write access to the USB node under ``/dev/bus/usb`` -- granting access to ``/dev/hidraw*`` - alone is **not** sufficient, and without USB access the device is enumerated without a product name. + If the SpaceMouse is not detected, you most likely need additional user permissions. Isaac Capture + reads the SpaceMouse from its HID node (``/dev/hidraw*``). Without the ``isaacteleop`` package, the + :class:`~isaaclab.devices.Se3SpaceMouse` reads it through the ``hidapi`` wheel's ``libusb`` + backend instead, which needs read and write access to the USB node under ``/dev/bus/usb``; without + it the device is enumerated without a product name. The rule below grants both. Grant the permission by installing a udev rule for the 3Dconnexion vendor id: diff --git a/docs/source/features/isaac_teleop.rst b/docs/source/features/isaac_teleop.rst index c5f492072e1b..48d0a981ad5a 100644 --- a/docs/source/features/isaac_teleop.rst +++ b/docs/source/features/isaac_teleop.rst @@ -303,7 +303,8 @@ interchangeable, and :func:`~isaaclab_teleop.poll_control_events` stays authorit A clientless CloudXR runtime advertises no HMD, so the standalone OpenXR session needs a device profile that reports a system without a connected client. The scripts default ``--cloudxr_env`` to :data:`~isaaclab_teleop.CLOUDXR_STANDALONE_ENV` when ``--xr`` is omitted (and to ``cloudxrjs`` - when it is passed); see :ref:`isaac-teleop-cloudxr-profiles`. + when it is passed); see :ref:`isaac-teleop-cloudxr-profiles`. Keyboard, gamepad and SpaceMouse + sessions use the same standalone profile (see :ref:`isaac-teleop-keyboard`). .. _isaac-teleop-so101-leader-example: @@ -748,13 +749,56 @@ These environments use the Isaac Capture XR pipeline with motion controllers or ideal for tasks that need a gripper or simple hand mapping. **Hand tracking** captures 26 wrist and finger joints per hand, required for dexterous retargeting to complex robot hands. -Keyboard and SpaceMouse Environments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _isaac-teleop-keyboard: + +Keyboard, SpaceMouse, and Gamepad Environments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. note:: - Keyboard and SpaceMouse teleoperation uses the legacy native Isaac Lab teleop stack - (``isaaclab.devices``), not Isaac Capture. These environments do not require an XR headset. + Keyboard, SpaceMouse, and Gamepad teleoperation run through Isaac Capture like every other + input device, driven by a keyboard, spacemouse, or gamepad + :class:`~isaaclab_teleop.IsaacTeleopDevice` pipeline instead of an XR headset -- see + :func:`~isaaclab_teleop.se3_keyboard_teleop_cfg`, :func:`~isaaclab_teleop.se3_spacemouse_teleop_cfg`, + and :func:`~isaaclab_teleop.se3_gamepad_teleop_cfg`. These environments do not require an XR + headset. An environment's ``teleop_devices`` entries (:class:`~isaaclab.devices.Se3KeyboardCfg`, + ...) run through the equivalent pipeline too. Without the ``isaacteleop`` package, the scripts + fall back to the :mod:`isaaclab.devices` keyboard, gamepad and SpaceMouse devices. + +**Keyboard input and window focus.** The keyboard runs inside the Isaac Lab process and reads the +keys typed into whichever visualizer window has focus, like any desktop app: click the window and +type. It needs no ``input`` group membership, ignores keys typed into other apps, and releases +held keys when the window loses focus. Like every Isaac Capture session, a keyboard session opens an +OpenXR session: without ``--xr`` the scripts launch a CloudXR runtime with the standalone profile +(see :ref:`isaac-teleop-standalone`), so no headset is needed. + +.. list-table:: + :header-rows: 1 + :widths: 25 75 + + * - Visualizer + - Keyboard input + * - ``--viz kit`` + - Keys typed into the Kit window, including keys typed into Kit's own UI (e.g. a text field). + * - ``--viz newton_gl`` / ``--viz newton_rtx`` + - Keys typed into the Newton viewer window, except keys typed into its UI. While a keyboard + pipeline runs, the viewer's WASD/QE and arrow-key camera movement is turned off; the mouse + still moves the camera. + * - ``--viz viser`` / ``--viz rerun`` / ``--viz none`` + - None. + +**SpaceMouse and Gamepad.** Both are read by an Isaac Capture plugin that their configs start +with the session, so they also run headset-free from a single command. The ``gamepad`` plugin +reads the first connected joystick (``/dev/input/jsN``), which udev lets the logged-in user read +without the ``input`` group. The ``spacemouse`` plugin reads the SpaceMouse's HID device +(``/dev/hidrawN``), which needs a udev rule (see +:ref:`SpaceMouse permissions `). Their buttons are sampled +once per frame, so a very quick tap (pressed and released within one frame, more likely when the +simulation runs slowly) can be missed; press again if the gripper or reset does not respond. + +The ``B`` / ``P`` / ``R`` start/pause/reset keys work with every pipeline, including XR, SpaceMouse +and Gamepad ones: :class:`~isaaclab_teleop.IsaacTeleopDevice` reads them straight from the focused +visualizer window, so each press fires once whatever the retargeting mode. The device button layouts below apply to all environments in this section. Per-environment differences (gripper enabled/disabled, sensitivity) are noted in the environment table that @@ -790,9 +834,15 @@ follows. * - Gripper toggle - ``K`` - Open / close gripper or suction (disabled in Reach envs). + * - Start/resume teleoperation + - ``B`` + - Resume after a pause. + * - Pause teleoperation + - ``P`` + - Pause without resetting. * - Reset - - ``L`` - - Clear accumulated delta pose and gripper state. + - ``R`` + - Reset the environment. **SpaceMouse** @@ -814,7 +864,7 @@ follows. - Open / close gripper or suction (disabled in Reach envs). * - Reset - Right button - - Clear accumulated delta pose and gripper state. + - Reset the environment. **Gamepad** (Reach environments only) @@ -838,7 +888,7 @@ follows. - Right stick (left/right) - Rotate about Z axis. * - Gripper toggle - - X button + - West button (X on Xbox, Square on PlayStation) - Open / close gripper (disabled in Reach envs). .. list-table:: @@ -1171,7 +1221,8 @@ times spike. This favors lower latency over CloudXR's pose-wait smoothing. :data:`~isaaclab_teleop.CLOUDXR_STANDALONE_ENV` is the default when running **without** ``--xr`` (see :ref:`isaac-teleop-standalone`). It pins an emulated ``quest3`` device profile so a CloudXR runtime with no client still advertises an OpenXR system, working around -``XR_ERROR_FORM_FACTOR_UNAVAILABLE``. +``XR_ERROR_FORM_FACTOR_UNAVAILABLE``. Keyboard, gamepad and SpaceMouse sessions without ``--xr`` +use it too. Override at launch time ~~~~~~~~~~~~~~~~~~~~~~~ @@ -1683,9 +1734,9 @@ the registered task ID, but not the command-line selector values: --dataset_file ./datasets/dataset.hdf5 \ physics=isaacsim_physx presets=diffik -Some environments use the legacy ``teleop_devices`` configuration instead of ``isaac_teleop`` -(e.g. the Galbot RmpFlow relative-mode tasks). For these, pass ``--teleop_device`` to select -the input device: +Some environments declare their input devices in ``teleop_devices`` instead of ``isaac_teleop`` +(e.g. the Galbot RmpFlow relative-mode tasks). For these, pass ``--teleop_device`` to select the +input device; keyboard and spacemouse run through Isaac Capture when ``isaacteleop`` is installed: .. tab-set:: @@ -1701,11 +1752,11 @@ the input device: The workflow is: #. Configure your environment with ``IsaacTeleopCfg`` (see :ref:`isaac-teleop-env-config`) - or ``teleop_devices`` for legacy devices (keyboard, spacemouse). + or ``teleop_devices`` for keyboard and spacemouse. #. Run ``record_demos.py`` with the task name and any ``physics=``, ``renderer=``, or ``presets=`` selectors required by the task. #. For XR tasks: start AR, connect your XR device, and teleoperate. - For legacy tasks: use the configured input device directly. + For keyboard/spacemouse: use the input device directly. #. Demonstrations are recorded to HDF5 files. #. Use the recorded data with Isaac Lab Mimic or other imitation learning frameworks. diff --git a/scripts/environments/teleoperation/teleop_se3_agent.py b/scripts/environments/teleoperation/teleop_se3_agent.py index c6f94f292381..4522f11aded7 100644 --- a/scripts/environments/teleoperation/teleop_se3_agent.py +++ b/scripts/environments/teleoperation/teleop_se3_agent.py @@ -42,9 +42,10 @@ type=str, default=None, help=( - "Legacy teleop device name. When omitted, the Isaac Capture pipeline is used if configured in the env," - " otherwise keyboard is used as fallback. When explicitly provided, the script uses the legacy" - " teleop_devices path and looks up this name in env_cfg.teleop_devices.devices." + "Teleop device name: an entry of env_cfg.teleop_devices.devices, or a built-in device (keyboard," + " spacemouse, gamepad). When omitted, the Isaac Capture pipeline is used if configured in the env," + " otherwise keyboard. Keyboard, gamepad and spacemouse run through Isaac Capture when the isaacteleop" + " package is installed, and through the isaaclab.devices devices otherwise." ), ) parser.add_argument("--task", type=str, default=None, help="Name of the task.") @@ -117,7 +118,7 @@ from isaaclab_physx.renderers import IsaacRtxRendererGlobalSettingsCfg from isaaclab_teleop import CLOUDXR_AVP_ENV, CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV -from isaaclab.devices import Se3GamepadCfg, Se3KeyboardCfg, Se3SpaceMouseCfg +from isaaclab.devices import DeviceCfg, Se3GamepadCfg, Se3KeyboardCfg, Se3SpaceMouseCfg from isaaclab.devices.openxr import remove_camera_configs from isaaclab.devices.teleop_device_factory import create_teleop_device from isaaclab.envs import ManagerBasedRLEnvCfg @@ -143,6 +144,15 @@ }.get(args_cli.cloudxr_env.lower(), args_cli.cloudxr_env) +def _builtin_device_cfgs(sensitivity: float) -> dict[str, DeviceCfg]: + """The built-in devices, by ``--teleop_device`` name, for names the environment does not declare.""" + return { + "keyboard": Se3KeyboardCfg(pos_sensitivity=0.05 * sensitivity, rot_sensitivity=0.05 * sensitivity), + "spacemouse": Se3SpaceMouseCfg(pos_sensitivity=0.05 * sensitivity, rot_sensitivity=0.05 * sensitivity), + "gamepad": Se3GamepadCfg(pos_sensitivity=0.1 * sensitivity, rot_sensitivity=0.1 * sensitivity), + } + + def main() -> None: # noqa: C901 """ Run teleoperation with an Isaac Lab manipulation environment. @@ -171,12 +181,16 @@ def main() -> None: # noqa: C901 # add termination condition for reaching the goal otherwise the environment won't reset env_cfg.terminations.object_reached_goal = DoneTerm(func=mdp.object_reached_goal) - # When --teleop_device is explicitly provided, use the legacy teleop_devices path - # even if isaac_teleop is configured. Otherwise prefer isaac_teleop when available. - teleop_device_explicitly_set = args_cli.teleop_device is not None - use_isaac_teleop = ( - not teleop_device_explicitly_set and hasattr(env_cfg, "isaac_teleop") and env_cfg.isaac_teleop is not None + from isaaclab_teleop.device_selection import select_isaac_teleop_cfg + + # Isaac Capture serves the request when it can: the env's isaac_teleop pipeline, or the + # keyboard, gamepad or spacemouse it names. Otherwise the isaaclab.devices device does. + isaac_teleop_cfg = select_isaac_teleop_cfg( + env_cfg, args_cli.teleop_device, _builtin_device_cfgs(args_cli.sensitivity), args_cli.device ) + use_isaac_teleop = isaac_teleop_cfg is not None + if use_isaac_teleop: + env_cfg.isaac_teleop = isaac_teleop_cfg # XR-rendering setup (camera removal + DLSS) is only needed for the Kit XR # path. Without --xr, Isaac Capture runs standalone (I/O only) and renders @@ -315,12 +329,8 @@ def stop_teleoperation() -> None: ) else: device_name = "keyboard" if args_cli.teleop_device is None else args_cli.teleop_device.lower() - device_cfgs = { - "keyboard": Se3KeyboardCfg(pos_sensitivity=0.05, rot_sensitivity=0.05), - "spacemouse": Se3SpaceMouseCfg(pos_sensitivity=0.05, rot_sensitivity=0.05), - "gamepad": Se3GamepadCfg(pos_sensitivity=0.1, rot_sensitivity=0.1), - } - if device_name not in device_cfgs: + device_cfg = _builtin_device_cfgs(args_cli.sensitivity).get(device_name) + if device_cfg is None: logger.error( f"--teleop_device={args_cli.teleop_device} was passed but no matching entry exists in" " env_cfg.teleop_devices and it is not a built-in device name. Either remove" @@ -330,9 +340,6 @@ def stop_teleoperation() -> None: ) env.close() return - device_cfg = device_cfgs[device_name] - device_cfg.pos_sensitivity *= args_cli.sensitivity - device_cfg.rot_sensitivity *= args_cli.sensitivity teleop_interface = instantiate(device_cfg) for key, callback in teleoperation_callbacks.items(): try: @@ -352,18 +359,9 @@ def stop_teleoperation() -> None: haptic_driver = create_haptic_feedback_driver(env.unwrapped, teleop_interface, env_cfg) - # Retain the keyboard so its input subscription lives for the whole session. - if use_isaac_teleop and env.sim.has_gui and not env.sim.get_setting("/isaaclab/xr/auto_start"): - try: - control_keyboard = instantiate(Se3KeyboardCfg(pos_sensitivity=0.0, rot_sensitivity=0.0)) - control_keyboard.add_callback("B", teleop_interface.request_start) - control_keyboard.add_callback("P", teleop_interface.request_stop) - # Inject a reset pulse; the loop owns the environment reset. - control_keyboard.add_callback("R", lambda: teleop_interface.reset(pause=True)) - print("Isaac Capture control keys: [B] start/resume [P] pause [R] reset") - except Exception as e: - logger.warning(f"Control keyboard unavailable ({e}); teleop still auto-starts without --xr") - control_keyboard = None + if use_isaac_teleop: + # the device reads these from the focused visualizer window + print("Isaac Capture control keys: [B] start/resume [P] pause [R] reset") def run_loop(): """Inner function to run the teleop loop with access to nonlocal variables.""" diff --git a/scripts/tools/record_demos.py b/scripts/tools/record_demos.py index 8c9f2446f039..81efced49093 100644 --- a/scripts/tools/record_demos.py +++ b/scripts/tools/record_demos.py @@ -20,8 +20,10 @@ optional arguments: -h, --help Show this help message and exit - --teleop_device Legacy teleop device name. When omitted, Isaac Capture is used if - configured, otherwise keyboard. When set, forces the legacy path. + --teleop_device Teleop device name: an env teleop_devices entry, or keyboard / + spacemouse. When omitted, Isaac Capture is used if configured, + otherwise keyboard. Keyboard and spacemouse run through Isaac Capture + when isaacteleop is installed, else through isaaclab.devices. --dataset_file File path to export recorded demos. (default: "./datasets/dataset.hdf5") --step_hz Environment stepping rate in Hz. (default: 30) --num_demos Number of demonstrations to record. (default: 0) @@ -62,9 +64,10 @@ type=str, default=None, help=( - "Legacy teleop device name. When omitted, the Isaac Capture pipeline is used if configured in the env," - " otherwise keyboard is used as fallback. When explicitly provided, the script uses the legacy" - " teleop_devices path and looks up this name in env_cfg.teleop_devices.devices." + "Teleop device name: an entry of env_cfg.teleop_devices.devices, or a built-in device (keyboard," + " spacemouse). When omitted, the Isaac Capture pipeline is used if configured in the env, otherwise" + " keyboard. Keyboard and spacemouse run through Isaac Capture when the isaacteleop package is installed," + " and through the isaaclab.devices devices otherwise." ), ) parser.add_argument( @@ -173,7 +176,7 @@ from isaaclab_physx.renderers import IsaacRtxRendererGlobalSettingsCfg from isaaclab_teleop import CLOUDXR_AVP_ENV, CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV -from isaaclab.devices import Se3KeyboardCfg, Se3SpaceMouseCfg +from isaaclab.devices import DeviceCfg, Se3KeyboardCfg, Se3SpaceMouseCfg from isaaclab.devices.openxr import remove_camera_configs from isaaclab.devices.teleop_device_factory import create_teleop_device from isaaclab.envs import DirectRLEnvCfg, ManagerBasedRLEnvCfg @@ -293,12 +296,14 @@ def create_environment_config( logger.error(f"Failed to parse environment configuration: {e}") exit(1) - # When --teleop_device is explicitly provided, use the legacy teleop_devices path - # even if isaac_teleop is configured. Otherwise prefer isaac_teleop when available. - teleop_device_explicitly_set = args_cli.teleop_device is not None - use_isaac_teleop = ( - not teleop_device_explicitly_set and hasattr(env_cfg, "isaac_teleop") and env_cfg.isaac_teleop is not None - ) + from isaaclab_teleop.device_selection import select_isaac_teleop_cfg + + # Isaac Capture serves the request when it can: the env's isaac_teleop pipeline, or the + # keyboard or spacemouse it names. Otherwise the isaaclab.devices device does. + isaac_teleop_cfg = select_isaac_teleop_cfg(env_cfg, args_cli.teleop_device, _builtin_device_cfgs(), args_cli.device) + use_isaac_teleop = isaac_teleop_cfg is not None + if use_isaac_teleop: + env_cfg.isaac_teleop = isaac_teleop_cfg # Extract the success condition for manual evaluation in the main loop. Keep # an inert term registered under the same name so rewards and other manager @@ -348,6 +353,14 @@ def create_environment_config( return env_cfg, success_term, use_isaac_teleop +def _builtin_device_cfgs() -> dict[str, DeviceCfg]: + """The built-in devices, by ``--teleop_device`` name, for names the environment does not declare.""" + return { + "keyboard": Se3KeyboardCfg(pos_sensitivity=0.2, rot_sensitivity=0.5), + "spacemouse": Se3SpaceMouseCfg(pos_sensitivity=0.2, rot_sensitivity=0.5), + } + + def setup_teleop_device( env_cfg: ManagerBasedRLEnvCfg, callbacks: dict[str, Callable], use_isaac_teleop: bool = False ) -> object: @@ -393,11 +406,8 @@ def setup_teleop_device( ) else: device_name = "keyboard" if args_cli.teleop_device is None else args_cli.teleop_device.lower() - device_cfgs = { - "keyboard": Se3KeyboardCfg(pos_sensitivity=0.2, rot_sensitivity=0.5), - "spacemouse": Se3SpaceMouseCfg(pos_sensitivity=0.2, rot_sensitivity=0.5), - } - if device_name not in device_cfgs: + device_cfg = _builtin_device_cfgs().get(device_name) + if device_cfg is None: logger.error( f"--teleop_device={args_cli.teleop_device} was passed but no matching entry exists in" " env_cfg.teleop_devices and it is not a built-in device name. Either remove" @@ -406,7 +416,7 @@ def setup_teleop_device( " Built-in devices: keyboard, spacemouse." ) exit(1) - teleop_interface = instantiate(device_cfgs[device_name]) + teleop_interface = instantiate(device_cfg) for key, callback in callbacks.items(): teleop_interface.add_callback(key, callback) except Exception as e: @@ -588,18 +598,9 @@ def stop_recording_instance(): haptic_driver = create_haptic_feedback_driver(env.unwrapped, teleop_interface, env.cfg) - # Retain the keyboard so its input subscription lives for the whole session. - if use_isaac_teleop and env.sim.has_gui and not env.sim.get_setting("/isaaclab/xr/auto_start"): - try: - control_keyboard = instantiate(Se3KeyboardCfg(pos_sensitivity=0.0, rot_sensitivity=0.0)) - control_keyboard.add_callback("B", teleop_interface.request_start) - control_keyboard.add_callback("P", teleop_interface.request_stop) - # Inject a reset pulse; the loop owns the environment reset. - control_keyboard.add_callback("R", lambda: teleop_interface.reset(pause=True)) - print("Isaac Capture control keys: [B] start/resume [P] pause [R] reset") - except Exception as e: - logger.warning(f"Control keyboard unavailable ({e}); recording still auto-starts without --xr") - control_keyboard = None + if use_isaac_teleop: + # the device reads these from the focused visualizer window + print("Isaac Capture control keys: [B] start/resume [P] pause [R] reset") label_text = f"Recorded {current_recorded_demo_count} successful demonstrations." instruction_display = setup_ui(label_text, env) From 7b7a6f5fb973a8a279072486304b291aa3881c65 Mon Sep 17 00:00:00 2001 From: Rafael Wiltz Date: Fri, 23 Oct 2026 11:36:04 -0400 Subject: [PATCH 3/6] Require Isaac Capture 1.7 for the keyboard, gamepad and SpaceMouse devices The keyboard, gamepad and SpaceMouse sources and the gamepad and spacemouse plugins first ship in Isaac Capture 1.7, published as isaaccapture (isaacteleop renamed; it still provides the isaacteleop import). The teleop extra now installs isaaccapture~=1.7.44a1 instead of isaacteleop~=1.4.145. uv.lock adds isaaccapture and cupy-cuda12x and moves the isaacteleop alias package to the same release. The docs point the release branch, web client and Haptikos plugin links at 1.7.x. --- docs/source/features/isaac_teleop.rst | 12 +-- docs/source/how-to/cloudxr_teleoperation.rst | 24 +++--- pyproject.toml | 2 +- ...rwiltz-keyboard-focus-surfaces.changed.rst | 3 + uv.lock | 79 ++++++++++++------- 5 files changed, 74 insertions(+), 46 deletions(-) diff --git a/docs/source/features/isaac_teleop.rst b/docs/source/features/isaac_teleop.rst index 48d0a981ad5a..5844ee94de9d 100644 --- a/docs/source/features/isaac_teleop.rst +++ b/docs/source/features/isaac_teleop.rst @@ -344,9 +344,9 @@ Prerequisites sudo apt-get update sudo apt-get install -y build-essential cmake libx11-dev clang-format-14 ccache patchelf - Then clone, check out the release branch matching the ``isaacteleop`` version Isaac Lab is - pinned to (``isaacteleop~=1.4.145`` in the ``teleop`` extra of the root ``pyproject.toml``, so the - plugin's wire format matches the ``isaacteleop`` package Isaac Lab installs), configure, build, + Then clone, check out the release branch matching the ``isaaccapture`` version Isaac Lab is + pinned to (``isaaccapture~=1.7.44a1`` in the ``teleop`` extra of the root ``pyproject.toml``, so the + plugin's wire format matches the ``isaaccapture`` package Isaac Lab installs), configure, build, and install. To target a specific Python version, pass ``-DISAAC_TELEOP_PYTHON_VERSION=3.12`` (or ``3.11``, ``3.13``) on the configure line; each version needs its own build directory if building multiple at once: @@ -355,7 +355,7 @@ Prerequisites git clone https://github.com/NVIDIA/IsaacCapture.git cd IsaacCapture - git checkout release/1.4.x + git checkout release/1.7.x cmake -B build # configure (default: Python 3.11) cmake --build build --parallel # build @@ -612,7 +612,7 @@ Source Nodes Available Retargeters ~~~~~~~~~~~~~~~~~~~~~ -Retargeters are provided by the ``isaacteleop`` package from the +Retargeters are provided by the ``isaaccapture`` package from the `Isaac Capture `_ repository. The retargeters listed below are those used by the built-in Isaac Lab environments. Isaac Capture may offer additional retargeters not listed here -- refer to the @@ -762,7 +762,7 @@ Keyboard, SpaceMouse, and Gamepad Environments :func:`~isaaclab_teleop.se3_keyboard_teleop_cfg`, :func:`~isaaclab_teleop.se3_spacemouse_teleop_cfg`, and :func:`~isaaclab_teleop.se3_gamepad_teleop_cfg`. These environments do not require an XR headset. An environment's ``teleop_devices`` entries (:class:`~isaaclab.devices.Se3KeyboardCfg`, - ...) run through the equivalent pipeline too. Without the ``isaacteleop`` package, the scripts + ...) run through the equivalent pipeline too. Without the ``isaaccapture`` package, the scripts fall back to the :mod:`isaaclab.devices` keyboard, gamepad and SpaceMouse devices. **Keyboard input and window focus.** The keyboard runs inside the Isaac Lab process and reads the diff --git a/docs/source/how-to/cloudxr_teleoperation.rst b/docs/source/how-to/cloudxr_teleoperation.rst index f80695e34da7..c5d99ba33858 100644 --- a/docs/source/how-to/cloudxr_teleoperation.rst +++ b/docs/source/how-to/cloudxr_teleoperation.rst @@ -382,12 +382,12 @@ choose the tab that matches your hardware. start automatically. #. Open the browser on your headset and navigate to the hosted CloudXR.js client: - ``_. + ``_. .. note:: - The web client URL is versioned. The ``release-1.4.x`` path corresponds to the - ``isaacteleop`` version pinned by Isaac Lab (``isaacteleop~=1.4.145`` in the ``teleop`` + The web client URL is versioned. The ``release-1.7.x`` path corresponds to the + ``isaaccapture`` version pinned by Isaac Lab (``isaaccapture~=1.7.44a1`` in the ``teleop`` extra of the root ``pyproject.toml``). When Isaac Lab updates its teleoperation dependency, update this link to the matching client release. @@ -594,16 +594,16 @@ Prerequisites ``install/plugins/manus/manus_hand_plugin`` does not exist in your Isaac Capture checkout, this step has not been completed. - Clone the repository and check out the release branch matching the ``isaacteleop`` version Isaac - Lab is pinned to (``isaacteleop~=1.4.145`` in the ``teleop`` extra of the root - ``pyproject.toml``), so the plugin's wire format matches the ``isaacteleop`` package Isaac Lab + Clone the repository and check out the release branch matching the ``isaaccapture`` version Isaac + Lab is pinned to (``isaaccapture~=1.7.44a1`` in the ``teleop`` extra of the root + ``pyproject.toml``), so the plugin's wire format matches the ``isaaccapture`` package Isaac Lab installs: .. code-block:: bash git clone https://github.com/NVIDIA/IsaacCapture.git cd IsaacCapture - git checkout release/1.4.x + git checkout release/1.7.x .. note:: @@ -765,7 +765,7 @@ time. Haptikos Exoskeletons with Quest -------------------------------- -The `Haptikos plugin `_ +The `Haptikos plugin `_ combines controller wrist poses with exoskeleton finger tracking from the Haptikos Core App and pushes hand joints into the OpenXR runtime. Isaac Lab receives them through Isaac Capture's standard hand-tracking input, so no Haptikos-specific Isaac Lab device is needed. The plugin @@ -776,18 +776,18 @@ Build the plugin ^^^^^^^^^^^^^^^^ The Haptikos plugin is built from Isaac Capture source; it is not included in Isaac Lab's -``teleop`` extra. Check out the release branch matching Isaac Lab's ``isaacteleop`` pin (currently -``1.4.x``), obtain the `Haptikos Robotics API `_, +``teleop`` extra. Check out the release branch matching Isaac Lab's ``isaaccapture`` pin (currently +``1.7.x``), obtain the `Haptikos Robotics API `_, and copy its ``HaptikosCpp_API_Shared`` directory into ``src/plugins/haptikos``. The C++ API is required to build the tracking plugin, even if you do not use haptic feedback. Follow the -`plugin's setup instructions `_ +`plugin's setup instructions `_ for Haptikos account and licensing requirements. .. code-block:: bash git clone https://github.com/NVIDIA/IsaacCapture.git cd IsaacCapture - git checkout release/1.4.x + git checkout release/1.7.x # Copy HaptikosCpp_API_Shared to src/plugins/haptikos before building. cmake -S . -B build -DENABLE_CLANG_FORMAT_CHECK=OFF cmake --build build --target haptikos_hands_plugin --parallel 4 diff --git a/pyproject.toml b/pyproject.toml index b143612a98e7..c2d452c40591 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,7 +196,7 @@ mimic = [ teleop = [ "isaaclab-teleop", "isaaclab-mimic", - "isaacteleop[retargeters,ui,cloudxr]~=1.4.145 ; platform_system == 'Linux' and platform_machine == 'x86_64'", + "isaaccapture[retargeters,ui,cloudxr]~=1.7.44a1 ; platform_system == 'Linux' and platform_machine == 'x86_64'", "dex-retargeting==0.5.0 ; platform_system == 'Linux' and platform_machine == 'x86_64'", ] rlinf = [ diff --git a/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.changed.rst b/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.changed.rst index bd02528c6e67..f85fc9c0115b 100644 --- a/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.changed.rst +++ b/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.changed.rst @@ -2,3 +2,6 @@ ``B`` / ``P`` / ``R`` keys typed into a visualizer window and on a SpaceMouse pipeline's right button, so scripts no longer bind them. While a pipeline reads the keyboard, the visualizers' keyboard camera movement is turned off. +* Changed the ``teleop`` extra to install ``isaaccapture~=1.7.44a1``, the Isaac Capture release + with the keyboard, gamepad and SpaceMouse devices, in place of ``isaacteleop~=1.4.145``. + ``isaaccapture`` is ``isaacteleop`` renamed and still provides the ``isaacteleop`` import. diff --git a/uv.lock b/uv.lock index e55477a12fe1..6d87eee9f2a3 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = "==3.12.*" resolution-markers = [ "platform_machine == 'x86_64' and sys_platform == 'linux'", @@ -885,6 +885,18 @@ nvtx = [ { name = "nvidia-nvtx" }, ] +[[package]] +name = "cupy-cuda12x" +version = "14.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-pathfinder" }, + { name = "numpy" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/5b/65124de2dbaf2e85109f611a41947e39acd6dd938751c04b4c4d7bf6fc82/cupy_cuda12x-14.2.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:f82141761f2c81905d49387464ae29438887956d99063381c93a1d5d1b7d32e8", size = 136111459, upload-time = "2026-08-20T02:40:04.909Z" }, +] + [[package]] name = "cycler" version = "0.12.1" @@ -1746,6 +1758,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl", hash = "sha256:ecaca67aed704a338f88f67b1181b58f821ab5dc89c1f0f5ef99db43c1c2921e", size = 139808, upload-time = "2025-11-01T21:18:10.956Z" }, ] +[[package]] +name = "isaaccapture" +version = "1.7.44a1" +source = { registry = "https://pypi.nvidia.com/" } +dependencies = [ + { name = "isaacteleop" }, + { name = "numpy" }, + { name = "pyyaml" }, +] +wheels = [ + { url = "https://pypi.nvidia.com/isaaccapture/isaaccapture-1.7.44a1-cp312-cp312-manylinux_2_35_x86_64.whl", hash = "sha256:bba80a67891474bc2cf9ddc4b6b0bd6ea78977f8d4d5cf38c4e7f2aae02c781c" }, +] + +[package.optional-dependencies] +cloudxr = [ + { name = "cryptography" }, + { name = "cupy-cuda12x" }, + { name = "pillow" }, + { name = "websockets" }, +] +retargeters = [ + { name = "cmeel-tinyxml2" }, + { name = "cmeel-urdfdom" }, + { name = "dex-retargeting" }, + { name = "nlopt" }, + { name = "pin" }, + { name = "pyyaml" }, + { name = "scipy" }, + { name = "torch" }, +] +ui = [ + { name = "imgui", extra = ["glfw"] }, +] + [[package]] name = "isaaclab" version = "41.0.3" @@ -1959,9 +2005,9 @@ skrl-jax = [ ] teleop = [ { name = "dex-retargeting", marker = "platform_machine == 'x86_64' and sys_platform != 'win32'" }, + { name = "isaaccapture", extra = ["cloudxr", "retargeters", "ui"], marker = "platform_machine == 'x86_64' and sys_platform != 'win32'" }, { name = "isaaclab-mimic" }, { name = "isaaclab-teleop" }, - { name = "isaacteleop", extra = ["cloudxr", "retargeters", "ui"], marker = "platform_machine == 'x86_64' and sys_platform != 'win32'" }, ] test = [ { name = "coverage" }, @@ -2015,6 +2061,7 @@ requires-dist = [ { name = "hydra-core" }, { name = "imgui-bundle", specifier = ">=1.92.5" }, { name = "ipywidgets", marker = "extra == 'mimic'", specifier = ">=8.1.5" }, + { name = "isaaccapture", extras = ["retargeters", "ui", "cloudxr"], marker = "platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'teleop'", specifier = "~=1.7.44a1" }, { name = "isaaclab", editable = "source/isaaclab" }, { name = "isaaclab-assets", editable = "source/isaaclab_assets" }, { name = "isaaclab-contrib", editable = "source/isaaclab_contrib" }, @@ -2035,7 +2082,6 @@ requires-dist = [ { name = "isaaclab-visualizers", editable = "source/isaaclab_visualizers" }, { name = "isaacsim", extras = ["all", "extscache"], marker = "extra == 'isaacsim'", specifier = "==6.2.0.0" }, { name = "isaacsim-asset-isolated", marker = "extra == 'importers'", specifier = "==6.2.0.0" }, - { name = "isaacteleop", extras = ["retargeters", "ui", "cloudxr"], marker = "platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'teleop'", specifier = "~=1.4.145" }, { name = "jax", extras = ["cuda13"], marker = "sys_platform == 'linux' and extra == 'skrl-jax'" }, { name = "jinja2" }, { name = "junitparser", marker = "extra == 'test'" }, @@ -2743,33 +2789,13 @@ wheels = [ [[package]] name = "isaacteleop" -version = "1.4.145" +version = "1!1.7.44a1" source = { registry = "https://pypi.nvidia.com/" } dependencies = [ - { name = "numpy" }, - { name = "pyyaml" }, + { name = "isaaccapture" }, ] wheels = [ - { url = "https://pypi.nvidia.com/isaacteleop/isaacteleop-1.4.145-cp312-cp312-manylinux_2_35_x86_64.whl", hash = "sha256:b18b210535b28516593bb6e85ffb70a6e8265d9547365c818e38920184ee7b40" }, -] - -[package.optional-dependencies] -cloudxr = [ - { name = "cryptography" }, - { name = "websockets" }, -] -retargeters = [ - { name = "cmeel-tinyxml2" }, - { name = "cmeel-urdfdom" }, - { name = "dex-retargeting" }, - { name = "nlopt" }, - { name = "pin" }, - { name = "pyyaml" }, - { name = "scipy" }, - { name = "torch" }, -] -ui = [ - { name = "imgui", extra = ["glfw"] }, + { url = "https://pypi.nvidia.com/isaacteleop/isaacteleop-1%211.7.44a1-py3-none-any.whl", hash = "sha256:d09008a3ece5ae4e2debb9b7f0fa8a58aaa4632c9f025c487bfbf946dd2829f8" }, ] [[package]] @@ -3829,7 +3855,6 @@ dependencies = [ { name = "configargparse" }, { name = "dash" }, { name = "flask" }, - { name = "ipywidgets" }, { name = "matplotlib" }, { name = "nbformat" }, { name = "numpy" }, From 37f932101518705da702cd627f8ff85a965020ca Mon Sep 17 00:00:00 2001 From: Rafael Wiltz Date: Fri, 23 Oct 2026 11:48:24 -0400 Subject: [PATCH 4/6] Keep protected demos on reset and attach windows opened later record_demos.py now applies the R callback's guard to an Isaac Capture reset: while a successful demo is still recording its post-success steps, the reset is refused with the same message as for the isaaclab.devices keyboard. Other resets, including the host reset after each episode, are handled as before. The session checks for unattached visualizer windows on every step, not only until the first one attaches, so a window that initializes later also gets keyboard input. --- scripts/tools/record_demos.py | 6 +++++- .../isaaclab_teleop/session_lifecycle.py | 18 +++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/scripts/tools/record_demos.py b/scripts/tools/record_demos.py index 81efced49093..01e8689e6347 100644 --- a/scripts/tools/record_demos.py +++ b/scripts/tools/record_demos.py @@ -641,7 +641,11 @@ def inner_loop(): if ctrl.is_active is not None: running_recording_instance = ctrl.is_active if ctrl.should_reset: - should_reset_recording_instance = True + if success_step_count > 0: + # same guard as the R callback: keep a demo whose success is still being recorded + reset_recording_instance() + else: + should_reset_recording_instance = True if action is None: env.sim.render() diff --git a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py index bf943de491a4..b6864fa28ade 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py +++ b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py @@ -363,6 +363,7 @@ def __init__( # retargeting mode. self._keyboard_detachers: list[Callable[[], None]] = [] self._attached_visualizers: list[Any] = [] + self._pipeline_keyboards: list[Any] | None = None # found once per session self._pressed_keys: list[int] = [] # CloudXR runtime launcher (created in start if configured, stopped in stop) @@ -595,6 +596,7 @@ def start(self) -> None: self._pipeline = self._build_combined_pipeline(user_pipeline) self._pressed_keys.clear() + self._pipeline_keyboards = None self._attach_keyboards() # Build the optional haptic sink. It reuses the button-controller @@ -731,9 +733,9 @@ def _build_combined_pipeline(self, user_pipeline): def _attach_keyboards(self) -> None: """Feed the pipeline's keyboards and the operator's key presses from the visualizer windows. - Called at start and, until a window is attached, on each step, since visualizer windows can - open after the session starts. While the pipeline has a keyboard, each visualizer's keyboard - camera movement is turned off. + Called at start and on each step, since visualizer windows can open after the session starts; + windows already attached are skipped. While the pipeline has a keyboard, each visualizer's + keyboard camera movement is turned off. """ if self._pipeline is None or self._is_replay: return @@ -745,9 +747,11 @@ def _attach_keyboards(self) -> None: return if sim is None: return - from isaacteleop.retargeting_engine.deviceio_source_nodes import KeyboardSource, find_sources + if self._pipeline_keyboards is None: + from isaacteleop.retargeting_engine.deviceio_source_nodes import KeyboardSource, find_sources - keyboards = find_sources(self._pipeline, KeyboardSource) + self._pipeline_keyboards = find_sources(self._pipeline, KeyboardSource) + keyboards = self._pipeline_keyboards for visualizer in sim.visualizers: # a visualizer without a window that reports keys has no source source = visualizer.key_event_source @@ -777,6 +781,7 @@ def _detach_keyboards(self) -> None: except Exception as e: logger.debug(f"Suppressed error detaching a visualizer keyboard: {e}") self._attached_visualizers = [] + self._pipeline_keyboards = None def drain_pressed_keys(self) -> list[int]: """Evdev codes of the keys pressed in the visualizer windows since the last call, in order. @@ -1302,8 +1307,7 @@ def step( return None # Visualizer windows can open after the session starts. - if not self._attached_visualizers: - self._attach_keyboards() + self._attach_keyboards() # Build external inputs (e.g. world-to-anchor transform) if the # pipeline contains ValueInput leaf nodes. From 4ca88dae7e07e19c92551db25b7b4ccd1b0fe428 Mon Sep 17 00:00:00 2001 From: Rafael Wiltz Date: Fri, 23 Oct 2026 12:02:57 -0400 Subject: [PATCH 5/6] Relock with the repository's uv and tidy review nits uv.lock is regenerated from main's lock with uv 0.13.5, which keeps lock revision 5 and viser's ipywidgets dependency; only isaaccapture, cupy-cuda12x and the isaacteleop alias package change. The B / P / R key codes are class constants at the top of IsaacTeleopDevice, and the device-selection test records the builders the routing table actually calls rather than names the test supplies. --- .../isaaclab_teleop/isaac_teleop_device.py | 10 ++++++---- source/isaaclab_teleop/test/test_device_selection.py | 5 ++--- uv.lock | 3 ++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py index aad98b878f27..e7546ee50777 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py +++ b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py @@ -9,7 +9,7 @@ import logging from collections.abc import Callable -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, ClassVar import numpy as np import torch @@ -109,6 +109,11 @@ class IsaacTeleopDevice: env.step(action.repeat(num_envs, 1)) """ + _START_KEY: ClassVar[int] = 48 # evdev code of B + _STOP_KEY: ClassVar[int] = 25 # P + _RESET_KEY: ClassVar[int] = 19 # R + _SPACEMOUSE_RIGHT_BUTTON: ClassVar[int] = 1 + def __init__( self, cfg: IsaacTeleopCfg, @@ -470,9 +475,6 @@ def _dispatch_control_callbacks(self) -> None: # Operator keys and buttons # ------------------------------------------------------------------ - _START_KEY, _STOP_KEY, _RESET_KEY = 48, 25, 19 # evdev codes of B, P and R - _SPACEMOUSE_RIGHT_BUTTON = 1 - def _handle_operator_input(self) -> None: """Start, pause or reset on the operator's ``B`` / ``P`` / ``R`` keys and SpaceMouse right button. diff --git a/source/isaaclab_teleop/test/test_device_selection.py b/source/isaaclab_teleop/test/test_device_selection.py index 8cf1dacffb57..a106c762e683 100644 --- a/source/isaaclab_teleop/test/test_device_selection.py +++ b/source/isaaclab_teleop/test/test_device_selection.py @@ -43,13 +43,12 @@ def recorded(monkeypatch): """Record which builder ``device_selection`` calls, and with what, without changing its routing.""" calls = [] - for cfg_type, name in BUILDERS.items(): + for cfg_type, (routed, fields) in list(device_selection._PIPELINES.items()): - def builder(name=name, **kwargs): + def builder(name=routed.__name__, **kwargs): calls.append((name, kwargs)) return f"pipeline:{name}" - fields = device_selection._PIPELINES[cfg_type][1] monkeypatch.setitem(device_selection._PIPELINES, cfg_type, (builder, fields)) monkeypatch.setattr(device_selection, "_isaacteleop_available", lambda: True) return calls diff --git a/uv.lock b/uv.lock index 6d87eee9f2a3..aa0c13546690 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = "==3.12.*" resolution-markers = [ "platform_machine == 'x86_64' and sys_platform == 'linux'", @@ -3855,6 +3855,7 @@ dependencies = [ { name = "configargparse" }, { name = "dash" }, { name = "flask" }, + { name = "ipywidgets" }, { name = "matplotlib" }, { name = "nbformat" }, { name = "numpy" }, From bfec2217931e220f68efc2a9e9b43849a1581a38 Mon Sep 17 00:00:00 2001 From: Octi Zhang Date: Sat, 24 Oct 2026 23:18:22 -0700 Subject: [PATCH 6/6] Centralize teleop setup and pass window inputs explicitly --- docs/source/features/isaac_teleop.rst | 9 ++ .../teleoperation/teleop_se3_agent.py | 100 +++++------- scripts/tools/record_demos.py | 145 ++++++------------ .../explicit-window-input.changed.rst | 2 + .../rwiltz-keyboard-focus-surfaces.added.rst | 5 +- .../isaaclab_teleop/device_selection.py | 101 +++++------- .../isaaclab_teleop/isaac_teleop_device.py | 17 +- .../isaaclab_teleop/session_lifecycle.py | 55 ++++--- .../test/test_device_selection.py | 112 +++++++++++--- .../test/test_keyboard_session.py | 28 +++- 10 files changed, 293 insertions(+), 281 deletions(-) create mode 100644 source/isaaclab_teleop/changelog.d/explicit-window-input.changed.rst diff --git a/docs/source/features/isaac_teleop.rst b/docs/source/features/isaac_teleop.rst index 5844ee94de9d..a2b66e69b138 100644 --- a/docs/source/features/isaac_teleop.rst +++ b/docs/source/features/isaac_teleop.rst @@ -772,6 +772,15 @@ held keys when the window loses focus. Like every Isaac Capture session, a keybo OpenXR session: without ``--xr`` the scripts launch a CloudXR runtime with the standalone profile (see :ref:`isaac-teleop-standalone`), so no headset is needed. +The scripts resolve the requested device once before scene startup through +``isaaclab_teleop.device_selection.select_teleop_cfg`` and create it through +``create_teleop_device`` after the environment initializes. Native compatibility stays at that +boundary; a Capture initialization error is reported without selecting a different device. +For direct use of :func:`~isaaclab_teleop.create_isaac_teleop_device` or +:class:`~isaaclab_teleop.IsaacTeleopDevice`, pass ``visualizers=sim.visualizers`` to supply +window input explicitly. The session keeps this live sequence so later windows can join; +omitting it creates a session without window input. + .. list-table:: :header-rows: 1 :widths: 25 75 diff --git a/scripts/environments/teleoperation/teleop_se3_agent.py b/scripts/environments/teleoperation/teleop_se3_agent.py index 4522f11aded7..7f62ee897f27 100644 --- a/scripts/environments/teleoperation/teleop_se3_agent.py +++ b/scripts/environments/teleoperation/teleop_se3_agent.py @@ -116,14 +116,13 @@ import gymnasium as gym import torch from isaaclab_physx.renderers import IsaacRtxRendererGlobalSettingsCfg -from isaaclab_teleop import CLOUDXR_AVP_ENV, CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV +from isaaclab_teleop import CLOUDXR_AVP_ENV, CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV, IsaacTeleopCfg +from isaaclab_teleop.device_selection import create_teleop_device, select_teleop_cfg from isaaclab.devices import DeviceCfg, Se3GamepadCfg, Se3KeyboardCfg, Se3SpaceMouseCfg from isaaclab.devices.openxr import remove_camera_configs -from isaaclab.devices.teleop_device_factory import create_teleop_device from isaaclab.envs import ManagerBasedRLEnvCfg from isaaclab.managers import TerminationTermCfg as DoneTerm -from isaaclab.utils import instantiate import isaaclab_tasks # noqa: F401 from isaaclab_tasks.core.lift import mdp @@ -144,15 +143,6 @@ }.get(args_cli.cloudxr_env.lower(), args_cli.cloudxr_env) -def _builtin_device_cfgs(sensitivity: float) -> dict[str, DeviceCfg]: - """The built-in devices, by ``--teleop_device`` name, for names the environment does not declare.""" - return { - "keyboard": Se3KeyboardCfg(pos_sensitivity=0.05 * sensitivity, rot_sensitivity=0.05 * sensitivity), - "spacemouse": Se3SpaceMouseCfg(pos_sensitivity=0.05 * sensitivity, rot_sensitivity=0.05 * sensitivity), - "gamepad": Se3GamepadCfg(pos_sensitivity=0.1 * sensitivity, rot_sensitivity=0.1 * sensitivity), - } - - def main() -> None: # noqa: C901 """ Run teleoperation with an Isaac Lab manipulation environment. @@ -181,16 +171,24 @@ def main() -> None: # noqa: C901 # add termination condition for reaching the goal otherwise the environment won't reset env_cfg.terminations.object_reached_goal = DoneTerm(func=mdp.object_reached_goal) - from isaaclab_teleop.device_selection import select_isaac_teleop_cfg - - # Isaac Capture serves the request when it can: the env's isaac_teleop pipeline, or the - # keyboard, gamepad or spacemouse it names. Otherwise the isaaclab.devices device does. - isaac_teleop_cfg = select_isaac_teleop_cfg( - env_cfg, args_cli.teleop_device, _builtin_device_cfgs(args_cli.sensitivity), args_cli.device + teleop_cfg = select_teleop_cfg( + env_cfg, + args_cli.teleop_device, + { + "keyboard": Se3KeyboardCfg( + pos_sensitivity=0.05 * args_cli.sensitivity, rot_sensitivity=0.05 * args_cli.sensitivity + ), + "spacemouse": Se3SpaceMouseCfg( + pos_sensitivity=0.05 * args_cli.sensitivity, rot_sensitivity=0.05 * args_cli.sensitivity + ), + "gamepad": Se3GamepadCfg( + pos_sensitivity=0.1 * args_cli.sensitivity, rot_sensitivity=0.1 * args_cli.sensitivity + ), + }, + args_cli.device, ) - use_isaac_teleop = isaac_teleop_cfg is not None - if use_isaac_teleop: - env_cfg.isaac_teleop = isaac_teleop_cfg + if isinstance(teleop_cfg, IsaacTeleopCfg): + env_cfg.isaac_teleop = teleop_cfg # XR-rendering setup (camera removal + DLSS) is only needed for the Kit XR # path. Without --xr, Isaac Capture runs standalone (I/O only) and renders @@ -202,13 +200,17 @@ def main() -> None: # noqa: C901 env_cfg = remove_camera_configs(env_cfg) with launch_simulation(env_cfg, args_cli), ExitStack() as cleanup: - run_teleoperation(env_cfg, use_isaac_teleop, cleanup) + run_teleoperation(env_cfg, teleop_cfg, cleanup) -def run_teleoperation(env_cfg: ManagerBasedRLEnvCfg, use_isaac_teleop: bool, cleanup: ExitStack) -> None: # noqa: C901 +def run_teleoperation( + env_cfg: ManagerBasedRLEnvCfg, teleop_cfg: IsaacTeleopCfg | DeviceCfg, cleanup: ExitStack +) -> None: # noqa: C901 """Create the environment and teleop device, then run the teleoperation loop until the app is closed.""" from isaaclab_physx.renderers.isaac_rtx_renderer_utils import apply_isaac_rtx_global_settings - from isaaclab_teleop import XrCameraFeedSession + from isaaclab_teleop import XrCameraFeedSession, poll_control_events + + use_isaac_teleop = isinstance(teleop_cfg, IsaacTeleopCfg) camera_feed_session = XrCameraFeedSession.prepare( env_cfg, @@ -307,45 +309,17 @@ def stop_teleoperation() -> None: teleoperation_active = True try: - if use_isaac_teleop: - from isaaclab_teleop import create_isaac_teleop_device, poll_control_events - - teleop_interface = create_isaac_teleop_device( - env_cfg.isaac_teleop, - sim_device=args_cli.device, - callbacks=teleoperation_callbacks, - cloudxr_env_file=args_cli.cloudxr_env, - auto_launch_cloudxr=args_cli.auto_launch_cloudxr, - enable_debug_visualization=args_cli.enable_debug_visualization, - use_kit_xr_bridge=args_cli.xr, - haptic_cfg=getattr(env_cfg, "haptic_feedback", None), - ) - - else: - # Task-declared devices take precedence only when explicitly selected. - if args_cli.teleop_device is not None and args_cli.teleop_device in env_cfg.teleop_devices.devices: - teleop_interface = create_teleop_device( - args_cli.teleop_device, env_cfg.teleop_devices.devices, teleoperation_callbacks - ) - else: - device_name = "keyboard" if args_cli.teleop_device is None else args_cli.teleop_device.lower() - device_cfg = _builtin_device_cfgs(args_cli.sensitivity).get(device_name) - if device_cfg is None: - logger.error( - f"--teleop_device={args_cli.teleop_device} was passed but no matching entry exists in" - " env_cfg.teleop_devices and it is not a built-in device name. Either remove" - " --teleop_device to use the Isaac Capture pipeline, or add a" - f" '{args_cli.teleop_device}' entry under teleop_devices in the environment config." - " Built-in devices: keyboard, spacemouse, gamepad." - ) - env.close() - return - teleop_interface = instantiate(device_cfg) - for key, callback in teleoperation_callbacks.items(): - try: - teleop_interface.add_callback(key, callback) - except (ValueError, TypeError) as e: - logger.warning(f"Failed to add callback for key {key}: {e}") + teleop_interface = create_teleop_device( + teleop_cfg, + sim_device=args_cli.device, + callbacks=teleoperation_callbacks, + visualizers=env.sim.visualizers, + cloudxr_env_file=args_cli.cloudxr_env, + auto_launch_cloudxr=args_cli.auto_launch_cloudxr, + enable_debug_visualization=args_cli.enable_debug_visualization, + use_kit_xr_bridge=args_cli.xr, + haptic_cfg=getattr(env_cfg, "haptic_feedback", None), + ) except Exception as e: logger.error(f"Failed to create teleop device: {e}") env.close() diff --git a/scripts/tools/record_demos.py b/scripts/tools/record_demos.py index 01e8689e6347..5975ac7a5ad6 100644 --- a/scripts/tools/record_demos.py +++ b/scripts/tools/record_demos.py @@ -169,21 +169,19 @@ import logging import os import time -from collections.abc import Callable import gymnasium as gym import torch from isaaclab_physx.renderers import IsaacRtxRendererGlobalSettingsCfg -from isaaclab_teleop import CLOUDXR_AVP_ENV, CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV +from isaaclab_teleop import CLOUDXR_AVP_ENV, CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV, IsaacTeleopCfg +from isaaclab_teleop.device_selection import create_teleop_device, select_teleop_cfg from isaaclab.devices import DeviceCfg, Se3KeyboardCfg, Se3SpaceMouseCfg from isaaclab.devices.openxr import remove_camera_configs -from isaaclab.devices.teleop_device_factory import create_teleop_device from isaaclab.envs import DirectRLEnvCfg, ManagerBasedRLEnvCfg from isaaclab.envs.mdp.recorders.recorders_cfg import ActionStateRecorderManagerCfg from isaaclab.envs.utils._manual_success import _prepare_success_term from isaaclab.managers import CommandTermCfg, DatasetExportMode -from isaaclab.utils import instantiate import isaaclab_mimic.envs # noqa: F401 @@ -267,7 +265,7 @@ def setup_output_directories() -> tuple[str, str]: def create_environment_config( output_dir: str, output_file_name: str -) -> tuple[ManagerBasedRLEnvCfg | DirectRLEnvCfg, object | None, bool]: +) -> tuple[ManagerBasedRLEnvCfg | DirectRLEnvCfg, object | None, IsaacTeleopCfg | DeviceCfg]: """Create and configure the environment configuration. Parses the environment configuration and makes necessary adjustments for demo recording. @@ -278,10 +276,10 @@ def create_environment_config( output_file_name: Name of the file to store the demonstrations Returns: - tuple[isaaclab_tasks.utils.parse_cfg.EnvCfg, Optional[object], bool]: A tuple containing: + tuple: A tuple containing: - env_cfg: The configured environment configuration - success_term: The success termination object or None if not available - - use_isaac_teleop: Whether Isaac Capture stack should be used + - teleop_cfg: The resolved Capture or native device configuration Raises: Exception: If parsing the environment configuration fails @@ -296,14 +294,17 @@ def create_environment_config( logger.error(f"Failed to parse environment configuration: {e}") exit(1) - from isaaclab_teleop.device_selection import select_isaac_teleop_cfg - - # Isaac Capture serves the request when it can: the env's isaac_teleop pipeline, or the - # keyboard or spacemouse it names. Otherwise the isaaclab.devices device does. - isaac_teleop_cfg = select_isaac_teleop_cfg(env_cfg, args_cli.teleop_device, _builtin_device_cfgs(), args_cli.device) - use_isaac_teleop = isaac_teleop_cfg is not None - if use_isaac_teleop: - env_cfg.isaac_teleop = isaac_teleop_cfg + teleop_cfg = select_teleop_cfg( + env_cfg, + args_cli.teleop_device, + { + "keyboard": Se3KeyboardCfg(pos_sensitivity=0.2, rot_sensitivity=0.5), + "spacemouse": Se3SpaceMouseCfg(pos_sensitivity=0.2, rot_sensitivity=0.5), + }, + args_cli.device, + ) + if isinstance(teleop_cfg, IsaacTeleopCfg): + env_cfg.isaac_teleop = teleop_cfg # Extract the success condition for manual evaluation in the main loop. Keep # an inert term registered under the same name so rewards and other manager @@ -350,80 +351,7 @@ def create_environment_config( env_cfg.recorders.dataset_filename = output_file_name env_cfg.recorders.dataset_export_mode = DatasetExportMode.EXPORT_SUCCEEDED_ONLY - return env_cfg, success_term, use_isaac_teleop - - -def _builtin_device_cfgs() -> dict[str, DeviceCfg]: - """The built-in devices, by ``--teleop_device`` name, for names the environment does not declare.""" - return { - "keyboard": Se3KeyboardCfg(pos_sensitivity=0.2, rot_sensitivity=0.5), - "spacemouse": Se3SpaceMouseCfg(pos_sensitivity=0.2, rot_sensitivity=0.5), - } - - -def setup_teleop_device( - env_cfg: ManagerBasedRLEnvCfg, callbacks: dict[str, Callable], use_isaac_teleop: bool = False -) -> object: - """Set up the teleoperation device based on configuration. - - Attempts to create a teleoperation device based on the environment configuration. - Falls back to default devices if the specified device is not found in the configuration. - - Args: - env_cfg: Environment configuration containing the selected teleop devices. - callbacks: Dictionary mapping callback keys to functions that will be - attached to the teleop device - use_isaac_teleop: Whether to use Isaac Capture stack instead of native stack - - Returns: - object: The configured teleoperation device interface - - Raises: - Exception: If teleop device creation fails - """ - try: - if use_isaac_teleop: - from isaaclab_teleop import create_isaac_teleop_device - - teleop_interface = create_isaac_teleop_device( - env_cfg.isaac_teleop, - sim_device=args_cli.device, - callbacks=callbacks, - cloudxr_env_file=args_cli.cloudxr_env, - auto_launch_cloudxr=args_cli.auto_launch_cloudxr, - use_kit_xr_bridge=args_cli.xr, - mcap_record_path=args_cli.mcap_record_path, - enable_debug_visualization=args_cli.enable_debug_visualization, - haptic_cfg=getattr(env_cfg, "haptic_feedback", None), - ) - if args_cli.mcap_record_path is not None: - logger.info("Recording live Isaac Capture session to MCAP (debug-only): %s", args_cli.mcap_record_path) - - else: - if args_cli.teleop_device is not None and args_cli.teleop_device in env_cfg.teleop_devices.devices: - teleop_interface = create_teleop_device( - args_cli.teleop_device, env_cfg.teleop_devices.devices, callbacks - ) - else: - device_name = "keyboard" if args_cli.teleop_device is None else args_cli.teleop_device.lower() - device_cfg = _builtin_device_cfgs().get(device_name) - if device_cfg is None: - logger.error( - f"--teleop_device={args_cli.teleop_device} was passed but no matching entry exists in" - " env_cfg.teleop_devices and it is not a built-in device name. Either remove" - " --teleop_device to use the Isaac Capture pipeline, or add a" - f" '{args_cli.teleop_device}' entry under teleop_devices in the environment config." - " Built-in devices: keyboard, spacemouse." - ) - exit(1) - teleop_interface = instantiate(device_cfg) - for key, callback in callbacks.items(): - teleop_interface.add_callback(key, callback) - except Exception as e: - logger.error(f"Failed to create teleop device: {e}") - exit(1) - - return teleop_interface + return env_cfg, success_term, teleop_cfg def setup_ui(label_text: str, env: gym.Env) -> "InstructionDisplay": @@ -531,7 +459,7 @@ def run_simulation_loop( # noqa: C901 success_term: object | None, rate_limiter: RateLimiter | None, camera_feed_session: "XrCameraFeedSession", - use_isaac_teleop: bool = False, + teleop_cfg: IsaacTeleopCfg | DeviceCfg, ) -> int: """Run the main simulation loop for collecting demonstrations. @@ -544,11 +472,12 @@ def run_simulation_loop( # noqa: C901 success_term: The success termination object or None if not available rate_limiter: Optional rate limiter to control simulation speed camera_feed_session: Shared XR camera-feed lifecycle - use_isaac_teleop: Whether to use Isaac Capture stack + teleop_cfg: The resolved Capture or native device configuration. Returns: int: Number of successful demonstrations recorded """ + use_isaac_teleop = isinstance(teleop_cfg, IsaacTeleopCfg) current_recorded_demo_count = 0 success_step_count = 0 should_reset_recording_instance = False @@ -590,7 +519,24 @@ def stop_recording_instance(): "RESET": reset_recording_instance, } - teleop_interface = setup_teleop_device(env.cfg, teleoperation_callbacks, use_isaac_teleop) + try: + teleop_interface = create_teleop_device( + teleop_cfg, + sim_device=args_cli.device, + callbacks=teleoperation_callbacks, + visualizers=env.sim.visualizers, + cloudxr_env_file=args_cli.cloudxr_env, + auto_launch_cloudxr=args_cli.auto_launch_cloudxr, + use_kit_xr_bridge=args_cli.xr, + mcap_record_path=args_cli.mcap_record_path, + enable_debug_visualization=args_cli.enable_debug_visualization, + haptic_cfg=getattr(env.cfg, "haptic_feedback", None), + ) + except Exception as e: + logger.error(f"Failed to create teleop device: {e}") + exit(1) + if use_isaac_teleop and args_cli.mcap_record_path is not None: + logger.info("Recording live Isaac Capture session to MCAP (debug-only): %s", args_cli.mcap_record_path) haptic_driver = None if use_isaac_teleop: @@ -746,19 +692,24 @@ def main() -> None: output_dir, output_file_name = setup_output_directories() # Create and configure environment - env_cfg, success_term, use_isaac_teleop = create_environment_config(output_dir, output_file_name) + env_cfg, success_term, teleop_cfg = create_environment_config(output_dir, output_file_name) with launch_simulation(env_cfg, args_cli), contextlib.ExitStack() as cleanup: - record_demos(env_cfg, success_term, use_isaac_teleop, cleanup) + record_demos(env_cfg, success_term, teleop_cfg, cleanup) def record_demos( - env_cfg: ManagerBasedRLEnvCfg, success_term: object | None, use_isaac_teleop: bool, cleanup: contextlib.ExitStack + env_cfg: ManagerBasedRLEnvCfg, + success_term: object | None, + teleop_cfg: IsaacTeleopCfg | DeviceCfg, + cleanup: contextlib.ExitStack, ) -> None: """Create the environment and record demonstrations until the session ends.""" from isaaclab_physx.renderers.isaac_rtx_renderer_utils import apply_isaac_rtx_global_settings from isaaclab_teleop import XrCameraFeedSession + use_isaac_teleop = isinstance(teleop_cfg, IsaacTeleopCfg) + # Apply the RTX/DLSS global settings when an RTX render pipeline will run (Kit visualizer, # external cameras, or XR). if not args_cli.disable_external_cameras or args_cli.kit_visualizer or args_cli.xr: @@ -799,9 +750,7 @@ def record_demos( exit(1) # Run simulation loop - current_recorded_demo_count = run_simulation_loop( - env, success_term, rate_limiter, camera_feed_session, use_isaac_teleop - ) + current_recorded_demo_count = run_simulation_loop(env, success_term, rate_limiter, camera_feed_session, teleop_cfg) # Clean up env.close() diff --git a/source/isaaclab_teleop/changelog.d/explicit-window-input.changed.rst b/source/isaaclab_teleop/changelog.d/explicit-window-input.changed.rst new file mode 100644 index 000000000000..d6ccfcd16ad3 --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/explicit-window-input.changed.rst @@ -0,0 +1,2 @@ +* Passed visualizer windows explicitly into Capture sessions and centralized script device + selection and creation. Window input attachments rolled back partial setup on failure. diff --git a/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.added.rst b/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.added.rst index 64c1bb7f43d6..79f309c41c49 100644 --- a/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.added.rst +++ b/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.added.rst @@ -8,5 +8,6 @@ ``gamepad`` and ``spacemouse`` plugins, which their configs start with the session. Without ``--xr`` they run headset-free on the standalone CloudXR profile. * Added :mod:`isaaclab_teleop.device_selection`, which translates an :mod:`isaaclab.devices` - keyboard, gamepad or SpaceMouse config into the equivalent pipeline and picks the pipeline for a - script's ``--teleop_device`` request, or ``None`` when ``isaacteleop`` is not installed. + keyboard, gamepad or SpaceMouse config into the equivalent pipeline and resolves each script's + ``--teleop_device`` request once. Device creation retained the native config and factory when + Capture was unavailable or the requested device required the native implementation. diff --git a/source/isaaclab_teleop/isaaclab_teleop/device_selection.py b/source/isaaclab_teleop/isaaclab_teleop/device_selection.py index 03f66d446acd..b378fbcc1f94 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/device_selection.py +++ b/source/isaaclab_teleop/isaaclab_teleop/device_selection.py @@ -3,34 +3,24 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Serve a script's teleop device request with an Isaac Capture pipeline when one can. - -An environment declares its keyboard, gamepad and SpaceMouse devices with the -:mod:`isaaclab.devices` configs (:class:`~isaaclab.devices.Se3KeyboardCfg`, ...). -:func:`select_isaac_teleop_cfg` translates such a config into the equivalent Isaac Capture -pipeline. Without the ``isaacteleop`` package, or for a device it has no pipeline for (e.g. OpenXR -hand tracking), it returns ``None`` and the caller uses the :mod:`isaaclab.devices` device. -""" +"""Resolve and create script teleop devices through one Capture/native selection boundary.""" from __future__ import annotations import importlib.util -from collections.abc import Mapping -from typing import TYPE_CHECKING, Any +from collections.abc import Callable, Mapping +from typing import Any +from isaaclab.devices import DeviceCfg from isaaclab.devices.gamepad import Se2GamepadCfg, Se3GamepadCfg from isaaclab.devices.keyboard import Se2KeyboardCfg, Se3KeyboardCfg from isaaclab.devices.spacemouse import Se2SpaceMouseCfg, Se3SpaceMouseCfg from .gamepad import se2_gamepad_teleop_cfg, se3_gamepad_teleop_cfg +from .isaac_teleop_cfg import IsaacTeleopCfg from .keyboard import se2_keyboard_teleop_cfg, se3_keyboard_teleop_cfg from .spacemouse import se2_spacemouse_teleop_cfg, se3_spacemouse_teleop_cfg -if TYPE_CHECKING: - from isaaclab.devices import DeviceCfg - - from .isaac_teleop_cfg import IsaacTeleopCfg - # Each device config type, the builder of its pipeline, and the config fields the builder takes. _PIPELINES = { Se3KeyboardCfg: (se3_keyboard_teleop_cfg, ("pos_sensitivity", "rot_sensitivity", "gripper_term")), @@ -45,59 +35,50 @@ } -def _isaacteleop_available() -> bool: - return importlib.util.find_spec("isaacteleop") is not None - - -def pipeline_from_device_cfg(device_cfg: DeviceCfg, sim_device: str) -> IsaacTeleopCfg | None: - """Build the Isaac Capture pipeline equivalent to a device config. - - Args: - device_cfg: A device config, e.g. :class:`~isaaclab.devices.Se3KeyboardCfg`. - sim_device: Torch device for the pipeline's output tensors. - - Returns: - The pipeline, or ``None`` when no pipeline serves this device config type. - """ - entry = _PIPELINES.get(type(device_cfg)) - if entry is None: - return None - builder, fields = entry - return builder(**{field: getattr(device_cfg, field) for field in fields}, sim_device=sim_device) - - -def select_isaac_teleop_cfg( +def select_teleop_cfg( env_cfg: Any, device_name: str | None, default_device_cfgs: Mapping[str, DeviceCfg], sim_device: str, -) -> IsaacTeleopCfg | None: - """Pick the Isaac Capture pipeline that serves a script's device request. +) -> IsaacTeleopCfg | DeviceCfg: + """Resolve a device once before scene startup, retaining native configs when Capture is unavailable. - Without ``device_name``, the environment's declared ``isaac_teleop`` pipeline is used, else - the ``"keyboard"`` entry of ``default_device_cfgs``. With it, the environment's - ``teleop_devices`` entry of that name is used, else the ``default_device_cfgs`` entry. - - Args: - env_cfg: The environment config, read for ``isaac_teleop`` and ``teleop_devices``. - device_name: The requested device (e.g. ``--teleop_device``), or ``None``. - default_device_cfgs: The script's own device configs, by name, for devices the - environment does not declare. - sim_device: Torch device for the pipeline's output tensors. - - Returns: - The pipeline, or ``None`` when ``isaacteleop`` is not installed or no pipeline serves the - requested device; the caller then uses the :mod:`isaaclab.devices` device. + An unnamed request prefers the environment's Capture pipeline, then its keyboard or the script's default. + A named request prefers the environment's device, then the script's built-ins. Unknown names raise + before launching the simulation. Native devices and their retargeters remain supported. """ - if not _isaacteleop_available(): - return None + capture_available = importlib.util.find_spec("isaacteleop") is not None if device_name is None: declared = getattr(env_cfg, "isaac_teleop", None) - if declared is not None: + if capture_available and declared is not None: return declared device_name = "keyboard" - env_devices = getattr(getattr(env_cfg, "teleop_devices", None), "devices", None) or {} - device_cfg = env_devices.get(device_name) or default_device_cfgs.get(device_name) + devices = getattr(env_cfg, "teleop_devices", None) + device_cfg = devices.devices.get(device_name) if devices is not None else None + if device_cfg is None: + device_cfg = default_device_cfgs.get(device_name.lower()) if device_cfg is None: - return None - return pipeline_from_device_cfg(device_cfg, sim_device) + raise ValueError(f"Unknown teleop device: {device_name!r}") + entry = _PIPELINES.get(type(device_cfg)) if capture_available else None + if entry is None: + return device_cfg + builder, fields = entry + return builder(**{field: getattr(device_cfg, field) for field in fields}, sim_device=sim_device) + + +def create_teleop_device( + cfg: IsaacTeleopCfg | DeviceCfg, callbacks: Mapping[str, Callable] | None = None, **capture_options +): + """Create the resolved device; Capture errors propagate instead of switching implementations. + + The existing native factory continues to construct native retargeters and register callbacks. + Capture options, including the explicit visualizer sequence, apply only to Capture devices. + """ + if isinstance(cfg, IsaacTeleopCfg): + from .isaac_teleop_device import create_isaac_teleop_device + + return create_isaac_teleop_device(cfg, callbacks=callbacks, **capture_options) + from .deprecated.teleop_device_factory import create_teleop_device as create_native_device + + name = type(cfg).__name__ + return create_native_device(name, {name: cfg}, callbacks) diff --git a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py index e7546ee50777..1d21c0e58aa0 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py +++ b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py @@ -8,7 +8,7 @@ from __future__ import annotations import logging -from collections.abc import Callable +from collections.abc import Callable, Sequence from typing import TYPE_CHECKING, ClassVar import numpy as np @@ -16,6 +16,7 @@ from scipy.spatial.transform import Rotation from isaaclab.sim.utils.stage import get_current_stage_id +from isaaclab.visualizers.base_visualizer import BaseVisualizer from .command_handler import CommandHandler from .control_events import ControlEvents @@ -89,20 +90,20 @@ class IsaacTeleopDevice: ) # Poses in world frame (default) - with IsaacTeleopDevice(cfg) as device: + with IsaacTeleopDevice(cfg, visualizers=env.sim.visualizers) as device: while running: action = device.advance() env.step(action.repeat(num_envs, 1)) # Config-driven rebase into robot base frame cfg.target_frame_prim_path = "/World/Robot/base_link" - with IsaacTeleopDevice(cfg) as device: + with IsaacTeleopDevice(cfg, visualizers=env.sim.visualizers) as device: while running: action = device.advance() env.step(action.repeat(num_envs, 1)) # Explicit rebase into robot base frame - with IsaacTeleopDevice(cfg) as device: + with IsaacTeleopDevice(cfg, visualizers=env.sim.visualizers) as device: while running: robot_T_world = get_robot_base_transform() action = device.advance(target_T_world=robot_T_world) @@ -124,6 +125,7 @@ def __init__( mcap_replay_path: str | None = None, enable_debug_visualization: bool = False, haptic_cfg: HapticFeedbackCfg | None = None, + visualizers: Sequence[BaseVisualizer] = (), ): """Initialize the Isaac Capture device. @@ -160,6 +162,8 @@ def __init__( the device renders per-hand output vectors pushed via :meth:`send_haptic` on the configured device (controller, glove, ...). ``None`` disables haptics. + visualizers: Live sequence of windows supplying keyboard input, usually ``sim.visualizers``. + The default empty sequence disables window input. """ self._cfg = cfg self._target_world_matrix = None @@ -175,6 +179,7 @@ def __init__( mcap_replay_path=mcap_replay_path, enable_debug_visualization=enable_debug_visualization, haptic_cfg=haptic_cfg, + visualizers=visualizers, ) self._prev_right_a_pressed = False @@ -548,6 +553,7 @@ def create_isaac_teleop_device( mcap_replay_path: str | None = None, enable_debug_visualization: bool = False, haptic_cfg: HapticFeedbackCfg | None = None, + visualizers: Sequence[BaseVisualizer] = (), ) -> IsaacTeleopDevice: """Create an :class:`IsaacTeleopDevice` with required Omniverse extension setup. @@ -602,6 +608,8 @@ def create_isaac_teleop_device( :class:`~isaaclab_teleop.HapticFeedbackReceiver` and renders per-hand output vectors on the configured device (controller, glove, ...). + visualizers: Live sequence of windows supplying keyboard input, usually ``sim.visualizers``. + Returns: A fully configured :class:`IsaacTeleopDevice` ready for use in a ``with`` block. @@ -635,6 +643,7 @@ def create_isaac_teleop_device( mcap_replay_path=mcap_replay_path, enable_debug_visualization=enable_debug_visualization, haptic_cfg=haptic_cfg, + visualizers=visualizers, ) if callbacks is not None: diff --git a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py index b6864fa28ade..6933813d7581 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py +++ b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py @@ -12,12 +12,15 @@ import logging import os import time -from collections.abc import Callable +from collections.abc import Callable, Sequence +from contextlib import ExitStack from typing import TYPE_CHECKING, Any, Protocol import numpy as np import torch +from isaaclab.visualizers.base_visualizer import BaseVisualizer + if TYPE_CHECKING: from isaacteleop.cloudxr import CloudXRLauncher from isaacteleop.oxr import OpenXRSessionHandles @@ -265,6 +268,7 @@ def __init__( mcap_replay_path: str | None = None, enable_debug_visualization: bool = False, haptic_cfg: HapticFeedbackCfg | None = None, + visualizers: Sequence[BaseVisualizer] = (), ): """Initialize the session lifecycle manager. @@ -305,6 +309,8 @@ def __init__( per-hand output vectors pushed via :meth:`push_haptic` are rendered on the configured device (controller, glove, ...). ``None`` disables haptics entirely. + visualizers: Live sequence of windows supplying keyboard input. Pass ``sim.visualizers``; + the default empty sequence disables window input. Raises: ValueError: If both *mcap_record_path* and *mcap_replay_path* @@ -358,11 +364,9 @@ def __init__( # Fallback for host-initiated resets when no control pipeline is configured self._pending_reset = False - # Keyboards: the visualizer windows feeding the pipeline's keyboards, and the operator's key - # presses, read straight from those windows so each press is consumed once whatever the - # retargeting mode. - self._keyboard_detachers: list[Callable[[], None]] = [] - self._attached_visualizers: list[Any] = [] + # Keep the supplied live sequence so windows initialized later can join the session. + self._visualizers = visualizers + self._keyboard_attachments: dict[BaseVisualizer, ExitStack | None] = {} self._pipeline_keyboards: list[Any] | None = None # found once per session self._pressed_keys: list[int] = [] @@ -739,34 +743,28 @@ def _attach_keyboards(self) -> None: """ if self._pipeline is None or self._is_replay: return - try: - from isaaclab.sim import SimulationContext - - sim = SimulationContext.instance() - except Exception: - return - if sim is None: - return if self._pipeline_keyboards is None: from isaacteleop.retargeting_engine.deviceio_source_nodes import KeyboardSource, find_sources self._pipeline_keyboards = find_sources(self._pipeline, KeyboardSource) keyboards = self._pipeline_keyboards - for visualizer in sim.visualizers: + for visualizer in self._visualizers: # a visualizer without a window that reports keys has no source source = visualizer.key_event_source - if source is None or any(visualizer is attached for attached in self._attached_visualizers): + if source is None or visualizer in self._keyboard_attachments: continue # attached even if this fails, so a failing window is not retried every step - self._attached_visualizers.append(visualizer) + self._keyboard_attachments[visualizer] = None try: - for keyboard in keyboards: - self._keyboard_detachers.append(keyboard.attach(source, capture=False).close) - if keyboards: - visualizer.set_camera_keys_enabled(False) - self._keyboard_detachers.append(lambda v=visualizer: v.set_camera_keys_enabled(True)) - listener = _ControlKeyListener(self._pressed_keys) - self._keyboard_detachers.append(source.add_key_listener(listener.on_key, listener.on_focus_lost).close) + with ExitStack() as attachments: + for keyboard in keyboards: + attachments.callback(keyboard.attach(source, capture=False).close) + if keyboards: + attachments.callback(visualizer.set_camera_keys_enabled, True) + visualizer.set_camera_keys_enabled(False) + listener = _ControlKeyListener(self._pressed_keys) + attachments.callback(source.add_key_listener(listener.on_key, listener.on_focus_lost).close) + self._keyboard_attachments[visualizer] = attachments.pop_all() except Exception as e: logger.warning(f"Keyboard input from {type(visualizer).__name__} is unavailable: {e}") continue @@ -774,13 +772,14 @@ def _attach_keyboards(self) -> None: def _detach_keyboards(self) -> None: """Detach every visualizer window, releasing held keys and restoring keyboard camera movement.""" - detachers, self._keyboard_detachers = self._keyboard_detachers, [] - for detach in detachers: + attachments, self._keyboard_attachments = self._keyboard_attachments, {} + for attachment in attachments.values(): + if attachment is None: + continue try: - detach() + attachment.close() except Exception as e: logger.debug(f"Suppressed error detaching a visualizer keyboard: {e}") - self._attached_visualizers = [] self._pipeline_keyboards = None def drain_pressed_keys(self) -> list[int]: diff --git a/source/isaaclab_teleop/test/test_device_selection.py b/source/isaaclab_teleop/test/test_device_selection.py index a106c762e683..971d33fc03ba 100644 --- a/source/isaaclab_teleop/test/test_device_selection.py +++ b/source/isaaclab_teleop/test/test_device_selection.py @@ -7,14 +7,17 @@ from __future__ import annotations +import ast +from pathlib import Path from types import SimpleNamespace import isaaclab_teleop.device_selection as device_selection import pytest from isaaclab_teleop import IsaacTeleopCfg -from isaaclab_teleop.device_selection import pipeline_from_device_cfg, select_isaac_teleop_cfg +from isaaclab_teleop.device_selection import create_teleop_device, select_teleop_cfg from isaaclab.devices import ( + DeviceBase, DeviceCfg, DevicesCfg, Se2GamepadCfg, @@ -50,7 +53,7 @@ def builder(name=routed.__name__, **kwargs): return f"pipeline:{name}" monkeypatch.setitem(device_selection._PIPELINES, cfg_type, (builder, fields)) - monkeypatch.setattr(device_selection, "_isaacteleop_available", lambda: True) + monkeypatch.setattr(device_selection.importlib.util, "find_spec", lambda _: object()) return calls @@ -76,7 +79,7 @@ class TestPipelineFromDeviceCfg: def test_every_legacy_device_reads_its_own_device(self, cfg_type, source_name): nodes = pytest.importorskip("isaacteleop.retargeting_engine.deviceio_source_nodes") - cfg = pipeline_from_device_cfg(cfg_type(), "cpu") + cfg = select_teleop_cfg(_env_cfg(), "device", {"device": cfg_type()}, "cpu") assert isinstance(cfg, IsaacTeleopCfg) assert len(nodes.find_sources(cfg.pipeline_builder(), getattr(nodes, source_name))) == 1 @@ -112,42 +115,115 @@ def test_every_legacy_device_reads_its_own_device(self, cfg_type, source_name): ids=lambda value: type(value).__name__ if not isinstance(value, dict) else "", ) def test_config_reaches_its_builder_with_every_setting(self, recorded, device_cfg, expected): - pipeline_from_device_cfg(device_cfg, "cuda:0") + select_teleop_cfg(_env_cfg(), "device", {"device": device_cfg}, "cuda:0") assert recorded == [(BUILDERS[type(device_cfg)], {**expected, "sim_device": "cuda:0"})] def test_unsupported_config_has_no_pipeline(self): - assert pipeline_from_device_cfg(DeviceCfg(), "cpu") is None + cfg = DeviceCfg() + assert select_teleop_cfg(_env_cfg(), "device", {"device": cfg}, "cpu") is cfg class TestSelectIsaacTeleopCfg: - def test_without_isaacteleop_selects_nothing(self, recorded, monkeypatch): - monkeypatch.setattr(device_selection, "_isaacteleop_available", lambda: False) - - assert select_isaac_teleop_cfg(_env_cfg(isaac_teleop="declared"), None, DEFAULTS, "cpu") is None + def test_without_isaacteleop_selects_native_keyboard(self, recorded, monkeypatch): + monkeypatch.setattr(device_selection.importlib.util, "find_spec", lambda _: None) + cfg = Se3KeyboardCfg(pos_sensitivity=0.9) + env_cfg = _env_cfg(isaac_teleop="declared", keyboard=cfg) + assert select_teleop_cfg(env_cfg, None, DEFAULTS, "cpu") is env_cfg.teleop_devices.devices["keyboard"] def test_declared_pipeline_wins_without_a_device_name(self, recorded): - assert select_isaac_teleop_cfg(_env_cfg(isaac_teleop="declared"), None, DEFAULTS, "cpu") == "declared" + assert select_teleop_cfg(_env_cfg(isaac_teleop="declared"), None, DEFAULTS, "cpu") == "declared" assert recorded == [] def test_default_keyboard_without_a_declared_pipeline(self, recorded): - assert select_isaac_teleop_cfg(_env_cfg(), None, DEFAULTS, "cpu") == "pipeline:se3_keyboard_teleop_cfg" + assert select_teleop_cfg(_env_cfg(), None, DEFAULTS, "cpu") == "pipeline:se3_keyboard_teleop_cfg" assert recorded[0][1]["pos_sensitivity"] == 0.2 + recorded.clear() + env_cfg = _env_cfg(keyboard=Se3KeyboardCfg(pos_sensitivity=0.9)) + select_teleop_cfg(env_cfg, None, DEFAULTS, "cpu") + assert recorded[0][1]["pos_sensitivity"] == 0.9 + def test_environment_device_wins_over_the_default(self, recorded): env_cfg = _env_cfg(isaac_teleop="declared", spacemouse=Se3SpaceMouseCfg(pos_sensitivity=0.9)) - assert select_isaac_teleop_cfg(env_cfg, "spacemouse", DEFAULTS, "cpu") == "pipeline:se3_spacemouse_teleop_cfg" + assert select_teleop_cfg(env_cfg, "spacemouse", DEFAULTS, "cpu") == "pipeline:se3_spacemouse_teleop_cfg" assert recorded[0][1]["pos_sensitivity"] == 0.9 def test_default_serves_a_name_the_environment_lacks(self, recorded): - assert ( - select_isaac_teleop_cfg(_env_cfg(), "spacemouse", DEFAULTS, "cpu") == "pipeline:se3_spacemouse_teleop_cfg" - ) + assert select_teleop_cfg(_env_cfg(), "spacemouse", DEFAULTS, "cpu") == "pipeline:se3_spacemouse_teleop_cfg" assert recorded[0][1]["pos_sensitivity"] == 0.3 - def test_unknown_or_unsupported_device_selects_nothing(self, recorded): + def test_unsupported_device_retains_its_config_and_unknown_name_raises(self, recorded): env_cfg = _env_cfg(handtracking=DeviceCfg()) - assert select_isaac_teleop_cfg(env_cfg, "handtracking", DEFAULTS, "cpu") is None - assert select_isaac_teleop_cfg(env_cfg, "joystick", DEFAULTS, "cpu") is None + assert ( + select_teleop_cfg(env_cfg, "handtracking", DEFAULTS, "cpu") + is env_cfg.teleop_devices.devices["handtracking"] + ) + with pytest.raises(ValueError, match="joystick"): + select_teleop_cfg(env_cfg, "joystick", DEFAULTS, "cpu") + + +class _NativeDevice(DeviceBase): + def __init__(self, cfg): + super().__init__() + self.cfg = cfg + self.callbacks = {} + + def reset(self): + pass + + def add_callback(self, key, callback): + self.callbacks[key] = callback + + +def test_creation_uses_the_resolved_config_for_both_implementations(): + """Native callbacks and explicitly supplied Capture windows reach the selected device.""" + cfg = DeviceCfg(class_type=_NativeDevice) + + def callback(): + pass + + native = create_teleop_device(cfg, callbacks={"R": callback}, use_kit_xr_bridge=False) + assert native.cfg is cfg and native.callbacks == {"R": callback} + + from isaaclab_teleop.isaac_teleop_device import IsaacTeleopDevice + + windows = [] + capture = create_teleop_device(IsaacTeleopCfg(), visualizers=windows, use_kit_xr_bridge=False) + assert isinstance(capture, IsaacTeleopDevice) + assert capture._session_lifecycle._visualizers is windows + + +def test_capture_creation_failure_does_not_switch_to_a_native_device(monkeypatch): + from isaaclab_teleop import isaac_teleop_device + + def fail(*args, **kwargs): + raise RuntimeError("Capture initialization failed") + + monkeypatch.setattr(isaac_teleop_device, "create_isaac_teleop_device", fail) + with pytest.raises(RuntimeError, match="Capture initialization failed"): + create_teleop_device(IsaacTeleopCfg()) + + +def test_window_dependencies_and_device_creation_have_one_owner(): + """Keep global window discovery and repeated device-selection branches out of the session/scripts.""" + root = Path(__file__).resolve().parents[3] + session = ast.parse((root / "source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py").read_text()) + assert not any(isinstance(node, ast.Name) and node.id == "SimulationContext" for node in ast.walk(session)) + assert not any( + isinstance(node, ast.Attribute) and node.attr in ("_attached_visualizers", "_keyboard_detachers") + for node in ast.walk(session) + ) + for script in ("scripts/environments/teleoperation/teleop_se3_agent.py", "scripts/tools/record_demos.py"): + tree = ast.parse((root / script).read_text()) + assert not any( + isinstance(node, ast.FunctionDef) and node.name in ("_builtin_device_cfgs", "setup_teleop_device") + for node in ast.walk(tree) + ) + calls = [node for node in ast.walk(tree) if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)] + for name in ("select_teleop_cfg", "create_teleop_device"): + assert sum(node.func.id == name for node in calls) == 1, (script, name) + factory = next(node for node in calls if node.func.id == "create_teleop_device") + assert any(keyword.arg == "visualizers" for keyword in factory.keywords), script diff --git a/source/isaaclab_teleop/test/test_keyboard_session.py b/source/isaaclab_teleop/test/test_keyboard_session.py index 781921324ff7..44eca25882f7 100644 --- a/source/isaaclab_teleop/test/test_keyboard_session.py +++ b/source/isaaclab_teleop/test/test_keyboard_session.py @@ -136,13 +136,11 @@ def blur(self) -> None: self.key_event_source._emit_focus_lost() -def _start(lifecycle, *visualizers): +def _start(lifecycle): """Start a lifecycle reading *visualizers*, without launching CloudXR, a system check or OpenXR.""" import isaacteleop.teleop_session_manager as tsm - sim = SimpleNamespace(visualizers=[*visualizers, SimpleNamespace(key_event_source=None)]) with ( - patch("isaaclab.sim.SimulationContext.instance", return_value=sim), patch.object(lifecycle, "_ensure_cloudxr_runtime") as ensure_cloudxr, patch.object(lifecycle, "_run_system_check"), patch.object(tsm, "TeleopSession", MagicMock()), @@ -152,8 +150,8 @@ def _start(lifecycle, *visualizers): def _started_lifecycle(cfg, *visualizers, **kwargs): - lifecycle = TeleopSessionLifecycle(cfg, use_kit_xr_bridge=False, **kwargs) - return lifecycle, _start(lifecycle, *visualizers) + lifecycle = TeleopSessionLifecycle(cfg, use_kit_xr_bridge=False, visualizers=list(visualizers), **kwargs) + return lifecycle, _start(lifecycle) @pytest.fixture @@ -177,11 +175,19 @@ def test_keyboard_session_takes_the_standard_path(visualizer): def test_visualizer_feeds_the_keyboard_and_the_control_keys(visualizer): """The pipeline's keyboard and the control keys read the window; its camera keys are off until stop.""" - lifecycle, _ = _started_lifecycle(_keyboard_cfg(), visualizer) + windows = [visualizer] + lifecycle = TeleopSessionLifecycle(_keyboard_cfg(), use_kit_xr_bridge=False, visualizers=windows) + _start(lifecycle) assert visualizer.listener_count == 2 assert visualizer.camera_keys == [False] + later = _Visualizer() + windows.append(later) + lifecycle._attach_keyboards() + lifecycle._attach_keyboards() + assert later.listener_count == 2 and later.camera_keys == [False] lifecycle.stop() + assert later.listener_count == 0 and later.camera_keys == [False, True] assert visualizer.listener_count == 0 assert visualizer.camera_keys == [False, True] @@ -220,7 +226,7 @@ def test_presses_do_not_carry_into_the_next_session(visualizer): lifecycle, _ = _started_lifecycle(_keyboard_cfg(), visualizer) visualizer.tap("KeyR") lifecycle.stop() - _start(lifecycle, visualizer) + _start(lifecycle) try: visualizer.press("KeyR") # the stale tap is gone; a new press counts assert lifecycle.drain_pressed_keys() == [KEY_R] @@ -244,9 +250,15 @@ def test_another_window_losing_focus_does_not_repeat_a_held_key(): def test_a_failing_window_does_not_stop_the_session(visualizer): """A window whose keyboard cannot be attached is skipped with a warning; the others still work.""" broken = _Visualizer() - broken.set_camera_keys_enabled = MagicMock(side_effect=RuntimeError("no window")) + + def set_camera_keys(enabled): + if not enabled: + raise RuntimeError("no window") + + broken.set_camera_keys_enabled = set_camera_keys lifecycle, _ = _started_lifecycle(_keyboard_cfg(), broken, visualizer) try: + assert broken.listener_count == 0 visualizer.tap("KeyB") assert lifecycle.drain_pressed_keys() == [KEY_B] finally: