diff --git a/docs/source/api/lab/isaaclab.visualizers.rst b/docs/source/api/lab/isaaclab.visualizers.rst index 2ebb4b75f9b8..f18253c6f9de 100644 --- a/docs/source/api/lab/isaaclab.visualizers.rst +++ b/docs/source/api/lab/isaaclab.visualizers.rst @@ -40,6 +40,8 @@ The following classes are part of the public :mod:`isaaclab.visualizers` API. .. autoclass:: KeyboardSubscription :members: closed, close +.. autofunction:: add_key_callbacks + .. autoclass:: VisualizerCfg :show-inheritance: diff --git a/docs/source/api/lab_teleop/isaaclab_teleop.rst b/docs/source/api/lab_teleop/isaaclab_teleop.rst index 370b9bc56cf1..48fcec1918c1 100644 --- a/docs/source/api/lab_teleop/isaaclab_teleop.rst +++ b/docs/source/api/lab_teleop/isaaclab_teleop.rst @@ -30,6 +30,12 @@ isaaclab_teleop create_isaac_teleop_device create_haptic_feedback_driver remove_camera_configs + se2_gamepad_teleop_cfg + se2_keyboard_teleop_cfg + se2_spacemouse_teleop_cfg + se3_gamepad_teleop_cfg + se3_keyboard_teleop_cfg + se3_spacemouse_teleop_cfg Configuration ------------- @@ -64,6 +70,27 @@ Device .. autofunction:: create_isaac_teleop_device +Keyboard +-------- + +.. autofunction:: se2_keyboard_teleop_cfg + +.. autofunction:: se3_keyboard_teleop_cfg + +Gamepad +------- + +.. autofunction:: se2_gamepad_teleop_cfg + +.. autofunction:: se3_gamepad_teleop_cfg + +SpaceMouse +---------- + +.. autofunction:: se2_spacemouse_teleop_cfg + +.. autofunction:: se3_spacemouse_teleop_cfg + Haptic Feedback --------------- diff --git a/docs/source/features/imitation-learning/teleop_imitation.rst b/docs/source/features/imitation-learning/teleop_imitation.rst index 42f6096c43ae..233c97737abb 100644 --- a/docs/source/features/imitation-learning/teleop_imitation.rst +++ b/docs/source/features/imitation-learning/teleop_imitation.rst @@ -88,7 +88,8 @@ Teleoperation Isaac Lab supports teleoperation of robots through a variety of input devices including keyboard, SpaceMouse, and XR headsets. -Run the following command to spin up the environment and teleoperate the robot using the keyboard. Familiarize yourself +Run the following command to spin up the environment and teleoperate the robot using the keyboard: click the +visualizer window and type (see :ref:`isaac-teleop-keyboard`). Familiarize yourself with the controls and perform the stacking task. The order of the stacked cubes should be blue (bottom), red (middle), green (top). Once you feel sufficiently comfortable with the controls, you may shutdown the environment by quitting the script with Ctrl+C. @@ -145,12 +146,15 @@ the key bindings are: Move arm along z-axis: Push or pull the SpaceMouse Rotate arm: Twist the SpaceMouse +.. _isaac-teleop-spacemouse-permissions: + .. tip:: - If the SpaceMouse is not detected, you most likely need additional user permissions. The ``hidapi`` - wheel installed by Isaac Lab bundles a backend that talks to the device over ``libusb``, so it needs - read and write access to the USB node under ``/dev/bus/usb`` -- granting access to ``/dev/hidraw*`` - alone is **not** sufficient, and without USB access the device is enumerated without a product name. + If the SpaceMouse is not detected, you most likely need additional user permissions. Isaac Capture + reads the SpaceMouse from its HID node (``/dev/hidraw*``). Without the ``isaacteleop`` package, the + :class:`~isaaclab.devices.Se3SpaceMouse` reads it through the ``hidapi`` wheel's ``libusb`` + backend instead, which needs read and write access to the USB node under ``/dev/bus/usb``; without + it the device is enumerated without a product name. The rule below grants both. Grant the permission by installing a udev rule for the 3Dconnexion vendor id: diff --git a/docs/source/features/isaac_teleop.rst b/docs/source/features/isaac_teleop.rst index c5f492072e1b..707cc9debcb4 100644 --- a/docs/source/features/isaac_teleop.rst +++ b/docs/source/features/isaac_teleop.rst @@ -303,7 +303,8 @@ interchangeable, and :func:`~isaaclab_teleop.poll_control_events` stays authorit A clientless CloudXR runtime advertises no HMD, so the standalone OpenXR session needs a device profile that reports a system without a connected client. The scripts default ``--cloudxr_env`` to :data:`~isaaclab_teleop.CLOUDXR_STANDALONE_ENV` when ``--xr`` is omitted (and to ``cloudxrjs`` - when it is passed); see :ref:`isaac-teleop-cloudxr-profiles`. + when it is passed); see :ref:`isaac-teleop-cloudxr-profiles`. Keyboard, gamepad and SpaceMouse + sessions use the same standalone profile (see :ref:`isaac-teleop-keyboard`). .. _isaac-teleop-so101-leader-example: @@ -343,9 +344,9 @@ Prerequisites sudo apt-get update sudo apt-get install -y build-essential cmake libx11-dev clang-format-14 ccache patchelf - Then clone, check out the release branch matching the ``isaacteleop`` version Isaac Lab is - pinned to (``isaacteleop~=1.4.145`` in the ``teleop`` extra of the root ``pyproject.toml``, so the - plugin's wire format matches the ``isaacteleop`` package Isaac Lab installs), configure, build, + Then clone, check out the release branch matching the ``isaaccapture`` version Isaac Lab is + pinned to (``isaaccapture~=1.7.44a1`` in the ``teleop`` extra of the root ``pyproject.toml``, so the + plugin's wire format matches the ``isaaccapture`` package Isaac Lab installs), configure, build, and install. To target a specific Python version, pass ``-DISAAC_TELEOP_PYTHON_VERSION=3.12`` (or ``3.11``, ``3.13``) on the configure line; each version needs its own build directory if building multiple at once: @@ -354,7 +355,7 @@ Prerequisites git clone https://github.com/NVIDIA/IsaacCapture.git cd IsaacCapture - git checkout release/1.4.x + git checkout release/1.7.x cmake -B build # configure (default: Python 3.11) cmake --build build --parallel # build @@ -611,7 +612,7 @@ Source Nodes Available Retargeters ~~~~~~~~~~~~~~~~~~~~~ -Retargeters are provided by the ``isaacteleop`` package from the +Retargeters are provided by the ``isaaccapture`` package from the `Isaac Capture `_ repository. The retargeters listed below are those used by the built-in Isaac Lab environments. Isaac Capture may offer additional retargeters not listed here -- refer to the @@ -748,13 +749,58 @@ These environments use the Isaac Capture XR pipeline with motion controllers or ideal for tasks that need a gripper or simple hand mapping. **Hand tracking** captures 26 wrist and finger joints per hand, required for dexterous retargeting to complex robot hands. -Keyboard and SpaceMouse Environments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _isaac-teleop-keyboard: + +Keyboard, SpaceMouse, and Gamepad Environments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. note:: - Keyboard and SpaceMouse teleoperation uses the legacy native Isaac Lab teleop stack - (``isaaclab.devices``), not Isaac Capture. These environments do not require an XR headset. + Keyboard, SpaceMouse, and Gamepad teleoperation run through Isaac Capture like every other + input device, driven by a keyboard, spacemouse, or gamepad + :class:`~isaaclab_teleop.IsaacTeleopDevice` pipeline instead of an XR headset -- see + :func:`~isaaclab_teleop.se3_keyboard_teleop_cfg`, :func:`~isaaclab_teleop.se3_spacemouse_teleop_cfg`, + and :func:`~isaaclab_teleop.se3_gamepad_teleop_cfg`. These environments do not require an XR + headset. An environment's ``teleop_devices`` entries (:class:`~isaaclab.devices.Se3KeyboardCfg`, + ...) run through the equivalent pipeline too. Without the ``isaaccapture`` package, the scripts + fall back to the :mod:`isaaclab.devices` keyboard, gamepad and SpaceMouse devices. + +**Keyboard input and window focus.** The keyboard runs inside the Isaac Lab process and reads the +keys typed into whichever visualizer window has focus, like any desktop app: click the window and +type. It needs no ``input`` group membership, ignores keys typed into other apps, and releases +held keys when the window loses focus. Like every Isaac Capture session, a keyboard session opens an +OpenXR session: without ``--xr`` the scripts launch a CloudXR runtime with the standalone profile +(see :ref:`isaac-teleop-standalone`), so no headset is needed. + +.. list-table:: + :header-rows: 1 + :widths: 25 75 + + * - Visualizer + - Keyboard input + * - ``--viz kit`` + - Keys typed into the Kit window, including keys typed into Kit's own UI (e.g. a text field). + * - ``--viz newton_gl`` / ``--viz newton_rtx`` + - Keys typed into the Newton viewer window, except keys typed into its UI. While a keyboard + pipeline runs, the viewer's WASD/QE and arrow-key camera movement is turned off; the mouse + still moves the camera. + * - ``--viz viser`` / ``--viz rerun`` / ``--viz none`` + - None. + +**SpaceMouse and Gamepad.** Both are read by an Isaac Capture plugin that their configs start +with the session, so they also run headset-free from a single command. The ``gamepad`` plugin +reads the first connected joystick (``/dev/input/jsN``), which udev lets the logged-in user read +without the ``input`` group. The ``spacemouse`` plugin reads the SpaceMouse's HID device +(``/dev/hidrawN``), which needs a udev rule (see +:ref:`SpaceMouse permissions `). Their buttons are sampled +once per frame, so a very quick tap (pressed and released within one frame, more likely when the +simulation runs slowly) can be missed; press again if the gripper or reset does not respond. + +The ``B`` / ``P`` / ``R`` start/pause/reset keys work with every pipeline, including XR, SpaceMouse +and Gamepad ones: :class:`~isaaclab_teleop.IsaacTeleopDevice` reads them straight from the focused +visualizer window, so each press fires once whatever the retargeting mode. Script hotkeys, such as the +``N`` / ``B`` / ``S`` keys of ``annotate_demos.py`` and ``replay_demos.py``, are read from the visualizer +windows the same way (see :func:`~isaaclab.visualizers.add_key_callbacks`). The device button layouts below apply to all environments in this section. Per-environment differences (gripper enabled/disabled, sensitivity) are noted in the environment table that @@ -790,9 +836,15 @@ follows. * - Gripper toggle - ``K`` - Open / close gripper or suction (disabled in Reach envs). + * - Start/resume teleoperation + - ``B`` + - Resume after a pause. + * - Pause teleoperation + - ``P`` + - Pause without resetting. * - Reset - - ``L`` - - Clear accumulated delta pose and gripper state. + - ``R`` + - Reset the environment. **SpaceMouse** @@ -814,7 +866,7 @@ follows. - Open / close gripper or suction (disabled in Reach envs). * - Reset - Right button - - Clear accumulated delta pose and gripper state. + - Reset the environment. **Gamepad** (Reach environments only) @@ -838,7 +890,7 @@ follows. - Right stick (left/right) - Rotate about Z axis. * - Gripper toggle - - X button + - West button (X on Xbox, Square on PlayStation) - Open / close gripper (disabled in Reach envs). .. list-table:: @@ -1171,7 +1223,8 @@ times spike. This favors lower latency over CloudXR's pose-wait smoothing. :data:`~isaaclab_teleop.CLOUDXR_STANDALONE_ENV` is the default when running **without** ``--xr`` (see :ref:`isaac-teleop-standalone`). It pins an emulated ``quest3`` device profile so a CloudXR runtime with no client still advertises an OpenXR system, working around -``XR_ERROR_FORM_FACTOR_UNAVAILABLE``. +``XR_ERROR_FORM_FACTOR_UNAVAILABLE``. Keyboard, gamepad and SpaceMouse sessions without ``--xr`` +use it too. Override at launch time ~~~~~~~~~~~~~~~~~~~~~~~ @@ -1683,9 +1736,9 @@ the registered task ID, but not the command-line selector values: --dataset_file ./datasets/dataset.hdf5 \ physics=isaacsim_physx presets=diffik -Some environments use the legacy ``teleop_devices`` configuration instead of ``isaac_teleop`` -(e.g. the Galbot RmpFlow relative-mode tasks). For these, pass ``--teleop_device`` to select -the input device: +Some environments declare their input devices in ``teleop_devices`` instead of ``isaac_teleop`` +(e.g. the Galbot RmpFlow relative-mode tasks). For these, pass ``--teleop_device`` to select the +input device; keyboard and spacemouse run through Isaac Capture when ``isaacteleop`` is installed: .. tab-set:: @@ -1701,11 +1754,11 @@ the input device: The workflow is: #. Configure your environment with ``IsaacTeleopCfg`` (see :ref:`isaac-teleop-env-config`) - or ``teleop_devices`` for legacy devices (keyboard, spacemouse). + or ``teleop_devices`` for keyboard and spacemouse. #. Run ``record_demos.py`` with the task name and any ``physics=``, ``renderer=``, or ``presets=`` selectors required by the task. #. For XR tasks: start AR, connect your XR device, and teleoperate. - For legacy tasks: use the configured input device directly. + For keyboard/spacemouse: use the input device directly. #. Demonstrations are recorded to HDF5 files. #. Use the recorded data with Isaac Lab Mimic or other imitation learning frameworks. diff --git a/docs/source/how-to/cloudxr_teleoperation.rst b/docs/source/how-to/cloudxr_teleoperation.rst index f80695e34da7..c5d99ba33858 100644 --- a/docs/source/how-to/cloudxr_teleoperation.rst +++ b/docs/source/how-to/cloudxr_teleoperation.rst @@ -382,12 +382,12 @@ choose the tab that matches your hardware. start automatically. #. Open the browser on your headset and navigate to the hosted CloudXR.js client: - ``_. + ``_. .. note:: - The web client URL is versioned. The ``release-1.4.x`` path corresponds to the - ``isaacteleop`` version pinned by Isaac Lab (``isaacteleop~=1.4.145`` in the ``teleop`` + The web client URL is versioned. The ``release-1.7.x`` path corresponds to the + ``isaaccapture`` version pinned by Isaac Lab (``isaaccapture~=1.7.44a1`` in the ``teleop`` extra of the root ``pyproject.toml``). When Isaac Lab updates its teleoperation dependency, update this link to the matching client release. @@ -594,16 +594,16 @@ Prerequisites ``install/plugins/manus/manus_hand_plugin`` does not exist in your Isaac Capture checkout, this step has not been completed. - Clone the repository and check out the release branch matching the ``isaacteleop`` version Isaac - Lab is pinned to (``isaacteleop~=1.4.145`` in the ``teleop`` extra of the root - ``pyproject.toml``), so the plugin's wire format matches the ``isaacteleop`` package Isaac Lab + Clone the repository and check out the release branch matching the ``isaaccapture`` version Isaac + Lab is pinned to (``isaaccapture~=1.7.44a1`` in the ``teleop`` extra of the root + ``pyproject.toml``), so the plugin's wire format matches the ``isaaccapture`` package Isaac Lab installs: .. code-block:: bash git clone https://github.com/NVIDIA/IsaacCapture.git cd IsaacCapture - git checkout release/1.4.x + git checkout release/1.7.x .. note:: @@ -765,7 +765,7 @@ time. Haptikos Exoskeletons with Quest -------------------------------- -The `Haptikos plugin `_ +The `Haptikos plugin `_ combines controller wrist poses with exoskeleton finger tracking from the Haptikos Core App and pushes hand joints into the OpenXR runtime. Isaac Lab receives them through Isaac Capture's standard hand-tracking input, so no Haptikos-specific Isaac Lab device is needed. The plugin @@ -776,18 +776,18 @@ Build the plugin ^^^^^^^^^^^^^^^^ The Haptikos plugin is built from Isaac Capture source; it is not included in Isaac Lab's -``teleop`` extra. Check out the release branch matching Isaac Lab's ``isaacteleop`` pin (currently -``1.4.x``), obtain the `Haptikos Robotics API `_, +``teleop`` extra. Check out the release branch matching Isaac Lab's ``isaaccapture`` pin (currently +``1.7.x``), obtain the `Haptikos Robotics API `_, and copy its ``HaptikosCpp_API_Shared`` directory into ``src/plugins/haptikos``. The C++ API is required to build the tracking plugin, even if you do not use haptic feedback. Follow the -`plugin's setup instructions `_ +`plugin's setup instructions `_ for Haptikos account and licensing requirements. .. code-block:: bash git clone https://github.com/NVIDIA/IsaacCapture.git cd IsaacCapture - git checkout release/1.4.x + git checkout release/1.7.x # Copy HaptikosCpp_API_Shared to src/plugins/haptikos before building. cmake -S . -B build -DENABLE_CLANG_FORMAT_CHECK=OFF cmake --build build --target haptikos_hands_plugin --parallel 4 diff --git a/examples/demos/pick_and_place.py b/examples/demos/pick_and_place.py index f4bfb67688ca..2ca657763979 100644 --- a/examples/demos/pick_and_place.py +++ b/examples/demos/pick_and_place.py @@ -8,6 +8,7 @@ from __future__ import annotations import argparse +import contextlib from collections.abc import Sequence import torch @@ -45,7 +46,7 @@ from isaaclab.sim import SimulationCfg from isaaclab.utils import configclass, index_fill_, instantiate, replace from isaaclab.utils.math import sample_uniform -from isaaclab.visualizers import VisualizerCfg +from isaaclab.visualizers import VisualizerCfg, add_key_callbacks from isaaclab_assets.robots.pick_and_place import PICK_AND_PLACE_CFG @@ -326,30 +327,46 @@ def main() -> None: pick_and_place.reset() actions = torch.zeros((pick_and_place.num_envs, 4), device=pick_and_place.device) teleop = None + teleop_device = contextlib.ExitStack() if pick_and_place.sim.has_gui: teleop_cfg = Se3KeyboardCfg(pos_sensitivity=10.0, sim_device=pick_and_place.device) - teleop = instantiate(teleop_cfg) - teleop.add_callback("N", lambda: pick_and_place.auto_aim(cube=True)) - teleop.add_callback("M", lambda: pick_and_place.auto_aim(cube=False)) + try: + from isaaclab_teleop.script_input import open_teleop_device + except ImportError: # without the teleop or mimic extra + teleop = instantiate(teleop_cfg) + else: + teleop = teleop_device.enter_context(open_teleop_device(teleop_cfg)) + add_key_callbacks( + {"N": lambda: pick_and_place.auto_aim(cube=True), "M": lambda: pick_and_place.auto_aim(cube=False)} + ) print(teleop) print("Pick up the purple cube and drop it on the red sphere, in ALL environments at once.") print("\tW/S and A/D move the gantries, Q/E latch them UP/DOWN, K toggles the grippers.") print("\tN/M make the grippers track the cube/target position.") step_count = 0 + teleop_active = True try: while pick_and_place.sim.is_running() and (args_cli.max_steps < 0 or step_count < args_cli.max_steps): - if teleop is not None: - cmd = teleop.advance() + cmd = teleop.advance() if teleop is not None else None + # P / B pause and resume an Isaac Capture device; paused, the scene renders without stepping + events = getattr(teleop, "last_control_events", None) + if events is not None and events.is_active is not None: + teleop_active = events.is_active + if not teleop_active: + pick_and_place.sim.render() + continue + if cmd is not None: actions[:, :2] = cmd[:2] # Latch the gantry height effort; the z joint moves up for negative effort if cmd[2] != 0: actions[:, 2] = -200.0 if cmd[2] > 0 else 100.0 - # Se3Keyboard reports +1 for open, the surface gripper uses -1 for open + # the keyboard reports +1 for open, the surface gripper uses -1 for open actions[:, 3] = -cmd[6] with torch.inference_mode(): pick_and_place.step(actions) step_count += 1 finally: + teleop_device.close() pick_and_place.close() diff --git a/pyproject.toml b/pyproject.toml index b143612a98e7..c2d452c40591 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,7 +196,7 @@ mimic = [ teleop = [ "isaaclab-teleop", "isaaclab-mimic", - "isaacteleop[retargeters,ui,cloudxr]~=1.4.145 ; platform_system == 'Linux' and platform_machine == 'x86_64'", + "isaaccapture[retargeters,ui,cloudxr]~=1.7.44a1 ; platform_system == 'Linux' and platform_machine == 'x86_64'", "dex-retargeting==0.5.0 ; platform_system == 'Linux' and platform_machine == 'x86_64'", ] rlinf = [ diff --git a/scripts/environments/teleoperation/teleop_se3_agent.py b/scripts/environments/teleoperation/teleop_se3_agent.py index c6f94f292381..4522f11aded7 100644 --- a/scripts/environments/teleoperation/teleop_se3_agent.py +++ b/scripts/environments/teleoperation/teleop_se3_agent.py @@ -42,9 +42,10 @@ type=str, default=None, help=( - "Legacy teleop device name. When omitted, the Isaac Capture pipeline is used if configured in the env," - " otherwise keyboard is used as fallback. When explicitly provided, the script uses the legacy" - " teleop_devices path and looks up this name in env_cfg.teleop_devices.devices." + "Teleop device name: an entry of env_cfg.teleop_devices.devices, or a built-in device (keyboard," + " spacemouse, gamepad). When omitted, the Isaac Capture pipeline is used if configured in the env," + " otherwise keyboard. Keyboard, gamepad and spacemouse run through Isaac Capture when the isaacteleop" + " package is installed, and through the isaaclab.devices devices otherwise." ), ) parser.add_argument("--task", type=str, default=None, help="Name of the task.") @@ -117,7 +118,7 @@ from isaaclab_physx.renderers import IsaacRtxRendererGlobalSettingsCfg from isaaclab_teleop import CLOUDXR_AVP_ENV, CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV -from isaaclab.devices import Se3GamepadCfg, Se3KeyboardCfg, Se3SpaceMouseCfg +from isaaclab.devices import DeviceCfg, Se3GamepadCfg, Se3KeyboardCfg, Se3SpaceMouseCfg from isaaclab.devices.openxr import remove_camera_configs from isaaclab.devices.teleop_device_factory import create_teleop_device from isaaclab.envs import ManagerBasedRLEnvCfg @@ -143,6 +144,15 @@ }.get(args_cli.cloudxr_env.lower(), args_cli.cloudxr_env) +def _builtin_device_cfgs(sensitivity: float) -> dict[str, DeviceCfg]: + """The built-in devices, by ``--teleop_device`` name, for names the environment does not declare.""" + return { + "keyboard": Se3KeyboardCfg(pos_sensitivity=0.05 * sensitivity, rot_sensitivity=0.05 * sensitivity), + "spacemouse": Se3SpaceMouseCfg(pos_sensitivity=0.05 * sensitivity, rot_sensitivity=0.05 * sensitivity), + "gamepad": Se3GamepadCfg(pos_sensitivity=0.1 * sensitivity, rot_sensitivity=0.1 * sensitivity), + } + + def main() -> None: # noqa: C901 """ Run teleoperation with an Isaac Lab manipulation environment. @@ -171,12 +181,16 @@ def main() -> None: # noqa: C901 # add termination condition for reaching the goal otherwise the environment won't reset env_cfg.terminations.object_reached_goal = DoneTerm(func=mdp.object_reached_goal) - # When --teleop_device is explicitly provided, use the legacy teleop_devices path - # even if isaac_teleop is configured. Otherwise prefer isaac_teleop when available. - teleop_device_explicitly_set = args_cli.teleop_device is not None - use_isaac_teleop = ( - not teleop_device_explicitly_set and hasattr(env_cfg, "isaac_teleop") and env_cfg.isaac_teleop is not None + from isaaclab_teleop.device_selection import select_isaac_teleop_cfg + + # Isaac Capture serves the request when it can: the env's isaac_teleop pipeline, or the + # keyboard, gamepad or spacemouse it names. Otherwise the isaaclab.devices device does. + isaac_teleop_cfg = select_isaac_teleop_cfg( + env_cfg, args_cli.teleop_device, _builtin_device_cfgs(args_cli.sensitivity), args_cli.device ) + use_isaac_teleop = isaac_teleop_cfg is not None + if use_isaac_teleop: + env_cfg.isaac_teleop = isaac_teleop_cfg # XR-rendering setup (camera removal + DLSS) is only needed for the Kit XR # path. Without --xr, Isaac Capture runs standalone (I/O only) and renders @@ -315,12 +329,8 @@ def stop_teleoperation() -> None: ) else: device_name = "keyboard" if args_cli.teleop_device is None else args_cli.teleop_device.lower() - device_cfgs = { - "keyboard": Se3KeyboardCfg(pos_sensitivity=0.05, rot_sensitivity=0.05), - "spacemouse": Se3SpaceMouseCfg(pos_sensitivity=0.05, rot_sensitivity=0.05), - "gamepad": Se3GamepadCfg(pos_sensitivity=0.1, rot_sensitivity=0.1), - } - if device_name not in device_cfgs: + device_cfg = _builtin_device_cfgs(args_cli.sensitivity).get(device_name) + if device_cfg is None: logger.error( f"--teleop_device={args_cli.teleop_device} was passed but no matching entry exists in" " env_cfg.teleop_devices and it is not a built-in device name. Either remove" @@ -330,9 +340,6 @@ def stop_teleoperation() -> None: ) env.close() return - device_cfg = device_cfgs[device_name] - device_cfg.pos_sensitivity *= args_cli.sensitivity - device_cfg.rot_sensitivity *= args_cli.sensitivity teleop_interface = instantiate(device_cfg) for key, callback in teleoperation_callbacks.items(): try: @@ -352,18 +359,9 @@ def stop_teleoperation() -> None: haptic_driver = create_haptic_feedback_driver(env.unwrapped, teleop_interface, env_cfg) - # Retain the keyboard so its input subscription lives for the whole session. - if use_isaac_teleop and env.sim.has_gui and not env.sim.get_setting("/isaaclab/xr/auto_start"): - try: - control_keyboard = instantiate(Se3KeyboardCfg(pos_sensitivity=0.0, rot_sensitivity=0.0)) - control_keyboard.add_callback("B", teleop_interface.request_start) - control_keyboard.add_callback("P", teleop_interface.request_stop) - # Inject a reset pulse; the loop owns the environment reset. - control_keyboard.add_callback("R", lambda: teleop_interface.reset(pause=True)) - print("Isaac Capture control keys: [B] start/resume [P] pause [R] reset") - except Exception as e: - logger.warning(f"Control keyboard unavailable ({e}); teleop still auto-starts without --xr") - control_keyboard = None + if use_isaac_teleop: + # the device reads these from the focused visualizer window + print("Isaac Capture control keys: [B] start/resume [P] pause [R] reset") def run_loop(): """Inner function to run the teleop loop with access to nonlocal variables.""" diff --git a/scripts/imitation_learning/isaaclab_mimic/annotate_demos.py b/scripts/imitation_learning/isaaclab_mimic/annotate_demos.py index 1659510f7412..375765e1cc14 100644 --- a/scripts/imitation_learning/isaaclab_mimic/annotate_demos.py +++ b/scripts/imitation_learning/isaaclab_mimic/annotate_demos.py @@ -283,15 +283,12 @@ def annotate_dataset(env_cfg, dataset_file_handler: HDF5DatasetFileHandler, succ # Only enables inputs if this script runs with a GUI if env.sim.has_gui: - from isaaclab.devices import Se3Keyboard, Se3KeyboardCfg + from isaaclab.visualizers import add_key_callbacks - keyboard_interface = Se3Keyboard(Se3KeyboardCfg(pos_sensitivity=0.1, rot_sensitivity=0.1)) - keyboard_interface.add_callback("N", play_cb) - keyboard_interface.add_callback("B", pause_cb) - keyboard_interface.add_callback("Q", skip_episode_cb) + key_callbacks = {"N": play_cb, "B": pause_cb, "Q": skip_episode_cb} if not args_cli.auto: - keyboard_interface.add_callback("S", mark_subtask_cb) - keyboard_interface.reset() + key_callbacks["S"] = mark_subtask_cb + add_key_callbacks(key_callbacks) # simulate environment -- run everything in inference mode exported_episode_count = 0 diff --git a/scripts/imitation_learning/isaaclab_mimic/consolidated_demo.py b/scripts/imitation_learning/isaaclab_mimic/consolidated_demo.py index 34ab9e282e09..4395d41c9d26 100644 --- a/scripts/imitation_learning/isaaclab_mimic/consolidated_demo.py +++ b/scripts/imitation_learning/isaaclab_mimic/consolidated_demo.py @@ -180,25 +180,29 @@ def pre_process_actions(delta_pose: torch.Tensor, gripper_command: bool) -> torc return torch.concat([delta_pose, gripper_vel], dim=1) +def teleop_device_cfg(): + """The ``--teleop_device`` config; imported once the simulation is launched.""" + from isaaclab.devices import Se3KeyboardCfg, Se3SpaceMouseCfg + + device_cfgs = { + "keyboard": Se3KeyboardCfg(pos_sensitivity=0.2, rot_sensitivity=0.5), + "spacemouse": Se3SpaceMouseCfg(pos_sensitivity=0.2, rot_sensitivity=0.5), + } + device_cfg = device_cfgs.get(args_cli.teleop_device.lower()) + if device_cfg is None: + raise ValueError(f"Invalid device interface '{args_cli.teleop_device}'. Supported: 'keyboard', 'spacemouse'.") + return device_cfg + + async def run_teleop_robot( - env, env_id, env_action_queue, shared_datagen_info_pool, success_term, exported_dataset_path, teleop_interface=None + env, env_id, env_action_queue, shared_datagen_info_pool, success_term, exported_dataset_path, teleop_interface ): """Run teleop robot.""" + from isaaclab_teleop import poll_control_events + global num_recorded should_reset_teleop_instance = False - # create controller if needed - if teleop_interface is None: - # the teleop devices need the Kit runtime, which is running by the time this coroutine executes - from isaaclab.devices import Se3Keyboard, Se3KeyboardCfg, Se3SpaceMouse, Se3SpaceMouseCfg - - if args_cli.teleop_device.lower() == "keyboard": - teleop_interface = Se3Keyboard(Se3KeyboardCfg(pos_sensitivity=0.2, rot_sensitivity=0.5)) - elif args_cli.teleop_device.lower() == "spacemouse": - teleop_interface = Se3SpaceMouse(Se3SpaceMouseCfg(pos_sensitivity=0.2, rot_sensitivity=0.5)) - else: - raise ValueError( - f"Invalid device interface '{args_cli.teleop_device}'. Supported: 'keyboard', 'spacemouse'." - ) + teleop_active = True # add teleoperation key for reset current recording instance def reset_teleop_instance(): @@ -223,10 +227,18 @@ def reset_teleop_instance(): should_reset_teleop_instance = False success_step_count = 0 - # get keyboard command - delta_pose, gripper_command = teleop_interface.advance() - # convert to torch - delta_pose = torch.tensor(delta_pose, dtype=torch.float, device=env.device).repeat(1, 1) + # get the device command [dx, dy, dz, drx, dry, drz, gripper]; every env submits one action per + # round (see env_loop), so hold still with the gripper open while an Isaac Capture session starts + # or is paused (P / B pause and resume it) + action = teleop_interface.advance() + ctrl = poll_control_events(teleop_interface) + if ctrl.is_active is not None: + teleop_active = ctrl.is_active + if action is None or not teleop_active: + action = torch.tensor([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0]) + delta_pose = action[:6].to(dtype=torch.float, device=env.device).repeat(1, 1) + # the gripper value is +1 to open and -1 to close; pre_process_actions takes "close" as True + gripper_command = bool(action[6] < 0) # compute actions based on environment teleop_action = pre_process_actions(delta_pose, gripper_command) @@ -440,6 +452,14 @@ def run_consolidated_demo(env_cfg, success_term): shared_datagen_info_pool.load_from_dataset_file(args_cli.input_file) print(f"Loaded {shared_datagen_info_pool.num_datagen_infos} to datagen info pool") + # the teleoperated env's device, closed after the env loop + teleop_device = contextlib.ExitStack() + teleop_interface = None + if args_cli.teleop_env_index is not None: + from isaaclab_teleop.script_input import open_teleop_device + + teleop_interface = teleop_device.enter_context(open_teleop_device(teleop_device_cfg())) + # make data generator object data_generator_asyncio_tasks = [] for i in range(num_envs): @@ -447,7 +467,13 @@ def run_consolidated_demo(env_cfg, success_term): data_generator_asyncio_tasks.append( asyncio_event_loop.create_task( run_teleop_robot( - env, i, env_action_queue, shared_datagen_info_pool, success_term, args_cli.output_file + env, + i, + env_action_queue, + shared_datagen_info_pool, + success_term, + args_cli.output_file, + teleop_interface, ) ) ) @@ -470,7 +496,8 @@ def run_consolidated_demo(env_cfg, success_term): except asyncio.CancelledError: print("Tasks were cancelled.") - env_loop(env, env_action_queue, shared_datagen_info_pool, asyncio_event_loop) + with teleop_device: + env_loop(env, env_action_queue, shared_datagen_info_pool, asyncio_event_loop) if __name__ == "__main__": diff --git a/scripts/tools/record_demos.py b/scripts/tools/record_demos.py index 8c9f2446f039..01e8689e6347 100644 --- a/scripts/tools/record_demos.py +++ b/scripts/tools/record_demos.py @@ -20,8 +20,10 @@ optional arguments: -h, --help Show this help message and exit - --teleop_device Legacy teleop device name. When omitted, Isaac Capture is used if - configured, otherwise keyboard. When set, forces the legacy path. + --teleop_device Teleop device name: an env teleop_devices entry, or keyboard / + spacemouse. When omitted, Isaac Capture is used if configured, + otherwise keyboard. Keyboard and spacemouse run through Isaac Capture + when isaacteleop is installed, else through isaaclab.devices. --dataset_file File path to export recorded demos. (default: "./datasets/dataset.hdf5") --step_hz Environment stepping rate in Hz. (default: 30) --num_demos Number of demonstrations to record. (default: 0) @@ -62,9 +64,10 @@ type=str, default=None, help=( - "Legacy teleop device name. When omitted, the Isaac Capture pipeline is used if configured in the env," - " otherwise keyboard is used as fallback. When explicitly provided, the script uses the legacy" - " teleop_devices path and looks up this name in env_cfg.teleop_devices.devices." + "Teleop device name: an entry of env_cfg.teleop_devices.devices, or a built-in device (keyboard," + " spacemouse). When omitted, the Isaac Capture pipeline is used if configured in the env, otherwise" + " keyboard. Keyboard and spacemouse run through Isaac Capture when the isaacteleop package is installed," + " and through the isaaclab.devices devices otherwise." ), ) parser.add_argument( @@ -173,7 +176,7 @@ from isaaclab_physx.renderers import IsaacRtxRendererGlobalSettingsCfg from isaaclab_teleop import CLOUDXR_AVP_ENV, CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV -from isaaclab.devices import Se3KeyboardCfg, Se3SpaceMouseCfg +from isaaclab.devices import DeviceCfg, Se3KeyboardCfg, Se3SpaceMouseCfg from isaaclab.devices.openxr import remove_camera_configs from isaaclab.devices.teleop_device_factory import create_teleop_device from isaaclab.envs import DirectRLEnvCfg, ManagerBasedRLEnvCfg @@ -293,12 +296,14 @@ def create_environment_config( logger.error(f"Failed to parse environment configuration: {e}") exit(1) - # When --teleop_device is explicitly provided, use the legacy teleop_devices path - # even if isaac_teleop is configured. Otherwise prefer isaac_teleop when available. - teleop_device_explicitly_set = args_cli.teleop_device is not None - use_isaac_teleop = ( - not teleop_device_explicitly_set and hasattr(env_cfg, "isaac_teleop") and env_cfg.isaac_teleop is not None - ) + from isaaclab_teleop.device_selection import select_isaac_teleop_cfg + + # Isaac Capture serves the request when it can: the env's isaac_teleop pipeline, or the + # keyboard or spacemouse it names. Otherwise the isaaclab.devices device does. + isaac_teleop_cfg = select_isaac_teleop_cfg(env_cfg, args_cli.teleop_device, _builtin_device_cfgs(), args_cli.device) + use_isaac_teleop = isaac_teleop_cfg is not None + if use_isaac_teleop: + env_cfg.isaac_teleop = isaac_teleop_cfg # Extract the success condition for manual evaluation in the main loop. Keep # an inert term registered under the same name so rewards and other manager @@ -348,6 +353,14 @@ def create_environment_config( return env_cfg, success_term, use_isaac_teleop +def _builtin_device_cfgs() -> dict[str, DeviceCfg]: + """The built-in devices, by ``--teleop_device`` name, for names the environment does not declare.""" + return { + "keyboard": Se3KeyboardCfg(pos_sensitivity=0.2, rot_sensitivity=0.5), + "spacemouse": Se3SpaceMouseCfg(pos_sensitivity=0.2, rot_sensitivity=0.5), + } + + def setup_teleop_device( env_cfg: ManagerBasedRLEnvCfg, callbacks: dict[str, Callable], use_isaac_teleop: bool = False ) -> object: @@ -393,11 +406,8 @@ def setup_teleop_device( ) else: device_name = "keyboard" if args_cli.teleop_device is None else args_cli.teleop_device.lower() - device_cfgs = { - "keyboard": Se3KeyboardCfg(pos_sensitivity=0.2, rot_sensitivity=0.5), - "spacemouse": Se3SpaceMouseCfg(pos_sensitivity=0.2, rot_sensitivity=0.5), - } - if device_name not in device_cfgs: + device_cfg = _builtin_device_cfgs().get(device_name) + if device_cfg is None: logger.error( f"--teleop_device={args_cli.teleop_device} was passed but no matching entry exists in" " env_cfg.teleop_devices and it is not a built-in device name. Either remove" @@ -406,7 +416,7 @@ def setup_teleop_device( " Built-in devices: keyboard, spacemouse." ) exit(1) - teleop_interface = instantiate(device_cfgs[device_name]) + teleop_interface = instantiate(device_cfg) for key, callback in callbacks.items(): teleop_interface.add_callback(key, callback) except Exception as e: @@ -588,18 +598,9 @@ def stop_recording_instance(): haptic_driver = create_haptic_feedback_driver(env.unwrapped, teleop_interface, env.cfg) - # Retain the keyboard so its input subscription lives for the whole session. - if use_isaac_teleop and env.sim.has_gui and not env.sim.get_setting("/isaaclab/xr/auto_start"): - try: - control_keyboard = instantiate(Se3KeyboardCfg(pos_sensitivity=0.0, rot_sensitivity=0.0)) - control_keyboard.add_callback("B", teleop_interface.request_start) - control_keyboard.add_callback("P", teleop_interface.request_stop) - # Inject a reset pulse; the loop owns the environment reset. - control_keyboard.add_callback("R", lambda: teleop_interface.reset(pause=True)) - print("Isaac Capture control keys: [B] start/resume [P] pause [R] reset") - except Exception as e: - logger.warning(f"Control keyboard unavailable ({e}); recording still auto-starts without --xr") - control_keyboard = None + if use_isaac_teleop: + # the device reads these from the focused visualizer window + print("Isaac Capture control keys: [B] start/resume [P] pause [R] reset") label_text = f"Recorded {current_recorded_demo_count} successful demonstrations." instruction_display = setup_ui(label_text, env) @@ -640,7 +641,11 @@ def inner_loop(): if ctrl.is_active is not None: running_recording_instance = ctrl.is_active if ctrl.should_reset: - should_reset_recording_instance = True + if success_step_count > 0: + # same guard as the R callback: keep a demo whose success is still being recorded + reset_recording_instance() + else: + should_reset_recording_instance = True if action is None: env.sim.render() diff --git a/scripts/tools/replay_demos.py b/scripts/tools/replay_demos.py index 389fc48b30d9..c05ab324301f 100644 --- a/scripts/tools/replay_demos.py +++ b/scripts/tools/replay_demos.py @@ -304,16 +304,14 @@ def replay_dataset( ): """Create the environment and replay the selected episodes of the dataset.""" # the keyboard device needs the Kit runtime, which is running at this point - from isaaclab.devices import Se3Keyboard, Se3KeyboardCfg + from isaaclab.visualizers import add_key_callbacks num_envs = args_cli.num_envs # create environment from loaded config env = gym.make(args_cli.task, cfg=env_cfg).unwrapped - teleop_interface = Se3Keyboard(Se3KeyboardCfg(pos_sensitivity=0.1, rot_sensitivity=0.1)) - teleop_interface.add_callback("N", play_cb) - teleop_interface.add_callback("B", pause_cb) + add_key_callbacks({"N": play_cb, "B": pause_cb}) print('Press "B" to pause and "N" to resume the replayed actions.') # Determine if state validation should be conducted @@ -331,7 +329,6 @@ def replay_dataset( # reset before starting env.reset() - teleop_interface.reset() episode_names = list(dataset_file_handler.get_episode_names()) replayed_episode_count, recorded_episode_count, failed_demo_ids = replay_episodes_loop( diff --git a/source/isaaclab/changelog.d/rwiltz-teleop-scripts-isaac-capture.added.rst b/source/isaaclab/changelog.d/rwiltz-teleop-scripts-isaac-capture.added.rst new file mode 100644 index 000000000000..07cf259bcb15 --- /dev/null +++ b/source/isaaclab/changelog.d/rwiltz-teleop-scripts-isaac-capture.added.rst @@ -0,0 +1,2 @@ +* Added :func:`~isaaclab.visualizers.add_key_callbacks`, which calls a script's hotkey callbacks for + keys pressed in a visualizer window. diff --git a/source/isaaclab/changelog.d/rwiltz-teleop-scripts-isaac-capture.deprecated.rst b/source/isaaclab/changelog.d/rwiltz-teleop-scripts-isaac-capture.deprecated.rst new file mode 100644 index 000000000000..181e0339cc93 --- /dev/null +++ b/source/isaaclab/changelog.d/rwiltz-teleop-scripts-isaac-capture.deprecated.rst @@ -0,0 +1,10 @@ +* Deprecated :class:`~isaaclab.devices.Se2Keyboard`, :class:`~isaaclab.devices.Se3Keyboard`, + :class:`~isaaclab.devices.Se2Gamepad`, :class:`~isaaclab.devices.Se3Gamepad`, + :class:`~isaaclab.devices.Se2SpaceMouse` and :class:`~isaaclab.devices.Se3SpaceMouse`; their + constructors now emit a :class:`DeprecationWarning`. Use the ``isaaclab_teleop`` builders + instead (:func:`~isaaclab_teleop.se3_keyboard_teleop_cfg`, + :func:`~isaaclab_teleop.se3_gamepad_teleop_cfg`, :func:`~isaaclab_teleop.se3_spacemouse_teleop_cfg` + and their SE(2) counterparts), which run the device through Isaac Capture. The device + configs (:class:`~isaaclab.devices.Se3KeyboardCfg`, ...) are not deprecated: the teleop scripts + run an environment's ``teleop_devices`` entries through the equivalent Isaac Capture pipeline + when the ``isaacteleop`` package is installed, and through these devices otherwise. diff --git a/source/isaaclab/changelog.d/rwiltz-teleop-scripts-isaac-capture.minor b/source/isaaclab/changelog.d/rwiltz-teleop-scripts-isaac-capture.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab/isaaclab/devices/gamepad/__init__.py b/source/isaaclab/isaaclab/devices/gamepad/__init__.py index b9032063c2a1..1f165b9eb647 100644 --- a/source/isaaclab/isaaclab/devices/gamepad/__init__.py +++ b/source/isaaclab/isaaclab/devices/gamepad/__init__.py @@ -3,7 +3,13 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Gamepad device for SE(2) and SE(3) control.""" +"""Gamepad device for SE(2) and SE(3) control. + +.. deprecated:: + Use :func:`isaaclab_teleop.se2_gamepad_teleop_cfg` / :func:`isaaclab_teleop.se3_gamepad_teleop_cfg`, which run + the device in process through Isaac Capture. Imports from this package keep working; + the device constructors emit a :class:`DeprecationWarning`. +""" from ...utils.module import lazy_export diff --git a/source/isaaclab/isaaclab/devices/gamepad/se2_gamepad.py b/source/isaaclab/isaaclab/devices/gamepad/se2_gamepad.py index aac6b99d2a79..d004b2ccbe9e 100644 --- a/source/isaaclab/isaaclab/devices/gamepad/se2_gamepad.py +++ b/source/isaaclab/isaaclab/devices/gamepad/se2_gamepad.py @@ -7,6 +7,7 @@ from __future__ import annotations +import warnings import weakref from collections.abc import Callable from typing import TYPE_CHECKING @@ -55,6 +56,11 @@ def __init__(self, cfg: Se2GamepadCfg): Args: cfg: Configuration object for gamepad settings. """ + warnings.warn( + "Se2Gamepad is deprecated. Please use isaaclab_teleop.se2_gamepad_teleop_cfg instead.", + DeprecationWarning, + stacklevel=2, + ) # turn off simulator gamepad control get_settings_manager().set("/persistent/app/omniverse/gamepadCameraControl", False) # store inputs diff --git a/source/isaaclab/isaaclab/devices/gamepad/se3_gamepad.py b/source/isaaclab/isaaclab/devices/gamepad/se3_gamepad.py index d433b6e18fab..4472a41488a4 100644 --- a/source/isaaclab/isaaclab/devices/gamepad/se3_gamepad.py +++ b/source/isaaclab/isaaclab/devices/gamepad/se3_gamepad.py @@ -7,6 +7,7 @@ from __future__ import annotations +import warnings import weakref from collections.abc import Callable from typing import TYPE_CHECKING @@ -62,6 +63,11 @@ def __init__(self, cfg: Se3GamepadCfg): Args: cfg: Configuration object for gamepad settings. """ + warnings.warn( + "Se3Gamepad is deprecated. Please use isaaclab_teleop.se3_gamepad_teleop_cfg instead.", + DeprecationWarning, + stacklevel=2, + ) # turn off simulator gamepad control get_settings_manager().set("/persistent/app/omniverse/gamepadCameraControl", False) # store inputs diff --git a/source/isaaclab/isaaclab/devices/keyboard/__init__.py b/source/isaaclab/isaaclab/devices/keyboard/__init__.py index 0e30128451b6..c4356b17fa40 100644 --- a/source/isaaclab/isaaclab/devices/keyboard/__init__.py +++ b/source/isaaclab/isaaclab/devices/keyboard/__init__.py @@ -3,7 +3,13 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Keyboard device for SE(2) and SE(3) control.""" +"""Keyboard device for SE(2) and SE(3) control. + +.. deprecated:: + Use :func:`isaaclab_teleop.se2_keyboard_teleop_cfg` / :func:`isaaclab_teleop.se3_keyboard_teleop_cfg`, which run + the device in process through Isaac Capture. Imports from this package keep working; + the device constructors emit a :class:`DeprecationWarning`. +""" from ...utils.module import lazy_export diff --git a/source/isaaclab/isaaclab/devices/keyboard/se2_keyboard.py b/source/isaaclab/isaaclab/devices/keyboard/se2_keyboard.py index e371b309509f..37ec5bf5af59 100644 --- a/source/isaaclab/isaaclab/devices/keyboard/se2_keyboard.py +++ b/source/isaaclab/isaaclab/devices/keyboard/se2_keyboard.py @@ -7,6 +7,7 @@ from __future__ import annotations +import warnings import weakref from collections.abc import Callable from typing import TYPE_CHECKING @@ -53,6 +54,11 @@ def __init__(self, cfg: Se2KeyboardCfg): Args: cfg: Configuration object for keyboard settings. """ + warnings.warn( + "Se2Keyboard is deprecated. Please use isaaclab_teleop.se2_keyboard_teleop_cfg instead.", + DeprecationWarning, + stacklevel=2, + ) # store inputs self.v_x_sensitivity = cfg.v_x_sensitivity self.v_y_sensitivity = cfg.v_y_sensitivity diff --git a/source/isaaclab/isaaclab/devices/keyboard/se3_keyboard.py b/source/isaaclab/isaaclab/devices/keyboard/se3_keyboard.py index 74a03cf4cd4f..8fcf12e46054 100644 --- a/source/isaaclab/isaaclab/devices/keyboard/se3_keyboard.py +++ b/source/isaaclab/isaaclab/devices/keyboard/se3_keyboard.py @@ -7,6 +7,7 @@ from __future__ import annotations +import warnings import weakref from collections.abc import Callable from typing import TYPE_CHECKING @@ -61,6 +62,11 @@ def __init__(self, cfg: Se3KeyboardCfg): Args: cfg: Configuration object for keyboard settings. """ + warnings.warn( + "Se3Keyboard is deprecated. Please use isaaclab_teleop.se3_keyboard_teleop_cfg instead.", + DeprecationWarning, + stacklevel=2, + ) # store inputs self.pos_sensitivity = cfg.pos_sensitivity self.rot_sensitivity = cfg.rot_sensitivity diff --git a/source/isaaclab/isaaclab/devices/spacemouse/__init__.py b/source/isaaclab/isaaclab/devices/spacemouse/__init__.py index fbf695f505d5..97473311624f 100644 --- a/source/isaaclab/isaaclab/devices/spacemouse/__init__.py +++ b/source/isaaclab/isaaclab/devices/spacemouse/__init__.py @@ -3,7 +3,13 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Spacemouse device for SE(2) and SE(3) control.""" +"""Spacemouse device for SE(2) and SE(3) control. + +.. deprecated:: + Use :func:`isaaclab_teleop.se2_spacemouse_teleop_cfg` / :func:`isaaclab_teleop.se3_spacemouse_teleop_cfg`, which run + the device in process through Isaac Capture. Imports from this package keep working; + the device constructors emit a :class:`DeprecationWarning`. +""" from ...utils.module import lazy_export diff --git a/source/isaaclab/isaaclab/devices/spacemouse/se2_spacemouse.py b/source/isaaclab/isaaclab/devices/spacemouse/se2_spacemouse.py index 8a4b9d801a7d..3e6597b5fdf7 100644 --- a/source/isaaclab/isaaclab/devices/spacemouse/se2_spacemouse.py +++ b/source/isaaclab/isaaclab/devices/spacemouse/se2_spacemouse.py @@ -9,6 +9,7 @@ import threading import time +import warnings from collections.abc import Callable from typing import TYPE_CHECKING @@ -52,6 +53,11 @@ def __init__(self, cfg: Se2SpaceMouseCfg): Args: cfg: Configuration for the spacemouse device. """ + warnings.warn( + "Se2SpaceMouse is deprecated. Please use isaaclab_teleop.se2_spacemouse_teleop_cfg instead.", + DeprecationWarning, + stacklevel=2, + ) # listener thread, started once the device is found self._thread = None # store inputs diff --git a/source/isaaclab/isaaclab/devices/spacemouse/se3_spacemouse.py b/source/isaaclab/isaaclab/devices/spacemouse/se3_spacemouse.py index 39aa314efe98..1c579355b628 100644 --- a/source/isaaclab/isaaclab/devices/spacemouse/se3_spacemouse.py +++ b/source/isaaclab/isaaclab/devices/spacemouse/se3_spacemouse.py @@ -9,6 +9,7 @@ import threading import time +import warnings from collections.abc import Callable from typing import TYPE_CHECKING @@ -61,6 +62,11 @@ def __init__(self, cfg: Se3SpaceMouseCfg): Args: cfg: Configuration object for space-mouse settings. """ + warnings.warn( + "Se3SpaceMouse is deprecated. Please use isaaclab_teleop.se3_spacemouse_teleop_cfg instead.", + DeprecationWarning, + stacklevel=2, + ) # store inputs self.pos_sensitivity = cfg.pos_sensitivity self.rot_sensitivity = cfg.rot_sensitivity diff --git a/source/isaaclab/isaaclab/visualizers/__init__.pyi b/source/isaaclab/isaaclab/visualizers/__init__.pyi index a98acaa7cda8..803df022f934 100644 --- a/source/isaaclab/isaaclab/visualizers/__init__.pyi +++ b/source/isaaclab/isaaclab/visualizers/__init__.pyi @@ -14,9 +14,10 @@ __all__ = [ "VisualizerCfg", "VisualizerBackendCfg", "WindowCfg", + "add_key_callbacks", ] from .base_visualizer import BaseVisualizer from .image_view import ImageView -from .key_event_source import KeyboardSubscription, KeyEventSource +from .key_event_source import KeyboardSubscription, KeyEventSource, add_key_callbacks from .visualizer_cfg import WindowCfg, ImageViewCfg, PerspectiveCameraCfg, SceneCameraCfg, VisualizerCfg, VisualizerBackendCfg diff --git a/source/isaaclab/isaaclab/visualizers/key_event_source.py b/source/isaaclab/isaaclab/visualizers/key_event_source.py index d2de10de7565..658ece00ccc2 100644 --- a/source/isaaclab/isaaclab/visualizers/key_event_source.py +++ b/source/isaaclab/isaaclab/visualizers/key_event_source.py @@ -8,7 +8,7 @@ from __future__ import annotations import logging -from collections.abc import Callable +from collections.abc import Callable, Mapping logger = logging.getLogger(__name__) @@ -121,6 +121,58 @@ def _emit_focus_lost(self) -> None: _call(subscription._on_focus_lost) +def add_key_callbacks(callbacks: Mapping[str, Callable[[], None]]) -> Callable[[], None]: + """Call ``callbacks[key]`` when ``key`` is pressed in a visualizer window, e.g. for a script's hotkeys. + + Keys are letters or digits (``"N"``, ``"1"``) or W3C ``KeyboardEvent.code`` strings (``"Escape"``). + Key repeat is not a press, and a press reported by several visualizers (Kit visualizers share the + app window) fires once. Reads every visualizer window open when called; the callbacks end when + the visualizers close, or when the returned function is called. + + Args: + callbacks: The function to call for each key. Takes no arguments. + + Returns: + A function that removes the callbacks. + """ + from isaaclab.sim import SimulationContext + + by_code = {_w3c_code(key): func for key, func in callbacks.items()} + sim = SimulationContext.instance() + # shared by every window, so a press several visualizers report fires once + held: set[str] = set() + + def on_key(code: str, pressed: bool) -> None: + if not pressed: + held.discard(code) + elif code not in held: + held.add(code) + if code in by_code: + by_code[code]() + + subscriptions = [] + for visualizer in sim.visualizers if sim is not None else (): + source = visualizer.key_event_source + if source is not None: + subscriptions.append(source.add_key_listener(on_key, held.clear)) + if not subscriptions: + logger.info("No visualizer window reports keys, so the key callbacks are inactive") + + def remove() -> None: + for subscription in subscriptions: + subscription.close() + + return remove + + +def _w3c_code(key: str) -> str: + if len(key) == 1 and key.isalpha(): + return f"Key{key.upper()}" + if len(key) == 1 and key.isdigit(): + return f"Digit{key}" + return key + + def _call(callback: Callable, *args) -> None: try: callback(*args) diff --git a/source/isaaclab/test/devices/test_device_constructors.py b/source/isaaclab/test/devices/test_device_constructors.py index 52854b517659..a8f5f0f26729 100644 --- a/source/isaaclab/test/devices/test_device_constructors.py +++ b/source/isaaclab/test/devices/test_device_constructors.py @@ -143,13 +143,14 @@ def mock_environment(mocker): def test_device_constructs_and_advance_shape( mock_environment, mocker, module_name, device_cls, cfg, patched_modules, report, expected_shape ): - """Each device applies its config and ``advance()`` returns a command of its documented size.""" + """Each deprecated device warns, applies its config, and ``advance()`` returns a command of its documented size.""" device_mod = importlib.import_module(module_name) mocker.patch.dict("sys.modules", {name: mock_environment[name] for name in patched_modules}) for name in patched_modules: mocker.patch.object(device_mod, name, mock_environment[name]) - device = device_cls(cfg) + with pytest.warns(DeprecationWarning, match=f"{device_cls.__name__} is deprecated"): + device = device_cls(cfg) # Verify configuration was applied correctly for field in ("v_x_sensitivity", "v_y_sensitivity", "omega_z_sensitivity", "pos_sensitivity", "rot_sensitivity"): diff --git a/source/isaaclab/test/visualizers/test_key_event_source.py b/source/isaaclab/test/visualizers/test_key_event_source.py index e91ed52e25bc..ed031e3c8301 100644 --- a/source/isaaclab/test/visualizers/test_key_event_source.py +++ b/source/isaaclab/test/visualizers/test_key_event_source.py @@ -11,10 +11,12 @@ from __future__ import annotations import logging +from types import SimpleNamespace +from unittest.mock import patch import pytest -from isaaclab.visualizers import KeyEventSource +from isaaclab.visualizers import KeyEventSource, add_key_callbacks pytestmark = [pytest.mark.unit] @@ -138,3 +140,43 @@ def test_source_does_not_own_backend_lifecycle_or_controller_state(): assert all(not hasattr(source, name) for name in forbidden) subscription = source.add_key_listener(lambda code, pressed: None, lambda: None) assert not hasattr(subscription, "_closed"), "Subscription lifetime is owned by its source reference." + + +@pytest.fixture +def source(): + """A visualizer window's keyboard, read by :func:`add_key_callbacks` through a fake simulation.""" + key_event_source = KeyEventSource() + sim = SimpleNamespace( + visualizers=[SimpleNamespace(key_event_source=key_event_source), SimpleNamespace(key_event_source=None)] + ) + with patch("isaaclab.sim.SimulationContext.instance", return_value=sim): + yield key_event_source + + +def test_key_callbacks_fire_once_per_press(source): + pressed = [] + add_key_callbacks({"N": lambda: pressed.append("N"), "1": lambda: pressed.append("1"), "Escape": lambda: 0}) + + source._emit_key("KeyN", True) + source._emit_key("KeyN", True) # key repeat + source._emit_key("KeyN", False) + source._emit_key("KeyN", True) + source._emit_key("Digit1", True) + source._emit_key("KeyM", True) + assert pressed == ["N", "N", "1"] + + +def test_focus_loss_ends_a_held_key(source): + pressed = [] + add_key_callbacks({"N": lambda: pressed.append("N")}) + source._emit_key("KeyN", True) + source._emit_focus_lost() + source._emit_key("KeyN", True) + assert pressed == ["N", "N"] + + +def test_removing_the_key_callbacks_unsubscribes(source): + remove = add_key_callbacks({"N": lambda: None}) + assert len(source._subscriptions) == 1 + remove() + assert source._subscriptions == [] diff --git a/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.added.rst b/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.added.rst new file mode 100644 index 000000000000..64c1bb7f43d6 --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.added.rst @@ -0,0 +1,12 @@ +* Added :func:`~isaaclab_teleop.se2_keyboard_teleop_cfg`, :func:`~isaaclab_teleop.se3_keyboard_teleop_cfg`, + :func:`~isaaclab_teleop.se2_gamepad_teleop_cfg`, :func:`~isaaclab_teleop.se3_gamepad_teleop_cfg`, + :func:`~isaaclab_teleop.se2_spacemouse_teleop_cfg` and :func:`~isaaclab_teleop.se3_spacemouse_teleop_cfg`, + which build an :class:`~isaaclab_teleop.IsaacTeleopCfg` for keyboard, gamepad or SpaceMouse + SE(2)/SE(3) teleoperation. The keyboard is read in process from whichever visualizer window has + focus (no ``input`` group, no keys from other apps). The gamepad is read from ``/dev/input/jsN`` + and the SpaceMouse from ``/dev/hidrawN`` (which needs a udev rule) by the Isaac Capture + ``gamepad`` and ``spacemouse`` plugins, which their configs start with the session. Without + ``--xr`` they run headset-free on the standalone CloudXR profile. +* Added :mod:`isaaclab_teleop.device_selection`, which translates an :mod:`isaaclab.devices` + keyboard, gamepad or SpaceMouse config into the equivalent pipeline and picks the pipeline for a + script's ``--teleop_device`` request, or ``None`` when ``isaacteleop`` is not installed. diff --git a/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.changed.rst b/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.changed.rst new file mode 100644 index 000000000000..f85fc9c0115b --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.changed.rst @@ -0,0 +1,7 @@ +* Changed :class:`~isaaclab_teleop.IsaacTeleopDevice` to start, pause and reset teleoperation on the + ``B`` / ``P`` / ``R`` keys typed into a visualizer window and on a SpaceMouse pipeline's right + button, so scripts no longer bind them. While a pipeline reads the keyboard, the visualizers' + keyboard camera movement is turned off. +* Changed the ``teleop`` extra to install ``isaaccapture~=1.7.44a1``, the Isaac Capture release + with the keyboard, gamepad and SpaceMouse devices, in place of ``isaacteleop~=1.4.145``. + ``isaaccapture`` is ``isaacteleop`` renamed and still provides the ``isaacteleop`` import. diff --git a/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.minor b/source/isaaclab_teleop/changelog.d/rwiltz-keyboard-focus-surfaces.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_teleop/changelog.d/rwiltz-teleop-scripts-isaac-capture.added.rst b/source/isaaclab_teleop/changelog.d/rwiltz-teleop-scripts-isaac-capture.added.rst new file mode 100644 index 000000000000..272954f472e6 --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/rwiltz-teleop-scripts-isaac-capture.added.rst @@ -0,0 +1,3 @@ +* Added :func:`~isaaclab_teleop.script_input.open_teleop_device`, which opens a keyboard, gamepad or + SpaceMouse through Isaac Capture when ``isaacteleop`` is installed and through + :mod:`isaaclab.devices` otherwise. diff --git a/source/isaaclab_teleop/changelog.d/rwiltz-teleop-scripts-isaac-capture.minor b/source/isaaclab_teleop/changelog.d/rwiltz-teleop-scripts-isaac-capture.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi index eb5c4f82838b..fe204a802a14 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi +++ b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi @@ -31,10 +31,17 @@ __all__ = [ "create_isaac_teleop_device", "poll_control_events", "remove_camera_configs", + "se2_gamepad_teleop_cfg", + "se2_keyboard_teleop_cfg", + "se2_spacemouse_teleop_cfg", + "se3_gamepad_teleop_cfg", + "se3_keyboard_teleop_cfg", + "se3_spacemouse_teleop_cfg", ] from .camera_feed import XrCameraFeedSession from .control_events import TELEOP_CONTROL_CHANNEL_UUID, ControlEvents, SupportsControlEvents, poll_control_events +from .gamepad import se2_gamepad_teleop_cfg, se3_gamepad_teleop_cfg from .haptic_feedback import ( ControllerHapticFeedbackCfg, GloveHapticFeedbackCfg, @@ -52,7 +59,9 @@ from .isaac_teleop_cfg import ( XrCameraFeedLayoutCfg, ) from .isaac_teleop_device import IsaacTeleopDevice, create_isaac_teleop_device +from .keyboard import se2_keyboard_teleop_cfg, se3_keyboard_teleop_cfg from .session_lifecycle import cloudxr_eula_accepted +from .spacemouse import se2_spacemouse_teleop_cfg, se3_spacemouse_teleop_cfg from .system_check import SystemCheckItem, SystemCheckResult, check_system_requirements from .xr_anchor_utils import XrAnchorSynchronizer from .xr_cfg import XrAnchorRotationMode, XrCfg, remove_camera_configs diff --git a/source/isaaclab_teleop/isaaclab_teleop/_plugins.py b/source/isaaclab_teleop/isaaclab_teleop/_plugins.py new file mode 100644 index 000000000000..a36e0ae3e021 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/_plugins.py @@ -0,0 +1,24 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""The Isaac Capture plugins that read the gamepad and the SpaceMouse for their teleop configs.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from isaacteleop.teleop_session_manager import PluginConfig + + +def bundled_plugin_configs(name: str) -> list[PluginConfig]: + """The Isaac Capture plugin ``name`` (``"gamepad"`` or ``"spacemouse"``), started with the session. + + It runs from the copy bundled with Isaac Capture and pushes to the collection the pipeline's + source of the same name reads. + """ + from isaacteleop.teleop_session_manager import BUNDLED_PLUGINS_DIR, PluginConfig + + return [PluginConfig(plugin_name=name, plugin_root_id=name, search_paths=[BUNDLED_PLUGINS_DIR], required=True)] diff --git a/source/isaaclab_teleop/isaaclab_teleop/device_selection.py b/source/isaaclab_teleop/isaaclab_teleop/device_selection.py new file mode 100644 index 000000000000..03f66d446acd --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/device_selection.py @@ -0,0 +1,103 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Serve a script's teleop device request with an Isaac Capture pipeline when one can. + +An environment declares its keyboard, gamepad and SpaceMouse devices with the +:mod:`isaaclab.devices` configs (:class:`~isaaclab.devices.Se3KeyboardCfg`, ...). +:func:`select_isaac_teleop_cfg` translates such a config into the equivalent Isaac Capture +pipeline. Without the ``isaacteleop`` package, or for a device it has no pipeline for (e.g. OpenXR +hand tracking), it returns ``None`` and the caller uses the :mod:`isaaclab.devices` device. +""" + +from __future__ import annotations + +import importlib.util +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any + +from isaaclab.devices.gamepad import Se2GamepadCfg, Se3GamepadCfg +from isaaclab.devices.keyboard import Se2KeyboardCfg, Se3KeyboardCfg +from isaaclab.devices.spacemouse import Se2SpaceMouseCfg, Se3SpaceMouseCfg + +from .gamepad import se2_gamepad_teleop_cfg, se3_gamepad_teleop_cfg +from .keyboard import se2_keyboard_teleop_cfg, se3_keyboard_teleop_cfg +from .spacemouse import se2_spacemouse_teleop_cfg, se3_spacemouse_teleop_cfg + +if TYPE_CHECKING: + from isaaclab.devices import DeviceCfg + + from .isaac_teleop_cfg import IsaacTeleopCfg + +# Each device config type, the builder of its pipeline, and the config fields the builder takes. +_PIPELINES = { + Se3KeyboardCfg: (se3_keyboard_teleop_cfg, ("pos_sensitivity", "rot_sensitivity", "gripper_term")), + Se2KeyboardCfg: (se2_keyboard_teleop_cfg, ("v_x_sensitivity", "v_y_sensitivity", "omega_z_sensitivity")), + Se3GamepadCfg: (se3_gamepad_teleop_cfg, ("pos_sensitivity", "rot_sensitivity", "dead_zone", "gripper_term")), + Se2GamepadCfg: ( + se2_gamepad_teleop_cfg, + ("v_x_sensitivity", "v_y_sensitivity", "omega_z_sensitivity", "dead_zone"), + ), + Se3SpaceMouseCfg: (se3_spacemouse_teleop_cfg, ("pos_sensitivity", "rot_sensitivity", "gripper_term")), + Se2SpaceMouseCfg: (se2_spacemouse_teleop_cfg, ("v_x_sensitivity", "v_y_sensitivity", "omega_z_sensitivity")), +} + + +def _isaacteleop_available() -> bool: + return importlib.util.find_spec("isaacteleop") is not None + + +def pipeline_from_device_cfg(device_cfg: DeviceCfg, sim_device: str) -> IsaacTeleopCfg | None: + """Build the Isaac Capture pipeline equivalent to a device config. + + Args: + device_cfg: A device config, e.g. :class:`~isaaclab.devices.Se3KeyboardCfg`. + sim_device: Torch device for the pipeline's output tensors. + + Returns: + The pipeline, or ``None`` when no pipeline serves this device config type. + """ + entry = _PIPELINES.get(type(device_cfg)) + if entry is None: + return None + builder, fields = entry + return builder(**{field: getattr(device_cfg, field) for field in fields}, sim_device=sim_device) + + +def select_isaac_teleop_cfg( + env_cfg: Any, + device_name: str | None, + default_device_cfgs: Mapping[str, DeviceCfg], + sim_device: str, +) -> IsaacTeleopCfg | None: + """Pick the Isaac Capture pipeline that serves a script's device request. + + Without ``device_name``, the environment's declared ``isaac_teleop`` pipeline is used, else + the ``"keyboard"`` entry of ``default_device_cfgs``. With it, the environment's + ``teleop_devices`` entry of that name is used, else the ``default_device_cfgs`` entry. + + Args: + env_cfg: The environment config, read for ``isaac_teleop`` and ``teleop_devices``. + device_name: The requested device (e.g. ``--teleop_device``), or ``None``. + default_device_cfgs: The script's own device configs, by name, for devices the + environment does not declare. + sim_device: Torch device for the pipeline's output tensors. + + Returns: + The pipeline, or ``None`` when ``isaacteleop`` is not installed or no pipeline serves the + requested device; the caller then uses the :mod:`isaaclab.devices` device. + """ + if not _isaacteleop_available(): + return None + if device_name is None: + declared = getattr(env_cfg, "isaac_teleop", None) + if declared is not None: + return declared + device_name = "keyboard" + env_devices = getattr(getattr(env_cfg, "teleop_devices", None), "devices", None) or {} + device_cfg = env_devices.get(device_name) or default_device_cfgs.get(device_name) + if device_cfg is None: + return None + return pipeline_from_device_cfg(device_cfg, sim_device) diff --git a/source/isaaclab_teleop/isaaclab_teleop/gamepad/__init__.py b/source/isaaclab_teleop/isaaclab_teleop/gamepad/__init__.py new file mode 100644 index 000000000000..2957e08ed477 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/gamepad/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Gamepad device for SE(2) and SE(3) control.""" + +from isaaclab.utils.module import lazy_export + +lazy_export() diff --git a/source/isaaclab_teleop/isaaclab_teleop/gamepad/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/gamepad/__init__.pyi new file mode 100644 index 000000000000..cd2a8c1216bf --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/gamepad/__init__.pyi @@ -0,0 +1,12 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "se2_gamepad_teleop_cfg", + "se3_gamepad_teleop_cfg", +] + +from .se2_gamepad import se2_gamepad_teleop_cfg +from .se3_gamepad import se3_gamepad_teleop_cfg diff --git a/source/isaaclab_teleop/isaaclab_teleop/gamepad/se2_gamepad.py b/source/isaaclab_teleop/isaaclab_teleop/gamepad/se2_gamepad.py new file mode 100644 index 000000000000..6dbaae854f26 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/gamepad/se2_gamepad.py @@ -0,0 +1,92 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Gamepad-driven IsaacTeleopCfg builder for SE(2) control.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..isaac_teleop_cfg import IsaacTeleopCfg + + +def se2_gamepad_teleop_cfg( + v_x_sensitivity: float = 1.0, + v_y_sensitivity: float = 1.0, + omega_z_sensitivity: float = 1.0, + dead_zone: float = 0.15, + sim_device: str = "cpu", +) -> IsaacTeleopCfg: + r"""Build an :class:`~isaaclab_teleop.IsaacTeleopCfg` for gamepad-driven SE(2) velocity control. + + Raw stick state is retargeted via :class:`~isaacteleop.retargeters.GamepadToSe2Retargeter`. + + The gamepad is read by the Isaac Capture ``gamepad`` plugin from the first connected Linux + joystick-API device (``/dev/input/jsN``); the config starts the plugin with the session, so no + headset or separate process is needed, and udev grants the logged-in user access without the + ``input`` group. + + Key bindings: + ====================== ========================= ======================== + Command Key (+ve axis) Key (-ve axis) + ====================== ========================= ======================== + Move along x-axis left stick up left stick down + Move along y-axis left stick right left stick left + Rotate along z-axis right stick right right stick left + ====================== ========================= ======================== + + No physical button drives session start/stop/reset -- those come from the teleop session's + control channel or an auxiliary control device, matching the legacy ``Se2Gamepad``. + + Args: + v_x_sensitivity: Linear x-velocity scale per step. + v_y_sensitivity: Linear y-velocity scale per step. + omega_z_sensitivity: Angular z-velocity scale per step. + dead_zone: Stick deflection magnitude below which input is treated as zero. + sim_device: Torch device string for the pipeline's output tensors. + + Returns: + IsaacTeleopCfg driving this pipeline. + """ + from .._plugins import bundled_plugin_configs + from ..isaac_teleop_cfg import IsaacTeleopCfg + + def build_pipeline(): + from isaacteleop.retargeters import GamepadToSe2Retargeter, GamepadToSe2RetargeterConfig, TensorReorderer + from isaacteleop.retargeting_engine.deviceio_source_nodes import GamepadSource + from isaacteleop.retargeting_engine.interface import OutputCombiner + + gamepad_source = GamepadSource("gamepad") + + se2 = GamepadToSe2Retargeter( + GamepadToSe2RetargeterConfig( + v_x_sensitivity=v_x_sensitivity, + v_y_sensitivity=v_y_sensitivity, + omega_z_sensitivity=omega_z_sensitivity, + dead_zone=dead_zone, + ), + name="se2", + ) + connected_se2 = se2.connect({"gamepad": gamepad_source.output("gamepad")}) + + base_command_elements = ["v_x", "v_y", "omega_z"] + reorderer = TensorReorderer( + input_config={"base_command": base_command_elements}, + output_order=base_command_elements, + name="action_reorderer", + input_types={"base_command": "array"}, + ) + connected_reorderer = reorderer.connect({"base_command": connected_se2.output("base_command")}) + + return OutputCombiner({"action": connected_reorderer.output("output")}) + + return IsaacTeleopCfg( + pipeline_builder=build_pipeline, + sim_device=sim_device, + teleoperation_active_default=True, + app_name="IsaacLabGamepadSe2", + plugins=bundled_plugin_configs("gamepad"), + ) diff --git a/source/isaaclab_teleop/isaaclab_teleop/gamepad/se3_gamepad.py b/source/isaaclab_teleop/isaaclab_teleop/gamepad/se3_gamepad.py new file mode 100644 index 000000000000..020df585c509 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/gamepad/se3_gamepad.py @@ -0,0 +1,113 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Gamepad-driven IsaacTeleopCfg builder for SE(3) control.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..isaac_teleop_cfg import IsaacTeleopCfg + + +def se3_gamepad_teleop_cfg( + pos_sensitivity: float = 1.0, + rot_sensitivity: float = 1.6, + dead_zone: float = 0.15, + gripper_term: bool = True, + sim_device: str = "cpu", +) -> IsaacTeleopCfg: + """Build an :class:`~isaaclab_teleop.IsaacTeleopCfg` for gamepad-driven SE(3) delta-pose control. + + Raw stick/button state is retargeted via + :class:`~isaacteleop.retargeters.GamepadToSe3RelRetargeter` / + :class:`~isaacteleop.retargeters.GamepadGripperRetargeter`. + + The gamepad is read by the Isaac Capture ``gamepad`` plugin from the first connected Linux + joystick-API device (``/dev/input/jsN``); the config starts the plugin with the session, so no + headset or separate process is needed, and udev grants the logged-in user access without the + ``input`` group. + + Stick and Button bindings: + ============================ ========================= ========================= + Description Stick/Button (+ve axis) Stick/Button (-ve axis) + ============================ ========================= ========================= + Toggle gripper(open/close) West Button (X, Square) West Button (X, Square) + Move along x-axis Left Stick Up Left Stick Down + Move along y-axis Left Stick Right Left Stick Left + Move along z-axis Right Stick Up Right Stick Down + Rotate along x-axis D-Pad Left D-Pad Right + Rotate along y-axis D-Pad Down D-Pad Up + Rotate along z-axis Right Stick Right Right Stick Left + ============================ ========================= ========================= + + No physical button drives session start/stop/reset -- those come from the teleop session's + control channel or an auxiliary control device, matching the legacy ``Se3Gamepad``. + + Args: + pos_sensitivity: Position delta scale per step [m]. + rot_sensitivity: Rotation delta scale per step [rad]. + dead_zone: Stick deflection magnitude below which input is treated as zero. + gripper_term: Whether to include a gripper open/close command as a 7th action element. + sim_device: Torch device string for the pipeline's output tensors. + + Returns: + IsaacTeleopCfg driving this pipeline. + """ + from .._plugins import bundled_plugin_configs + from ..isaac_teleop_cfg import IsaacTeleopCfg + + def build_pipeline(): + from isaacteleop.retargeters import ( + GamepadGripperRetargeter, + GamepadToSe3RelRetargeter, + GamepadToSe3RelRetargeterConfig, + TensorReorderer, + ) + from isaacteleop.retargeting_engine.deviceio_source_nodes import GamepadSource + from isaacteleop.retargeting_engine.interface import OutputCombiner + + gamepad_source = GamepadSource("gamepad") + + se3 = GamepadToSe3RelRetargeter( + GamepadToSe3RelRetargeterConfig( + pos_sensitivity=pos_sensitivity, rot_sensitivity=rot_sensitivity, dead_zone=dead_zone + ), + name="se3", + ) + connected_se3 = se3.connect({"gamepad": gamepad_source.output("gamepad")}) + + ee_delta_elements = ["dx", "dy", "dz", "drx", "dry", "drz"] + input_config = {"ee_delta": ee_delta_elements} + input_types = {"ee_delta": "array"} + reorder_inputs = {"ee_delta": connected_se3.output("ee_delta")} + output_order = list(ee_delta_elements) + + if gripper_term: + gripper = GamepadGripperRetargeter(name="gripper") + connected_gripper = gripper.connect({"gamepad": gamepad_source.output("gamepad")}) + input_config["gripper_command"] = ["gripper_value"] + input_types["gripper_command"] = "scalar" + reorder_inputs["gripper_command"] = connected_gripper.output("gripper_command") + output_order.append("gripper_value") + + reorderer = TensorReorderer( + input_config=input_config, + output_order=output_order, + name="action_reorderer", + input_types=input_types, + ) + connected_reorderer = reorderer.connect(reorder_inputs) + + return OutputCombiner({"action": connected_reorderer.output("output")}) + + return IsaacTeleopCfg( + pipeline_builder=build_pipeline, + sim_device=sim_device, + teleoperation_active_default=True, + app_name="IsaacLabGamepadSe3", + plugins=bundled_plugin_configs("gamepad"), + ) diff --git a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py index 2170a29207db..e7546ee50777 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py +++ b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py @@ -9,7 +9,7 @@ import logging from collections.abc import Callable -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, ClassVar import numpy as np import torch @@ -75,7 +75,10 @@ class IsaacTeleopDevice: Teleop commands: The device supports callbacks for START, STOP, and RESET commands that can be triggered via the message-channel control pipeline or - registered directly via :meth:`add_callback`. + registered directly via :meth:`add_callback`. Without a headset, the + ``B`` / ``P`` / ``R`` keys typed into a visualizer window start, pause + and reset teleoperation, and a SpaceMouse pipeline's right button + resets it. Example: .. code-block:: python @@ -106,6 +109,11 @@ class IsaacTeleopDevice: env.step(action.repeat(num_envs, 1)) """ + _START_KEY: ClassVar[int] = 48 # evdev code of B + _STOP_KEY: ClassVar[int] = 25 # P + _RESET_KEY: ClassVar[int] = 19 # R + _SPACEMOUSE_RIGHT_BUTTON: ClassVar[int] = 1 + def __init__( self, cfg: IsaacTeleopCfg, @@ -170,6 +178,7 @@ def __init__( ) self._prev_right_a_pressed = False + self._prev_spacemouse_right_pressed = False self._prev_control_is_active: bool | None = None # Each visualizer is created lazily on the first frame with matching @@ -361,6 +370,7 @@ def advance(self, target_T_world: np.ndarray | torch.Tensor | SupportsDLPack | N if self._enable_debug_visualization: self._update_debug_visualization() + self._handle_operator_input() self._dispatch_control_callbacks() return action @@ -461,6 +471,33 @@ def _dispatch_control_callbacks(self) -> None: self._command_handler.fire("START" if events.is_active else "STOP") self._prev_control_is_active = events.is_active + # ------------------------------------------------------------------ + # Operator keys and buttons + # ------------------------------------------------------------------ + + def _handle_operator_input(self) -> None: + """Start, pause or reset on the operator's ``B`` / ``P`` / ``R`` keys and SpaceMouse right button. + + An operator reset also pauses, so teleop resumes on ``B``. The keys come from the visualizer + windows, each press once whatever the retargeting mode. + """ + for code in self._session_lifecycle.drain_pressed_keys(): + if code == self._START_KEY: + self.request_start() + elif code == self._STOP_KEY: + self.request_stop() + elif code == self._RESET_KEY: + self.reset(pause=True) + + result = self._session_lifecycle.last_step_result + buttons = None if result is None else result.get("spacemouse_buttons") + if buttons is None or buttons.is_none: + return + pressed = bool(buttons[0][self._SPACEMOUSE_RIGHT_BUTTON]) + if pressed and not self._prev_spacemouse_right_pressed: + self.reset(pause=True) + self._prev_spacemouse_right_pressed = pressed + # ------------------------------------------------------------------ # Controller button polling (glue between session and anchor manager) # ------------------------------------------------------------------ diff --git a/source/isaaclab_teleop/isaaclab_teleop/keyboard/__init__.py b/source/isaaclab_teleop/isaaclab_teleop/keyboard/__init__.py new file mode 100644 index 000000000000..d5a91b84fdd1 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/keyboard/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Keyboard device for SE(2) and SE(3) control.""" + +from isaaclab.utils.module import lazy_export + +lazy_export() diff --git a/source/isaaclab_teleop/isaaclab_teleop/keyboard/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/keyboard/__init__.pyi new file mode 100644 index 000000000000..87898b0a4ce5 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/keyboard/__init__.pyi @@ -0,0 +1,12 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "se2_keyboard_teleop_cfg", + "se3_keyboard_teleop_cfg", +] + +from .se2_keyboard import se2_keyboard_teleop_cfg +from .se3_keyboard import se3_keyboard_teleop_cfg diff --git a/source/isaaclab_teleop/isaaclab_teleop/keyboard/se2_keyboard.py b/source/isaaclab_teleop/isaaclab_teleop/keyboard/se2_keyboard.py new file mode 100644 index 000000000000..d226dcbd9ac0 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/keyboard/se2_keyboard.py @@ -0,0 +1,85 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Keyboard-driven IsaacTeleopCfg builder for SE(2) control.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..isaac_teleop_cfg import IsaacTeleopCfg + + +def se2_keyboard_teleop_cfg( + v_x_sensitivity: float = 0.8, + v_y_sensitivity: float = 0.4, + omega_z_sensitivity: float = 1.0, + sim_device: str = "cpu", +) -> IsaacTeleopCfg: + r"""Build an :class:`~isaaclab_teleop.IsaacTeleopCfg` for keyboard-driven SE(2) velocity control. + + Keys come from an in-process :class:`~isaacteleop.retargeting_engine.deviceio_source_nodes.KeyboardSource`, + fed by whichever visualizer window has focus (see :ref:`isaac-teleop-keyboard`), and are + retargeted via :class:`~isaacteleop.retargeters.KeyboardToSe2Retargeter`. The pipeline needs + no plugin process or headset. + + Key bindings: + ====================== ========================= ======================== + Command Key (+ve axis) Key (-ve axis) + ====================== ========================= ======================== + Move along x-axis Numpad 8 / Arrow Up Numpad 2 / Arrow Down + Move along y-axis Numpad 4 / Arrow Left Numpad 6 / Arrow Right + Rotate along z-axis Numpad 7 / Z Numpad 9 / X + ====================== ========================= ======================== + + ``B`` / ``P`` / ``R`` start, pause and reset teleoperation (see + :class:`~isaaclab_teleop.IsaacTeleopDevice`). + + Args: + v_x_sensitivity: Linear x-velocity scale per step. + v_y_sensitivity: Linear y-velocity scale per step. + omega_z_sensitivity: Angular z-velocity scale per step. + sim_device: Torch device string for the pipeline's output tensors. + + Returns: + IsaacTeleopCfg driving this pipeline. + """ + from ..isaac_teleop_cfg import IsaacTeleopCfg + + def build_pipeline(): + from isaacteleop.retargeters import KeyboardToSe2Retargeter, KeyboardToSe2RetargeterConfig, TensorReorderer + from isaacteleop.retargeting_engine.deviceio_source_nodes import KeyboardSource + from isaacteleop.retargeting_engine.interface import OutputCombiner + + keyboard_source = KeyboardSource("keyboard") + + se2 = KeyboardToSe2Retargeter( + KeyboardToSe2RetargeterConfig( + v_x_sensitivity=v_x_sensitivity, + v_y_sensitivity=v_y_sensitivity, + omega_z_sensitivity=omega_z_sensitivity, + ), + name="se2", + ) + connected_se2 = se2.connect({"keyboard_held": keyboard_source.output("keyboard_held")}) + + base_command_elements = ["v_x", "v_y", "omega_z"] + reorderer = TensorReorderer( + input_config={"base_command": base_command_elements}, + output_order=base_command_elements, + name="action_reorderer", + input_types={"base_command": "array"}, + ) + connected_reorderer = reorderer.connect({"base_command": connected_se2.output("base_command")}) + + return OutputCombiner({"action": connected_reorderer.output("output")}) + + return IsaacTeleopCfg( + pipeline_builder=build_pipeline, + sim_device=sim_device, + teleoperation_active_default=True, + app_name="IsaacLabKeyboardSe2", + ) diff --git a/source/isaaclab_teleop/isaaclab_teleop/keyboard/se3_keyboard.py b/source/isaaclab_teleop/isaaclab_teleop/keyboard/se3_keyboard.py new file mode 100644 index 000000000000..8d1fd6f1da2f --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/keyboard/se3_keyboard.py @@ -0,0 +1,104 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Keyboard-driven IsaacTeleopCfg builder for SE(3) control.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..isaac_teleop_cfg import IsaacTeleopCfg + + +def se3_keyboard_teleop_cfg( + pos_sensitivity: float = 0.4, + rot_sensitivity: float = 0.8, + gripper_term: bool = True, + sim_device: str = "cpu", +) -> IsaacTeleopCfg: + """Build an :class:`~isaaclab_teleop.IsaacTeleopCfg` for keyboard-driven SE(3) delta-pose control. + + Keys come from an in-process :class:`~isaacteleop.retargeting_engine.deviceio_source_nodes.KeyboardSource`, + fed by whichever visualizer window has focus (see :ref:`isaac-teleop-keyboard`), and are + retargeted via :class:`~isaacteleop.retargeters.KeyboardToSe3RelRetargeter` / + :class:`~isaacteleop.retargeters.KeyboardGripperRetargeter`. The pipeline needs no plugin + process or headset. + + Key bindings: + ============================== ================= ================= + Description Key (+ve axis) Key (-ve axis) + ============================== ================= ================= + Toggle gripper (open/close) K + Move along x-axis W S + Move along y-axis A D + Move along z-axis Q E + Rotate along x-axis Z X + Rotate along y-axis T G + Rotate along z-axis C V + ============================== ================= ================= + + ``B`` / ``P`` / ``R`` start, pause and reset teleoperation (see + :class:`~isaaclab_teleop.IsaacTeleopDevice`). + + Args: + pos_sensitivity: Position delta scale per step [m]. + rot_sensitivity: Rotation delta scale per step [rad]. + gripper_term: Whether to include a gripper open/close command as a 7th action element. + sim_device: Torch device string for the pipeline's output tensors. + + Returns: + IsaacTeleopCfg driving this pipeline. + """ + from ..isaac_teleop_cfg import IsaacTeleopCfg + + def build_pipeline(): + from isaacteleop.retargeters import ( + KeyboardGripperRetargeter, + KeyboardToSe3RelRetargeter, + KeyboardToSe3RelRetargeterConfig, + TensorReorderer, + ) + from isaacteleop.retargeting_engine.deviceio_source_nodes import KeyboardSource + from isaacteleop.retargeting_engine.interface import OutputCombiner + + keyboard_source = KeyboardSource("keyboard") + + se3 = KeyboardToSe3RelRetargeter( + KeyboardToSe3RelRetargeterConfig(pos_sensitivity=pos_sensitivity, rot_sensitivity=rot_sensitivity), + name="se3", + ) + connected_se3 = se3.connect({"keyboard_held": keyboard_source.output("keyboard_held")}) + + ee_delta_elements = ["dx", "dy", "dz", "drx", "dry", "drz"] + input_config = {"ee_delta": ee_delta_elements} + input_types = {"ee_delta": "array"} + reorder_inputs = {"ee_delta": connected_se3.output("ee_delta")} + output_order = list(ee_delta_elements) + + if gripper_term: + gripper = KeyboardGripperRetargeter(name="gripper") + connected_gripper = gripper.connect({"keyboard_pressed": keyboard_source.output("keyboard_pressed")}) + input_config["gripper_command"] = ["gripper_value"] + input_types["gripper_command"] = "scalar" + reorder_inputs["gripper_command"] = connected_gripper.output("gripper_command") + output_order.append("gripper_value") + + reorderer = TensorReorderer( + input_config=input_config, + output_order=output_order, + name="action_reorderer", + input_types=input_types, + ) + connected_reorderer = reorderer.connect(reorder_inputs) + + return OutputCombiner({"action": connected_reorderer.output("output")}) + + return IsaacTeleopCfg( + pipeline_builder=build_pipeline, + sim_device=sim_device, + teleoperation_active_default=True, + app_name="IsaacLabKeyboardSe3", + ) diff --git a/source/isaaclab_teleop/isaaclab_teleop/script_input.py b/source/isaaclab_teleop/isaaclab_teleop/script_input.py new file mode 100644 index 000000000000..1342378d9c0c --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/script_input.py @@ -0,0 +1,49 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Teleop devices for scripts, through Isaac Capture when it is installed.""" + +from __future__ import annotations + +import contextlib +from collections.abc import Iterator +from typing import TYPE_CHECKING, Any + +from .device_selection import _isaacteleop_available, pipeline_from_device_cfg + +if TYPE_CHECKING: + from isaaclab.devices import DeviceCfg + + +@contextlib.contextmanager +def open_teleop_device(device_cfg: DeviceCfg) -> Iterator[Any]: + """Open the keyboard, gamepad or SpaceMouse a device config describes, through Isaac Capture when installed. + + With the ``isaacteleop`` package the device runs as an Isaac Capture pipeline, started without a + headset on the standalone CloudXR profile, and ``B`` / ``P`` / ``R`` start, pause and reset it (see + :class:`~isaaclab_teleop.IsaacTeleopDevice`). Without it, the :mod:`isaaclab.devices` device is used. + Either way ``advance()`` returns the device command (``None`` while an Isaac Capture session starts) + and ``add_callback("R", func)`` registers a reset callback. + + Args: + device_cfg: A keyboard, gamepad or SpaceMouse config, e.g. :class:`~isaaclab.devices.Se3KeyboardCfg`. + + Yields: + The open device. + """ + pipeline = pipeline_from_device_cfg(device_cfg, device_cfg.sim_device) if _isaacteleop_available() else None + if pipeline is None: + from isaaclab.utils import instantiate + + yield instantiate(device_cfg) + return + from .isaac_teleop_cfg import CLOUDXR_STANDALONE_ENV + from .isaac_teleop_device import create_isaac_teleop_device + + with create_isaac_teleop_device( + pipeline, use_kit_xr_bridge=False, cloudxr_env_file=CLOUDXR_STANDALONE_ENV + ) as device: + device.request_start() + yield device diff --git a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py index 75f34b9310ae..b6864fa28ade 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py +++ b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py @@ -177,6 +177,33 @@ def _execution_events_to_control(ee: ExecutionEvents) -> ControlEvents: return ControlEvents(is_active=is_active, should_reset=ee.reset) +class _ControlKeyListener: + """Queues one visualizer window's key presses for :meth:`TeleopSessionLifecycle.drain_pressed_keys`. + + Keeps that window's held keys, so key repeat is not a press and a focus loss on one window + leaves the others alone. + """ + + def __init__(self, pressed: list[int]) -> None: + self._pressed = pressed + self._held: set[int] = set() + + def on_key(self, code: str, pressed: bool) -> None: + from isaacteleop.deviceio_trackers import evdev_code_from_w3c + + evdev_code = evdev_code_from_w3c(code) + if evdev_code is None: + return + if not pressed: + self._held.discard(evdev_code) + elif evdev_code not in self._held: + self._held.add(evdev_code) + self._pressed.append(evdev_code) + + def on_focus_lost(self) -> None: + self._held.clear() + + class TeleopSessionLifecycle: """Manages the Isaac Capture session lifecycle. @@ -191,6 +218,7 @@ class TeleopSessionLifecycle: 6. Building external inputs for pipeline leaf nodes (e.g. world-to-anchor transform) 7. Stepping the session and extracting the flattened action tensor 8. Managing the optional retargeting tuning UI + 9. Feeding the pipeline's keyboards from the focused visualizer windows """ WORLD_T_ANCHOR_INPUT_NAME = "world_T_anchor" @@ -330,6 +358,14 @@ def __init__( # Fallback for host-initiated resets when no control pipeline is configured self._pending_reset = False + # Keyboards: the visualizer windows feeding the pipeline's keyboards, and the operator's key + # presses, read straight from those windows so each press is consumed once whatever the + # retargeting mode. + self._keyboard_detachers: list[Callable[[], None]] = [] + self._attached_visualizers: list[Any] = [] + self._pipeline_keyboards: list[Any] | None = None # found once per session + self._pressed_keys: list[int] = [] + # CloudXR runtime launcher (created in start if configured, stopped in stop) self._cloudxr_launcher: CloudXRLauncher | None = None @@ -559,6 +595,9 @@ def start(self) -> None: self._last_step_result = None self._pipeline = self._build_combined_pipeline(user_pipeline) + self._pressed_keys.clear() + self._pipeline_keyboards = None + self._attach_keyboards() # Build the optional haptic sink. It reuses the button-controller # tracker so no additional ControllersSource (and thus no duplicate @@ -607,6 +646,9 @@ def stop(self, exc_type=None, exc_val=None, exc_tb=None) -> None: exc_val: Exception value. exc_tb: Exception traceback. """ + self._detach_keyboards() + self._pressed_keys.clear() + # Close the retargeting tuning UI first if self._retargeting_ui_ctx is not None: self._retargeting_ui_ctx.__exit__(exc_type, exc_val, exc_tb) @@ -671,18 +713,86 @@ def _build_combined_pipeline(self, user_pipeline): from isaacteleop.retargeting_engine.deviceio_source_nodes import ControllersSource from isaacteleop.retargeting_engine.interface import OutputCombiner + # Forward every output the user pipeline declares, not just "action", so the + # device can read the others (e.g. the SpaceMouse buttons) from last_step_result. + pipeline_outputs: dict[str, Any] = {name: user_pipeline.output(name) for name in user_pipeline.output_types()} + # Stored on self so start() can reuse this ControllersSource's tracker for # the haptic sink, avoiding a second controller action-set attachment. self._button_controllers = ControllersSource("_button_controllers") - pipeline_outputs: dict[str, Any] = { - "action": user_pipeline.output("action"), - self._CONTROLLER_RIGHT_KEY: self._button_controllers.output(ControllersSource.RIGHT), - } + pipeline_outputs[self._CONTROLLER_RIGHT_KEY] = self._button_controllers.output(ControllersSource.RIGHT) if self._enable_debug_visualization: pipeline_outputs[self._CONTROLLER_LEFT_KEY] = self._button_controllers.output(ControllersSource.LEFT) self._chain_hand_debug_outputs(user_pipeline, pipeline_outputs) return OutputCombiner(pipeline_outputs) + # ------------------------------------------------------------------ + # Keyboards + # ------------------------------------------------------------------ + + def _attach_keyboards(self) -> None: + """Feed the pipeline's keyboards and the operator's key presses from the visualizer windows. + + Called at start and on each step, since visualizer windows can open after the session starts; + windows already attached are skipped. While the pipeline has a keyboard, each visualizer's + keyboard camera movement is turned off. + """ + if self._pipeline is None or self._is_replay: + return + try: + from isaaclab.sim import SimulationContext + + sim = SimulationContext.instance() + except Exception: + return + if sim is None: + return + if self._pipeline_keyboards is None: + from isaacteleop.retargeting_engine.deviceio_source_nodes import KeyboardSource, find_sources + + self._pipeline_keyboards = find_sources(self._pipeline, KeyboardSource) + keyboards = self._pipeline_keyboards + for visualizer in sim.visualizers: + # a visualizer without a window that reports keys has no source + source = visualizer.key_event_source + if source is None or any(visualizer is attached for attached in self._attached_visualizers): + continue + # attached even if this fails, so a failing window is not retried every step + self._attached_visualizers.append(visualizer) + try: + for keyboard in keyboards: + self._keyboard_detachers.append(keyboard.attach(source, capture=False).close) + if keyboards: + visualizer.set_camera_keys_enabled(False) + self._keyboard_detachers.append(lambda v=visualizer: v.set_camera_keys_enabled(True)) + listener = _ControlKeyListener(self._pressed_keys) + self._keyboard_detachers.append(source.add_key_listener(listener.on_key, listener.on_focus_lost).close) + except Exception as e: + logger.warning(f"Keyboard input from {type(visualizer).__name__} is unavailable: {e}") + continue + logger.info(f"Isaac Capture keyboard input from {type(visualizer).__name__}") + + def _detach_keyboards(self) -> None: + """Detach every visualizer window, releasing held keys and restoring keyboard camera movement.""" + detachers, self._keyboard_detachers = self._keyboard_detachers, [] + for detach in detachers: + try: + detach() + except Exception as e: + logger.debug(f"Suppressed error detaching a visualizer keyboard: {e}") + self._attached_visualizers = [] + self._pipeline_keyboards = None + + def drain_pressed_keys(self) -> list[int]: + """Evdev codes of the keys pressed in the visualizer windows since the last call, in order. + + Each press is returned once, whatever the retargeting execution mode; key repeat is not a + press. Presses from before the session started or stopped are dropped. + """ + pressed = list(self._pressed_keys) + self._pressed_keys.clear() + return pressed + @staticmethod def _chain_hand_debug_outputs(user_pipeline, pipeline_outputs: dict) -> None: """Auto-discover a ``HandsSource`` and chain its raw outputs. @@ -1196,6 +1306,9 @@ def step( if not self._try_start_session(): return None + # Visualizer windows can open after the session starts. + self._attach_keyboards() + # Build external inputs (e.g. world-to-anchor transform) if the # pipeline contains ValueInput leaf nodes. external_inputs = self._build_external_inputs(anchor_world_matrix_fn, target_T_world) diff --git a/source/isaaclab_teleop/isaaclab_teleop/spacemouse/__init__.py b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/__init__.py new file mode 100644 index 000000000000..9e7710143178 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""SpaceMouse device for SE(2) and SE(3) control.""" + +from isaaclab.utils.module import lazy_export + +lazy_export() diff --git a/source/isaaclab_teleop/isaaclab_teleop/spacemouse/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/__init__.pyi new file mode 100644 index 000000000000..68cd178c261f --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/__init__.pyi @@ -0,0 +1,12 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "se2_spacemouse_teleop_cfg", + "se3_spacemouse_teleop_cfg", +] + +from .se2_spacemouse import se2_spacemouse_teleop_cfg +from .se3_spacemouse import se3_spacemouse_teleop_cfg diff --git a/source/isaaclab_teleop/isaaclab_teleop/spacemouse/se2_spacemouse.py b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/se2_spacemouse.py new file mode 100644 index 000000000000..48e0b8961c24 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/se2_spacemouse.py @@ -0,0 +1,99 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Spacemouse-driven IsaacTeleopCfg builder for SE(2) control.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..isaac_teleop_cfg import IsaacTeleopCfg + + +def se2_spacemouse_teleop_cfg( + v_x_sensitivity: float = 0.8, + v_y_sensitivity: float = 0.4, + omega_z_sensitivity: float = 1.0, + sim_device: str = "cpu", +) -> IsaacTeleopCfg: + r"""Build an :class:`~isaaclab_teleop.IsaacTeleopCfg` for spacemouse-driven SE(2) velocity control. + + Raw axis/button state is retargeted via + :class:`~isaacteleop.retargeters.SpaceMouseToSe2Retargeter`. + + The SpaceMouse is read by the Isaac Capture ``spacemouse`` plugin from its HID device + (``/dev/hidrawN``); the config starts the plugin with the session, so no headset or separate + process is needed. Reading ``/dev/hidraw*`` needs a udev rule granting the user access (see + :ref:`SpaceMouse permissions `). + + The interface finds and uses the first supported device connected to the computer. + Currently tested for the SpaceMouse Compact + (https://3dconnexion.com/de/product/spacemouse-compact/). + + The pipeline also outputs ``"spacemouse_buttons"``, from which + :class:`~isaaclab_teleop.IsaacTeleopDevice` resets teleoperation on the right button. + + Args: + v_x_sensitivity: Linear x-velocity scale per step. + v_y_sensitivity: Linear y-velocity scale per step. + omega_z_sensitivity: Angular z-velocity scale per step. + sim_device: Torch device string for the pipeline's output tensors. + + Returns: + IsaacTeleopCfg driving this pipeline. + """ + from .._plugins import bundled_plugin_configs + from ..isaac_teleop_cfg import IsaacTeleopCfg + + def build_pipeline(): + from isaacteleop.retargeters import ( + SpaceMouseToSe2Retargeter, + SpaceMouseToSe2RetargeterConfig, + TensorReorderer, + ) + from isaacteleop.retargeting_engine.deviceio_source_nodes import SpaceMouseSource + from isaacteleop.retargeting_engine.interface import OutputCombiner + + spacemouse_source = SpaceMouseSource("spacemouse") + + se2 = SpaceMouseToSe2Retargeter( + SpaceMouseToSe2RetargeterConfig( + v_x_sensitivity=v_x_sensitivity, + v_y_sensitivity=v_y_sensitivity, + omega_z_sensitivity=omega_z_sensitivity, + ), + name="se2", + ) + connected_se2 = se2.connect( + { + "spacemouse_translation": spacemouse_source.output("spacemouse_translation"), + "spacemouse_rotation": spacemouse_source.output("spacemouse_rotation"), + } + ) + + base_command_elements = ["v_x", "v_y", "omega_z"] + reorderer = TensorReorderer( + input_config={"base_command": base_command_elements}, + output_order=base_command_elements, + name="action_reorderer", + input_types={"base_command": "array"}, + ) + connected_reorderer = reorderer.connect({"base_command": connected_se2.output("base_command")}) + + return OutputCombiner( + { + "action": connected_reorderer.output("output"), + "spacemouse_buttons": spacemouse_source.output("spacemouse_buttons"), + } + ) + + return IsaacTeleopCfg( + pipeline_builder=build_pipeline, + sim_device=sim_device, + teleoperation_active_default=True, + app_name="IsaacLabSpaceMouseSe2", + plugins=bundled_plugin_configs("spacemouse"), + ) diff --git a/source/isaaclab_teleop/isaaclab_teleop/spacemouse/se3_spacemouse.py b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/se3_spacemouse.py new file mode 100644 index 000000000000..56750c3fc524 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/spacemouse/se3_spacemouse.py @@ -0,0 +1,111 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Spacemouse-driven IsaacTeleopCfg builder for SE(3) control.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..isaac_teleop_cfg import IsaacTeleopCfg + + +def se3_spacemouse_teleop_cfg( + pos_sensitivity: float = 0.4, + rot_sensitivity: float = 0.8, + gripper_term: bool = True, + sim_device: str = "cpu", +) -> IsaacTeleopCfg: + """Build an :class:`~isaaclab_teleop.IsaacTeleopCfg` for spacemouse-driven SE(3) delta-pose control. + + Raw axis/button state is retargeted via + :class:`~isaacteleop.retargeters.SpaceMouseToSe3RelRetargeter` / + :class:`~isaacteleop.retargeters.SpaceMouseGripperRetargeter`. + + The SpaceMouse is read by the Isaac Capture ``spacemouse`` plugin from its HID device + (``/dev/hidrawN``); the config starts the plugin with the session, so no headset or separate + process is needed. Reading ``/dev/hidraw*`` needs a udev rule granting the user access (see + :ref:`SpaceMouse permissions `). + + The interface finds and uses the first supported device connected to the computer. + Currently tested for the SpaceMouse Compact + (https://3dconnexion.com/de/product/spacemouse-compact/). + + The left button's gripper toggle is handled inside the pipeline by + ``SpaceMouseGripperRetargeter``. The pipeline also outputs ``"spacemouse_buttons"``, from which + :class:`~isaaclab_teleop.IsaacTeleopDevice` resets teleoperation on the right button. + + Args: + pos_sensitivity: Position delta scale per step [m]. + rot_sensitivity: Rotation delta scale per step [rad]. + gripper_term: Whether to include a gripper open/close command as a 7th action element. + sim_device: Torch device string for the pipeline's output tensors. + + Returns: + IsaacTeleopCfg driving this pipeline. + """ + from .._plugins import bundled_plugin_configs + from ..isaac_teleop_cfg import IsaacTeleopCfg + + def build_pipeline(): + from isaacteleop.retargeters import ( + SpaceMouseGripperRetargeter, + SpaceMouseToSe3RelRetargeter, + SpaceMouseToSe3RelRetargeterConfig, + TensorReorderer, + ) + from isaacteleop.retargeting_engine.deviceio_source_nodes import SpaceMouseSource + from isaacteleop.retargeting_engine.interface import OutputCombiner + + spacemouse_source = SpaceMouseSource("spacemouse") + + se3 = SpaceMouseToSe3RelRetargeter( + SpaceMouseToSe3RelRetargeterConfig(pos_sensitivity=pos_sensitivity, rot_sensitivity=rot_sensitivity), + name="se3", + ) + connected_se3 = se3.connect( + { + "spacemouse_translation": spacemouse_source.output("spacemouse_translation"), + "spacemouse_rotation": spacemouse_source.output("spacemouse_rotation"), + } + ) + + ee_delta_elements = ["dx", "dy", "dz", "drx", "dry", "drz"] + input_config = {"ee_delta": ee_delta_elements} + input_types = {"ee_delta": "array"} + reorder_inputs = {"ee_delta": connected_se3.output("ee_delta")} + output_order = list(ee_delta_elements) + + if gripper_term: + gripper = SpaceMouseGripperRetargeter(name="gripper") + connected_gripper = gripper.connect({"spacemouse_buttons": spacemouse_source.output("spacemouse_buttons")}) + input_config["gripper_command"] = ["gripper_value"] + input_types["gripper_command"] = "scalar" + reorder_inputs["gripper_command"] = connected_gripper.output("gripper_command") + output_order.append("gripper_value") + + reorderer = TensorReorderer( + input_config=input_config, + output_order=output_order, + name="action_reorderer", + input_types=input_types, + ) + connected_reorderer = reorderer.connect(reorder_inputs) + + return OutputCombiner( + { + "action": connected_reorderer.output("output"), + "spacemouse_buttons": spacemouse_source.output("spacemouse_buttons"), + } + ) + + return IsaacTeleopCfg( + pipeline_builder=build_pipeline, + sim_device=sim_device, + teleoperation_active_default=True, + app_name="IsaacLabSpaceMouseSe3", + plugins=bundled_plugin_configs("spacemouse"), + ) diff --git a/source/isaaclab_teleop/test/test_device_selection.py b/source/isaaclab_teleop/test/test_device_selection.py new file mode 100644 index 000000000000..a106c762e683 --- /dev/null +++ b/source/isaaclab_teleop/test/test_device_selection.py @@ -0,0 +1,153 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for :mod:`isaaclab_teleop.device_selection`: device configs -> Isaac Capture pipelines.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import isaaclab_teleop.device_selection as device_selection +import pytest +from isaaclab_teleop import IsaacTeleopCfg +from isaaclab_teleop.device_selection import pipeline_from_device_cfg, select_isaac_teleop_cfg + +from isaaclab.devices import ( + DeviceCfg, + DevicesCfg, + Se2GamepadCfg, + Se2KeyboardCfg, + Se2SpaceMouseCfg, + Se3GamepadCfg, + Se3KeyboardCfg, + Se3SpaceMouseCfg, +) + +pytestmark = pytest.mark.unit + + +BUILDERS = { + Se3KeyboardCfg: "se3_keyboard_teleop_cfg", + Se2KeyboardCfg: "se2_keyboard_teleop_cfg", + Se3GamepadCfg: "se3_gamepad_teleop_cfg", + Se2GamepadCfg: "se2_gamepad_teleop_cfg", + Se3SpaceMouseCfg: "se3_spacemouse_teleop_cfg", + Se2SpaceMouseCfg: "se2_spacemouse_teleop_cfg", +} +"""Each device config type and the builder that must serve it.""" + + +@pytest.fixture +def recorded(monkeypatch): + """Record which builder ``device_selection`` calls, and with what, without changing its routing.""" + calls = [] + for cfg_type, (routed, fields) in list(device_selection._PIPELINES.items()): + + def builder(name=routed.__name__, **kwargs): + calls.append((name, kwargs)) + return f"pipeline:{name}" + + monkeypatch.setitem(device_selection._PIPELINES, cfg_type, (builder, fields)) + monkeypatch.setattr(device_selection, "_isaacteleop_available", lambda: True) + return calls + + +def _env_cfg(isaac_teleop=None, **devices): + return SimpleNamespace(isaac_teleop=isaac_teleop, teleop_devices=DevicesCfg(devices=devices)) + + +DEFAULTS = {"keyboard": Se3KeyboardCfg(pos_sensitivity=0.2), "spacemouse": Se3SpaceMouseCfg(pos_sensitivity=0.3)} + + +class TestPipelineFromDeviceCfg: + @pytest.mark.parametrize( + ("cfg_type", "source_name"), + [ + (Se2KeyboardCfg, "KeyboardSource"), + (Se3KeyboardCfg, "KeyboardSource"), + (Se2GamepadCfg, "GamepadSource"), + (Se3GamepadCfg, "GamepadSource"), + (Se2SpaceMouseCfg, "SpaceMouseSource"), + (Se3SpaceMouseCfg, "SpaceMouseSource"), + ], + ) + def test_every_legacy_device_reads_its_own_device(self, cfg_type, source_name): + nodes = pytest.importorskip("isaacteleop.retargeting_engine.deviceio_source_nodes") + + cfg = pipeline_from_device_cfg(cfg_type(), "cpu") + + assert isinstance(cfg, IsaacTeleopCfg) + assert len(nodes.find_sources(cfg.pipeline_builder(), getattr(nodes, source_name))) == 1 + + @pytest.mark.parametrize( + ("device_cfg", "expected"), + [ + ( + Se3KeyboardCfg(pos_sensitivity=0.7, rot_sensitivity=0.6, gripper_term=False), + {"pos_sensitivity": 0.7, "rot_sensitivity": 0.6, "gripper_term": False}, + ), + ( + Se2KeyboardCfg(v_x_sensitivity=0.1, v_y_sensitivity=0.2, omega_z_sensitivity=0.3), + {"v_x_sensitivity": 0.1, "v_y_sensitivity": 0.2, "omega_z_sensitivity": 0.3}, + ), + ( + Se3GamepadCfg(pos_sensitivity=0.7, rot_sensitivity=0.6, dead_zone=0.05, gripper_term=False), + {"pos_sensitivity": 0.7, "rot_sensitivity": 0.6, "dead_zone": 0.05, "gripper_term": False}, + ), + ( + Se2GamepadCfg(v_x_sensitivity=0.1, v_y_sensitivity=0.2, omega_z_sensitivity=0.3, dead_zone=0.05), + {"v_x_sensitivity": 0.1, "v_y_sensitivity": 0.2, "omega_z_sensitivity": 0.3, "dead_zone": 0.05}, + ), + ( + Se3SpaceMouseCfg(pos_sensitivity=0.7, rot_sensitivity=0.6, gripper_term=False), + {"pos_sensitivity": 0.7, "rot_sensitivity": 0.6, "gripper_term": False}, + ), + ( + Se2SpaceMouseCfg(v_x_sensitivity=0.1, v_y_sensitivity=0.2, omega_z_sensitivity=0.3), + {"v_x_sensitivity": 0.1, "v_y_sensitivity": 0.2, "omega_z_sensitivity": 0.3}, + ), + ], + ids=lambda value: type(value).__name__ if not isinstance(value, dict) else "", + ) + def test_config_reaches_its_builder_with_every_setting(self, recorded, device_cfg, expected): + pipeline_from_device_cfg(device_cfg, "cuda:0") + + assert recorded == [(BUILDERS[type(device_cfg)], {**expected, "sim_device": "cuda:0"})] + + def test_unsupported_config_has_no_pipeline(self): + assert pipeline_from_device_cfg(DeviceCfg(), "cpu") is None + + +class TestSelectIsaacTeleopCfg: + def test_without_isaacteleop_selects_nothing(self, recorded, monkeypatch): + monkeypatch.setattr(device_selection, "_isaacteleop_available", lambda: False) + + assert select_isaac_teleop_cfg(_env_cfg(isaac_teleop="declared"), None, DEFAULTS, "cpu") is None + + def test_declared_pipeline_wins_without_a_device_name(self, recorded): + assert select_isaac_teleop_cfg(_env_cfg(isaac_teleop="declared"), None, DEFAULTS, "cpu") == "declared" + assert recorded == [] + + def test_default_keyboard_without_a_declared_pipeline(self, recorded): + assert select_isaac_teleop_cfg(_env_cfg(), None, DEFAULTS, "cpu") == "pipeline:se3_keyboard_teleop_cfg" + assert recorded[0][1]["pos_sensitivity"] == 0.2 + + def test_environment_device_wins_over_the_default(self, recorded): + env_cfg = _env_cfg(isaac_teleop="declared", spacemouse=Se3SpaceMouseCfg(pos_sensitivity=0.9)) + + assert select_isaac_teleop_cfg(env_cfg, "spacemouse", DEFAULTS, "cpu") == "pipeline:se3_spacemouse_teleop_cfg" + assert recorded[0][1]["pos_sensitivity"] == 0.9 + + def test_default_serves_a_name_the_environment_lacks(self, recorded): + assert ( + select_isaac_teleop_cfg(_env_cfg(), "spacemouse", DEFAULTS, "cpu") == "pipeline:se3_spacemouse_teleop_cfg" + ) + assert recorded[0][1]["pos_sensitivity"] == 0.3 + + def test_unknown_or_unsupported_device_selects_nothing(self, recorded): + env_cfg = _env_cfg(handtracking=DeviceCfg()) + + assert select_isaac_teleop_cfg(env_cfg, "handtracking", DEFAULTS, "cpu") is None + assert select_isaac_teleop_cfg(env_cfg, "joystick", DEFAULTS, "cpu") is None diff --git a/source/isaaclab_teleop/test/test_gamepad_constructors.py b/source/isaaclab_teleop/test/test_gamepad_constructors.py new file mode 100644 index 000000000000..b5b901fdbb3f --- /dev/null +++ b/source/isaaclab_teleop/test/test_gamepad_constructors.py @@ -0,0 +1,65 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the gamepad :class:`~isaaclab_teleop.IsaacTeleopCfg` builders.""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("isaacteleop") + +from isaaclab_teleop.gamepad.se2_gamepad import se2_gamepad_teleop_cfg +from isaaclab_teleop.gamepad.se3_gamepad import se3_gamepad_teleop_cfg + + +def _has_gamepad_source(cfg) -> bool: + """Whether the config's pipeline reads the gamepad.""" + from isaacteleop.retargeting_engine.deviceio_source_nodes import GamepadSource, find_sources + + return len(find_sources(cfg.pipeline_builder(), GamepadSource)) == 1 + + +def _starts_the_gamepad_plugin(cfg) -> bool: + """Whether the config starts the Isaac Capture gamepad plugin that feeds its source.""" + from isaacteleop.teleop_session_manager import BUNDLED_PLUGINS_DIR + + return [(p.plugin_name, p.plugin_root_id, p.search_paths) for p in cfg.plugins] == [ + ("gamepad", "gamepad", [BUNDLED_PLUGINS_DIR]) + ] + + +class TestSe3GamepadTeleopCfg: + def test_defaults(self): + cfg = se3_gamepad_teleop_cfg() + + assert cfg.sim_device == "cpu" + assert cfg.teleoperation_active_default is True + assert cfg.app_name == "IsaacLabGamepadSe3" + assert _starts_the_gamepad_plugin(cfg) + assert _has_gamepad_source(cfg) + assert callable(cfg.pipeline_builder) + + def test_custom_sim_device(self): + cfg = se3_gamepad_teleop_cfg(sim_device="cuda:0") + + assert cfg.sim_device == "cuda:0" + + +class TestSe2GamepadTeleopCfg: + def test_defaults(self): + cfg = se2_gamepad_teleop_cfg() + + assert cfg.sim_device == "cpu" + assert cfg.teleoperation_active_default is True + assert cfg.app_name == "IsaacLabGamepadSe2" + assert _starts_the_gamepad_plugin(cfg) + assert _has_gamepad_source(cfg) + assert callable(cfg.pipeline_builder) + + def test_custom_sim_device(self): + cfg = se2_gamepad_teleop_cfg(sim_device="cuda:0") + + assert cfg.sim_device == "cuda:0" diff --git a/source/isaaclab_teleop/test/test_hand_debug_viz.py b/source/isaaclab_teleop/test/test_hand_debug_viz.py index 15e22b494ff9..c67f6d8da4cf 100644 --- a/source/isaaclab_teleop/test/test_hand_debug_viz.py +++ b/source/isaaclab_teleop/test/test_hand_debug_viz.py @@ -201,6 +201,9 @@ def get_leaf_nodes(self) -> list: def output(self, key: str) -> str: return f"selector:user_pipeline:{key}" + def output_types(self) -> dict: + return {"action": None} + class _FakeTensorGroup: """Stand-in for an ``OptionalTensorGroup`` carrying joint positions.""" diff --git a/source/isaaclab_teleop/test/test_keyboard_constructors.py b/source/isaaclab_teleop/test/test_keyboard_constructors.py new file mode 100644 index 000000000000..50dbf6fbeec4 --- /dev/null +++ b/source/isaaclab_teleop/test/test_keyboard_constructors.py @@ -0,0 +1,72 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the keyboard ``IsaacTeleopCfg`` builders.""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("isaacteleop") + +from isaaclab_teleop.keyboard.se2_keyboard import se2_keyboard_teleop_cfg +from isaaclab_teleop.keyboard.se3_keyboard import se3_keyboard_teleop_cfg + +# --------------------------------------------------------------------------- +# se2_keyboard_teleop_cfg / se3_keyboard_teleop_cfg +# --------------------------------------------------------------------------- + + +def _keyboard_sources(pipeline): + from isaacteleop.retargeting_engine.deviceio_source_nodes import KeyboardSource, find_sources + + return find_sources(pipeline, KeyboardSource) + + +class TestSe3KeyboardTeleopCfg: + def test_defaults(self): + cfg = se3_keyboard_teleop_cfg() + + assert cfg.sim_device == "cpu" + assert cfg.teleoperation_active_default is True + assert cfg.app_name == "IsaacLabKeyboardSe3" + # the keyboard runs in process: no plugin + assert cfg.plugins == [] + assert callable(cfg.pipeline_builder) + + def test_custom_sim_device(self): + cfg = se3_keyboard_teleop_cfg(sim_device="cuda:0") + + assert cfg.sim_device == "cuda:0" + + def test_pipeline_reads_one_keyboard(self): + assert len(_keyboard_sources(se3_keyboard_teleop_cfg().pipeline_builder())) == 1 + + @pytest.mark.parametrize(("gripper_term", "action_size"), [(True, 7), (False, 6)]) + def test_action_is_the_pose_delta_plus_an_optional_gripper(self, gripper_term, action_size): + """The action is ``[dx, dy, dz, drx, dry, drz]``, plus the gripper command when ``gripper_term``.""" + pipeline = se3_keyboard_teleop_cfg(gripper_term=gripper_term).pipeline_builder() + + (action,) = pipeline.output_types()["action"].types + assert action.shape == (action_size,) + + +class TestSe2KeyboardTeleopCfg: + def test_defaults(self): + cfg = se2_keyboard_teleop_cfg() + + assert cfg.sim_device == "cpu" + assert cfg.teleoperation_active_default is True + assert cfg.app_name == "IsaacLabKeyboardSe2" + assert cfg.plugins == [] + assert callable(cfg.pipeline_builder) + + def test_custom_sim_device(self): + cfg = se2_keyboard_teleop_cfg(sim_device="cuda:0") + + assert cfg.sim_device == "cuda:0" + + def test_pipeline_reads_one_keyboard(self): + assert len(_keyboard_sources(se2_keyboard_teleop_cfg().pipeline_builder())) == 1 diff --git a/source/isaaclab_teleop/test/test_keyboard_session.py b/source/isaaclab_teleop/test/test_keyboard_session.py new file mode 100644 index 000000000000..781921324ff7 --- /dev/null +++ b/source/isaaclab_teleop/test/test_keyboard_session.py @@ -0,0 +1,253 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# pyright: reportPrivateUsage=none + +"""Tests for keyboard teleop sessions fed by scripted key events in place of a visualizer window. + +The ``se3_keyboard_teleop_cfg`` pipeline runs in a real Isaac Capture session. The in-process +keyboard never touches the OpenXR session handles, so a placeholder OpenXR session stands in for a +CloudXR runtime. The session lifecycle is checked without stepping, since its combined pipeline +also polls the controllers through OpenXR. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +pytest.importorskip("isaacteleop") + +from isaaclab_teleop.keyboard import se3_keyboard_teleop_cfg +from isaaclab_teleop.session_lifecycle import TeleopSessionLifecycle +from isaacteleop.retargeting_engine.deviceio_source_nodes import KeyboardSource, find_sources + +_SETTLE_STEPS = 3 + + +def _keyboard_cfg(**kwargs): + """The SE(3) keyboard config, retargeting synchronously so each step reflects the current keys.""" + from isaacteleop.teleop_session_manager import RetargetingExecutionConfig + + cfg = se3_keyboard_teleop_cfg(**kwargs) + cfg.retargeting_execution = RetargetingExecutionConfig(mode="sync") + return cfg + + +class _PlaceholderOpenXRSession: + """Non-null handles that only OpenXR-backed tracker impls would dereference.""" + + def __init__(self, *args, **kwargs): + pass + + def get_handles(self): + from isaacteleop.oxr import OpenXRSessionHandles + + return OpenXRSessionHandles(1, 1, 1, 1) + + def get_provider_snapshot(self): + return SimpleNamespace(state="AVAILABLE", headset_state="CONNECTED", reason="NONE", result_code=None, error="") + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +@pytest.fixture +def keyboard_session(visualizer, monkeypatch): + """The keyboard config's pipeline in a real Isaac Capture session, fed by *visualizer*.""" + from isaacteleop import oxr + from isaacteleop.teleop_session_manager import TeleopSession, TeleopSessionConfig + + monkeypatch.setattr(oxr, "OpenXRSession", _PlaceholderOpenXRSession) + cfg = _keyboard_cfg(pos_sensitivity=0.5, rot_sensitivity=0.5) + pipeline = cfg.pipeline_builder() + source = visualizer.key_event_source + attachments = [keyboard.attach(source, capture=False) for keyboard in find_sources(pipeline, KeyboardSource)] + session_config = TeleopSessionConfig( + app_name="KeyboardSessionTest", pipeline=pipeline, retargeting_execution=cfg.retargeting_execution + ) + with TeleopSession(session_config) as session: + yield session + for attachment in attachments: + attachment.close() + + +def _settle(session): + result = None + for _ in range(_SETTLE_STEPS): + result = session.step() + return result["action"][0] + + +def test_held_key_drives_the_action(keyboard_session, visualizer): + visualizer.press("KeyW") + action = _settle(keyboard_session) + assert action[0] > 0.0 + + visualizer.release("KeyW") + action = _settle(keyboard_session) + assert action[0] == 0.0 + + +def test_focus_loss_releases_held_keys(keyboard_session, visualizer): + visualizer.press("KeyW") + _settle(keyboard_session) + + visualizer.blur() + action = _settle(keyboard_session) + + assert action[0] == 0.0 + + +class _Visualizer: + """A running visualizer: an Isaac Lab keyboard source with key helpers, and its camera-keys toggle.""" + + def __init__(self) -> None: + from isaaclab.visualizers import KeyEventSource + + self.key_event_source = KeyEventSource() + self.camera_keys: list[bool] = [] + + def set_camera_keys_enabled(self, enabled: bool) -> None: + self.camera_keys.append(enabled) + + @property + def listener_count(self) -> int: + return len(self.key_event_source._subscriptions) + + def press(self, code: str) -> None: + self.key_event_source._emit_key(code, True) + + def release(self, code: str) -> None: + self.key_event_source._emit_key(code, False) + + def tap(self, code: str) -> None: + self.press(code) + self.release(code) + + def blur(self) -> None: + self.key_event_source._emit_focus_lost() + + +def _start(lifecycle, *visualizers): + """Start a lifecycle reading *visualizers*, without launching CloudXR, a system check or OpenXR.""" + import isaacteleop.teleop_session_manager as tsm + + sim = SimpleNamespace(visualizers=[*visualizers, SimpleNamespace(key_event_source=None)]) + with ( + patch("isaaclab.sim.SimulationContext.instance", return_value=sim), + patch.object(lifecycle, "_ensure_cloudxr_runtime") as ensure_cloudxr, + patch.object(lifecycle, "_run_system_check"), + patch.object(tsm, "TeleopSession", MagicMock()), + ): + lifecycle.start() + return ensure_cloudxr + + +def _started_lifecycle(cfg, *visualizers, **kwargs): + lifecycle = TeleopSessionLifecycle(cfg, use_kit_xr_bridge=False, **kwargs) + return lifecycle, _start(lifecycle, *visualizers) + + +@pytest.fixture +def visualizer(): + return _Visualizer() + + +def test_keyboard_session_takes_the_standard_path(visualizer): + """A keyboard session launches the CloudXR profile and uses the control channel, like any other.""" + lifecycle, ensure_cloudxr = _started_lifecycle( + _keyboard_cfg(), visualizer, cloudxr_env_file="/nonexistent/cloudxr.env" + ) + try: + ensure_cloudxr.assert_called_once() + assert lifecycle._message_processor is not None + outputs = lifecycle._pipeline.output_types() + assert TeleopSessionLifecycle._CONTROLLER_RIGHT_KEY in outputs + finally: + lifecycle.stop() + + +def test_visualizer_feeds_the_keyboard_and_the_control_keys(visualizer): + """The pipeline's keyboard and the control keys read the window; its camera keys are off until stop.""" + lifecycle, _ = _started_lifecycle(_keyboard_cfg(), visualizer) + assert visualizer.listener_count == 2 + assert visualizer.camera_keys == [False] + + lifecycle.stop() + assert visualizer.listener_count == 0 + assert visualizer.camera_keys == [False, True] + + +KEY_R, KEY_B = 19, 48 + + +def test_each_control_key_press_is_drained_once(visualizer): + """Presses reach the app once each, with no session step in between: repeats of a held key are dropped.""" + lifecycle, _ = _started_lifecycle(_keyboard_cfg(), visualizer) + try: + visualizer.press("KeyR") + visualizer.press("KeyR") # still held: a repeat, not a press + visualizer.release("KeyR") + visualizer.tap("KeyB") + assert lifecycle.drain_pressed_keys() == [KEY_R, KEY_B] + assert lifecycle.drain_pressed_keys() == [] + finally: + lifecycle.stop() + + +def test_control_keys_work_without_a_pipeline_keyboard(visualizer): + """A pipeline with no keyboard still gets B/P/R, and the window keeps its camera keys.""" + from isaaclab_teleop import se3_spacemouse_teleop_cfg + + lifecycle, _ = _started_lifecycle(se3_spacemouse_teleop_cfg(), visualizer) + try: + assert visualizer.camera_keys == [] + visualizer.tap("KeyB") + assert lifecycle.drain_pressed_keys() == [KEY_B] + finally: + lifecycle.stop() + + +def test_presses_do_not_carry_into_the_next_session(visualizer): + lifecycle, _ = _started_lifecycle(_keyboard_cfg(), visualizer) + visualizer.tap("KeyR") + lifecycle.stop() + _start(lifecycle, visualizer) + try: + visualizer.press("KeyR") # the stale tap is gone; a new press counts + assert lifecycle.drain_pressed_keys() == [KEY_R] + finally: + lifecycle.stop() + + +def test_another_window_losing_focus_does_not_repeat_a_held_key(): + """Each window tracks its own held keys, so one window's blur leaves another's repeat suppression intact.""" + first, second = _Visualizer(), _Visualizer() + lifecycle, _ = _started_lifecycle(_keyboard_cfg(), first, second) + try: + first.press("KeyR") + second.blur() + first.press("KeyR") # an autorepeat of the key the first window still holds + assert lifecycle.drain_pressed_keys() == [KEY_R] + finally: + lifecycle.stop() + + +def test_a_failing_window_does_not_stop_the_session(visualizer): + """A window whose keyboard cannot be attached is skipped with a warning; the others still work.""" + broken = _Visualizer() + broken.set_camera_keys_enabled = MagicMock(side_effect=RuntimeError("no window")) + lifecycle, _ = _started_lifecycle(_keyboard_cfg(), broken, visualizer) + try: + visualizer.tap("KeyB") + assert lifecycle.drain_pressed_keys() == [KEY_B] + finally: + lifecycle.stop() diff --git a/source/isaaclab_teleop/test/test_operator_input.py b/source/isaaclab_teleop/test/test_operator_input.py new file mode 100644 index 000000000000..3b0fa9132031 --- /dev/null +++ b/source/isaaclab_teleop/test/test_operator_input.py @@ -0,0 +1,88 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# pyright: reportPrivateUsage=none + +"""Tests for the operator's ``B`` / ``P`` / ``R`` keys and SpaceMouse right button on +:class:`~isaaclab_teleop.IsaacTeleopDevice`, against a mocked session lifecycle.""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import numpy as np +import pytest +from isaaclab_teleop.isaac_teleop_device import IsaacTeleopDevice + +KEY_R, KEY_P, KEY_B, KEY_N = 19, 25, 48, 49 + + +class _Buttons: + """Stand-in for the ``spacemouse_buttons`` entry of a pipeline step result.""" + + def __init__(self, *pressed: int) -> None: + self.is_none = False + self._bitmap = np.zeros(8, dtype=np.uint8) + self._bitmap[list(pressed)] = 1 + + def __getitem__(self, index): + return self._bitmap + + +def _device(pressed_keys=(), step_result=None) -> IsaacTeleopDevice: + device = object.__new__(IsaacTeleopDevice) + device._session_lifecycle = MagicMock() + device._session_lifecycle.drain_pressed_keys.return_value = list(pressed_keys) + device._session_lifecycle.last_step_result = step_result + device._prev_spacemouse_right_pressed = False + device.request_start = MagicMock() + device.request_stop = MagicMock() + device.reset = MagicMock() + return device + + +@pytest.mark.parametrize(("code", "action"), [(KEY_B, "request_start"), (KEY_P, "request_stop"), (KEY_R, "reset")]) +def test_control_keys(code, action): + device = _device(pressed_keys=[code]) + device._handle_operator_input() + + for name in ("request_start", "request_stop", "reset"): + assert getattr(device, name).call_count == (name == action) + if action == "reset": + device.reset.assert_called_once_with(pause=True) + + +def test_other_keys_do_nothing(): + device = _device(pressed_keys=[KEY_N]) + device._handle_operator_input() + + device.request_start.assert_not_called() + device.reset.assert_not_called() + + +def test_spacemouse_right_button_resets_once_per_press(): + device = _device(step_result={"spacemouse_buttons": _Buttons(1)}) + device._handle_operator_input() + device._handle_operator_input() # still held + device.reset.assert_called_once_with(pause=True) + + device._session_lifecycle.last_step_result = {"spacemouse_buttons": _Buttons()} + device._handle_operator_input() + device._session_lifecycle.last_step_result = {"spacemouse_buttons": _Buttons(1)} + device._handle_operator_input() + assert device.reset.call_count == 2 + + +def test_spacemouse_left_button_does_not_reset(): + device = _device(step_result={"spacemouse_buttons": _Buttons(0)}) + device._handle_operator_input() + device.reset.assert_not_called() + + +@pytest.mark.parametrize("step_result", [None, {"action": object()}]) +def test_without_spacemouse_buttons_nothing_resets(step_result): + device = _device(step_result=step_result) + device._handle_operator_input() + device.reset.assert_not_called() diff --git a/source/isaaclab_teleop/test/test_script_input.py b/source/isaaclab_teleop/test/test_script_input.py new file mode 100644 index 000000000000..a7a7d147047f --- /dev/null +++ b/source/isaaclab_teleop/test/test_script_input.py @@ -0,0 +1,45 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# pyright: reportPrivateUsage=none + +"""Tests for :mod:`isaaclab_teleop.script_input`: teleop devices for scripts.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import isaaclab_teleop.script_input as script_input +import pytest +from isaaclab_teleop.isaac_teleop_cfg import CLOUDXR_STANDALONE_ENV +from isaaclab_teleop.script_input import open_teleop_device + +pytestmark = pytest.mark.unit + + +def test_without_isaacteleop_the_device_config_is_instantiated(monkeypatch): + monkeypatch.setattr(script_input, "_isaacteleop_available", lambda: False) + device = object() + with patch("isaaclab.utils.instantiate", return_value=device) as instantiate: + with open_teleop_device("cfg") as opened: + assert opened is device + instantiate.assert_called_once_with("cfg") + + +def test_with_isaacteleop_the_pipeline_runs_headset_free(monkeypatch): + monkeypatch.setattr(script_input, "_isaacteleop_available", lambda: True) + monkeypatch.setattr(script_input, "pipeline_from_device_cfg", lambda cfg, sim_device: f"pipeline:{cfg.name}") + device = MagicMock() + device.__enter__.return_value = device + with patch("isaaclab_teleop.isaac_teleop_device.create_isaac_teleop_device", return_value=device) as create: + with open_teleop_device(SimpleNamespace(name="keyboard", sim_device="cpu")) as opened: + assert opened is device + device.request_start.assert_called_once() + device.__exit__.assert_not_called() + create.assert_called_once_with( + "pipeline:keyboard", use_kit_xr_bridge=False, cloudxr_env_file=CLOUDXR_STANDALONE_ENV + ) + device.__exit__.assert_called_once() diff --git a/source/isaaclab_teleop/test/test_spacemouse_constructors.py b/source/isaaclab_teleop/test/test_spacemouse_constructors.py new file mode 100644 index 000000000000..724de0f86e1d --- /dev/null +++ b/source/isaaclab_teleop/test/test_spacemouse_constructors.py @@ -0,0 +1,72 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# pyright: reportPrivateUsage=none + +"""Tests for the SpaceMouse ``IsaacTeleopCfg`` builders.""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("isaacteleop") + +from isaaclab_teleop.spacemouse.se2_spacemouse import se2_spacemouse_teleop_cfg +from isaaclab_teleop.spacemouse.se3_spacemouse import se3_spacemouse_teleop_cfg + + +def _has_spacemouse_source(cfg) -> bool: + """Whether the config's pipeline reads the SpaceMouse.""" + from isaacteleop.retargeting_engine.deviceio_source_nodes import SpaceMouseSource, find_sources + + return len(find_sources(cfg.pipeline_builder(), SpaceMouseSource)) == 1 + + +def _starts_the_spacemouse_plugin(cfg) -> bool: + """Whether the config starts the Isaac Capture spacemouse plugin that feeds its source.""" + from isaacteleop.teleop_session_manager import BUNDLED_PLUGINS_DIR + + return [(p.plugin_name, p.plugin_root_id, p.search_paths) for p in cfg.plugins] == [ + ("spacemouse", "spacemouse", [BUNDLED_PLUGINS_DIR]) + ] + + +# --------------------------------------------------------------------------- +# se2_spacemouse_teleop_cfg / se3_spacemouse_teleop_cfg +# --------------------------------------------------------------------------- + + +class TestSe3SpaceMouseTeleopCfg: + def test_defaults(self): + cfg = se3_spacemouse_teleop_cfg() + + assert cfg.sim_device == "cpu" + assert cfg.teleoperation_active_default is True + assert cfg.app_name == "IsaacLabSpaceMouseSe3" + assert _starts_the_spacemouse_plugin(cfg) + assert _has_spacemouse_source(cfg) + assert callable(cfg.pipeline_builder) + + def test_custom_sim_device(self): + cfg = se3_spacemouse_teleop_cfg(sim_device="cuda:0") + + assert cfg.sim_device == "cuda:0" + + +class TestSe2SpaceMouseTeleopCfg: + def test_defaults(self): + cfg = se2_spacemouse_teleop_cfg() + + assert cfg.sim_device == "cpu" + assert cfg.teleoperation_active_default is True + assert cfg.app_name == "IsaacLabSpaceMouseSe2" + assert _starts_the_spacemouse_plugin(cfg) + assert _has_spacemouse_source(cfg) + assert callable(cfg.pipeline_builder) + + def test_custom_sim_device(self): + cfg = se2_spacemouse_teleop_cfg(sim_device="cuda:0") + + assert cfg.sim_device == "cuda:0" diff --git a/uv.lock b/uv.lock index e55477a12fe1..aa0c13546690 100644 --- a/uv.lock +++ b/uv.lock @@ -885,6 +885,18 @@ nvtx = [ { name = "nvidia-nvtx" }, ] +[[package]] +name = "cupy-cuda12x" +version = "14.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-pathfinder" }, + { name = "numpy" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/5b/65124de2dbaf2e85109f611a41947e39acd6dd938751c04b4c4d7bf6fc82/cupy_cuda12x-14.2.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:f82141761f2c81905d49387464ae29438887956d99063381c93a1d5d1b7d32e8", size = 136111459, upload-time = "2026-08-20T02:40:04.909Z" }, +] + [[package]] name = "cycler" version = "0.12.1" @@ -1746,6 +1758,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl", hash = "sha256:ecaca67aed704a338f88f67b1181b58f821ab5dc89c1f0f5ef99db43c1c2921e", size = 139808, upload-time = "2025-11-01T21:18:10.956Z" }, ] +[[package]] +name = "isaaccapture" +version = "1.7.44a1" +source = { registry = "https://pypi.nvidia.com/" } +dependencies = [ + { name = "isaacteleop" }, + { name = "numpy" }, + { name = "pyyaml" }, +] +wheels = [ + { url = "https://pypi.nvidia.com/isaaccapture/isaaccapture-1.7.44a1-cp312-cp312-manylinux_2_35_x86_64.whl", hash = "sha256:bba80a67891474bc2cf9ddc4b6b0bd6ea78977f8d4d5cf38c4e7f2aae02c781c" }, +] + +[package.optional-dependencies] +cloudxr = [ + { name = "cryptography" }, + { name = "cupy-cuda12x" }, + { name = "pillow" }, + { name = "websockets" }, +] +retargeters = [ + { name = "cmeel-tinyxml2" }, + { name = "cmeel-urdfdom" }, + { name = "dex-retargeting" }, + { name = "nlopt" }, + { name = "pin" }, + { name = "pyyaml" }, + { name = "scipy" }, + { name = "torch" }, +] +ui = [ + { name = "imgui", extra = ["glfw"] }, +] + [[package]] name = "isaaclab" version = "41.0.3" @@ -1959,9 +2005,9 @@ skrl-jax = [ ] teleop = [ { name = "dex-retargeting", marker = "platform_machine == 'x86_64' and sys_platform != 'win32'" }, + { name = "isaaccapture", extra = ["cloudxr", "retargeters", "ui"], marker = "platform_machine == 'x86_64' and sys_platform != 'win32'" }, { name = "isaaclab-mimic" }, { name = "isaaclab-teleop" }, - { name = "isaacteleop", extra = ["cloudxr", "retargeters", "ui"], marker = "platform_machine == 'x86_64' and sys_platform != 'win32'" }, ] test = [ { name = "coverage" }, @@ -2015,6 +2061,7 @@ requires-dist = [ { name = "hydra-core" }, { name = "imgui-bundle", specifier = ">=1.92.5" }, { name = "ipywidgets", marker = "extra == 'mimic'", specifier = ">=8.1.5" }, + { name = "isaaccapture", extras = ["retargeters", "ui", "cloudxr"], marker = "platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'teleop'", specifier = "~=1.7.44a1" }, { name = "isaaclab", editable = "source/isaaclab" }, { name = "isaaclab-assets", editable = "source/isaaclab_assets" }, { name = "isaaclab-contrib", editable = "source/isaaclab_contrib" }, @@ -2035,7 +2082,6 @@ requires-dist = [ { name = "isaaclab-visualizers", editable = "source/isaaclab_visualizers" }, { name = "isaacsim", extras = ["all", "extscache"], marker = "extra == 'isaacsim'", specifier = "==6.2.0.0" }, { name = "isaacsim-asset-isolated", marker = "extra == 'importers'", specifier = "==6.2.0.0" }, - { name = "isaacteleop", extras = ["retargeters", "ui", "cloudxr"], marker = "platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'teleop'", specifier = "~=1.4.145" }, { name = "jax", extras = ["cuda13"], marker = "sys_platform == 'linux' and extra == 'skrl-jax'" }, { name = "jinja2" }, { name = "junitparser", marker = "extra == 'test'" }, @@ -2743,33 +2789,13 @@ wheels = [ [[package]] name = "isaacteleop" -version = "1.4.145" +version = "1!1.7.44a1" source = { registry = "https://pypi.nvidia.com/" } dependencies = [ - { name = "numpy" }, - { name = "pyyaml" }, + { name = "isaaccapture" }, ] wheels = [ - { url = "https://pypi.nvidia.com/isaacteleop/isaacteleop-1.4.145-cp312-cp312-manylinux_2_35_x86_64.whl", hash = "sha256:b18b210535b28516593bb6e85ffb70a6e8265d9547365c818e38920184ee7b40" }, -] - -[package.optional-dependencies] -cloudxr = [ - { name = "cryptography" }, - { name = "websockets" }, -] -retargeters = [ - { name = "cmeel-tinyxml2" }, - { name = "cmeel-urdfdom" }, - { name = "dex-retargeting" }, - { name = "nlopt" }, - { name = "pin" }, - { name = "pyyaml" }, - { name = "scipy" }, - { name = "torch" }, -] -ui = [ - { name = "imgui", extra = ["glfw"] }, + { url = "https://pypi.nvidia.com/isaacteleop/isaacteleop-1%211.7.44a1-py3-none-any.whl", hash = "sha256:d09008a3ece5ae4e2debb9b7f0fa8a58aaa4632c9f025c487bfbf946dd2829f8" }, ] [[package]]