From 33b568fe0054d311e84b20a41c23db6cb6b86f77 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Wed, 30 Sep 2026 02:01:59 -0700 Subject: [PATCH 1/7] Record video from the Warp environments The Warp environment classes never wired the VideoRecorder hook, so env_cfg.video_recorders was dropped: ManagerBasedEnvWarp logged a warning and DirectRLEnvWarp ignored it. Create the recorders before the simulation starts, advance them once per step after rendering and outside the captured stages, and flush them on close, mirroring DirectRLEnv, ManagerBasedEnv and ManagerBasedRLEnv. The test builds both hierarchies through WarpFrontend.build_env, so the scene is InteractiveSceneWarp, and runs kitless from a Newton SimulationCfg. It steps fewer times than one clip, so the clip reaches disk only through close() and its frame count pins one tick per step. --- .../jichuanh-warp-video-recording.added.rst | 4 + .../jichuanh-warp-video-recording.minor | 0 .../envs/direct_rl_env_warp.py | 11 +++ .../envs/manager_based_env_warp.py | 19 +++-- .../envs/manager_based_rl_env_warp.py | 4 + .../test/envs/test_video_recording_warp.py | 81 +++++++++++++++++++ 6 files changed, 111 insertions(+), 8 deletions(-) create mode 100644 source/isaaclab_experimental/changelog.d/jichuanh-warp-video-recording.added.rst create mode 100644 source/isaaclab_experimental/changelog.d/jichuanh-warp-video-recording.minor create mode 100644 source/isaaclab_experimental/test/envs/test_video_recording_warp.py diff --git a/source/isaaclab_experimental/changelog.d/jichuanh-warp-video-recording.added.rst b/source/isaaclab_experimental/changelog.d/jichuanh-warp-video-recording.added.rst new file mode 100644 index 000000000000..4e413cd11447 --- /dev/null +++ b/source/isaaclab_experimental/changelog.d/jichuanh-warp-video-recording.added.rst @@ -0,0 +1,4 @@ +* Added video recording to the Warp environments. :class:`~isaaclab_experimental.envs.ManagerBasedEnvWarp`, + :class:`~isaaclab_experimental.envs.ManagerBasedRLEnvWarp` and :class:`~isaaclab_experimental.envs.DirectRLEnvWarp` + now create the recorders in ``env_cfg.video_recorders``, advance them once per step and flush them on close. + Previously the setting was ignored, with a warning only on the manager-based environments. diff --git a/source/isaaclab_experimental/changelog.d/jichuanh-warp-video-recording.minor b/source/isaaclab_experimental/changelog.d/jichuanh-warp-video-recording.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_experimental/isaaclab_experimental/envs/direct_rl_env_warp.py b/source/isaaclab_experimental/isaaclab_experimental/envs/direct_rl_env_warp.py index 7fdd1ffe3562..359e10f7c6c1 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/envs/direct_rl_env_warp.py +++ b/source/isaaclab_experimental/isaaclab_experimental/envs/direct_rl_env_warp.py @@ -27,6 +27,7 @@ from isaaclab.envs.direct_rl_env import DirectRLEnv from isaaclab.envs.direct_rl_env_cfg import DirectRLEnvCfg from isaaclab.envs.utils.spaces import sample_space, spec_to_gym_space +from isaaclab.envs.utils.video_recorder import VideoRecorder from isaaclab.managers import EventManager from isaaclab.sim import SimulationContext from isaaclab.sim.utils import use_stage @@ -174,6 +175,8 @@ def __init__(self, cfg: DirectRLEnvCfg, render_mode: str | None = None, **kwargs if "prestartup" in self.event_manager.available_modes: self.event_manager.apply(mode="prestartup") + self.video_recorders: list[VideoRecorder] = [VideoRecorder(cfg, self) for cfg in self.cfg.video_recorders] + # play the simulator to activate physics handles # note: this activates the physics simulation view that exposes TensorAPIs # note: when started in extension mode, first call sim.reset_async() and then initialize the managers @@ -445,6 +448,10 @@ def step(self, action: torch.Tensor) -> VecEnvStepReturn: with Timer(name="visualize", msg="Visualize took:", enable=DEBUG_TIMERS): self._post_step_visualize() + # advance video recorders (after render and visualization, outside the captured graphs) + for recorder in self.video_recorders: + recorder.step() + # return observations, rewards, resets and extras # store the returned buffer so RslRlVecEnvWrapper.get_observations() can read env.obs_buf self.obs_buf = {"policy": self.torch_obs_buf.clone()} @@ -570,6 +577,10 @@ def render(self, recompute: bool = False) -> np.ndarray | None: def close(self): """Cleanup for the environment.""" if not self._is_closed: + # flush any buffered video frames + for recorder in getattr(self, "video_recorders", []): + recorder.close() + # close entities related to the environment # note: this is order-sensitive to avoid any dangling references if self.cfg.events: diff --git a/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_env_warp.py b/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_env_warp.py index 3e94486af610..04318c7dfd6a 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_env_warp.py +++ b/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_env_warp.py @@ -33,6 +33,7 @@ export_articulations_data, export_scene_data, ) +from isaaclab.envs.utils.video_recorder import VideoRecorder from isaaclab.sim import SimulationContext from isaaclab.sim.utils import use_stage from isaaclab.utils import validate @@ -79,14 +80,6 @@ def __init__(self, cfg: ManagerBasedEnvCfg): validate(cfg) # store inputs to class self.cfg = cfg - # Video recording is not supported on Warp environments. - if getattr(cfg, "video_recorders", None): - import logging as _logging - - _logging.getLogger(__name__).warning( - "cfg.video_recorders is set but ManagerBasedEnvWarp does not support VideoRecorder. " - "No clips will be written. Use ManagerBasedEnv for video recording support." - ) # initialize internal variables self._is_closed = False # temporary debug runtime config for manager source/call switching. @@ -176,6 +169,8 @@ def __init__(self, cfg: ManagerBasedEnvCfg): if "prestartup" in self.event_manager.available_modes: self.event_manager.apply(mode="prestartup") + self.video_recorders: list[VideoRecorder] = [VideoRecorder(cfg, self) for cfg in self.cfg.video_recorders] + # play the simulator to activate physics handles # note: this activates the physics simulation view that exposes TensorAPIs # note: when started in extension mode, first call sim.reset_async() and then initialize the managers @@ -579,6 +574,10 @@ def step(self, action: torch.Tensor) -> tuple[VecEnvObs, dict]: if "interval" in self.event_manager.available_modes: self.event_manager.apply(mode="interval", dt=self.step_dt) + # advance video recorders (after render, before obs) + for recorder in self.video_recorders: + recorder.step() + # -- compute observations self.obs_buf = self.observation_manager.compute(update_history=True) self.recorder_manager.record_post_step() @@ -601,6 +600,10 @@ def seed(seed: int = -1) -> int: def close(self): """Cleanup for the environment.""" if not self._is_closed: + # flush any buffered video frames + for recorder in getattr(self, "video_recorders", []): + recorder.close() + # destructor is order-sensitive del self.action_manager del self.observation_manager diff --git a/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_rl_env_warp.py b/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_rl_env_warp.py index b6b56abc09d5..adcae2c1a9eb 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_rl_env_warp.py +++ b/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_rl_env_warp.py @@ -376,6 +376,10 @@ def step(self, action: torch.Tensor) -> VecEnvStepReturn: timer=DEBUG_TIMER_STEP, ) + # -- advance video recorders (after render and resets, before final obs) + for recorder in self.video_recorders: + recorder.step() + # -- compute observations # note: done after reset to get the correct observations for reset envs self.obs_buf = self._manager_call_switch.call_stage( diff --git a/source/isaaclab_experimental/test/envs/test_video_recording_warp.py b/source/isaaclab_experimental/test/envs/test_video_recording_warp.py new file mode 100644 index 000000000000..66b49d340332 --- /dev/null +++ b/source/isaaclab_experimental/test/envs/test_video_recording_warp.py @@ -0,0 +1,81 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Video recording on the Warp environments. + +``DirectRLEnvWarp`` and the manager-based Warp envs are separate implementations of the recorder +lifecycle, so both are built through :meth:`WarpFrontend.build_env`, the adapter the CLI uses, and +record from a headless Newton GL visualizer without Kit. +""" + +# Check for moviepy before launching so a missing dependency produces a clean collection-time skip. +import pytest + +try: + from moviepy.editor import VideoFileClip +except ImportError: + pytest.skip("moviepy is not installed; install with: pip install 'moviepy<2'", allow_module_level=True) + +from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg + +from isaaclab.sim import SimulationCfg +from isaaclab.test.utils import launch_test_simulation + +launch_test_simulation(SimulationCfg(physics=NewtonCfg(solver_cfg=MJWarpSolverCfg()))) + +import os +import tempfile + +import torch +from isaaclab_experimental.envs.frontend import WarpFrontend +from isaaclab_visualizers.newton import NewtonGLVisualizerCfg + +import isaaclab.sim as sim_utils +from isaaclab.envs.utils.video_recorder_cfg import VideoRecorderCfg + +import isaaclab_tasks # noqa: F401 +from isaaclab_tasks.core.cartpole.cartpole_direct_env_cfg import CartpoleEnvCfg as CartpoleDirectEnvCfg +from isaaclab_tasks.core.cartpole.cartpole_manager_env_cfg import CartpoleEnvCfg as CartpoleManagerEnvCfg + +_CLIP = 20 # frames per clip +_STEPS = _CLIP // 2 # shorter than one clip, so only close() can write it + + +@pytest.mark.parametrize( + ("cfg_class", "task_id"), + [(CartpoleDirectEnvCfg, "Isaac-Cartpole-Direct"), (CartpoleManagerEnvCfg, "Isaac-Cartpole")], + ids=["direct", "manager"], +) +def test_warp_env_records_and_flushes_clip(cfg_class: type, task_id: str): + """A Warp env records one frame per step and flushes the partial clip on close.""" + env_cfg = cfg_class() + env_cfg.seed = 42 + env_cfg.scene.num_envs = 1 + env_cfg.sim.physics = NewtonCfg(solver_cfg=MJWarpSolverCfg()) + env_cfg.sim.visualizer_cfgs = [NewtonGLVisualizerCfg(headless=True, window_width=320, window_height=240)] + with tempfile.TemporaryDirectory() as output_dir: + env_cfg.video_recorders = [ + VideoRecorderCfg( + source="visualizer:newton_gl", output_dir=output_dir, video_length=_CLIP, video_interval=0, fps=10 + ) + ] + sim_utils.create_new_stage() + env = WarpFrontend.build_env(env_cfg, task_id) + try: + env.reset() + actions = torch.zeros(env.num_envs, *env.action_space.shape[1:], device=env.device) + for _ in range(_STEPS): + env.step(actions) + finally: + env.close() + + clip_path = os.path.join(output_dir, "clip_0000.mp4") + assert os.path.isfile(clip_path), "close() did not flush the partial clip" + clip = VideoFileClip(clip_path) + frames = list(clip.iter_frames()) + clip.close() + + # decoded frame counts round by one against the clip duration; a missing or doubled tick lands outside + assert _STEPS <= len(frames) <= _STEPS + 1, f"expected one frame per env step, got {len(frames)}" From 45d3e896b8beb71f8cbb2425cbc2050004c63931 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Wed, 30 Sep 2026 16:20:41 -0700 Subject: [PATCH 2/7] Allow --video on the Warp frontend apply_video_recording rejected --video on any non-torch frontend because the Warp environments did not drive recorders. They do now, so drop the guard. Make the visualizer pre_launch_video_config adds for --video without --viz frontend-aware: a headless Kit visualizer on the torch frontend, as before, and a headless Newton GL visualizer on the Warp frontend. The Warp frontend always runs Newton physics and is commonly run without the Isaac Sim install that Kit needs. Update the Warp environments guide, which still said --video is rejected, and the 3.0 migration guide, which described the Kit default as unconditional. --- docs/source/concepts/warp_environments.rst | 5 ++- .../migration/migrating_to_isaaclab_3-0.rst | 8 +++-- .../jichuanh-warp-video-recording.changed.rst | 3 ++ .../jichuanh-warp-video-recording.minor | 0 .../isaaclab_rl/entrypoints/common.py | 32 +++++++++---------- .../test/test_apply_video_recording.py | 13 ++++++++ 6 files changed, 38 insertions(+), 23 deletions(-) create mode 100644 source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.changed.rst create mode 100644 source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.minor diff --git a/docs/source/concepts/warp_environments.rst b/docs/source/concepts/warp_environments.rst index 59e56aeddb1d..10d8849b6bc5 100644 --- a/docs/source/concepts/warp_environments.rst +++ b/docs/source/concepts/warp_environments.rst @@ -107,9 +107,8 @@ Stable-Baselines3. .. note:: - ``--video`` is rejected on the warp path, for both ``train`` and ``play``: video - recording requires the standard torch frontend. To record a rollout, replay the same - checkpoint with ``--frontend torch``; see :ref:`how_to_record_video`. + ``--video`` records on the warp path, for both ``train`` and ``play``. Without ``--viz``, + a headless Newton GL visualizer is created to record from; see :ref:`how_to_record_video`. Performance Comparison diff --git a/docs/source/migration/migrating_to_isaaclab_3-0.rst b/docs/source/migration/migrating_to_isaaclab_3-0.rst index 953faad1be85..8548623347f8 100644 --- a/docs/source/migration/migrating_to_isaaclab_3-0.rst +++ b/docs/source/migration/migrating_to_isaaclab_3-0.rst @@ -2584,12 +2584,14 @@ In Isaac Lab 3.0, the equivalent is: --viz kit --enable_cameras --headless --video As a convenience, passing ``--video`` without ``--viz`` still works: Isaac Lab -auto-creates a headless Kit visualizer (falling back to Newton GL if Kit is unavailable) -and sets ``source="visualizer:kit"`` on the default recorder, printing: +auto-creates a headless visualizer and points the default recorder at it. The visualizer +depends on the frontend: Kit (``source="visualizer:kit"``) with ``--frontend torch``, and +Newton GL (``source="visualizer:newton_gl"``) with ``--frontend warp``, which is commonly +run without a Kit install. With ``--frontend torch`` it prints: .. code-block:: text - [INFO] --video specified without --viz: adding a headless Kit visualizer to record + [INFO] --video specified without --viz: adding a headless kit visualizer to record from. Pass --viz to choose a different visualizer, or set video_recorders in your env config to record from a scene sensor instead. diff --git a/source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.changed.rst b/source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.changed.rst new file mode 100644 index 000000000000..b632d06302ad --- /dev/null +++ b/source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.changed.rst @@ -0,0 +1,3 @@ +* Allowed ``--video`` with ``--frontend warp``; it previously raised a :class:`ValueError`. + Without ``--viz``, the Warp frontend records from an auto-created headless Newton GL visualizer + instead of Kit, which needs a full Isaac Sim install. The torch frontend still gets Kit. diff --git a/source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.minor b/source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_rl/isaaclab_rl/entrypoints/common.py b/source/isaaclab_rl/isaaclab_rl/entrypoints/common.py index a2576b9a5feb..e28fa6c65ead 100644 --- a/source/isaaclab_rl/isaaclab_rl/entrypoints/common.py +++ b/source/isaaclab_rl/isaaclab_rl/entrypoints/common.py @@ -857,12 +857,13 @@ def wrap_sensor_capture(env: gym.Env, log_dir: str, args_cli: argparse.Namespace def pre_launch_video_config(env_cfg: Any, args_cli: argparse.Namespace) -> None: - """Add a headless Kit visualizer to *env_cfg* for ``--video`` to record from when it names none. + """Add a headless visualizer to *env_cfg* for ``--video`` to record from when it names none. - Must be called before :func:`~isaaclab.app.launch_simulation`, so the launch includes the Kit runtime. - Only acts when ``--video`` is set and neither the environment config nor the command line names a + Must be called before :func:`~isaaclab.app.launch_simulation`, so the launch includes the visualizer's + runtime. Only acts when ``--video`` is set and neither the environment config nor the command line names a visualizer or a video recorder to record from; :func:`apply_video_recording` wires the recorder after - the launch. + the launch. The torch frontend gets Kit. The warp frontend gets Newton GL: it always runs Newton physics + and is commonly run without the Isaac Sim install that Kit needs. Args: env_cfg: Isaac Lab environment config to modify in-place. @@ -881,15 +882,19 @@ def pre_launch_video_config(env_cfg: Any, args_cli: argparse.Namespace) -> None: if any(cfg.visualizer_type for cfg in visualizer_cfgs): return try: - from isaaclab_visualizers.kit import KitVisualizerCfg + if getattr(args_cli, "frontend", "torch") == "warp": + from isaaclab_visualizers.newton import NewtonGLVisualizerCfg as HeadlessVisualizerCfg + else: + from isaaclab_visualizers.kit import KitVisualizerCfg as HeadlessVisualizerCfg except ImportError: # isaaclab_visualizers is optional; apply_video_recording reports the missing visualizer return - sim_cfg.visualizer_cfgs = [*visualizer_cfgs, KitVisualizerCfg(headless=True)] + visualizer_cfg = HeadlessVisualizerCfg(headless=True) + sim_cfg.visualizer_cfgs = [*visualizer_cfgs, visualizer_cfg] print( - "[INFO] --video specified without --viz: adding a headless Kit visualizer to record from. Pass " - "--viz to choose a different visualizer, or set video_recorders in your env config to record " - "from a scene sensor instead." + f"[INFO] --video specified without --viz: adding a headless {visualizer_cfg.visualizer_type} visualizer to " + "record from. Pass --viz to choose a different visualizer, or set video_recorders in your env " + "config to record from a scene sensor instead." ) @@ -929,17 +934,10 @@ def apply_video_recording( numeric id, the checkpoint stem is appended to play video names. Raises: - ValueError: If recording is requested with the warp frontend or without a visualizer to record from. + ValueError: If recording is requested without a visualizer to record from. """ if not getattr(args_cli, "video", False): return - frontend = getattr(args_cli, "frontend", "torch") or "torch" - if frontend != "torch": - raise ValueError( - f"--video is not supported with --frontend {frontend!r}. " - "Video recording requires the standard torch frontend. " - "Remove --video or switch to --frontend torch." - ) if not getattr(env_cfg, "video_recorders", None): source = _resolve_video_source(env_cfg) diff --git a/source/isaaclab_rl/test/test_apply_video_recording.py b/source/isaaclab_rl/test/test_apply_video_recording.py index f4a361255afa..2622fa11473f 100644 --- a/source/isaaclab_rl/test/test_apply_video_recording.py +++ b/source/isaaclab_rl/test/test_apply_video_recording.py @@ -13,6 +13,7 @@ import pytest from isaaclab_visualizers.kit import KitVisualizerCfg +from isaaclab_visualizers.newton import NewtonGLVisualizerCfg from isaaclab.envs import ManagerBasedRLEnvCfg from isaaclab.envs.utils.video_recorder_cfg import VideoRecorderCfg @@ -74,6 +75,18 @@ def test_apply_video_recording_creates_default_recorder( assert env_cfg.sim.visualizer_cfgs[0].headless +def test_apply_video_recording_injects_newton_gl_visualizer_on_warp_frontend(): + """The warp frontend records too; without ``--viz`` it gets a headless Newton GL visualizer instead of Kit.""" + args = _args(frontend="warp") + env_cfg = _launched_env_cfg(args) + apply_video_recording(env_cfg, "/my/log", args) + + assert [recorder.source for recorder in env_cfg.video_recorders] == ["visualizer:newton_gl"] + assert len(env_cfg.sim.visualizer_cfgs) == 1 + assert isinstance(env_cfg.sim.visualizer_cfgs[0], NewtonGLVisualizerCfg) + assert env_cfg.sim.visualizer_cfgs[0].headless + + @pytest.mark.parametrize( ("subdir", "existing_prefix", "checkpoint_name", "expected_prefix"), [ From f1f57cf19b365d242175a98c435069cbc970d0f0 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Wed, 30 Sep 2026 02:17:41 -0700 Subject: [PATCH 3/7] Load Newton visual shapes for headless model visualizers A headless Newton GL visualizer declared in sim.visualizer_cfgs recorded a single flat colour. Headless visualizers do not render continuously, so is_rendering is False, and has_active_visualizers only sees --viz. The Newton cloner therefore resolved load_visual_shapes to False and built the model without the shapes the viewer draws. This is the visualizer --video injects on the Warp frontend, so every Warp recording without --viz was empty. The same config is empty on the torch frontend; --viz newton_gl was unaffected. Have each configured visualizer that draws the Newton model request the visual shapes before cloning, the contract camera sensors already use. Assert the Warp recording test's frames show the scene and change over the clip; without the fix both cases fail with a flat frame. --- .../jichuanh-warp-video-recording.fixed.rst | 3 +++ .../isaaclab/isaaclab/sim/simulation_context.py | 3 +++ .../sim/test_simulation_context_visualizers.py | 15 +++++++++++++++ .../test/envs/test_video_recording_warp.py | 13 ++++++++++--- 4 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 source/isaaclab/changelog.d/jichuanh-warp-video-recording.fixed.rst diff --git a/source/isaaclab/changelog.d/jichuanh-warp-video-recording.fixed.rst b/source/isaaclab/changelog.d/jichuanh-warp-video-recording.fixed.rst new file mode 100644 index 000000000000..edc3a1a56c0c --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-warp-video-recording.fixed.rst @@ -0,0 +1,3 @@ +* Fixed headless Newton visualizers declared in :attr:`~isaaclab.sim.SimulationCfg.visualizer_cfgs` capturing + empty frames, for example a video recorded from ``NewtonGLVisualizerCfg(headless=True)``. A headless visualizer + does not render continuously, so the Newton model was built without its visual shapes. diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index c9e3137ce484..4dc1c26f4ef8 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -444,6 +444,9 @@ def _create_visualizers(self) -> None: requires_stage, requires_model = REQUIRES_STAGE_AND_MODEL[cfg.visualizer_type] self.requires_usd_stage |= requires_stage self.requires_newton_model |= requires_model + # a model visualizer draws visual-only shapes even when headless, where it is not rendering + if requires_model: + self.require_visual_shapes() self._render_context.clone_contexts.update(cfg.cloning_contexts) self._pending_visualizers.append(instantiate(cfg)) diff --git a/source/isaaclab/test/sim/test_simulation_context_visualizers.py b/source/isaaclab/test/sim/test_simulation_context_visualizers.py index 44369af29f1f..d73bf1c3336a 100644 --- a/source/isaaclab/test/sim/test_simulation_context_visualizers.py +++ b/source/isaaclab/test/sim/test_simulation_context_visualizers.py @@ -836,6 +836,7 @@ def _make_context_with_settings( ctx._scene_data_provider = _FakeProvider() ctx.requires_usd_stage = False ctx.requires_newton_model = False + ctx._visual_shapes_required = False ctx._clone_plan = None ctx._viz_dt = 0.01 ctx.get_setting = lambda name: settings.get(name) @@ -991,6 +992,20 @@ def test_visualizer_init_keeps_requirements_published_before_reset(): assert ctx.requires_usd_stage +@pytest.mark.parametrize(("visualizer_type", "expected"), [("newton_gl", True), ("kit", False)]) +def test_configured_model_visualizer_requires_visual_shapes(visualizer_type: str, expected: bool): + """A configured visualizer that draws the Newton model requests its visual shapes, also when headless. + + A headless visualizer does not render continuously, so without the request the Newton model is + built without visual shapes and every captured frame is empty. + """ + ctx = _make_context_with_settings({}, visualizer_cfgs=[_FakeVisualizerCfg(visualizer_type)]) + + ctx._create_visualizers() + + assert ctx.visual_shapes_required is expected + + @pytest.mark.parametrize("cli_explicit", [False, True]) @pytest.mark.parametrize("fail_construct", [False, True]) def test_visualizer_failures_propagate_and_retain_constructed_instances(cli_explicit, fail_construct): diff --git a/source/isaaclab_experimental/test/envs/test_video_recording_warp.py b/source/isaaclab_experimental/test/envs/test_video_recording_warp.py index 66b49d340332..b34d5ea52043 100644 --- a/source/isaaclab_experimental/test/envs/test_video_recording_warp.py +++ b/source/isaaclab_experimental/test/envs/test_video_recording_warp.py @@ -28,6 +28,7 @@ import os import tempfile +import numpy as np import torch from isaaclab_experimental.envs.frontend import WarpFrontend from isaaclab_visualizers.newton import NewtonGLVisualizerCfg @@ -41,6 +42,7 @@ _CLIP = 20 # frames per clip _STEPS = _CLIP // 2 # shorter than one clip, so only close() can write it +_MIN_SPATIAL_STD = 5.0 # per-channel pixel std; a flat frame is ~0, the rendered scene is ~40 @pytest.mark.parametrize( @@ -49,7 +51,7 @@ ids=["direct", "manager"], ) def test_warp_env_records_and_flushes_clip(cfg_class: type, task_id: str): - """A Warp env records one frame per step and flushes the partial clip on close.""" + """A Warp env records the moving scene once per step and flushes the partial clip on close.""" env_cfg = cfg_class() env_cfg.seed = 42 env_cfg.scene.num_envs = 1 @@ -65,7 +67,8 @@ def test_warp_env_records_and_flushes_clip(cfg_class: type, task_id: str): env = WarpFrontend.build_env(env_cfg, task_id) try: env.reset() - actions = torch.zeros(env.num_envs, *env.action_space.shape[1:], device=env.device) + # push the cart so the clip shows motion + actions = torch.ones(env.num_envs, *env.action_space.shape[1:], device=env.device) for _ in range(_STEPS): env.step(actions) finally: @@ -74,8 +77,12 @@ def test_warp_env_records_and_flushes_clip(cfg_class: type, task_id: str): clip_path = os.path.join(output_dir, "clip_0000.mp4") assert os.path.isfile(clip_path), "close() did not flush the partial clip" clip = VideoFileClip(clip_path) - frames = list(clip.iter_frames()) + frames = np.stack(list(clip.iter_frames())).astype(np.float32) clip.close() # decoded frame counts round by one against the clip duration; a missing or doubled tick lands outside assert _STEPS <= len(frames) <= _STEPS + 1, f"expected one frame per env step, got {len(frames)}" + # a viewer that draws no geometry yields a single flat background colour + spatial_std = frames.reshape(len(frames), -1, 3).std(axis=1).max(axis=1) + assert spatial_std.min() > _MIN_SPATIAL_STD, f"a frame shows no scene (spatial std {spatial_std.min():.2f})" + assert not np.array_equal(frames[0], frames[-1]), "the clip does not change over time" From 136817657d64b1f6363adffa523c229c56bbc878 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Sun, 4 Oct 2026 02:19:40 -0700 Subject: [PATCH 4/7] Drop the visual-shape request develop's capture check covers Develop's SimulationContext.can_render_rgb_array() now returns True whenever SimulationCfg.visualizer_cfgs lists a visualizer (#8198), so Newton cloning loads visual shapes for a headless visualizer without the explicit require_visual_shapes() call. The Warp video test records non-flat frames without it, and records flat frames once that develop clause is removed. --- .../changelog.d/jichuanh-warp-video-recording.fixed.rst | 3 --- source/isaaclab/isaaclab/sim/simulation_context.py | 3 --- 2 files changed, 6 deletions(-) delete mode 100644 source/isaaclab/changelog.d/jichuanh-warp-video-recording.fixed.rst diff --git a/source/isaaclab/changelog.d/jichuanh-warp-video-recording.fixed.rst b/source/isaaclab/changelog.d/jichuanh-warp-video-recording.fixed.rst deleted file mode 100644 index edc3a1a56c0c..000000000000 --- a/source/isaaclab/changelog.d/jichuanh-warp-video-recording.fixed.rst +++ /dev/null @@ -1,3 +0,0 @@ -* Fixed headless Newton visualizers declared in :attr:`~isaaclab.sim.SimulationCfg.visualizer_cfgs` capturing - empty frames, for example a video recorded from ``NewtonGLVisualizerCfg(headless=True)``. A headless visualizer - does not render continuously, so the Newton model was built without its visual shapes. diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index 6e42fb0ba74d..e47ff9cf4b84 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -447,9 +447,6 @@ def _create_visualizers(self) -> None: requires_stage, requires_model = REQUIRES_STAGE_AND_MODEL[cfg.visualizer_type] self.requires_usd_stage |= requires_stage self.requires_newton_model |= requires_model - # a model visualizer draws visual-only shapes even when headless, where it is not rendering - if requires_model: - self.require_visual_shapes() self._render_context.clone_contexts.update(cfg.cloning_contexts) self._pending_visualizers.append(instantiate(cfg)) From 3e2eb74dd4ba8265c08756e3dab37599a403f618 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Sun, 4 Oct 2026 02:19:52 -0700 Subject: [PATCH 5/7] Allow --video on the Warp frontend with the --video source design Develop moved the Warp rejection into pre_launch_video_config when #8198 made --video take a source that launch_simulation resolves. Remove it there: the default source already records from a headless newton_gl when --viz selects no capture-capable visualizer, which is what Part 5 injected for the Warp frontend by hand. The source-resolution test now pins that --frontend warp resolves like the torch frontend, the Warp test uses the canonical viz:newton_gl source, and the docs and fragment drop the Kit-vs-Newton GL frontend split that no longer exists. --- docs/source/concepts/warp_environments.rst | 3 +-- .../test/envs/test_video_recording_warp.py | 2 +- .../jichuanh-warp-video-recording.changed.rst | 2 -- source/isaaclab_rl/isaaclab_rl/entrypoints/common.py | 10 ---------- source/isaaclab_rl/test/test_apply_video_recording.py | 7 ++++--- 5 files changed, 6 insertions(+), 18 deletions(-) diff --git a/docs/source/concepts/warp_environments.rst b/docs/source/concepts/warp_environments.rst index 10d8849b6bc5..010b7e0f79db 100644 --- a/docs/source/concepts/warp_environments.rst +++ b/docs/source/concepts/warp_environments.rst @@ -107,8 +107,7 @@ Stable-Baselines3. .. note:: - ``--video`` records on the warp path, for both ``train`` and ``play``. Without ``--viz``, - a headless Newton GL visualizer is created to record from; see :ref:`how_to_record_video`. + ``--video`` records on the warp path too, for both ``train`` and ``play``; see :ref:`how_to_record_video`. Performance Comparison diff --git a/source/isaaclab_experimental/test/envs/test_video_recording_warp.py b/source/isaaclab_experimental/test/envs/test_video_recording_warp.py index b34d5ea52043..ca556a5c73fb 100644 --- a/source/isaaclab_experimental/test/envs/test_video_recording_warp.py +++ b/source/isaaclab_experimental/test/envs/test_video_recording_warp.py @@ -60,7 +60,7 @@ def test_warp_env_records_and_flushes_clip(cfg_class: type, task_id: str): with tempfile.TemporaryDirectory() as output_dir: env_cfg.video_recorders = [ VideoRecorderCfg( - source="visualizer:newton_gl", output_dir=output_dir, video_length=_CLIP, video_interval=0, fps=10 + source="viz:newton_gl", output_dir=output_dir, video_length=_CLIP, video_interval=0, fps=10 ) ] sim_utils.create_new_stage() diff --git a/source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.changed.rst b/source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.changed.rst index b632d06302ad..c6fa07ee244a 100644 --- a/source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.changed.rst +++ b/source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.changed.rst @@ -1,3 +1 @@ * Allowed ``--video`` with ``--frontend warp``; it previously raised a :class:`ValueError`. - Without ``--viz``, the Warp frontend records from an auto-created headless Newton GL visualizer - instead of Kit, which needs a full Isaac Sim install. The torch frontend still gets Kit. diff --git a/source/isaaclab_rl/isaaclab_rl/entrypoints/common.py b/source/isaaclab_rl/isaaclab_rl/entrypoints/common.py index 50c95cd15274..159895bc81f6 100644 --- a/source/isaaclab_rl/isaaclab_rl/entrypoints/common.py +++ b/source/isaaclab_rl/isaaclab_rl/entrypoints/common.py @@ -902,19 +902,9 @@ def pre_launch_video_config(env_cfg: Any, args_cli: argparse.Namespace) -> None: Args: env_cfg: Isaac Lab environment config to modify in-place. args_cli: Parsed command-line arguments. - - Raises: - ValueError: If recording is requested with the warp frontend. """ if not getattr(args_cli, "video", None): return - frontend = getattr(args_cli, "frontend", "torch") or "torch" - if frontend != "torch": - raise ValueError( - f"--video is not supported with --frontend {frontend!r}. " - "Video recording requires the standard torch frontend. " - "Remove --video or switch to --frontend torch." - ) if not env_cfg.video_recorders: env_cfg.video_recorders = [VideoRecorderCfg(source=args_cli.video, video_interval=2000)] diff --git a/source/isaaclab_rl/test/test_apply_video_recording.py b/source/isaaclab_rl/test/test_apply_video_recording.py index abdee5958d89..d4665fd413c9 100644 --- a/source/isaaclab_rl/test/test_apply_video_recording.py +++ b/source/isaaclab_rl/test/test_apply_video_recording.py @@ -106,9 +106,10 @@ def start(launcher_args: dict) -> SimulationLauncher: (dict(video="kit"), ("viz:kit", [("kit", True)])), # a scene sensor needs no visualizer (dict(video="sensor:wrist_camera:depth"), ("sensor:wrist_camera:depth", [])), - # streaming visualizers have no frame capture, and recording needs the torch frontend + # the warp frontend records the same way + (dict(frontend="warp"), ("viz:newton_gl", [("newton_gl", True)])), + # streaming visualizers have no frame capture (dict(video="viser", visualizer=["viser"]), "has no frame capture"), - (dict(frontend="warp"), "--frontend 'warp'"), ], ids=[ "viz-selected", @@ -118,8 +119,8 @@ def start(launcher_args: dict) -> SimulationLauncher: "type-added", "kit-added", "sensor", + "warp-frontend", "streaming-rejected", - "warp-frontend-rejected", ], ) def test_video_source_resolves_against_the_visualizer_selection(launches, cli, expected): From b25c6c013f0a8549b46eda786e3421ab2f8c6caa Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Tue, 20 Oct 2026 10:11:13 -0700 Subject: [PATCH 6/7] Create the Warp environments' video recorders after the simulation starts --- .../isaaclab_experimental/envs/direct_rl_env_warp.py | 3 +-- .../isaaclab_experimental/envs/manager_based_env_warp.py | 3 +-- .../test/envs/test_video_recording_warp.py | 3 ++- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/source/isaaclab_experimental/isaaclab_experimental/envs/direct_rl_env_warp.py b/source/isaaclab_experimental/isaaclab_experimental/envs/direct_rl_env_warp.py index ba41a10db558..5283b845327d 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/envs/direct_rl_env_warp.py +++ b/source/isaaclab_experimental/isaaclab_experimental/envs/direct_rl_env_warp.py @@ -177,8 +177,6 @@ def __init__(self, cfg: DirectRLEnvCfg, render_mode: str | None = None, **kwargs if "prestartup" in self.event_manager.available_modes: self.event_manager.apply(mode="prestartup") - self.video_recorders: list[VideoRecorder] = [VideoRecorder(cfg, self) for cfg in self.cfg.video_recorders] - # play the simulator to activate physics handles # note: this activates the physics simulation view that exposes TensorAPIs # note: when started in extension mode, first call sim.reset_async() and then initialize the managers @@ -194,6 +192,7 @@ def __init__(self, cfg: DirectRLEnvCfg, render_mode: str | None = None, **kwargs # this shouldn't cause an issue since later on, users do a reset over all the # environments so the lazy buffers would be reset. self.scene.update(dt=self.physics_dt) + self.video_recorders: list[VideoRecorder] = [VideoRecorder(cfg, self) for cfg in self.cfg.video_recorders] # check if debug visualization is has been implemented by the environment source_code = inspect.getsource(self._set_debug_vis_impl) diff --git a/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_env_warp.py b/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_env_warp.py index 14ab185c0093..db97b5fdfedd 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_env_warp.py +++ b/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_env_warp.py @@ -171,8 +171,6 @@ def __init__(self, cfg: ManagerBasedEnvCfg): if "prestartup" in self.event_manager.available_modes: self.event_manager.apply(mode="prestartup") - self.video_recorders: list[VideoRecorder] = [VideoRecorder(cfg, self) for cfg in self.cfg.video_recorders] - # play the simulator to activate physics handles # note: this activates the physics simulation view that exposes TensorAPIs # note: when started in extension mode, first call sim.reset_async() and then initialize the managers @@ -188,6 +186,7 @@ def __init__(self, cfg: ManagerBasedEnvCfg): # this shouldn't cause an issue since later on, users do a reset over all the environments so the lazy # buffers would be reset. self.scene.update(dt=self.physics_dt) + self.video_recorders: list[VideoRecorder] = [VideoRecorder(cfg, self) for cfg in self.cfg.video_recorders] # TODO(jichuanh): This is a temporary solution for event_manager only, but it should be general for all managers # Resolve SceneEntityCfg-dependent term params once before any captured event paths. diff --git a/source/isaaclab_experimental/test/envs/test_video_recording_warp.py b/source/isaaclab_experimental/test/envs/test_video_recording_warp.py index ca556a5c73fb..8b4aca60f00e 100644 --- a/source/isaaclab_experimental/test/envs/test_video_recording_warp.py +++ b/source/isaaclab_experimental/test/envs/test_video_recording_warp.py @@ -35,6 +35,7 @@ import isaaclab.sim as sim_utils from isaaclab.envs.utils.video_recorder_cfg import VideoRecorderCfg +from isaaclab.visualizers import WindowCfg import isaaclab_tasks # noqa: F401 from isaaclab_tasks.core.cartpole.cartpole_direct_env_cfg import CartpoleEnvCfg as CartpoleDirectEnvCfg @@ -56,7 +57,7 @@ def test_warp_env_records_and_flushes_clip(cfg_class: type, task_id: str): env_cfg.seed = 42 env_cfg.scene.num_envs = 1 env_cfg.sim.physics = NewtonCfg(solver_cfg=MJWarpSolverCfg()) - env_cfg.sim.visualizer_cfgs = [NewtonGLVisualizerCfg(headless=True, window_width=320, window_height=240)] + env_cfg.sim.visualizer_cfgs = [NewtonGLVisualizerCfg(headless=True, window=WindowCfg(size=(320, 240)))] with tempfile.TemporaryDirectory() as output_dir: env_cfg.video_recorders = [ VideoRecorderCfg( From d43923df36e23d6af47abb8edff8126129496314 Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Tue, 27 Oct 2026 12:08:27 -0700 Subject: [PATCH 7/7] Drop the video source row that repeats the no-visualizer case No launch code reads --frontend any more, so the warp-frontend row ran the same path as viz-none with the same expected source. The Warp environments' own recording test covers recording on the warp frontend. --- source/isaaclab_rl/test/test_apply_video_recording.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/isaaclab_rl/test/test_apply_video_recording.py b/source/isaaclab_rl/test/test_apply_video_recording.py index 5b6144eaa230..d256864c40a1 100644 --- a/source/isaaclab_rl/test/test_apply_video_recording.py +++ b/source/isaaclab_rl/test/test_apply_video_recording.py @@ -114,8 +114,6 @@ def start(launcher_args: dict) -> SimulationLauncher: ), # a scene sensor needs no visualizer (dict(video="sensor:wrist_camera:depth"), ("sensor:wrist_camera:depth", [])), - # the warp frontend records the same way - (dict(frontend="warp"), ("viz:newton_gl", [("newton_gl", True)])), # streaming visualizers have no frame capture (dict(video="viser", visualizer=["viser"]), "has no frame capture"), ], @@ -129,7 +127,6 @@ def start(launcher_args: dict) -> SimulationLauncher: "camera-view-headless", "camera-view-windowed", "sensor", - "warp-frontend", "streaming-rejected", ], )