[Warp] Part 8/12: Allow --video on the Warp frontend

docs/source/concepts/warp_environments.rst

@@ -39,9 +39,7 @@ Stable-Baselines3.
 
 .. note::
 
-   ``--video`` is rejected on the warp path, for both ``train`` and ``play``: video
-   recording requires the standard torch frontend. To record a rollout, replay the same
-   checkpoint with ``--frontend torch``; see :ref:`how_to_record_video`.
+   ``--video`` records on the warp path too, for both ``train`` and ``play``; see :ref:`how_to_record_video`.
 
 
 Workflows

source/isaaclab_experimental/changelog.d/jichuanh-warp-video-recording.added.rst

@@ -0,0 +1,4 @@
+* Added video recording to the Warp environments. :class:`~isaaclab_experimental.envs.ManagerBasedEnvWarp`,
+  :class:`~isaaclab_experimental.envs.ManagerBasedRLEnvWarp` and :class:`~isaaclab_experimental.envs.DirectRLEnvWarp`
+  now create the recorders in ``env_cfg.video_recorders``, advance them once per step and flush them on close.
+  Previously the setting was ignored, with a warning only on the manager-based environments.

source/isaaclab_experimental/isaaclab_experimental/envs/direct_rl_env_warp.py

@@ -27,6 +27,7 @@
 from isaaclab.envs.direct_rl_env import DirectRLEnv
 from isaaclab.envs.direct_rl_env_cfg import DirectRLEnvCfg
 from isaaclab.envs.utils.spaces import sample_space, spec_to_gym_space
+from isaaclab.envs.utils.video_recorder import VideoRecorder
 from isaaclab.managers import EventManager
 from isaaclab.sim import SimulationContext
 from isaaclab.sim.utils import use_stage
@@ -191,6 +192,7 @@ def __init__(self, cfg: DirectRLEnvCfg, render_mode: str | None = None, **kwargs
             # this shouldn't cause an issue since later on, users do a reset over all the
             # environments so the lazy buffers would be reset.
             self.scene.update(dt=self.physics_dt)
+        self.video_recorders: list[VideoRecorder] = [VideoRecorder(cfg, self) for cfg in self.cfg.video_recorders]
 
         # check if debug visualization is has been implemented by the environment
         source_code = inspect.getsource(self._set_debug_vis_impl)
@@ -447,6 +449,10 @@ def step(self, action: torch.Tensor) -> VecEnvStepReturn:
         with Timer(name="visualize", msg="Visualize took:", enable=DEBUG_TIMERS):
             self._post_step_visualize()
 
+        # advance video recorders (after render and visualization, outside the captured graphs)
+        for recorder in self.video_recorders:
+            recorder.step()
+
         # return observations, rewards, resets and extras
         # store the returned buffer so RslRlVecEnvWrapper.get_observations() can read env.obs_buf
         self.obs_buf = {"policy": self.torch_obs_buf.clone()}
@@ -572,6 +578,10 @@ def render(self, recompute: bool = False) -> np.ndarray | None:
     def close(self):
         """Cleanup for the environment."""
         if not self._is_closed:
+            # flush any buffered video frames
+            for recorder in getattr(self, "video_recorders", []):
+                recorder.close()
+
             # close entities related to the environment
             # note: this is order-sensitive to avoid any dangling references
             if self.cfg.events:

source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_env_warp.py

@@ -33,6 +33,7 @@
     export_articulations_data,
     export_scene_data,
 )
+from isaaclab.envs.utils.video_recorder import VideoRecorder
 from isaaclab.sim import SimulationContext
 from isaaclab.sim.utils import use_stage
 from isaaclab.utils import validate
@@ -79,14 +80,6 @@ def __init__(self, cfg: ManagerBasedEnvCfg):
         validate(cfg)
         # store inputs to class
         self.cfg = cfg
-        # Video recording is not supported on Warp environments.
-        if getattr(cfg, "video_recorders", None):
-            import logging as _logging
-
-            _logging.getLogger(__name__).warning(
-                "cfg.video_recorders is set but ManagerBasedEnvWarp does not support VideoRecorder. "
-                "No clips will be written. Use ManagerBasedEnv for video recording support."
-            )
         # initialize internal variables
         self._is_closed = False
         # temporary debug runtime config for manager source/call switching.
@@ -193,6 +186,7 @@ def __init__(self, cfg: ManagerBasedEnvCfg):
             # this shouldn't cause an issue since later on, users do a reset over all the environments so the lazy
             # buffers would be reset.
             self.scene.update(dt=self.physics_dt)
+        self.video_recorders: list[VideoRecorder] = [VideoRecorder(cfg, self) for cfg in self.cfg.video_recorders]
 
         # TODO(jichuanh): This is a temporary solution for event_manager only, but it should be general for all managers
         # Resolve SceneEntityCfg-dependent term params once before any captured event paths.
@@ -581,6 +575,10 @@ def step(self, action: torch.Tensor) -> tuple[VecEnvObs, dict]:
         if "interval" in self.event_manager.available_modes:
             self.event_manager.apply(mode="interval", dt=self.step_dt)
 
+        # advance video recorders (after render, before obs)
+        for recorder in self.video_recorders:
+            recorder.step()
+
         # -- compute observations
         self.obs_buf = self.observation_manager.compute(update_history=True)
         self.recorder_manager.record_post_step()
@@ -603,6 +601,10 @@ def seed(seed: int = -1) -> int:
     def close(self):
         """Cleanup for the environment."""
         if not self._is_closed:
+            # flush any buffered video frames
+            for recorder in getattr(self, "video_recorders", []):
+                recorder.close()
+
             # destructor is order-sensitive
             del self.action_manager
             del self.observation_manager

source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_rl_env_warp.py

@@ -379,6 +379,10 @@ def step(self, action: torch.Tensor) -> VecEnvStepReturn:
                 timer=DEBUG_TIMER_STEP,
             )
 
+        # -- advance video recorders (after render and resets, before final obs)
+        for recorder in self.video_recorders:
+            recorder.step()
+
         # -- compute observations
         # note: done after reset to get the correct observations for reset envs
         self.obs_buf = self._manager_call_switch.call_stage(

source/isaaclab_experimental/test/envs/test_video_recording_warp.py

@@ -0,0 +1,89 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Video recording on the Warp environments.
+
+``DirectRLEnvWarp`` and the manager-based Warp envs are separate implementations of the recorder
+lifecycle, so both are built through :meth:`WarpFrontend.build_env`, the adapter the CLI uses, and
+record from a headless Newton GL visualizer without Kit.
+"""
+
+# Check for moviepy before launching so a missing dependency produces a clean collection-time skip.
+import pytest
+
+try:
+    from moviepy.editor import VideoFileClip
+except ImportError:
+    pytest.skip("moviepy is not installed; install with: pip install 'moviepy<2'", allow_module_level=True)
+
+from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg
+
+from isaaclab.sim import SimulationCfg
+from isaaclab.test.utils import launch_test_simulation
+
+launch_test_simulation(SimulationCfg(physics=NewtonCfg(solver_cfg=MJWarpSolverCfg())))
+
+import os
+import tempfile
+
+import numpy as np
+import torch
+from isaaclab_experimental.envs.frontend import WarpFrontend
+from isaaclab_visualizers.newton import NewtonGLVisualizerCfg
+
+import isaaclab.sim as sim_utils
+from isaaclab.envs.utils.video_recorder_cfg import VideoRecorderCfg
+from isaaclab.visualizers import WindowCfg
+
+import isaaclab_tasks  # noqa: F401
+from isaaclab_tasks.core.cartpole.cartpole_direct_env_cfg import CartpoleEnvCfg as CartpoleDirectEnvCfg
+from isaaclab_tasks.core.cartpole.cartpole_manager_env_cfg import CartpoleEnvCfg as CartpoleManagerEnvCfg
+
+_CLIP = 20  # frames per clip
+_STEPS = _CLIP // 2  # shorter than one clip, so only close() can write it
+_MIN_SPATIAL_STD = 5.0  # per-channel pixel std; a flat frame is ~0, the rendered scene is ~40
+
+
+@pytest.mark.parametrize(
+    ("cfg_class", "task_id"),
+    [(CartpoleDirectEnvCfg, "Isaac-Cartpole-Direct"), (CartpoleManagerEnvCfg, "Isaac-Cartpole")],
+    ids=["direct", "manager"],
+)
+def test_warp_env_records_and_flushes_clip(cfg_class: type, task_id: str):
+    """A Warp env records the moving scene once per step and flushes the partial clip on close."""
+    env_cfg = cfg_class()
+    env_cfg.seed = 42
+    env_cfg.scene.num_envs = 1
+    env_cfg.sim.physics = NewtonCfg(solver_cfg=MJWarpSolverCfg())
+    env_cfg.sim.visualizer_cfgs = [NewtonGLVisualizerCfg(headless=True, window=WindowCfg(size=(320, 240)))]
+    with tempfile.TemporaryDirectory() as output_dir:
+        env_cfg.video_recorders = [
+            VideoRecorderCfg(
+                source="viz:newton_gl", output_dir=output_dir, video_length=_CLIP, video_interval=0, fps=10
+            )
+        ]
+        sim_utils.create_new_stage()
+        env = WarpFrontend.build_env(env_cfg, task_id)
+        try:
+            env.reset()
+            # push the cart so the clip shows motion
+            actions = torch.ones(env.num_envs, *env.action_space.shape[1:], device=env.device)
+            for _ in range(_STEPS):
+                env.step(actions)
+        finally:
+            env.close()
+
+        clip_path = os.path.join(output_dir, "clip_0000.mp4")
+        assert os.path.isfile(clip_path), "close() did not flush the partial clip"
+        clip = VideoFileClip(clip_path)
+        frames = np.stack(list(clip.iter_frames())).astype(np.float32)
+        clip.close()
+
+    # decoded frame counts round by one against the clip duration; a missing or doubled tick lands outside
+    assert _STEPS <= len(frames) <= _STEPS + 1, f"expected one frame per env step, got {len(frames)}"
+    # a viewer that draws no geometry yields a single flat background colour
+    spatial_std = frames.reshape(len(frames), -1, 3).std(axis=1).max(axis=1)
+    assert spatial_std.min() > _MIN_SPATIAL_STD, f"a frame shows no scene (spatial std {spatial_std.min():.2f})"
+    assert not np.array_equal(frames[0], frames[-1]), "the clip does not change over time"

source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.changed.rst

@@ -0,0 +1 @@
+* Allowed ``--video`` with ``--frontend warp``; it previously raised a :class:`ValueError`.

source/isaaclab_rl/isaaclab_rl/entrypoints/common.py

@@ -902,19 +902,9 @@ def pre_launch_video_config(env_cfg: Any, args_cli: argparse.Namespace) -> None:
     Args:
         env_cfg: Isaac Lab environment config to modify in-place.
         args_cli: Parsed command-line arguments.
-
-    Raises:
-        ValueError: If recording is requested with the warp frontend.
     """
     if not getattr(args_cli, "video", None):
         return
-    frontend = getattr(args_cli, "frontend", "torch") or "torch"
-    if frontend != "torch":
-        raise ValueError(
-            f"--video is not supported with --frontend {frontend!r}. "
-            "Video recording requires the standard torch frontend. "
-            "Remove --video or switch to --frontend torch."
-        )
     if not env_cfg.video_recorders:
         env_cfg.video_recorders = [VideoRecorderCfg(source=args_cli.video, video_interval=2000)]
 

source/isaaclab_rl/test/test_apply_video_recording.py

@@ -114,9 +114,8 @@ def start(launcher_args: dict) -> SimulationLauncher:
         ),
         # a scene sensor needs no visualizer
         (dict(video="sensor:wrist_camera:depth"), ("sensor:wrist_camera:depth", [])),
-        # streaming visualizers have no frame capture, and recording needs the torch frontend
+        # streaming visualizers have no frame capture
         (dict(video="viser", visualizer=["viser"]), "has no frame capture"),
-        (dict(frontend="warp"), "--frontend 'warp'"),
     ],
     ids=[
         "viz-selected",
@@ -129,7 +128,6 @@ def start(launcher_args: dict) -> SimulationLauncher:
         "camera-view-windowed",
         "sensor",
         "streaming-rejected",
-        "warp-frontend-rejected",
     ],
 )
 def test_video_source_resolves_against_the_visualizer_selection(launches, cli, expected):