diff --git a/docs/source/concepts/warp_environments.rst b/docs/source/concepts/warp_environments.rst index 1debce871a09..55943cf2ebb4 100644 --- a/docs/source/concepts/warp_environments.rst +++ b/docs/source/concepts/warp_environments.rst @@ -39,9 +39,7 @@ Stable-Baselines3. .. note:: - ``--video`` is rejected on the warp path, for both ``train`` and ``play``: video - recording requires the standard torch frontend. To record a rollout, replay the same - checkpoint with ``--frontend torch``; see :ref:`how_to_record_video`. + ``--video`` records on the warp path too, for both ``train`` and ``play``; see :ref:`how_to_record_video`. Workflows diff --git a/source/isaaclab_experimental/changelog.d/jichuanh-warp-video-recording.added.rst b/source/isaaclab_experimental/changelog.d/jichuanh-warp-video-recording.added.rst new file mode 100644 index 000000000000..4e413cd11447 --- /dev/null +++ b/source/isaaclab_experimental/changelog.d/jichuanh-warp-video-recording.added.rst @@ -0,0 +1,4 @@ +* Added video recording to the Warp environments. :class:`~isaaclab_experimental.envs.ManagerBasedEnvWarp`, + :class:`~isaaclab_experimental.envs.ManagerBasedRLEnvWarp` and :class:`~isaaclab_experimental.envs.DirectRLEnvWarp` + now create the recorders in ``env_cfg.video_recorders``, advance them once per step and flush them on close. + Previously the setting was ignored, with a warning only on the manager-based environments. diff --git a/source/isaaclab_experimental/changelog.d/jichuanh-warp-video-recording.minor b/source/isaaclab_experimental/changelog.d/jichuanh-warp-video-recording.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_experimental/isaaclab_experimental/envs/direct_rl_env_warp.py b/source/isaaclab_experimental/isaaclab_experimental/envs/direct_rl_env_warp.py index 2c032cea3c22..5283b845327d 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/envs/direct_rl_env_warp.py +++ b/source/isaaclab_experimental/isaaclab_experimental/envs/direct_rl_env_warp.py @@ -27,6 +27,7 @@ from isaaclab.envs.direct_rl_env import DirectRLEnv from isaaclab.envs.direct_rl_env_cfg import DirectRLEnvCfg from isaaclab.envs.utils.spaces import sample_space, spec_to_gym_space +from isaaclab.envs.utils.video_recorder import VideoRecorder from isaaclab.managers import EventManager from isaaclab.sim import SimulationContext from isaaclab.sim.utils import use_stage @@ -191,6 +192,7 @@ def __init__(self, cfg: DirectRLEnvCfg, render_mode: str | None = None, **kwargs # this shouldn't cause an issue since later on, users do a reset over all the # environments so the lazy buffers would be reset. self.scene.update(dt=self.physics_dt) + self.video_recorders: list[VideoRecorder] = [VideoRecorder(cfg, self) for cfg in self.cfg.video_recorders] # check if debug visualization is has been implemented by the environment source_code = inspect.getsource(self._set_debug_vis_impl) @@ -447,6 +449,10 @@ def step(self, action: torch.Tensor) -> VecEnvStepReturn: with Timer(name="visualize", msg="Visualize took:", enable=DEBUG_TIMERS): self._post_step_visualize() + # advance video recorders (after render and visualization, outside the captured graphs) + for recorder in self.video_recorders: + recorder.step() + # return observations, rewards, resets and extras # store the returned buffer so RslRlVecEnvWrapper.get_observations() can read env.obs_buf self.obs_buf = {"policy": self.torch_obs_buf.clone()} @@ -572,6 +578,10 @@ def render(self, recompute: bool = False) -> np.ndarray | None: def close(self): """Cleanup for the environment.""" if not self._is_closed: + # flush any buffered video frames + for recorder in getattr(self, "video_recorders", []): + recorder.close() + # close entities related to the environment # note: this is order-sensitive to avoid any dangling references if self.cfg.events: diff --git a/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_env_warp.py b/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_env_warp.py index 8b6710633ae4..db97b5fdfedd 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_env_warp.py +++ b/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_env_warp.py @@ -33,6 +33,7 @@ export_articulations_data, export_scene_data, ) +from isaaclab.envs.utils.video_recorder import VideoRecorder from isaaclab.sim import SimulationContext from isaaclab.sim.utils import use_stage from isaaclab.utils import validate @@ -79,14 +80,6 @@ def __init__(self, cfg: ManagerBasedEnvCfg): validate(cfg) # store inputs to class self.cfg = cfg - # Video recording is not supported on Warp environments. - if getattr(cfg, "video_recorders", None): - import logging as _logging - - _logging.getLogger(__name__).warning( - "cfg.video_recorders is set but ManagerBasedEnvWarp does not support VideoRecorder. " - "No clips will be written. Use ManagerBasedEnv for video recording support." - ) # initialize internal variables self._is_closed = False # temporary debug runtime config for manager source/call switching. @@ -193,6 +186,7 @@ def __init__(self, cfg: ManagerBasedEnvCfg): # this shouldn't cause an issue since later on, users do a reset over all the environments so the lazy # buffers would be reset. self.scene.update(dt=self.physics_dt) + self.video_recorders: list[VideoRecorder] = [VideoRecorder(cfg, self) for cfg in self.cfg.video_recorders] # TODO(jichuanh): This is a temporary solution for event_manager only, but it should be general for all managers # Resolve SceneEntityCfg-dependent term params once before any captured event paths. @@ -581,6 +575,10 @@ def step(self, action: torch.Tensor) -> tuple[VecEnvObs, dict]: if "interval" in self.event_manager.available_modes: self.event_manager.apply(mode="interval", dt=self.step_dt) + # advance video recorders (after render, before obs) + for recorder in self.video_recorders: + recorder.step() + # -- compute observations self.obs_buf = self.observation_manager.compute(update_history=True) self.recorder_manager.record_post_step() @@ -603,6 +601,10 @@ def seed(seed: int = -1) -> int: def close(self): """Cleanup for the environment.""" if not self._is_closed: + # flush any buffered video frames + for recorder in getattr(self, "video_recorders", []): + recorder.close() + # destructor is order-sensitive del self.action_manager del self.observation_manager diff --git a/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_rl_env_warp.py b/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_rl_env_warp.py index be522e073a2b..fcec930fecc5 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_rl_env_warp.py +++ b/source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_rl_env_warp.py @@ -379,6 +379,10 @@ def step(self, action: torch.Tensor) -> VecEnvStepReturn: timer=DEBUG_TIMER_STEP, ) + # -- advance video recorders (after render and resets, before final obs) + for recorder in self.video_recorders: + recorder.step() + # -- compute observations # note: done after reset to get the correct observations for reset envs self.obs_buf = self._manager_call_switch.call_stage( diff --git a/source/isaaclab_experimental/test/envs/test_video_recording_warp.py b/source/isaaclab_experimental/test/envs/test_video_recording_warp.py new file mode 100644 index 000000000000..8b4aca60f00e --- /dev/null +++ b/source/isaaclab_experimental/test/envs/test_video_recording_warp.py @@ -0,0 +1,89 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Video recording on the Warp environments. + +``DirectRLEnvWarp`` and the manager-based Warp envs are separate implementations of the recorder +lifecycle, so both are built through :meth:`WarpFrontend.build_env`, the adapter the CLI uses, and +record from a headless Newton GL visualizer without Kit. +""" + +# Check for moviepy before launching so a missing dependency produces a clean collection-time skip. +import pytest + +try: + from moviepy.editor import VideoFileClip +except ImportError: + pytest.skip("moviepy is not installed; install with: pip install 'moviepy<2'", allow_module_level=True) + +from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg + +from isaaclab.sim import SimulationCfg +from isaaclab.test.utils import launch_test_simulation + +launch_test_simulation(SimulationCfg(physics=NewtonCfg(solver_cfg=MJWarpSolverCfg()))) + +import os +import tempfile + +import numpy as np +import torch +from isaaclab_experimental.envs.frontend import WarpFrontend +from isaaclab_visualizers.newton import NewtonGLVisualizerCfg + +import isaaclab.sim as sim_utils +from isaaclab.envs.utils.video_recorder_cfg import VideoRecorderCfg +from isaaclab.visualizers import WindowCfg + +import isaaclab_tasks # noqa: F401 +from isaaclab_tasks.core.cartpole.cartpole_direct_env_cfg import CartpoleEnvCfg as CartpoleDirectEnvCfg +from isaaclab_tasks.core.cartpole.cartpole_manager_env_cfg import CartpoleEnvCfg as CartpoleManagerEnvCfg + +_CLIP = 20 # frames per clip +_STEPS = _CLIP // 2 # shorter than one clip, so only close() can write it +_MIN_SPATIAL_STD = 5.0 # per-channel pixel std; a flat frame is ~0, the rendered scene is ~40 + + +@pytest.mark.parametrize( + ("cfg_class", "task_id"), + [(CartpoleDirectEnvCfg, "Isaac-Cartpole-Direct"), (CartpoleManagerEnvCfg, "Isaac-Cartpole")], + ids=["direct", "manager"], +) +def test_warp_env_records_and_flushes_clip(cfg_class: type, task_id: str): + """A Warp env records the moving scene once per step and flushes the partial clip on close.""" + env_cfg = cfg_class() + env_cfg.seed = 42 + env_cfg.scene.num_envs = 1 + env_cfg.sim.physics = NewtonCfg(solver_cfg=MJWarpSolverCfg()) + env_cfg.sim.visualizer_cfgs = [NewtonGLVisualizerCfg(headless=True, window=WindowCfg(size=(320, 240)))] + with tempfile.TemporaryDirectory() as output_dir: + env_cfg.video_recorders = [ + VideoRecorderCfg( + source="viz:newton_gl", output_dir=output_dir, video_length=_CLIP, video_interval=0, fps=10 + ) + ] + sim_utils.create_new_stage() + env = WarpFrontend.build_env(env_cfg, task_id) + try: + env.reset() + # push the cart so the clip shows motion + actions = torch.ones(env.num_envs, *env.action_space.shape[1:], device=env.device) + for _ in range(_STEPS): + env.step(actions) + finally: + env.close() + + clip_path = os.path.join(output_dir, "clip_0000.mp4") + assert os.path.isfile(clip_path), "close() did not flush the partial clip" + clip = VideoFileClip(clip_path) + frames = np.stack(list(clip.iter_frames())).astype(np.float32) + clip.close() + + # decoded frame counts round by one against the clip duration; a missing or doubled tick lands outside + assert _STEPS <= len(frames) <= _STEPS + 1, f"expected one frame per env step, got {len(frames)}" + # a viewer that draws no geometry yields a single flat background colour + spatial_std = frames.reshape(len(frames), -1, 3).std(axis=1).max(axis=1) + assert spatial_std.min() > _MIN_SPATIAL_STD, f"a frame shows no scene (spatial std {spatial_std.min():.2f})" + assert not np.array_equal(frames[0], frames[-1]), "the clip does not change over time" diff --git a/source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.changed.rst b/source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.changed.rst new file mode 100644 index 000000000000..c6fa07ee244a --- /dev/null +++ b/source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.changed.rst @@ -0,0 +1 @@ +* Allowed ``--video`` with ``--frontend warp``; it previously raised a :class:`ValueError`. diff --git a/source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.minor b/source/isaaclab_rl/changelog.d/jichuanh-warp-video-recording.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_rl/isaaclab_rl/entrypoints/common.py b/source/isaaclab_rl/isaaclab_rl/entrypoints/common.py index 63a24dc1b2f0..5d7f937316b4 100644 --- a/source/isaaclab_rl/isaaclab_rl/entrypoints/common.py +++ b/source/isaaclab_rl/isaaclab_rl/entrypoints/common.py @@ -902,19 +902,9 @@ def pre_launch_video_config(env_cfg: Any, args_cli: argparse.Namespace) -> None: Args: env_cfg: Isaac Lab environment config to modify in-place. args_cli: Parsed command-line arguments. - - Raises: - ValueError: If recording is requested with the warp frontend. """ if not getattr(args_cli, "video", None): return - frontend = getattr(args_cli, "frontend", "torch") or "torch" - if frontend != "torch": - raise ValueError( - f"--video is not supported with --frontend {frontend!r}. " - "Video recording requires the standard torch frontend. " - "Remove --video or switch to --frontend torch." - ) if not env_cfg.video_recorders: env_cfg.video_recorders = [VideoRecorderCfg(source=args_cli.video, video_interval=2000)] diff --git a/source/isaaclab_rl/test/test_apply_video_recording.py b/source/isaaclab_rl/test/test_apply_video_recording.py index c59a97efb1ba..d256864c40a1 100644 --- a/source/isaaclab_rl/test/test_apply_video_recording.py +++ b/source/isaaclab_rl/test/test_apply_video_recording.py @@ -114,9 +114,8 @@ def start(launcher_args: dict) -> SimulationLauncher: ), # a scene sensor needs no visualizer (dict(video="sensor:wrist_camera:depth"), ("sensor:wrist_camera:depth", [])), - # streaming visualizers have no frame capture, and recording needs the torch frontend + # streaming visualizers have no frame capture (dict(video="viser", visualizer=["viser"]), "has no frame capture"), - (dict(frontend="warp"), "--frontend 'warp'"), ], ids=[ "viz-selected", @@ -129,7 +128,6 @@ def start(launcher_args: dict) -> SimulationLauncher: "camera-view-windowed", "sensor", "streaming-rejected", - "warp-frontend-rejected", ], ) def test_video_source_resolves_against_the_visualizer_selection(launches, cli, expected):