diff --git a/source/isaaclab/changelog.d/moving-camera-pose-refresh.rst b/source/isaaclab/changelog.d/moving-camera-pose-refresh.rst new file mode 100644 index 000000000000..d653c8a48cd7 --- /dev/null +++ b/source/isaaclab/changelog.d/moving-camera-pose-refresh.rst @@ -0,0 +1,7 @@ +Fixed +^^^^^ + +* Documented the need for ``CameraCfg.update_latest_camera_pose=True`` on moving cameras when the renderer + receives explicit camera poses. +* Clarified that camera render specifications can describe logical instances whose clones are absent + from the authored USD stage. diff --git a/source/isaaclab/isaaclab/renderers/camera_render_spec.py b/source/isaaclab/isaaclab/renderers/camera_render_spec.py index 526b90121713..a76d24b59000 100644 --- a/source/isaaclab/isaaclab/renderers/camera_render_spec.py +++ b/source/isaaclab/isaaclab/renderers/camera_render_spec.py @@ -23,8 +23,10 @@ class CameraRenderSpec: cfg: Camera configuration (data types, resolution, filters, etc.). device: Torch device string (e.g. ``"cuda:0"``) used by GPU annotators and Warp. num_instances: Number of tiled camera instances (environments). - camera_prim_paths: Absolute USD paths for each environment's camera prim. - view_count: Number of camera prims (must match ``len(camera_prim_paths)``). + camera_prim_paths: Absolute paths of the authored camera prims. When the renderer + clones environments internally, this may contain only the source camera path; + the renderer resolves its logical per-environment paths during registration. + view_count: Number of logical camera instances in the sensor view. camera_path_relative_to_env_0: Camera prim path with ``/World/envs/env_0/`` prefix stripped; required by OVRTX. Empty string if the first camera is not under env 0. """ diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py index ff1196f1c437..22f2c50903aa 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py @@ -104,6 +104,9 @@ class OffsetCfg: If True, the latest camera pose is updated in the camera's data which will slow down performance due to the use of :class:`FrameView`. If False, the pose of the camera during initialization is returned. + + Renderers that receive explicit camera poses also use this cached pose for rendering. Set this + to True for cameras attached to moving bodies so their rendered views follow the bodies. """ semantic_filter: str | list[str] = "*:*" diff --git a/source/isaaclab_ov/changelog.d/ovrtx-cloned-camera-bindings.rst b/source/isaaclab_ov/changelog.d/ovrtx-cloned-camera-bindings.rst new file mode 100644 index 000000000000..9ea601461f58 --- /dev/null +++ b/source/isaaclab_ov/changelog.d/ovrtx-cloned-camera-bindings.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Bound OVRTX camera poses and intrinsics to every logical environment when Newton authored only + the source camera in USD, including additional cameras sharing the renderer. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index 439895d06b9c..9cd7aba234d9 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -27,6 +27,7 @@ import sys import weakref from collections.abc import Iterator, Sequence +from dataclasses import replace from pathlib import Path from typing import TYPE_CHECKING, Any, NoReturn, cast @@ -921,6 +922,20 @@ def create_render_data(self, spec: CameraRenderSpec) -> OVRTXCameraRenderData: Performs OVRTX initialization (stage export, USD load, bindings) on first call, matching the interface of Isaac RTX and Newton Warp which need no separate initialize(). """ + # Newton authors only the source camera in USD; OVRTX clones it for every environment. + # Bind later cameras and calibration writes to the same paths as the initial camera. + if ( + len(spec.camera_prim_paths) == 1 + and spec.num_instances > 1 + and spec.camera_path_relative_to_env_0 + and spec.camera_prim_paths[0] == f"/World/envs/env_0/{spec.camera_path_relative_to_env_0}" + ): + spec = replace( + spec, + camera_prim_paths=tuple( + f"/World/envs/env_{i}/{spec.camera_path_relative_to_env_0}" for i in range(spec.num_instances) + ), + ) # Normalize aliases such as "cuda" before comparing cameras sharing this renderer. warp_device = wp.get_device(spec.device) if self._initialized_scene and str(warp_device) != self._device: diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index 3b3c4ac63cea..70a286f156e6 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -249,7 +249,8 @@ def test_ovrtx_render_empty_sequence_does_not_require_initialized_backend(): @pytest.mark.integration @pytest.mark.rendering @pytest.mark.parametrize("use_ovstage", [False, True]) -def test_ovrtx_multiple_cameras_render_independent_views(monkeypatch, use_ovstage, tmp_path): +@pytest.mark.parametrize("source_only", [False, True]) +def test_ovrtx_multiple_cameras_render_independent_views(monkeypatch, use_ovstage, source_only, tmp_path): """Check independent camera batches and save RGB/depth frames under pytest's temporary directory. Use ``--basetemp=/tmp/ovrtx-camera-frames`` to choose where pytest writes the captures. @@ -337,7 +338,7 @@ def check_depth(rd, data, expected, label): cfg=cfg, device="cuda:0", num_instances=2, - camera_prim_paths=tuple(f"/World/envs/env_{i}/cam{index}" for i in range(2)), + camera_prim_paths=tuple(f"/World/envs/env_{i}/cam{index}" for i in range(1 if source_only else 2)), view_count=2, camera_path_relative_to_env_0=f"cam{index}", ) @@ -352,6 +353,13 @@ def check_depth(rd, data, expected, label): ) renderer.set_outputs(rd, data.output) cameras.append((rd, data)) + # Calibration writes must also cover clones absent from the authored USD. + parameters = wp.array( + [[24.0, 24.0], [20.0, 20.0], [20.0, 20.0], [0.0, 0.0], [0.0, 0.0]], + dtype=wp.float32, + device="cuda:0", + ) + renderer.update_camera_intrinsics(rd, wp.zeros(2, dtype=wp.mat33f, device="cuda:0"), parameters) # Register the next camera after rendering has already started. renderer.render([rd]) check_depth(rd, data, 5.0 - index - 0.5, f"initial_cam{index}") @@ -871,8 +879,11 @@ def test_ovrtx_cleanup_without_render_data_keeps_renderer_state(): @pytest.mark.parametrize("use_ovstage", [False, True]) -def test_intrinsic_updates_target_the_given_camera(monkeypatch, use_ovstage): - """Cameras sharing a renderer must bind and update distinct native camera paths.""" +@pytest.mark.parametrize("source_only", [False, True]) +def test_intrinsic_updates_target_the_given_camera(monkeypatch, use_ovstage, source_only): + """Bindings target every logical camera, including clones absent from the source USD.""" + from isaaclab.renderers.camera_render_spec import CameraRenderSpec + renderer = _make_ovrtx_renderer_without_backend() renderer._initialized_scene = True renderer._device = "cpu" @@ -888,15 +899,16 @@ def test_intrinsic_updates_target_the_given_camera(monkeypatch, use_ovstage): renderer.backend.stage.query_from_path_list.side_effect = lambda paths: contextlib.nullcontext(object()) for name in ("add_usd_reference_from_string", "apply_usd_changes", "remove_usd"): monkeypatch.setattr(ovrtx_renderer_module.ovstage.population, name, MagicMock()) - paths = [[f"/World/envs/env_{i}/{name}" for i in range(2)] for name in ("CameraA", "CameraB")] + paths = [[f"/World/envs/env_{i}/{name}" for i in range(2)] for name in ("CameraA", "Robot/wrist/CameraB")] cameras = [ renderer.create_render_data( - types.SimpleNamespace( + CameraRenderSpec( cfg=_make_camera_cfg(["depth"]), device="cpu", num_instances=2, - camera_prim_paths=camera_paths, - camera_path_relative_to_env_0=camera_paths[0].rsplit("/", 1)[1], + camera_prim_paths=tuple(camera_paths[:1] if source_only else camera_paths), + view_count=2, + camera_path_relative_to_env_0=camera_paths[0].removeprefix("/World/envs/env_0/"), ) ) for camera_paths in paths diff --git a/source/isaaclab_tasks/changelog.d/moving-camera-pose-refresh.rst b/source/isaaclab_tasks/changelog.d/moving-camera-pose-refresh.rst new file mode 100644 index 000000000000..489cc6162a5a --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/moving-camera-pose-refresh.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Fixed stale rendered views from Kuka-Allegro wrist cameras and Galbot wrist and head cameras by + enabling camera pose refresh. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/galbot/stack_rmp_rel_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/galbot/stack_rmp_rel_env_cfg.py index 4db125e1482d..1baeef467520 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/galbot/stack_rmp_rel_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/galbot/stack_rmp_rel_env_cfg.py @@ -171,6 +171,7 @@ def __post_init__(self): # Set left and right wrist cameras for VLA policy training self.scene.right_wrist_cam = CameraCfg( prim_path="{ENV_REGEX_NS}/Robot/right_arm_camera_sim_view_frame/right_camera", + update_latest_camera_pose=True, update_period=0.0333, height=256, width=256, @@ -184,6 +185,7 @@ def __post_init__(self): self.scene.left_wrist_cam = CameraCfg( prim_path="{ENV_REGEX_NS}/Robot/left_arm_camera_sim_view_frame/left_camera", + update_latest_camera_pose=True, update_period=0.0333, height=256, width=256, @@ -198,6 +200,7 @@ def __post_init__(self): # Set ego view camera self.scene.ego_cam = CameraCfg( prim_path="{ENV_REGEX_NS}/Robot/head_camera_sim_view_frame/head_camera", + update_latest_camera_pose=True, update_period=0.0333, height=256, width=256, diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/lift/config/kuka_allegro/camera_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/core/lift/config/kuka_allegro/camera_cfg.py index c32cb427ae75..d0832914019f 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/lift/config/kuka_allegro/camera_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/lift/config/kuka_allegro/camera_cfg.py @@ -44,6 +44,7 @@ WRIST_CAMERA_CFG = CameraCfg( prim_path="{ENV_REGEX_NS}/Robot/ee_link/palm_link/Camera", + update_latest_camera_pose=True, offset=CameraCfg.OffsetCfg( pos=(0.038, -0.38, -0.18), rot=(0.641, 0.641, -0.299, 0.299),