From fad3ae06c32dcea03d886d2b638a29409db9f6ce Mon Sep 17 00:00:00 2001 From: Alexander Temirov Date: Tue, 6 Oct 2026 16:40:29 -0700 Subject: [PATCH 01/20] Add an rgb_radiance camera output to all renderers Expose scene-linear RGB before camera exposure and response as a camera output on Isaac RTX, OVRTX, and Newton Warp, without changing CameraCfg.isp_cfg or the renderer-owned PPISP path. Isaac RTX and OVRTX read it from the HdrColor source and neutralize exposure on the camera prim, using the exposure override now shared from isaaclab.renderers. Newton applies no exposure, so it matches rgb_hdr. When both HDR outputs are requested they share one buffer. OVRTX also disables Gaussian skip-tonemapping on HDR-consuming render products so Gaussian pixels reach HdrColor. Signed-off-by: Alexander Temirov --- docs/source/concepts/sensors/camera.rst | 21 ++++- ...atemirov-rgb-radiance-groundwork.added.rst | 5 + .../atemirov-rgb-radiance-groundwork.minor | 0 .../isaaclab/renderers/output_contract.py | 7 ++ .../renderers/rtx_camera_overrides.py | 54 +++++++++++ .../isaaclab/sensors/camera/camera.py | 18 ++-- .../isaaclab/sensors/camera/camera_data.py | 11 +++ .../renderers/test_camera_output_contract.py | 26 ++++-- .../renderers}/test_rtx_camera_overrides.py | 10 +- ...atemirov-rgb-radiance-groundwork.added.rst | 2 + .../atemirov-rgb-radiance-groundwork.minor | 0 .../renderers/newton_warp_renderer.py | 6 +- .../renderers/newton_warp_renderer_cfg.py | 1 + ...atemirov-rgb-radiance-groundwork.added.rst | 2 + ...atemirov-rgb-radiance-groundwork.fixed.rst | 2 + .../atemirov-rgb-radiance-groundwork.minor | 0 .../isaaclab_ov/renderers/ovrtx_renderer.py | 29 +++--- .../renderers/ovrtx_renderer_cfg.py | 1 + .../isaaclab_ov/renderers/ovrtx_usd.py | 9 +- .../test/test_ovrtx_renderer_contract.py | 42 +++++++++ source/isaaclab_ov/test/test_ovrtx_usd.py | 39 ++++++-- ...atemirov-rgb-radiance-groundwork.added.rst | 2 + .../atemirov-rgb-radiance-groundwork.minor | 0 .../renderers/isaac_rtx_renderer.py | 42 +++++---- .../renderers/isaac_rtx_renderer_cfg.py | 1 + .../test_isaac_rtx_renderer_contract.py | 73 +++++++++++++++ ...emirov-rgb-radiance-groundwork.changed.rst | 2 + .../isaaclab_ppisp/rtx_camera_overrides.py | 92 +------------------ 28 files changed, 341 insertions(+), 156 deletions(-) create mode 100644 source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst create mode 100644 source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.minor create mode 100644 source/isaaclab/isaaclab/renderers/rtx_camera_overrides.py rename source/{isaaclab_ppisp/test => isaaclab/test/renderers}/test_rtx_camera_overrides.py (84%) create mode 100644 source/isaaclab_newton/changelog.d/atemirov-rgb-radiance-groundwork.added.rst create mode 100644 source/isaaclab_newton/changelog.d/atemirov-rgb-radiance-groundwork.minor create mode 100644 source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.added.rst create mode 100644 source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst create mode 100644 source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.minor create mode 100644 source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.added.rst create mode 100644 source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.minor create mode 100644 source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst diff --git a/docs/source/concepts/sensors/camera.rst b/docs/source/concepts/sensors/camera.rst index 2b2193b7b850..94fa76c0e8fb 100644 --- a/docs/source/concepts/sensors/camera.rst +++ b/docs/source/concepts/sensors/camera.rst @@ -138,7 +138,10 @@ by its :class:`~isaaclab.renderers.RenderBufferSpec`. - Low-dynamic-range color * - ``rgb_hdr`` - 3, ``float32`` - - Scene-linear high-dynamic-range color + - High-dynamic-range RGB using the active camera settings + * - ``rgb_radiance`` + - 3, ``float32`` + - Scene-linear RGB before exposure and response, in renderer-relative intensity units * - ``albedo`` - 4, ``uint8`` - Material base color @@ -168,6 +171,20 @@ by its :class:`~isaaclab.renderers.RenderBufferSpec`. non-colorized segmentation uses one ``int32`` ID channel. Label and prim-path mappings are stored in ``camera_data.info[output_name]``. +Requesting ``rgb_hdr`` alone preserves the renderer's existing camera settings. ``rgb_radiance`` is +the input for image processing that applies its own exposure and camera response, such as PPISP. +If both are requested, they share one buffer. + +.. note:: + + On Isaac RTX and OVRTX, ``rgb_radiance`` is not a native renderer output. These renderers derive + it from HDR color by authoring neutral exposure on the camera prim, so every output rendered from + that prim, including ``rgb``, ``rgba``, and ``rgb_hdr``, loses the authored exposure. This also + applies to other camera sensors that share the prim. A single render product cannot return both + authored-exposure color and ``rgb_radiance``; use a separate camera prim when both are required. + Newton Warp has no exposure model and is not affected. A native pre-exposure radiance output has + been requested from the RTX team (NVBug 6858736). + .. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/camera-renderer-isaac-rtx.webp :align: center :figwidth: 100% @@ -200,7 +217,7 @@ current support matrix is: - Isaac RTX - OVRTX - Newton Warp - * - ``rgb``, ``rgba``, ``rgb_hdr`` + * - ``rgb``, ``rgba``, ``rgb_hdr``, ``rgb_radiance`` - Yes - Yes - Yes diff --git a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst new file mode 100644 index 000000000000..247ec70a745e --- /dev/null +++ b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst @@ -0,0 +1,5 @@ +* Added the ``rgb_radiance`` camera output: scene-linear RGB before camera exposure and response, + in renderer-relative intensity units. When ``rgb_hdr`` and ``rgb_radiance`` are both requested, + they share one buffer. +* Added :func:`isaaclab.renderers.rtx_camera_overrides.apply_rtx_exposure_overrides`, shared by the + RTX renderers to author neutral camera exposure. diff --git a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.minor b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab/isaaclab/renderers/output_contract.py b/source/isaaclab/isaaclab/renderers/output_contract.py index fdd7d9e9f248..0095243e4090 100644 --- a/source/isaaclab/isaaclab/renderers/output_contract.py +++ b/source/isaaclab/isaaclab/renderers/output_contract.py @@ -26,6 +26,13 @@ class RenderBufferKind(StrEnum): RGB = "rgb" RGBA = "rgba" RGB_HDR = "rgb_hdr" + """HDR RGB using the camera's active rendering settings.""" + RGB_RADIANCE = "rgb_radiance" + """Scene-linear RGB before camera exposure and response, in renderer-relative intensity units. + + Isaac RTX and OVRTX derive it by neutralizing exposure on the camera prim, which also affects the + prim's other color outputs. See the camera documentation. + """ ALBEDO = "albedo" DEPTH = "depth" DISTANCE_TO_IMAGE_PLANE = "distance_to_image_plane" diff --git a/source/isaaclab/isaaclab/renderers/rtx_camera_overrides.py b/source/isaaclab/isaaclab/renderers/rtx_camera_overrides.py new file mode 100644 index 000000000000..8e1c63ee5266 --- /dev/null +++ b/source/isaaclab/isaaclab/renderers/rtx_camera_overrides.py @@ -0,0 +1,54 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared RTX camera exposure overrides for scene-linear sensor inputs. + +RTX has no native pre-exposure radiance output, so ``rgb_radiance`` is derived by authoring neutral +exposure on the camera prim, which affects every output rendered from it. These overrides can be +removed once RTX provides a native radiance output (NVBug 6858736). +""" + +from __future__ import annotations + +from typing import Any + +# Both RTX backends consume these schemas without requiring the schema plugin at authoring time. +_CAMERA_EXPOSURE_SCHEMAS = ("OmniRtxCameraAutoExposureAPI_1", "OmniRtxCameraExposureAPI_1") +_NEUTRAL_CAMERA_EXPOSURE: tuple[tuple[str, str, Any], ...] = ( + ("exposure", "Float", 0.0), + ("exposure:fStop", "Float", 1.0), + ("exposure:iso", "Float", 0.0), + ("exposure:responsivity", "Float", 1.0), + ("exposure:time", "Float", 1.0), + ("omni:rtx:autoExposure:enabled", "Bool", False), +) + + +def apply_rtx_exposure_overrides(stage: Any, prim_paths: list[str]) -> None: + """Disable RTX camera exposure so downstream processors receive scene-linear HDR. + + Existing API schemas are preserved. Schema metadata is authored directly so the RTX schema + plugin does not need to be loaded yet. + + Args: + stage: USD stage containing the camera prims. + prim_paths: Resolved camera prim paths. + """ + from pxr import Sdf + + for prim_path in prim_paths: + prim = stage.GetPrimAtPath(prim_path) + if not prim or not prim.IsValid(): + continue + current = prim.GetMetadata("apiSchemas") or Sdf.TokenListOp() + existing = list(current.prependedItems) + list(current.explicitItems) + list(current.appendedItems) + missing = [schema for schema in _CAMERA_EXPOSURE_SCHEMAS if schema not in existing] + if missing: + prim.SetMetadata("apiSchemas", Sdf.TokenListOp.Create(prependedItems=[*missing, *existing])) + for attr_name, sdf_type_name, value in _NEUTRAL_CAMERA_EXPOSURE: + attr = prim.GetAttribute(attr_name) + if not attr: + attr = prim.CreateAttribute(attr_name, getattr(Sdf.ValueTypeNames, sdf_type_name), custom=False) + attr.Set(value) diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 18280035e2ec..e4a8ce7a0e4c 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -246,9 +246,12 @@ def __init__(self, cfg: CameraCfg): sim_ctx.require_visual_shapes() # An ISP (any ``isp_cfg`` other than ``None``) requires the HDR AOV; - # an explicit ``"rgb_hdr"`` in ``data_types`` also requires the - # HDR-routing flag flipped on the RTX-bearing backends. - require_hdr_output = "rgb_hdr" in self.cfg.data_types or self.cfg.isp_cfg is not None + # an explicit ``"rgb_hdr"`` or ``"rgb_radiance"`` in ``data_types`` also + # requires the HDR-routing flag flipped on Isaac RTX. OVRTX routes HDR + # on each render product instead. + require_hdr_output = ( + not {"rgb_hdr", "rgb_radiance"}.isdisjoint(self.cfg.data_types) or self.cfg.isp_cfg is not None + ) # TODO(follow-up PR): move this flag flip out of Camera. The cleanest path is # an apply_pre_reset_settings() hook on RendererCfg (default no-op) that @@ -264,15 +267,6 @@ def __init__(self, cfg: CameraCfg): settings.set("/physics/fabricUpdateTransformations", True) if require_hdr_output: settings.set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) - elif renderer_type == "ovrtx" and require_hdr_output: - from ...app.settings_manager import get_settings_manager - - get_settings_manager().set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) - # FIXME: settings.set is a no-op for ovrtx - # warning only since it affects only ParticleField3DGaussianSplat scene - logger.warning( - "OVRTX backend with PPISP/HDR requires /rtx/rtpt/gaussian/skipTonemapping/enabled to be false." - ) # UsdGeom Camera prim for the sensor self._sensor_prims: list[UsdGeom.Camera] = [] diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_data.py b/source/isaaclab/isaaclab/sensors/camera/camera_data.py index 78933793a57e..92d6d613c02e 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_data.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_data.py @@ -145,6 +145,9 @@ def allocate( wrapped in a :class:`~isaaclab.utils.warp.ProxyArray`; call ``.torch`` on the result to obtain a zero-copy :class:`torch.Tensor` view. + When both are requested with matching layouts, ``rgb_hdr`` aliases ``rgb_radiance``: + radiance selects the camera's active HDR signal before exposure and response. + Args: data_types: Requested output names (typically :attr:`CameraCfg.data_types`). Every name must be a member of :class:`RenderBufferKind`. @@ -179,6 +182,11 @@ def allocate( allocated = requested.intersection(supported_specs) if rgb_alias: allocated.remove(RenderBufferKind.RGB) + hdr_alias = {RenderBufferKind.RGB_HDR, RenderBufferKind.RGB_RADIANCE} <= allocated and ( + supported_specs[RenderBufferKind.RGB_HDR] == supported_specs[RenderBufferKind.RGB_RADIANCE] + ) + if hdr_alias: + allocated.remove(RenderBufferKind.RGB_HDR) buffers: dict[str, ProxyArray] = {} for name, spec in supported_specs.items(): @@ -187,6 +195,9 @@ def allocate( shape = (num_views, height, width, spec.channels) buffers[str(name)] = ProxyArray(wp.zeros(shape, dtype=spec.dtype, device=device)) + if hdr_alias: + buffers[str(RenderBufferKind.RGB_HDR)] = buffers[str(RenderBufferKind.RGB_RADIANCE)] + if rgb_alias: # Zero-copy strided view into rgba: shape (N, H, W, 3), skipping the alpha channel. # Byte strides for a contiguous (N, H, W, 4) uint8 array are (H*W*4, W*4, 4, 1). diff --git a/source/isaaclab/test/renderers/test_camera_output_contract.py b/source/isaaclab/test/renderers/test_camera_output_contract.py index 4d2e3cca5ce3..09297e703a0a 100644 --- a/source/isaaclab/test/renderers/test_camera_output_contract.py +++ b/source/isaaclab/test/renderers/test_camera_output_contract.py @@ -136,6 +136,7 @@ def test_newton_warp_supported_output_types_key_set(): RenderBufferKind.RGB, RenderBufferKind.RGBA, RenderBufferKind.RGB_HDR, + RenderBufferKind.RGB_RADIANCE, RenderBufferKind.ALBEDO, RenderBufferKind.DEPTH, RenderBufferKind.DISTANCE_TO_CAMERA, @@ -145,6 +146,7 @@ def test_newton_warp_supported_output_types_key_set(): RenderBufferKind.INSTANCE_SEGMENTATION, } assert specs[RenderBufferKind.RGB_HDR] == RenderBufferSpec(3, wp.float32) + assert specs[RenderBufferKind.RGB_RADIANCE] == specs[RenderBufferKind.RGB_HDR] def test_camera_cfg_rejects_outputs_unsupported_by_renderer(): @@ -190,8 +192,9 @@ def test_newton_warp_segmentation_spec_follows_colorize_flags(colorize): assert specs[kind] == expected -def test_newton_warp_wraps_requested_rgb_hdr_output(): - """NewtonWarpRenderer wires requested RGB_HDR proxies to the Newton HDR output slot.""" +@pytest.mark.parametrize("data_type", [RenderBufferKind.RGB_HDR, RenderBufferKind.RGB_RADIANCE]) +def test_newton_warp_wraps_requested_hdr_output(data_type): + """Newton wires both linear color outputs to its HDR output slot.""" pytest.importorskip("isaaclab_newton") pytest.importorskip("newton") wp.init() @@ -205,10 +208,10 @@ def test_newton_warp_wraps_requested_rgb_hdr_output(): render_data = RenderData(fake_sensor, SimpleNamespace(cfg=camera_cfg)) hdr_proxy = ProxyArray(wp.zeros((2, 3, 4, 3), dtype=wp.float32, device="cpu")) - render_data.set_outputs({str(RenderBufferKind.RGB_HDR): hdr_proxy}) + render_data.set_outputs({str(data_type): hdr_proxy}) assert render_data.outputs.hdr_color_image is not None - assert render_data.get_output(RenderBufferKind.RGB_HDR) is render_data.outputs.hdr_color_image + assert render_data.get_output(data_type) is render_data.outputs.hdr_color_image def _make_camera_cfg(data_types: list[str]) -> CameraCfg: @@ -221,12 +224,14 @@ def _make_camera_cfg(data_types: list[str]) -> CameraCfg: ) -def test_camera_data_allocates_supported_subset_and_aliases_rgb(): - """CameraData allocates the intersection of requested + supported and aliases rgb into rgba.""" - cfg = _make_camera_cfg(["rgb", "rgba", "depth", "instance_segmentation"]) +def test_camera_data_allocates_supported_subset_and_aliases_color(): + """CameraData allocates requested + supported outputs, aliasing rgb into rgba and rgb_hdr into rgb_radiance.""" + cfg = _make_camera_cfg(["rgb", "rgba", "rgb_hdr", "rgb_radiance", "depth", "instance_segmentation"]) specs = { RenderBufferKind.RGBA: RenderBufferSpec(4, wp.uint8), RenderBufferKind.RGB: RenderBufferSpec(3, wp.uint8), + RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), + RenderBufferKind.RGB_RADIANCE: RenderBufferSpec(3, wp.float32), RenderBufferKind.DEPTH: RenderBufferSpec(1, wp.float32), RenderBufferKind.NORMALS: RenderBufferSpec(3, wp.float32), } @@ -234,14 +239,17 @@ def test_camera_data_allocates_supported_subset_and_aliases_rgb(): data_types=cfg.data_types, height=8, width=16, num_views=2, device="cpu", supported_specs=specs ) - assert set(data.output.keys()) == {"rgba", "rgb", "depth"} + assert set(data.output.keys()) == {"rgba", "rgb", "rgb_hdr", "rgb_radiance", "depth"} assert data.output["rgba"].shape == (2, 8, 16, 4) assert data.output["rgba"].dtype == wp.uint8 assert data.output["depth"].shape == (2, 8, 16, 1) assert data.output["depth"].dtype == wp.float32 assert data.output["rgb"].warp.ptr == data.output["rgba"].warp.ptr + assert data.output["rgb_hdr"] is data.output["rgb_radiance"] + assert data.output["rgb_radiance"].shape == (2, 8, 16, 3) + assert data.output["rgb_radiance"].dtype == wp.float32 assert data.image_shape == (8, 16) - assert data.info == {"rgba": None, "rgb": None, "depth": None} + assert data.info == dict.fromkeys(data.output) def test_camera_data_allocate_raises_on_unknown_name(): diff --git a/source/isaaclab_ppisp/test/test_rtx_camera_overrides.py b/source/isaaclab/test/renderers/test_rtx_camera_overrides.py similarity index 84% rename from source/isaaclab_ppisp/test/test_rtx_camera_overrides.py rename to source/isaaclab/test/renderers/test_rtx_camera_overrides.py index 77aafda82ef7..8cc5b09e8f49 100644 --- a/source/isaaclab_ppisp/test/test_rtx_camera_overrides.py +++ b/source/isaaclab/test/renderers/test_rtx_camera_overrides.py @@ -3,15 +3,15 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Tests for the RTX-side camera-exposure overrides applied when PPISP is active.""" - -from isaaclab_ppisp import apply_rtx_exposure_overrides +"""Tests for the RTX camera-exposure overrides used for scene-linear color outputs.""" from pxr import Sdf, Usd, UsdGeom +from isaaclab.renderers.rtx_camera_overrides import apply_rtx_exposure_overrides + def test_apply_rtx_exposure_overrides_resets_authored_exposure_values(): - """When PPISP is the ISP authority, the camera prim's RTX-side exposure must be neutral.""" + """Scene-linear outputs require neutral RTX-side exposure on the camera prim.""" stage = Usd.Stage.CreateInMemory() cam = UsdGeom.Camera.Define(stage, "/World/Camera") prim = cam.GetPrim() @@ -31,7 +31,7 @@ def test_apply_rtx_exposure_overrides_resets_authored_exposure_values(): assert prim.GetAttribute("exposure:responsivity").Get() == 1.0 assert prim.GetAttribute("exposure:time").Get() == 1.0 assert prim.GetAttribute("omni:rtx:autoExposure:enabled").Get() is False - # The PPISP API schemas must also be applied — that is what makes Kit's + # The RTX exposure API schemas must also be applied — that is what makes Kit's # USD watcher route the camera prim's ``exposure:*`` and # ``omni:rtx:autoExposure:*`` attributes into the per-camera carb # settings RTX consumes. diff --git a/source/isaaclab_newton/changelog.d/atemirov-rgb-radiance-groundwork.added.rst b/source/isaaclab_newton/changelog.d/atemirov-rgb-radiance-groundwork.added.rst new file mode 100644 index 000000000000..73cdcc547da4 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/atemirov-rgb-radiance-groundwork.added.rst @@ -0,0 +1,2 @@ +* Added ``rgb_radiance`` support to :class:`~isaaclab_newton.renderers.NewtonWarpRenderer`. Newton + applies no camera exposure, so it matches ``rgb_hdr``. diff --git a/source/isaaclab_newton/changelog.d/atemirov-rgb-radiance-groundwork.minor b/source/isaaclab_newton/changelog.d/atemirov-rgb-radiance-groundwork.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py index d39e2d7c2375..5afc87a901f8 100644 --- a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py +++ b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py @@ -89,6 +89,8 @@ class RenderData: _OUTPUT_MAP: dict[str, tuple[str, type]] = { str(RenderBufferKind.RGBA): ("color_image", wp.uint32), str(RenderBufferKind.RGB_HDR): ("hdr_color_image", wp.vec3f), + # Newton applies no camera exposure, so its HDR color is already pre-exposure radiance. + str(RenderBufferKind.RGB_RADIANCE): ("hdr_color_image", wp.vec3f), str(RenderBufferKind.ALBEDO): ("albedo_image", wp.uint32), str(RenderBufferKind.NORMALS): ("normals_image", wp.vec3f), } @@ -272,6 +274,8 @@ def set_outputs(self, output_data: dict[str, ProxyArray]): " Camera.cfg.data_types when isp_cfg is set." ) hdr_proxy = output_data.get(str(RenderBufferKind.RGB_HDR)) + if hdr_proxy is None: + hdr_proxy = output_data.get(str(RenderBufferKind.RGB_RADIANCE)) self._ppisp_hdr_source = hdr_proxy.warp if hdr_proxy is not None else self._hdr_scratch_wp self._ppisp_rgba_dest = output_data[str(RenderBufferKind.RGBA)].warp @@ -282,7 +286,7 @@ def get_output(self, output_name: str) -> wp.array: return self._seg_dests[output_name][0] elif output_name == RenderBufferKind.RGBA: return self.outputs.color_image - elif output_name == RenderBufferKind.RGB_HDR: + elif output_name in (RenderBufferKind.RGB_HDR, RenderBufferKind.RGB_RADIANCE): return self.outputs.hdr_color_image elif output_name == RenderBufferKind.ALBEDO: return self.outputs.albedo_image diff --git a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py index cb828f3cee92..6b1d93e4c1c4 100644 --- a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py +++ b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py @@ -136,6 +136,7 @@ def segmentation_spec(colorize: bool) -> RenderBufferSpec: RenderBufferKind.RGBA: RenderBufferSpec(4, wp.uint8), RenderBufferKind.RGB: RenderBufferSpec(3, wp.uint8), RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), + RenderBufferKind.RGB_RADIANCE: RenderBufferSpec(3, wp.float32), RenderBufferKind.ALBEDO: RenderBufferSpec(4, wp.uint8), RenderBufferKind.DEPTH: RenderBufferSpec(1, wp.float32), RenderBufferKind.DISTANCE_TO_CAMERA: RenderBufferSpec(1, wp.float32), diff --git a/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.added.rst b/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.added.rst new file mode 100644 index 000000000000..7e5255120d85 --- /dev/null +++ b/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.added.rst @@ -0,0 +1,2 @@ +* Added ``rgb_radiance`` support to :class:`~isaaclab_ov.renderers.OVRTXRenderer`. Requesting it + neutralizes exposure on the camera prim, which also affects the prim's other color outputs. diff --git a/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst b/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst new file mode 100644 index 000000000000..99dc1553081d --- /dev/null +++ b/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst @@ -0,0 +1,2 @@ +* Fixed black Gaussian HDR inputs by disabling Gaussian skip-tonemapping on HDR-consuming + OVRTX render products, including HDR inputs requested by camera ISP. diff --git a/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.minor b/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index bf8ee7d6d1f7..ba7abbae427d 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -74,6 +74,7 @@ from isaaclab.cloner import ClonePlan from isaaclab.cloner import path as cloner_path from isaaclab.renderers import BaseRenderer, RenderBufferKind, RenderBufferSpec +from isaaclab.renderers.rtx_camera_overrides import apply_rtx_exposure_overrides from isaaclab.scene_data import SceneDataFormat from isaaclab.sim import SimulationContext from isaaclab.utils.warp import ProxyArray @@ -454,20 +455,20 @@ def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: :mod:`isaaclab` does not need to know about PPISP. Then pins ``exposure:*`` to neutral and applies ``OmniRtxCameraExposureAPI_1`` so the RTX exposure model OVRTX embeds does not compound on top of the - ISP. Without an ISP, the camera prim's authored exposure is left alone. + ISP. Exposure is also neutralized when ``rgb_radiance`` is requested. Otherwise, the + camera prim's authored exposure is left alone. """ - if spec.cfg.isp_cfg is None: - return - try: - from isaaclab_ppisp import apply_rtx_exposure_overrides, resolve_and_normalize - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) + if spec.cfg.isp_cfg is not None: + try: + from isaaclab_ppisp import resolve_and_normalize + except ModuleNotFoundError as exc: + _raise_missing_ppisp_error(exc) - camera_prim_path = spec.camera_prim_paths[0] if spec.camera_prim_paths else None - spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) - if spec.cfg.isp_cfg is None or not spec.camera_prim_paths: - return - apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) + camera_prim_path = spec.camera_prim_paths[0] if spec.camera_prim_paths else None + spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) + needs_neutral_exposure = spec.cfg.isp_cfg is not None or "rgb_radiance" in spec.cfg.data_types + if needs_neutral_exposure and spec.camera_prim_paths: + apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) def prepare_stage(self, stage: Any, num_envs: int) -> None: """Prepare the USD stage for OVRTX before :meth:`create_render_data`. @@ -927,6 +928,10 @@ def set_outputs(self, render_data: OVRTXCameraRenderData, output_data: dict[str, render_data.warp_buffers = { name: proxy.warp for name, proxy in output_data.items() if name != str(RenderBufferKind.RGB) } + # Radiance reads the same HdrColor var; exposure is neutralized in :meth:`prepare_cameras`. + radiance = render_data.warp_buffers.get(str(RenderBufferKind.RGB_RADIANCE)) + if radiance is not None: + render_data.warp_buffers.setdefault(str(RenderBufferKind.RGB_HDR), radiance) # When PPISP is composed but the user did not request the raw HDR AOV, # allocate an internal HDR scratch buffer under "rgb_hdr" so both the # HdrColor extractor and PPISP dispatch can use the same buffer map. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py index 6ad6dbeb59a2..8c8eedc86c7e 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py @@ -131,6 +131,7 @@ def segmentation_spec(colorize: bool) -> RenderBufferSpec: RenderBufferKind.RGBA: RenderBufferSpec(4, wp.uint8), RenderBufferKind.RGB: RenderBufferSpec(3, wp.uint8), RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), + RenderBufferKind.RGB_RADIANCE: RenderBufferSpec(3, wp.float32), RenderBufferKind.ALBEDO: RenderBufferSpec(4, wp.uint8), RenderBufferKind.SIMPLE_SHADING_CONSTANT_DIFFUSE: RenderBufferSpec(3, wp.uint8), RenderBufferKind.SIMPLE_SHADING_DIFFUSE_MDL: RenderBufferSpec(3, wp.uint8), diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py index c5261f638ae7..8b258bb64b12 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py @@ -33,6 +33,7 @@ "simple_shading_diffuse_mdl": ("LdrColor", "LdrColor"), "simple_shading_full_mdl": ("LdrColor", "LdrColor"), "rgb_hdr": ("HdrColor", "HdrColor"), + "rgb_radiance": ("HdrColor", "HdrColor"), "albedo": ("albedo", "DiffuseAlbedoSD"), "depth": ("depth", "DistanceToImagePlaneSD"), "distance_to_image_plane": ("depth", "DistanceToImagePlaneSD"), @@ -241,6 +242,12 @@ def build_render_scope_usd( f"bool omni:rtx:minimal:castShadows = {'true' if enable_shadows else 'false'}", ] + api_schemas = ["OmniRtxSettingsCommonAdvancedAPI_1"] + if not {"rgb_hdr", "rgb_radiance"}.isdisjoint(data_types): + # OVRTX 0.5 reads this per product. Its default bypasses HdrColor for Gaussian pixels. + api_schemas.append("OmniRtxSettingsParticleFieldAPI_1") + render_mode_lines.append("bool omni:rtx:rtpt:gaussian:skipTonemapping:enabled = false") + api_schemas_block = ", ".join(f'"{schema}"' for schema in api_schemas) render_mode_block = "\n ".join(render_mode_lines) ordered_vars = ", ".join(f"<{path}>" for path, _, _ in render_var_configs) render_var_defs = "\n".join( @@ -255,7 +262,7 @@ def build_render_scope_usd( def Scope "{render_data.render_scope_name}" {{ def RenderProduct "{render_data.render_product_name}" ( - prepend apiSchemas = ["OmniRtxSettingsCommonAdvancedAPI_1"] + prepend apiSchemas = [{api_schemas_block}] ) {{ rel camera = [<{camera_path}>]{device_ids_line} {bg_type_line} diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index 7430cf04f169..00afab30e006 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -531,6 +531,48 @@ def capture(value, indices): assert renderer._camera_render_data == [] +@pytest.mark.parametrize("data_types", [["rgb_hdr"], ["rgb_radiance"], ["rgb_hdr", "rgb_radiance"]]) +def test_ovrtx_hdr_color_fills_requested_hdr_outputs(monkeypatch, data_types): + """Radiance and HDR read one HdrColor var into the camera's own buffer.""" + renderer = _make_ovrtx_renderer_without_backend() + renderer._device = "cpu" + cfg = _make_camera_cfg(data_types) + data = CameraData.allocate( + data_types=cfg.data_types, + height=8, + width=16, + num_views=2, + device="cpu", + supported_specs=renderer.supported_output_types(), + ) + render_data = _make_ovrtx_camera_render_data() + renderer.set_outputs(render_data, data.output) + + monkeypatch.setattr(renderer, "_map_render_var_to_dlpack", lambda source: contextlib.nullcontext(source)) + source = wp.full((8, 32, 4), 2.0, dtype=wp.float32, device="cpu") + frame = types.SimpleNamespace(render_vars={render_data.render_var_keys["HdrColor"]: source}) + renderer._process_render_frame(render_data, frame, render_data.warp_buffers) + + for name in data_types: + np.testing.assert_array_equal(data.output[name].warp.numpy(), 2.0) + + +@pytest.mark.parametrize("data_types", [["rgb_hdr"], ["rgb_radiance"]]) +def test_ovrtx_prepare_cameras_neutralizes_exposure_for_radiance(data_types): + """Radiance neutralizes camera exposure; standalone HDR keeps the authored camera settings.""" + from pxr import Sdf, Usd, UsdGeom + + stage = Usd.Stage.CreateInMemory() + camera = UsdGeom.Camera.Define(stage, "/World/Camera").GetPrim() + camera.CreateAttribute("exposure:iso", Sdf.ValueTypeNames.Float).Set(100.0) + spec = types.SimpleNamespace( + cfg=types.SimpleNamespace(data_types=data_types, isp_cfg=None), camera_prim_paths=("/World/Camera",) + ) + _make_ovrtx_renderer_without_backend().prepare_cameras(stage, spec) + + assert camera.GetAttribute("exposure:iso").Get() == (0.0 if "rgb_radiance" in data_types else 100.0) + + def test_ovrtx_set_outputs_wraps_caller_torch_zero_copy(): """OVRTXRenderer.set_outputs publishes warp views over the caller's warp storage.""" renderer = _make_ovrtx_renderer_without_backend() diff --git a/source/isaaclab_ov/test/test_ovrtx_usd.py b/source/isaaclab_ov/test/test_ovrtx_usd.py index 5c4c46232f66..0f890945be75 100644 --- a/source/isaaclab_ov/test/test_ovrtx_usd.py +++ b/source/isaaclab_ov/test/test_ovrtx_usd.py @@ -126,6 +126,14 @@ def test_render_product_solid_background_color(camera_spec, render_data): assert 'token omni:rtx:background:source:type = "domeLight"' not in render_scope +@pytest.mark.parametrize("data_types", [["rgb_hdr"], ["rgb_radiance"], ["rgb_hdr", "rgb_radiance"]]) +def test_ovrtx_hdr_outputs_use_one_hdr_color_render_var(data_types): + """HDR and radiance share one HdrColor native source.""" + assert get_render_var_configs(data_types, render_scope_name="RenderCamera_0") == [ + ("/RenderCamera_0/Vars/HdrColor", "HdrColor", "HdrColor"), + ] + + def test_render_var_prim_names_are_read_only(): with pytest.raises(TypeError): render_var_prim_names_by_source()["LdrColor"] = "mutated" # type: ignore[index] @@ -312,20 +320,33 @@ def test_render_product_omits_device_ids_when_no_device_is_given(camera_spec, re assert "deviceIds" not in render_product -@pytest.mark.parametrize("data_types", [["rgb"], ["rgb", "rgb_hdr"], []]) -def test_render_product_isp_requests_hdr_without_mutating_camera_outputs(camera_spec, render_data, data_types): - """ISP receives one HDR source while the camera's requested outputs remain unchanged.""" +@pytest.mark.parametrize( + ("data_types", "use_isp"), + [(["rgb"], True), (["rgb", "rgb_hdr"], False), (["rgb", "rgb_radiance"], False), ([], True), (["rgb"], False)], +) +def test_render_product_isp_requests_hdr_without_mutating_camera_outputs(camera_spec, render_data, data_types, use_isp): + """Route Gaussian HDR for explicit or ISP inputs without changing public outputs.""" camera_spec.cfg.data_types = data_types.copy() - camera_spec.cfg.isp_cfg = object() + camera_spec.cfg.isp_cfg = object() if use_isp else None render_product = build_render_product_as_string(camera_spec, render_data) layer = Sdf.Layer.CreateAnonymous(".usda") assert layer.ImportFromString(render_product) - ordered_vars = layer.GetRelationshipAtPath("/RenderCamera_0/RenderProduct.orderedVars") - assert list(ordered_vars.targetPathList.explicitItems) == [ - Sdf.Path("/RenderCamera_0/Vars/LdrColor"), - Sdf.Path("/RenderCamera_0/Vars/HdrColor"), - ] + product_path = "/RenderCamera_0/RenderProduct" + ordered_vars = layer.GetRelationshipAtPath(product_path + ".orderedVars") + expected_vars = [Sdf.Path("/RenderCamera_0/Vars/LdrColor")] + needs_hdr = use_isp or not {"rgb_hdr", "rgb_radiance"}.isdisjoint(data_types) + if needs_hdr: + expected_vars.append(Sdf.Path("/RenderCamera_0/Vars/HdrColor")) + assert list(ordered_vars.targetPathList.explicitItems) == expected_vars assert camera_spec.cfg.data_types == data_types + schemas = layer.GetPrimAtPath(product_path).GetInfo("apiSchemas").prependedItems + gaussian_setting = layer.GetAttributeAtPath(product_path + ".omni:rtx:rtpt:gaussian:skipTonemapping:enabled") + if needs_hdr: + assert "OmniRtxSettingsParticleFieldAPI_1" in schemas + assert gaussian_setting.default is False + else: + assert "OmniRtxSettingsParticleFieldAPI_1" not in schemas + assert gaussian_setting is None def test_ovrtx_rgb_and_rgb_hdr_author_both_render_vars(camera_spec, render_data): diff --git a/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.added.rst b/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.added.rst new file mode 100644 index 000000000000..79a4627d1afb --- /dev/null +++ b/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.added.rst @@ -0,0 +1,2 @@ +* Added ``rgb_radiance`` support to :class:`~isaaclab_physx.renderers.IsaacRtxRenderer`. Requesting + it neutralizes exposure on the camera prim, which also affects the prim's other color outputs. diff --git a/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.minor b/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index d5636fe5e676..97e0f4015321 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -24,6 +24,7 @@ from isaaclab.app.settings_manager import get_settings_manager from isaaclab.renderers import BaseRenderer, RenderBufferKind, RenderBufferSpec from isaaclab.renderers.camera_render_spec import CameraRenderSpec +from isaaclab.renderers.rtx_camera_overrides import apply_rtx_exposure_overrides from isaaclab.sim import SimulationContext from isaaclab.sim.utils import enable_extension from isaaclab.utils.version import get_isaac_sim_version @@ -66,6 +67,9 @@ def _raise_missing_ppisp_error(exc: ModuleNotFoundError) -> NoReturn: raise ModuleNotFoundError(_PPISP_IMPORT_ERROR_MESSAGE, name="isaaclab_ppisp") from exc +# Outputs read from the HdrColor AOV. Radiance differs only by the neutral camera exposure. +_HDR_DATA_TYPES = frozenset({str(RenderBufferKind.RGB_HDR), str(RenderBufferKind.RGB_RADIANCE)}) + # RTX simple-shading constants. # # Simple shading requires Kit's RTX "Minimal" render mode. Its shading level is @@ -217,23 +221,23 @@ def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: :mod:`isaaclab` does not need to know about PPISP. Then pins ``exposure:*`` to neutral and applies ``OmniRtxCameraExposureAPI_1`` so RTX's physical-camera exposure model does not compound on top of the - ISP. Without an ISP, the camera prim's authored exposure is left alone. + ISP. Exposure is also neutralized when ``rgb_radiance`` is requested. Otherwise, the + camera prim's authored exposure is left alone. :attr:`~isaaclab.sensors.camera.CameraCfg.background_color` is applied per-render-product in :meth:`create_render_data` via USD attributes. """ - if spec.cfg.isp_cfg is None: - return - try: - from isaaclab_ppisp import apply_rtx_exposure_overrides, resolve_and_normalize - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - camera_prim_path = spec.camera_prim_paths[0] if spec.camera_prim_paths else None - spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) - if spec.cfg.isp_cfg is None or not spec.camera_prim_paths: - return - apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) + if spec.cfg.isp_cfg is not None: + try: + from isaaclab_ppisp import resolve_and_normalize + except ModuleNotFoundError as exc: + _raise_missing_ppisp_error(exc) + + camera_prim_path = spec.camera_prim_paths[0] if spec.camera_prim_paths else None + spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) + needs_neutral_exposure = spec.cfg.isp_cfg is not None or "rgb_radiance" in spec.cfg.data_types + if needs_neutral_exposure and spec.camera_prim_paths: + apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: """Publish the per-output Replicator layout this RTX backend writes. @@ -319,7 +323,8 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: if isaac_sim_version.major >= 6: simple_shading_mode = self._resolve_simple_shading_mode(spec) needs_color_render = any( - data_type in spec.cfg.data_types for data_type in ("rgb", "rgba", str(RenderBufferKind.RGB_HDR)) + data_type in spec.cfg.data_types + for data_type in ("rgb", "rgba", str(RenderBufferKind.RGB_HDR), str(RenderBufferKind.RGB_RADIANCE)) ) has_gui = settings.get("/isaaclab/has_gui") if simple_shading_mode is None and (not needs_color_render or has_gui): @@ -399,7 +404,7 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: aov=SIMPLE_SHADING_AOV, output_data_type=np.uint8, output_channels=4 ) - needs_hdr_color = str(RenderBufferKind.RGB_HDR) in spec.cfg.data_types or ( + needs_hdr_color = not _HDR_DATA_TYPES.isdisjoint(spec.cfg.data_types) or ( spec.cfg.isp_cfg is not None and any(data_type in ("rgb", "rgba") for data_type in spec.cfg.data_types) ) if needs_hdr_color: @@ -420,7 +425,9 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: else: annotator = rep.AnnotatorRegistry.get_annotator("rgb", device=spec.device, do_array_copy=False) annotators["rgba"] = annotator - elif annotator_type == str(RenderBufferKind.RGB_HDR): + elif annotator_type in _HDR_DATA_TYPES: + # Radiance reads the same HdrColor AOV through the ``rgb_hdr`` alias set in + # :meth:`set_outputs`; exposure is neutralized in :meth:`prepare_cameras`. if str(RenderBufferKind.RGB_HDR) not in annotators: annotator = rep.AnnotatorRegistry.get_annotator("HdrColor", device=spec.device, do_array_copy=False) annotators[str(RenderBufferKind.RGB_HDR)] = annotator @@ -563,6 +570,9 @@ def set_outputs(self, render_data: IsaacRtxRenderData, output_data: dict[str, Pr " LDR output destination, but neither was provided. Add 'rgb' or 'rgba' to" " Camera.cfg.data_types when isp_cfg is set." ) + radiance = output_data.get(str(RenderBufferKind.RGB_RADIANCE)) + if radiance is not None and str(RenderBufferKind.RGB_HDR) not in output_data: + output_data = {**output_data, str(RenderBufferKind.RGB_HDR): radiance} render_data.output_data = output_data # Allocate an internal HDR scratch buffer when PPISP is composed but # the user did not request the raw HDR AOV in ``data_types`` — the diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py index b3efb5f8dff0..644f677e79d3 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py @@ -221,6 +221,7 @@ def segmentation_spec(colorize: bool) -> RenderBufferSpec: RenderBufferKind.RGBA: RenderBufferSpec(4, wp.uint8), RenderBufferKind.RGB: RenderBufferSpec(3, wp.uint8), RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), + RenderBufferKind.RGB_RADIANCE: RenderBufferSpec(3, wp.float32), RenderBufferKind.ALBEDO: RenderBufferSpec(4, wp.uint8), RenderBufferKind.DEPTH: RenderBufferSpec(1, wp.float32), RenderBufferKind.DISTANCE_TO_IMAGE_PLANE: RenderBufferSpec(1, wp.float32), diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py index 0a75262bc7f3..1eac6b1d4b6c 100644 --- a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py @@ -64,6 +64,7 @@ def test_isaac_rtx_supported_output_types_include_rgb_hdr(monkeypatch, isaac_sim specs = renderer.supported_output_types() assert specs[RenderBufferKind.RGB_HDR] == RenderBufferSpec(3, wp.float32) + assert specs[RenderBufferKind.RGB_RADIANCE] == specs[RenderBufferKind.RGB_HDR] requires_6_0 = [ RenderBufferKind.ALBEDO, RenderBufferKind.SIMPLE_SHADING_CONSTANT_DIFFUSE, @@ -86,6 +87,77 @@ def test_native_fabric_geometry_needs_no_separate_publication(monkeypatch): provider.get_geometry_points.assert_not_called() +@pytest.mark.parametrize("data_types", [["rgb_hdr"], ["rgb_radiance"]]) +def test_prepare_cameras_neutralizes_exposure_for_radiance(monkeypatch, data_types): + """Radiance neutralizes camera exposure; standalone HDR keeps the authored camera settings.""" + _install_omni_stubs(monkeypatch) + from isaaclab_physx.renderers.isaac_rtx_renderer import IsaacRtxRenderer + + from pxr import Sdf, Usd, UsdGeom + + stage = Usd.Stage.CreateInMemory() + camera = UsdGeom.Camera.Define(stage, "/World/Camera").GetPrim() + camera.CreateAttribute("exposure:iso", Sdf.ValueTypeNames.Float).Set(100.0) + spec = SimpleNamespace( + cfg=SimpleNamespace(data_types=data_types, isp_cfg=None), camera_prim_paths=("/World/Camera",) + ) + IsaacRtxRenderer.__new__(IsaacRtxRenderer).prepare_cameras(stage, spec) + + assert camera.GetAttribute("exposure:iso").Get() == (0.0 if "rgb_radiance" in data_types else 100.0) + + +@pytest.mark.parametrize("data_types", [["rgb_radiance"], ["rgb_radiance", "rgb_hdr"]]) +def test_radiance_reads_the_hdr_annotator_into_its_output(monkeypatch, data_types): + """Radiance and HDR attach one HdrColor annotator and extract it into the camera's own buffers.""" + replicator, syntheticdata = _install_omni_stubs(monkeypatch) + monkeypatch.setattr(syntheticdata, "SyntheticData", MagicMock(), raising=False) + + import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + from pxr import UsdGeom + + import isaaclab.sim.utils.stage as stage_utils + from isaaclab.sensors.camera.camera_data import CameraData + + settings = MagicMock() + settings.get.return_value = False + stage = MagicMock() + stage.SelectPrims.return_value.GetCount.return_value = 1 + stage.GetPrimAtPath.return_value.IsA.side_effect = lambda typ: typ is UsdGeom.Camera + product = SimpleNamespace(path="/Render/Test") + replicator.create = SimpleNamespace(render_product_tiled=MagicMock(return_value=product)) + annotator = MagicMock() + registry = MagicMock() + registry.get_annotator.return_value = annotator + replicator.AnnotatorRegistry = registry + spec = SimpleNamespace( + camera_prim_paths=("/World/Camera",), + device="cpu", + view_count=1, + cfg=SimpleNamespace(data_types=data_types, width=3, height=2, isp_cfg=None), + ) + renderer = rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg() + with ( + patch.object(rtx_renderer, "get_settings_manager", return_value=settings), + patch.object(rtx_renderer, "get_isaac_sim_version", return_value=version.parse("6.0")), + patch.object(stage_utils, "get_current_stage", return_value=stage), + ): + render_data = renderer.create_render_data(spec) + registry.get_annotator.assert_called_once_with("HdrColor", device="cpu", do_array_copy=False) + annotator.attach.assert_called_once_with([product.path]) + + camera_data = CameraData.allocate(data_types, 2, 3, 1, "cpu", supported_specs=renderer.cfg.supported_output_types()) + renderer.set_outputs(render_data, camera_data.output) + annotator.get_data.return_value = np.full((2, 3, 4), 2.0, dtype=np.float32) + renderer._read_annotator_output(render_data) + + assert set(camera_data.output) == set(data_types) + for name in data_types: + np.testing.assert_array_equal(camera_data.output[name].warp.numpy(), 2.0) + + def test_create_render_data_uses_unique_sdf_safe_render_product_name(monkeypatch): """Each tiled render product gets a fresh ``rp_`` name. @@ -184,6 +256,7 @@ def test_create_render_data_uses_unique_sdf_safe_render_product_name(monkeypatch pytest.param(["rgb", "simple_shading_full_mdl"], 3, False, id="rgb_keeps_path_tracing"), pytest.param(["rgba", "simple_shading_full_mdl"], 3, False, id="rgba_keeps_path_tracing"), pytest.param(["rgb_hdr", "simple_shading_full_mdl"], 3, False, id="rgb_hdr_keeps_path_tracing"), + pytest.param(["rgb_radiance", "simple_shading_full_mdl"], 3, False, id="radiance_keeps_path_tracing"), ], ) def test_simple_shading_configures_its_render_product( diff --git a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst new file mode 100644 index 000000000000..4320eaf8144d --- /dev/null +++ b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst @@ -0,0 +1,2 @@ +* Changed :func:`~isaaclab_ppisp.apply_rtx_exposure_overrides` to re-export + :func:`isaaclab.renderers.rtx_camera_overrides.apply_rtx_exposure_overrides`. diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py b/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py index 5ef5c6bfebf1..ea7ecf5c016b 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py @@ -3,94 +3,8 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""RTX-specific per-camera-prim USD overrides authored when PPISP is active. +"""RTX camera exposure overrides, shared with :mod:`isaaclab.renderers.rtx_camera_overrides`.""" -Kept in a Kit-free module (no ``omni.*`` imports) so it can be unit-tested -without booting Isaac Sim. Both Kit-driven Isaac RTX and Kit-less OVRTX embed -the same RTX core and author the same exposure schema, so the helper lives -here in :mod:`isaaclab_ppisp` and is called from both backends' -``prepare_cameras`` hook. -""" +from isaaclab.renderers.rtx_camera_overrides import apply_rtx_exposure_overrides -from __future__ import annotations - -from typing import Any - -# Neutral RTX-side values authored on the camera prim when PPISP is the ISP -# authority. RTX otherwise compounds its own physical-camera exposure model -# on top of PPISP, leaving the HDR AOV under- or over-exposed. PPISP supplies -# its own ``responsivity`` / ``exposureOffset``; pinning these stage-side -# values (``iso=0`` plus ``autoExposure:enabled=False``) keeps RTX's exposure -# stage a no-op. -# -# Each entry is ``(attribute_name, sdf_type_name, value)`` where -# ``sdf_type_name`` is the suffix of ``Sdf.ValueTypeNames``; values are -# coerced inside :func:`apply_rtx_exposure_overrides`. -_NEUTRAL_CAMERA_EXPOSURE: tuple[tuple[str, str, Any], ...] = ( - ("exposure", "Float", 0.0), - ("exposure:fStop", "Float", 1.0), - ("exposure:iso", "Float", 0.0), - ("exposure:responsivity", "Float", 1.0), - ("exposure:time", "Float", 1.0), - ("omni:rtx:autoExposure:enabled", "Bool", False), -) - -# API schemas applied (prepended) on each camera prim so Kit's USD watcher -# routes the camera's ``exposure:*`` and ``omni:rtx:autoExposure:*`` -# attributes into the per-camera carb settings RTX consumes. -_PPISP_CAMERA_API_SCHEMAS: tuple[str, ...] = ( - "OmniRtxCameraAutoExposureAPI_1", - "OmniRtxCameraExposureAPI_1", -) - - -def apply_rtx_exposure_overrides(stage: Any, prim_paths: list[str]) -> None: - """Pin RTX-side exposure to neutral and apply the API schemas on each camera prim. - - On every prim in ``prim_paths``: - - 1. Author the neutral values from :data:`_NEUTRAL_CAMERA_EXPOSURE` — - ``exposure:*`` knobs disabling RTX's physical-camera exposure stage - plus ``omni:rtx:autoExposure:enabled=False``. - 2. Apply the :data:`_PPISP_CAMERA_API_SCHEMAS` API schemas via - ``SetMetadata("apiSchemas", ...)``. In Kit, ``omni.kit``'s USD - watcher pumps the authored attributes into the per-camera carb - settings RTX consumes. ``Sdf.TokenListOp`` metadata is used rather - than ``Usd.Prim.ApplyAPI`` so this does not require the USD - ``PlugRegistry`` to have discovered the RTX-settings schema plugin - yet. - - Existing API schemas on the prim are preserved — the RTX schemas are - prepended, not replaced. - - Args: - stage: USD stage containing the camera prim(s). - prim_paths: Resolved camera prim paths (typically one per env). - """ - from pxr import Sdf - - sdf_value_types = { - "Bool": Sdf.ValueTypeNames.Bool, - "Float": Sdf.ValueTypeNames.Float, - } - - def coerce(value: Any, sdf_type_name: str) -> Any: - if sdf_type_name == "Float": - return float(value) - return value - - for prim_path in prim_paths: - prim = stage.GetPrimAtPath(prim_path) - if not prim or not prim.IsValid(): - continue - current = prim.GetMetadata("apiSchemas") or Sdf.TokenListOp() - existing = list(current.prependedItems) + list(current.explicitItems) + list(current.appendedItems) - missing = [s for s in _PPISP_CAMERA_API_SCHEMAS if s not in existing] - if missing: - merged = Sdf.TokenListOp.Create(prependedItems=[*missing, *existing]) - prim.SetMetadata("apiSchemas", merged) - for attr_name, sdf_type_name, value in _NEUTRAL_CAMERA_EXPOSURE: - attr = prim.GetAttribute(attr_name) - if not attr: - attr = prim.CreateAttribute(attr_name, sdf_value_types[sdf_type_name], custom=False) - attr.Set(coerce(value, sdf_type_name)) +__all__ = ["apply_rtx_exposure_overrides"] From af528a522ad88c727fc47de8e2a25f3822f345ef Mon Sep 17 00:00:00 2001 From: Alexander Temirov Date: Tue, 6 Oct 2026 17:10:36 -0700 Subject: [PATCH 02/20] Remove the PPISP exposure-override re-export Renderers import apply_rtx_exposure_overrides from isaaclab.renderers.rtx_camera_overrides, so isaaclab_ppisp no longer re-exports it. Signed-off-by: Alexander Temirov --- .../atemirov-rgb-radiance-groundwork.changed.rst | 2 -- .../changelog.d/atemirov-rgb-radiance-groundwork.major | 0 .../atemirov-rgb-radiance-groundwork.removed.rst | 2 ++ source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi | 2 -- .../isaaclab_ppisp/rtx_camera_overrides.py | 10 ---------- 5 files changed, 2 insertions(+), 14 deletions(-) delete mode 100644 source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst create mode 100644 source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.major create mode 100644 source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst delete mode 100644 source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py diff --git a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst deleted file mode 100644 index 4320eaf8144d..000000000000 --- a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst +++ /dev/null @@ -1,2 +0,0 @@ -* Changed :func:`~isaaclab_ppisp.apply_rtx_exposure_overrides` to re-export - :func:`isaaclab.renderers.rtx_camera_overrides.apply_rtx_exposure_overrides`. diff --git a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.major b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst new file mode 100644 index 000000000000..69a0756ffc2c --- /dev/null +++ b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst @@ -0,0 +1,2 @@ +* Removed ``isaaclab_ppisp.apply_rtx_exposure_overrides``. Import + :func:`isaaclab.renderers.rtx_camera_overrides.apply_rtx_exposure_overrides` instead. diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi index 2d69e60325e9..476c537b9420 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi +++ b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi @@ -10,7 +10,6 @@ __all__ = [ "PpispPipeline", "apply_ppisp_to_rgba", "apply_ppisp_to_rgba_with_controller_params", - "apply_rtx_exposure_overrides", "auto_any_ppisp_cfg", "auto_camera_ppisp_cfg", "compute_ppisp_controller_params", @@ -37,4 +36,3 @@ from .cfg import ( ) from .kernels import apply_ppisp_to_rgba, apply_ppisp_to_rgba_with_controller_params, compute_ppisp_controller_params from .pipeline import PpispPipeline -from .rtx_camera_overrides import apply_rtx_exposure_overrides diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py b/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py deleted file mode 100644 index ea7ecf5c016b..000000000000 --- a/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""RTX camera exposure overrides, shared with :mod:`isaaclab.renderers.rtx_camera_overrides`.""" - -from isaaclab.renderers.rtx_camera_overrides import apply_rtx_exposure_overrides - -__all__ = ["apply_rtx_exposure_overrides"] From b470d7bb0aba418589a05bf5a0ad44b9816a5ea1 Mon Sep 17 00:00:00 2001 From: Alexander Temirov Date: Tue, 6 Oct 2026 17:10:37 -0700 Subject: [PATCH 03/20] Move camera renderer settings into the renderer config Camera no longer branches on the renderer type. IsaacRtxRendererCfg sets the RTX sensor and Gaussian HDR routing settings in a new RendererCfg.apply_pre_reset_settings hook, which cameras call before sim.reset(). Signed-off-by: Alexander Temirov --- ...atemirov-rgb-radiance-groundwork.added.rst | 2 ++ ...emirov-rgb-radiance-groundwork.changed.rst | 2 ++ .../isaaclab/renderers/renderer_cfg.py | 12 ++++++++++ .../isaaclab/sensors/camera/camera.py | 24 ++----------------- ...emirov-rgb-radiance-groundwork.changed.rst | 3 +++ .../renderers/isaac_rtx_renderer_cfg.py | 17 +++++++++++++ .../test_isaac_rtx_renderer_contract.py | 18 ++++++++++++++ 7 files changed, 56 insertions(+), 22 deletions(-) create mode 100644 source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst create mode 100644 source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst diff --git a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst index 247ec70a745e..4d200a0c9e80 100644 --- a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst +++ b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst @@ -3,3 +3,5 @@ they share one buffer. * Added :func:`isaaclab.renderers.rtx_camera_overrides.apply_rtx_exposure_overrides`, shared by the RTX renderers to author neutral camera exposure. +* Added :meth:`~isaaclab.renderers.RendererCfg.apply_pre_reset_settings` so renderer configurations + apply the process-wide settings their cameras need before ``sim.reset()``. diff --git a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst new file mode 100644 index 000000000000..53ba96788094 --- /dev/null +++ b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst @@ -0,0 +1,2 @@ +* Changed :class:`~isaaclab.sensors.camera.Camera` to apply renderer-specific settings through + :meth:`~isaaclab.renderers.RendererCfg.apply_pre_reset_settings` instead of checking the renderer type. diff --git a/source/isaaclab/isaaclab/renderers/renderer_cfg.py b/source/isaaclab/isaaclab/renderers/renderer_cfg.py index 8ed1d9f73204..aabcefd0da4f 100644 --- a/source/isaaclab/isaaclab/renderers/renderer_cfg.py +++ b/source/isaaclab/isaaclab/renderers/renderer_cfg.py @@ -14,6 +14,7 @@ from .output_contract import RenderBufferKind, RenderBufferSpec if TYPE_CHECKING: + from ..sensors.camera.camera_cfg import CameraCfg from .base_renderer import BaseRenderer @@ -44,3 +45,14 @@ def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec] | N renderer is selected dynamically. """ return None + + def apply_pre_reset_settings(self, camera_cfg: CameraCfg) -> None: + """Apply process-wide settings that a camera rendered by this renderer needs. + + Cameras call this when constructed, before ``sim.reset()``, because environments read some + of these settings before the renderer is created. The default applies no settings. + + Args: + camera_cfg: Configuration of the camera being constructed. + """ + pass diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index e4a8ce7a0e4c..3f3ab1c85c13 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -245,28 +245,8 @@ def __init__(self, cfg: CameraCfg): if sim_ctx is not None: sim_ctx.require_visual_shapes() - # An ISP (any ``isp_cfg`` other than ``None``) requires the HDR AOV; - # an explicit ``"rgb_hdr"`` or ``"rgb_radiance"`` in ``data_types`` also - # requires the HDR-routing flag flipped on Isaac RTX. OVRTX routes HDR - # on each render product instead. - require_hdr_output = ( - not {"rgb_hdr", "rgb_radiance"}.isdisjoint(self.cfg.data_types) or self.cfg.isp_cfg is not None - ) - - # TODO(follow-up PR): move this flag flip out of Camera. The cleanest path is - # an apply_pre_reset_settings() hook on RendererCfg (default no-op) that - # IsaacRtxRendererCfg overrides to flip /isaaclab/render/rtx_sensors. The - # flag must be set pre-sim.reset() because SimulationContext.is_rendering - # and several env classes read it before the renderer's __init__ runs. - renderer_type = getattr(self.cfg.renderer_cfg, "renderer_type", None) - if renderer_type == "isaac_rtx": - from ...app.settings_manager import get_settings_manager - - settings = get_settings_manager() - settings.set("/isaaclab/render/rtx_sensors", True) - settings.set("/physics/fabricUpdateTransformations", True) - if require_hdr_output: - settings.set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) + # Simulation setup and environments read some renderer settings before the renderer exists. + self.cfg.renderer_cfg.apply_pre_reset_settings(self.cfg) # UsdGeom Camera prim for the sensor self._sensor_prims: list[UsdGeom.Camera] = [] diff --git a/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst b/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst new file mode 100644 index 000000000000..786857a47825 --- /dev/null +++ b/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst @@ -0,0 +1,3 @@ +* Changed :class:`~isaaclab_physx.renderers.IsaacRtxRendererCfg` to set the RTX sensor and HDR + routing settings for its cameras in + :meth:`~isaaclab_physx.renderers.IsaacRtxRendererCfg.apply_pre_reset_settings`. diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py index 644f677e79d3..7bb6ba1277a9 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py @@ -18,6 +18,8 @@ from isaaclab.utils.renderers import isaac_rtx_per_env_scene_partition_enabled if TYPE_CHECKING: + from isaaclab.sensors.camera import CameraCfg + from .isaac_rtx_renderer import IsaacRtxRenderer @@ -211,6 +213,21 @@ class IsaacRtxRendererCfg(RendererCfg): - ``"none"``: No clipping is applied. Values will be returned as ``inf``. """ + def apply_pre_reset_settings(self, camera_cfg: CameraCfg) -> None: + """Mark the run as rendering RTX sensors and route HDR color for linear camera outputs. + + See :meth:`~isaaclab.renderers.renderer_cfg.RendererCfg.apply_pre_reset_settings`. + """ + from isaaclab.app.settings_manager import get_settings_manager + + settings = get_settings_manager() + settings.set("/isaaclab/render/rtx_sensors", True) + settings.set("/physics/fabricUpdateTransformations", True) + needs_hdr = camera_cfg.isp_cfg is not None or not {"rgb_hdr", "rgb_radiance"}.isdisjoint(camera_cfg.data_types) + if needs_hdr: + # Gaussian splats otherwise skip tonemapping and never reach the HdrColor AOV. + settings.set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) + def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: """Return the per-output layouts supported by current Isaac RTX releases.""" diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py index 1eac6b1d4b6c..6bc93fbc0122 100644 --- a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py @@ -87,6 +87,24 @@ def test_native_fabric_geometry_needs_no_separate_publication(monkeypatch): provider.get_geometry_points.assert_not_called() +@pytest.mark.parametrize( + ("data_types", "isp_cfg", "routes_hdr"), + [(["rgb"], None, False), (["rgb_hdr"], None, True), (["rgb_radiance"], None, True), (["rgb"], object(), True)], +) +def test_pre_reset_settings_mark_rtx_sensors_and_route_hdr(data_types, isp_cfg, routes_hdr): + """Isaac RTX cameras flag RTX sensors before reset and route Gaussian HDR only when linear color is used.""" + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + settings = MagicMock() + with patch("isaaclab.app.settings_manager.get_settings_manager", return_value=settings): + IsaacRtxRendererCfg().apply_pre_reset_settings(SimpleNamespace(data_types=data_types, isp_cfg=isp_cfg)) + + settings.set.assert_any_call("/isaaclab/render/rtx_sensors", True) + settings.set.assert_any_call("/physics/fabricUpdateTransformations", True) + hdr_call = call("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) + assert (hdr_call in settings.set.call_args_list) is routes_hdr + + @pytest.mark.parametrize("data_types", [["rgb_hdr"], ["rgb_radiance"]]) def test_prepare_cameras_neutralizes_exposure_for_radiance(monkeypatch, data_types): """Radiance neutralizes camera exposure; standalone HDR keeps the authored camera settings.""" From 537d6b9a429fe16f2ea0df0e994767f5ea0ce8d2 Mon Sep 17 00:00:00 2001 From: Alexander Temirov Date: Wed, 7 Oct 2026 07:21:25 -0700 Subject: [PATCH 04/20] Prepare every camera before a renderer exports the stage OVRTX exports the stage once, when the first camera initializes, so overrides authored later by other cameras, such as the neutral exposure for rgb_radiance, never reached it. Cameras now apply their renderer overrides in a PHYSICS_READY callback that runs before any sensor initializes. Also address review comments: map the exposure attribute types explicitly, merge the HDR case into the existing OVRTX render-var test, state the expected HDR routing per test row, and point the rtx_sensors comment at IsaacRtxRendererCfg.apply_pre_reset_settings. Signed-off-by: Alexander Temirov --- ...atemirov-rgb-radiance-groundwork.fixed.rst | 2 + .../renderers/rtx_camera_overrides.py | 3 +- .../isaaclab/sensors/camera/camera.py | 73 ++++++++++++---- .../renderers/test_camera_output_contract.py | 83 +++++++++++++++++++ source/isaaclab_ov/test/test_ovrtx_usd.py | 37 +++++---- .../renderers/isaac_rtx_renderer.py | 3 +- 6 files changed, 165 insertions(+), 36 deletions(-) create mode 100644 source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst diff --git a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst new file mode 100644 index 000000000000..54a0fe5ec933 --- /dev/null +++ b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst @@ -0,0 +1,2 @@ +* Fixed cameras after the first one on a renderer that exports the stage once, such as OVRTX, missing their + renderer camera overrides. Every camera now applies them before any camera initializes. diff --git a/source/isaaclab/isaaclab/renderers/rtx_camera_overrides.py b/source/isaaclab/isaaclab/renderers/rtx_camera_overrides.py index 8e1c63ee5266..f3937ab13663 100644 --- a/source/isaaclab/isaaclab/renderers/rtx_camera_overrides.py +++ b/source/isaaclab/isaaclab/renderers/rtx_camera_overrides.py @@ -38,6 +38,7 @@ def apply_rtx_exposure_overrides(stage: Any, prim_paths: list[str]) -> None: """ from pxr import Sdf + value_types = {"Bool": Sdf.ValueTypeNames.Bool, "Float": Sdf.ValueTypeNames.Float} for prim_path in prim_paths: prim = stage.GetPrimAtPath(prim_path) if not prim or not prim.IsValid(): @@ -50,5 +51,5 @@ def apply_rtx_exposure_overrides(stage: Any, prim_paths: list[str]) -> None: for attr_name, sdf_type_name, value in _NEUTRAL_CAMERA_EXPOSURE: attr = prim.GetAttribute(attr_name) if not attr: - attr = prim.CreateAttribute(attr_name, getattr(Sdf.ValueTypeNames, sdf_type_name), custom=False) + attr = prim.CreateAttribute(attr_name, value_types[sdf_type_name], custom=False) attr.Set(value) diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 3f3ab1c85c13..1eb0cacbdc94 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -7,6 +7,7 @@ import logging import sys +import weakref from collections.abc import Sequence from typing import TYPE_CHECKING, Any, Literal, cast @@ -18,6 +19,7 @@ from ... import sim as sim_utils from ...app.logging_utils import force_log_level +from ...physics import PhysicsEvent, PhysicsManager from ...renderers import BaseRenderer, CameraRenderSpec from ...sim.views import FrameView from ...utils.math import ( @@ -623,22 +625,12 @@ def _initialize_impl(self): if self._renderer is None: self._renderer = sim_ctx.get_or_create_backend(self.cfg.renderer_cfg) - # Build the render spec early — both the wrapper ISP (which delegates - # any renderer-side per-camera setup) and ``create_render_data`` consume - # it, and the prims are already authored at this point. - cam_paths = tuple(str(p.GetPath()) for p in sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) - render_spec = CameraRenderSpec( - cfg=self.cfg, - device=str(self._device), - num_instances=self._num_envs, - camera_prim_paths=cam_paths, - view_count=self._num_envs, - ) - - # Delegate per-camera USD setup to the renderer — must run **before** - # ``ensure_prepare_stage`` so renderers that snapshot the stage - # (ovrtx's ``stage.Export``) capture the resulting overrides in their - # exported USD. + # Per-camera USD setup must run **before** ``ensure_prepare_stage`` so renderers that snapshot + # the stage (ovrtx's ``stage.Export``) capture the overrides in their exported USD. Every camera + # already did this in :meth:`_prepare_cameras_callback`; repeating it is harmless and covers + # cameras initialized without that callback. + render_spec = self._make_render_spec(self._num_envs) + cam_paths = render_spec.camera_prim_paths self._renderer.prepare_cameras(self.stage, render_spec) # Stage preprocessing must happen before creating the view because the view keeps @@ -677,6 +669,17 @@ def _initialize_impl(self): # Create internal buffers (includes intrinsic matrix and pose init) self._create_buffers() + def _make_render_spec(self, num_views: int) -> CameraRenderSpec: + """Describe this camera's prims and requested outputs to the renderer.""" + cam_paths = tuple(str(p.GetPath()) for p in sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) + return CameraRenderSpec( + cfg=self.cfg, + device=str(sim_utils.SimulationContext.instance().device), + num_instances=num_views, + camera_prim_paths=cam_paths, + view_count=num_views, + ) + def _prepare_camera(self, env_mask: wp.array) -> None: """Advance capture frames and refresh requested poses before rendering.""" if self.cfg.update_latest_camera_pose: @@ -1009,6 +1012,44 @@ def _env_mask_has_any(self, env_mask: wp.array) -> bool: Internal simulation callbacks. """ + def _register_callbacks(self): + super()._register_callbacks() + physics_mgr_cls = sim_utils.SimulationContext.instance().physics_manager + obj_ref = weakref.proxy(self) + + def _invoke(event): + obj_ref._prepare_cameras_callback(event) + + # Runs before every sensor's initialization (order 10), so a renderer that snapshots the stage + # when the first camera initializes sees the overrides of all cameras. + self._prepare_cameras_handle = physics_mgr_cls.register_callback( + lambda payload: PhysicsManager.safe_callback_invoke(_invoke, payload, physics_manager=physics_mgr_cls), + PhysicsEvent.PHYSICS_READY, + order=9, + ) + + def _clear_callbacks(self) -> None: + handle = getattr(self, "_prepare_cameras_handle", None) + if handle is not None: + handle.deregister() + self._prepare_cameras_handle = None + super()._clear_callbacks() + + def _prepare_cameras_callback(self, event): + """Apply this camera's renderer USD overrides before any camera initializes.""" + if self._is_initialized: + return + sim_ctx = sim_utils.SimulationContext.instance() + if self._renderer is None: + self._renderer = sim_ctx.get_or_create_backend(self.cfg.renderer_cfg) + # Matches the environment count that sensor initialization derives next. + clone_plan = sim_ctx.get_clone_plan() + if clone_plan is not None: + num_views = len(clone_plan.topology.world_prototype_layout) + else: + num_views = len(sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) + self._renderer.prepare_cameras(self.stage, self._make_render_spec(num_views)) + def _invalidate_initialize_callback(self, event): """Invalidates the scene elements.""" if self._renderer is not None and self._render_data is not None: diff --git a/source/isaaclab/test/renderers/test_camera_output_contract.py b/source/isaaclab/test/renderers/test_camera_output_contract.py index 09297e703a0a..22dabde3a2b9 100644 --- a/source/isaaclab/test/renderers/test_camera_output_contract.py +++ b/source/isaaclab/test/renderers/test_camera_output_contract.py @@ -266,3 +266,86 @@ def test_camera_data_allocate_raises_on_unknown_name(): ) assert "not_a_real_type" in str(exc_info.value) assert "RenderBufferKind" in str(exc_info.value) + + +def test_every_camera_prepares_before_the_shared_stage_export(monkeypatch): + """A renderer that snapshots the stage at the first camera's initialization sees every camera's overrides.""" + import numpy as np + + from pxr import Sdf, Usd, UsdGeom + + from isaaclab.physics import PhysicsEvent, PhysicsManager + from isaaclab.renderers.rtx_camera_overrides import apply_rtx_exposure_overrides + from isaaclab.sensors.camera import Camera + from isaaclab.sensors.camera import camera as camera_module + from isaaclab.sensors.sensor_base import SensorBase + from isaaclab.sim import SimulationContext + + class CameraPhysicsManager(PhysicsManager): + _callbacks = {} + + stage = Usd.Stage.CreateInMemory() + prims = [UsdGeom.Camera.Define(stage, path).GetPrim() for path in ("/World/First", "/World/Second")] + prims[1].CreateAttribute("exposure:iso", Sdf.ValueTypeNames.Float).Set(100.0) + specs = { + RenderBufferKind.RGB: RenderBufferSpec(3, wp.uint8), + RenderBufferKind.RGBA: RenderBufferSpec(4, wp.uint8), + RenderBufferKind.RGB_RADIANCE: RenderBufferSpec(3, wp.float32), + } + exported_iso = [] + + def prepare_cameras(stage, spec): + if "rgb_radiance" in spec.cfg.data_types: + apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) + + def export_stage(stage, num_envs): + if not exported_iso: + exported_iso.append(prims[1].GetAttribute("exposure:iso").Get()) + + renderer = SimpleNamespace( + supported_output_types=lambda: specs, + prepare_cameras=prepare_cameras, + create_render_data=lambda spec: SimpleNamespace(spec=spec), + set_outputs=lambda data, outputs: None, + cleanup=lambda data: None, + ) + sim = SimpleNamespace( + device="cpu", + physics_manager=CameraPhysicsManager, + get_clone_plan=lambda: SimpleNamespace(topology=SimpleNamespace(world_prototype_layout=np.zeros(2))), + render_context=SimpleNamespace(ensure_prepare_stage=export_stage), + vis_marker_registry=SimpleNamespace(clear_debug_vis_callback=lambda sensor: None), + ) + monkeypatch.setattr(SimulationContext, "instance", staticmethod(lambda: sim)) + monkeypatch.setattr(SensorBase, "_initialize_impl", lambda self: None) + monkeypatch.setattr(Camera, "_initialize_intrinsics", lambda self: None) + monkeypatch.setattr(Camera, "_update_poses", lambda self: None) + monkeypatch.setattr( + camera_module, + "FrameView", + lambda path, **kwargs: SimpleNamespace(count=2, prims=[stage.GetPrimAtPath(path)] * 2, close=lambda: None), + ) + cameras = [] + for prim, data_types in zip(prims, (["rgb"], ["rgb_radiance"])): + camera = Camera.__new__(Camera) + camera.cfg = SimpleNamespace(prim_path=str(prim.GetPath()), data_types=data_types, height=2, width=3) + camera.stage = stage + camera._device = "cpu" + camera._num_envs = 2 + camera._is_initialized = False + camera._sensor_prims = [] + camera._renderer = renderer + camera._render_data = None + camera._view = None + camera._register_callbacks() + cameras.append(camera) + + try: + CameraPhysicsManager.dispatch_event(PhysicsEvent.PHYSICS_READY) + + assert all(camera.is_initialized for camera in cameras) + assert exported_iso == [0.0] + finally: + for camera in cameras: + camera.__del__() + assert not CameraPhysicsManager._callbacks diff --git a/source/isaaclab_ov/test/test_ovrtx_usd.py b/source/isaaclab_ov/test/test_ovrtx_usd.py index 0f890945be75..081f536df4ad 100644 --- a/source/isaaclab_ov/test/test_ovrtx_usd.py +++ b/source/isaaclab_ov/test/test_ovrtx_usd.py @@ -126,14 +126,6 @@ def test_render_product_solid_background_color(camera_spec, render_data): assert 'token omni:rtx:background:source:type = "domeLight"' not in render_scope -@pytest.mark.parametrize("data_types", [["rgb_hdr"], ["rgb_radiance"], ["rgb_hdr", "rgb_radiance"]]) -def test_ovrtx_hdr_outputs_use_one_hdr_color_render_var(data_types): - """HDR and radiance share one HdrColor native source.""" - assert get_render_var_configs(data_types, render_scope_name="RenderCamera_0") == [ - ("/RenderCamera_0/Vars/HdrColor", "HdrColor", "HdrColor"), - ] - - def test_render_var_prim_names_are_read_only(): with pytest.raises(TypeError): render_var_prim_names_by_source()["LdrColor"] = "mutated" # type: ignore[index] @@ -223,14 +215,17 @@ def test_ovrtx_authors_one_render_var_per_requested_data_type(): def test_ovrtx_data_types_sharing_a_source_author_one_render_var(): - """``rgb``/``rgba`` and ``depth``/``distance_to_image_plane`` collapse onto one render var each.""" + """``rgb``/``rgba``, ``depth``/``distance_to_image_plane``, and ``rgb_hdr``/``rgb_radiance`` collapse onto one + render var each.""" render_var_configs = get_render_var_configs( - ["rgb", "rgba", "depth", "distance_to_image_plane"], render_scope_name="RenderCamera_0" + ["rgb", "rgba", "depth", "distance_to_image_plane", "rgb_hdr", "rgb_radiance"], + render_scope_name="RenderCamera_0", ) assert render_var_configs == [ ("/RenderCamera_0/Vars/LdrColor", "LdrColor", "LdrColor"), ("/RenderCamera_0/Vars/depth", "depth", "DistanceToImagePlaneSD"), + ("/RenderCamera_0/Vars/HdrColor", "HdrColor", "HdrColor"), ] @@ -321,10 +316,18 @@ def test_render_product_omits_device_ids_when_no_device_is_given(camera_spec, re @pytest.mark.parametrize( - ("data_types", "use_isp"), - [(["rgb"], True), (["rgb", "rgb_hdr"], False), (["rgb", "rgb_radiance"], False), ([], True), (["rgb"], False)], + ("data_types", "use_isp", "routes_hdr"), + [ + (["rgb"], True, True), + (["rgb", "rgb_hdr"], False, True), + (["rgb", "rgb_radiance"], False, True), + ([], True, True), + (["rgb"], False, False), + ], ) -def test_render_product_isp_requests_hdr_without_mutating_camera_outputs(camera_spec, render_data, data_types, use_isp): +def test_render_product_isp_requests_hdr_without_mutating_camera_outputs( + camera_spec, render_data, data_types, use_isp, routes_hdr +): """Route Gaussian HDR for explicit or ISP inputs without changing public outputs.""" camera_spec.cfg.data_types = data_types.copy() camera_spec.cfg.isp_cfg = object() if use_isp else None @@ -334,18 +337,16 @@ def test_render_product_isp_requests_hdr_without_mutating_camera_outputs(camera_ product_path = "/RenderCamera_0/RenderProduct" ordered_vars = layer.GetRelationshipAtPath(product_path + ".orderedVars") expected_vars = [Sdf.Path("/RenderCamera_0/Vars/LdrColor")] - needs_hdr = use_isp or not {"rgb_hdr", "rgb_radiance"}.isdisjoint(data_types) - if needs_hdr: + if routes_hdr: expected_vars.append(Sdf.Path("/RenderCamera_0/Vars/HdrColor")) assert list(ordered_vars.targetPathList.explicitItems) == expected_vars assert camera_spec.cfg.data_types == data_types schemas = layer.GetPrimAtPath(product_path).GetInfo("apiSchemas").prependedItems gaussian_setting = layer.GetAttributeAtPath(product_path + ".omni:rtx:rtpt:gaussian:skipTonemapping:enabled") - if needs_hdr: - assert "OmniRtxSettingsParticleFieldAPI_1" in schemas + assert ("OmniRtxSettingsParticleFieldAPI_1" in schemas) is routes_hdr + if routes_hdr: assert gaussian_setting.default is False else: - assert "OmniRtxSettingsParticleFieldAPI_1" not in schemas assert gaussian_setting is None diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index 97e0f4015321..2108f66605ad 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -200,7 +200,8 @@ def __init__(self, cfg: IsaacRtxRendererCfg): if settings.get("/isaaclab/render/deterministic", False): apply_isaac_rtx_determinism_settings(settings) ensure_rtx_hydra_engine_attached() - # ``/isaaclab/render/rtx_sensors`` is owned by ``Camera.__init__`` (must be set pre-``sim.reset()``). + # ``/isaaclab/render/rtx_sensors`` is set pre-``sim.reset()`` by + # ``IsaacRtxRendererCfg.apply_pre_reset_settings``. def initialize(self) -> None: """Bind shared Fabric destinations after scene creation.""" From 6dddd523494841b6a57a3beb5c9dd1d108f65657 Mon Sep 17 00:00:00 2001 From: Alexander Temirov Date: Wed, 7 Oct 2026 09:20:45 -0700 Subject: [PATCH 05/20] Prepare sensors through a SensorBase hook and document rgb_radiance support in the matrix SensorBase registers a PHYSICS_READY callback that runs every sensor's _prepare_initialize_impl() before any sensor initializes, through its existing weak reference. Camera overrides the hook to apply its renderer overrides, so it no longer registers its own callback or imports weakref. The rgb_radiance enum docstring keeps only the definition. The camera documentation's renderer support matrix now lists rgb_radiance as derived on Isaac RTX and OVRTX, with the note below it. Signed-off-by: Alexander Temirov --- docs/source/concepts/sensors/camera.rst | 26 ++++++++------ .../isaaclab/renderers/output_contract.py | 6 +--- .../isaaclab/sensors/camera/camera.py | 35 ++++--------------- .../isaaclab/isaaclab/sensors/sensor_base.py | 27 +++++++++++++- .../test/sensors/test_opencv_distortion.py | 1 + 5 files changed, 49 insertions(+), 46 deletions(-) diff --git a/docs/source/concepts/sensors/camera.rst b/docs/source/concepts/sensors/camera.rst index 94fa76c0e8fb..51d8d7af0d7c 100644 --- a/docs/source/concepts/sensors/camera.rst +++ b/docs/source/concepts/sensors/camera.rst @@ -175,16 +175,6 @@ Requesting ``rgb_hdr`` alone preserves the renderer's existing camera settings. the input for image processing that applies its own exposure and camera response, such as PPISP. If both are requested, they share one buffer. -.. note:: - - On Isaac RTX and OVRTX, ``rgb_radiance`` is not a native renderer output. These renderers derive - it from HDR color by authoring neutral exposure on the camera prim, so every output rendered from - that prim, including ``rgb``, ``rgba``, and ``rgb_hdr``, loses the authored exposure. This also - applies to other camera sensors that share the prim. A single render product cannot return both - authored-exposure color and ``rgb_radiance``; use a separate camera prim when both are required. - Newton Warp has no exposure model and is not affected. A native pre-exposure radiance output has - been requested from the RTX team (NVBug 6858736). - .. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/camera-renderer-isaac-rtx.webp :align: center :figwidth: 100% @@ -217,9 +207,13 @@ current support matrix is: - Isaac RTX - OVRTX - Newton Warp - * - ``rgb``, ``rgba``, ``rgb_hdr``, ``rgb_radiance`` + * - ``rgb``, ``rgba``, ``rgb_hdr`` + - Yes - Yes - Yes + * - ``rgb_radiance`` + - Derived (see below) + - Derived (see below) - Yes * - ``depth`` and both distance outputs - Yes @@ -250,6 +244,16 @@ current support matrix is: - Yes - No +.. note:: + + On Isaac RTX and OVRTX, ``rgb_radiance`` is not a native renderer output. These renderers derive + it from HDR color by authoring neutral exposure on the camera prim, so every output rendered from + that prim, including ``rgb``, ``rgba``, and ``rgb_hdr``, loses the authored exposure. This also + applies to other camera sensors that share the prim. A single render product cannot return both + authored-exposure color and ``rgb_radiance``; use a separate camera prim when both are required. + Newton Warp has no exposure model and is not affected. A native pre-exposure radiance output has + been requested from the RTX team (NVBug 6858736). + Querying an unsupported output fails during camera initialization. Renderer configuration controls semantic filters, segmentation colorization, and depth clipping where those options are backend-specific. diff --git a/source/isaaclab/isaaclab/renderers/output_contract.py b/source/isaaclab/isaaclab/renderers/output_contract.py index 0095243e4090..f62f4a5ef357 100644 --- a/source/isaaclab/isaaclab/renderers/output_contract.py +++ b/source/isaaclab/isaaclab/renderers/output_contract.py @@ -28,11 +28,7 @@ class RenderBufferKind(StrEnum): RGB_HDR = "rgb_hdr" """HDR RGB using the camera's active rendering settings.""" RGB_RADIANCE = "rgb_radiance" - """Scene-linear RGB before camera exposure and response, in renderer-relative intensity units. - - Isaac RTX and OVRTX derive it by neutralizing exposure on the camera prim, which also affects the - prim's other color outputs. See the camera documentation. - """ + """Scene-linear RGB before camera exposure and response, in renderer-relative intensity units.""" ALBEDO = "albedo" DEPTH = "depth" DISTANCE_TO_IMAGE_PLANE = "distance_to_image_plane" diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 1eb0cacbdc94..a69bcbf018af 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -7,7 +7,6 @@ import logging import sys -import weakref from collections.abc import Sequence from typing import TYPE_CHECKING, Any, Literal, cast @@ -19,7 +18,6 @@ from ... import sim as sim_utils from ...app.logging_utils import force_log_level -from ...physics import PhysicsEvent, PhysicsManager from ...renderers import BaseRenderer, CameraRenderSpec from ...sim.views import FrameView from ...utils.math import ( @@ -627,7 +625,7 @@ def _initialize_impl(self): # Per-camera USD setup must run **before** ``ensure_prepare_stage`` so renderers that snapshot # the stage (ovrtx's ``stage.Export``) capture the overrides in their exported USD. Every camera - # already did this in :meth:`_prepare_cameras_callback`; repeating it is harmless and covers + # already did this in :meth:`_prepare_initialize_impl`; repeating it is harmless and covers # cameras initialized without that callback. render_spec = self._make_render_spec(self._num_envs) cam_paths = render_spec.camera_prim_paths @@ -1012,33 +1010,12 @@ def _env_mask_has_any(self, env_mask: wp.array) -> bool: Internal simulation callbacks. """ - def _register_callbacks(self): - super()._register_callbacks() - physics_mgr_cls = sim_utils.SimulationContext.instance().physics_manager - obj_ref = weakref.proxy(self) + def _prepare_initialize_impl(self): + """Apply this camera's renderer USD overrides before any camera initializes. - def _invoke(event): - obj_ref._prepare_cameras_callback(event) - - # Runs before every sensor's initialization (order 10), so a renderer that snapshots the stage - # when the first camera initializes sees the overrides of all cameras. - self._prepare_cameras_handle = physics_mgr_cls.register_callback( - lambda payload: PhysicsManager.safe_callback_invoke(_invoke, payload, physics_manager=physics_mgr_cls), - PhysicsEvent.PHYSICS_READY, - order=9, - ) - - def _clear_callbacks(self) -> None: - handle = getattr(self, "_prepare_cameras_handle", None) - if handle is not None: - handle.deregister() - self._prepare_cameras_handle = None - super()._clear_callbacks() - - def _prepare_cameras_callback(self, event): - """Apply this camera's renderer USD overrides before any camera initializes.""" - if self._is_initialized: - return + A renderer that exports the stage when the first camera initializes, such as OVRTX, then sees + the overrides of every camera. + """ sim_ctx = sim_utils.SimulationContext.instance() if self._renderer is None: self._renderer = sim_ctx.get_or_create_backend(self.cfg.renderer_cfg) diff --git a/source/isaaclab/isaaclab/sensors/sensor_base.py b/source/isaaclab/isaaclab/sensors/sensor_base.py index 65a72c169d37..b683515f28f7 100644 --- a/source/isaaclab/isaaclab/sensors/sensor_base.py +++ b/source/isaaclab/isaaclab/sensors/sensor_base.py @@ -55,6 +55,7 @@ def __init__(self, cfg: SensorBaseCfg): cfg: The configuration parameters for the sensor. """ # start with unset callback handles so cleanup is safe if construction fails part way + self._prepare_initialize_handle = None self._initialize_handle = None self._invalidate_initialize_handle = None self._prim_deletion_handle = None @@ -343,7 +344,15 @@ def _register_callbacks(self): def _invoke(callback_name, event): getattr(obj_ref, callback_name)(event) - # Backend-agnostic: PHYSICS_READY (init) and STOP (invalidate) + # Backend-agnostic: PHYSICS_READY (prepare, then init) and STOP (invalidate). Every sensor prepares + # before any sensor initializes, so sensors can set up shared state, such as a renderer's stage, together. + self._prepare_initialize_handle = physics_mgr_cls.register_callback( + lambda payload: PhysicsManager.safe_callback_invoke( + _invoke, "_prepare_initialize_callback", payload, physics_manager=physics_mgr_cls + ), + PhysicsEvent.PHYSICS_READY, + order=9, + ) self._initialize_handle = physics_mgr_cls.register_callback( lambda payload: PhysicsManager.safe_callback_invoke( _invoke, "_initialize_callback", payload, physics_manager=physics_mgr_cls @@ -372,6 +381,19 @@ def _invoke(callback_name, event): IsaacEvents.PRIM_DELETION, ) + def _prepare_initialize_callback(self, event): + """Prepares the sensor before any sensor initializes.""" + if not self._is_initialized: + self._prepare_initialize_impl() + + def _prepare_initialize_impl(self): + """Prepare state that the initialization of other sensors depends on. + + Runs for every sensor on :attr:`PhysicsEvent.PHYSICS_READY`, before any sensor's + :meth:`_initialize_impl`. The default does nothing. + """ + pass + def _initialize_callback(self, event): """Initializes the scene elements. @@ -403,6 +425,9 @@ def _on_prim_deletion(self, event) -> None: def _clear_callbacks(self) -> None: """Clears the callbacks.""" + if self._prepare_initialize_handle is not None: + self._prepare_initialize_handle.deregister() + self._prepare_initialize_handle = None if self._initialize_handle is not None: self._initialize_handle.deregister() self._initialize_handle = None diff --git a/source/isaaclab/test/sensors/test_opencv_distortion.py b/source/isaaclab/test/sensors/test_opencv_distortion.py index 6d276bc0f451..ef53a991fde3 100644 --- a/source/isaaclab/test/sensors/test_opencv_distortion.py +++ b/source/isaaclab/test/sensors/test_opencv_distortion.py @@ -192,6 +192,7 @@ def _camera_for_prims(prims, width=640, height=480, device="cpu"): fake._ALL_INDICES = wp.array(np.arange(len(prims)), dtype=wp.int32, device=device) fake._ALL_ENV_MASK = wp.ones(len(prims), dtype=wp.bool, device=device) # attributes touched by ``__del__``/``_clear_callbacks`` when the fake object is garbage collected + fake._prepare_initialize_handle = None fake._initialize_handle = None fake._invalidate_initialize_handle = None fake._prim_deletion_handle = None From de8961aeb35e0023f066496f826f11ec14c58fd8 Mon Sep 17 00:00:00 2001 From: Alexander Temirov Date: Wed, 7 Oct 2026 09:30:33 -0700 Subject: [PATCH 06/20] Apply camera renderer settings through the renderer, not its config Replace RendererCfg.apply_pre_reset_settings() with BaseRenderer.apply_camera_settings(), which cameras call right after creating their renderer in __init__, still before sim.reset(). Renderer configurations stay data only. IsaacRtxRenderer implements the hook. Signed-off-by: Alexander Temirov --- .../atemirov-rgb-radiance-groundwork.added.rst | 4 ++-- ...atemirov-rgb-radiance-groundwork.changed.rst | 2 +- .../isaaclab/renderers/base_renderer.py | 12 ++++++++++++ .../isaaclab/isaaclab/renderers/renderer_cfg.py | 12 ------------ .../isaaclab/isaaclab/sensors/camera/camera.py | 5 ++--- ...atemirov-rgb-radiance-groundwork.changed.rst | 5 ++--- .../renderers/isaac_rtx_renderer.py | 16 ++++++++++++++-- .../renderers/isaac_rtx_renderer_cfg.py | 17 ----------------- .../test_isaac_rtx_renderer_contract.py | 11 +++++++---- 9 files changed, 40 insertions(+), 44 deletions(-) diff --git a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst index 4d200a0c9e80..9a84f5317c78 100644 --- a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst +++ b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst @@ -3,5 +3,5 @@ they share one buffer. * Added :func:`isaaclab.renderers.rtx_camera_overrides.apply_rtx_exposure_overrides`, shared by the RTX renderers to author neutral camera exposure. -* Added :meth:`~isaaclab.renderers.RendererCfg.apply_pre_reset_settings` so renderer configurations - apply the process-wide settings their cameras need before ``sim.reset()``. +* Added :meth:`~isaaclab.renderers.BaseRenderer.apply_camera_settings` so renderers apply the + process-wide settings their cameras need before ``sim.reset()``. diff --git a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst index 53ba96788094..d4c2cb2f8dd1 100644 --- a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst +++ b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst @@ -1,2 +1,2 @@ * Changed :class:`~isaaclab.sensors.camera.Camera` to apply renderer-specific settings through - :meth:`~isaaclab.renderers.RendererCfg.apply_pre_reset_settings` instead of checking the renderer type. + :meth:`~isaaclab.renderers.BaseRenderer.apply_camera_settings` instead of checking the renderer type. diff --git a/source/isaaclab/isaaclab/renderers/base_renderer.py b/source/isaaclab/isaaclab/renderers/base_renderer.py index 3cea8c3f1db2..d42828eae7b6 100644 --- a/source/isaaclab/isaaclab/renderers/base_renderer.py +++ b/source/isaaclab/isaaclab/renderers/base_renderer.py @@ -20,6 +20,7 @@ import torch import warp as wp + from ..sensors.camera.camera_cfg import CameraCfg from ..sensors.camera.camera_data import CameraData from ..utils.warp import ProxyArray @@ -51,6 +52,17 @@ def visual_material_writer(self) -> Callable[[tuple[VisualMaterialBatch, ...]], """ return None + def apply_camera_settings(self, camera_cfg: CameraCfg) -> None: + """Apply process-wide settings that a camera rendered by this renderer needs. + + Cameras call this when constructed, before ``sim.reset()``, because the simulation and + environments read some of these settings during setup. The default applies no settings. + + Args: + camera_cfg: Configuration of the camera being constructed. + """ + return + def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: """Pre-render per-camera setup the backend needs. diff --git a/source/isaaclab/isaaclab/renderers/renderer_cfg.py b/source/isaaclab/isaaclab/renderers/renderer_cfg.py index aabcefd0da4f..8ed1d9f73204 100644 --- a/source/isaaclab/isaaclab/renderers/renderer_cfg.py +++ b/source/isaaclab/isaaclab/renderers/renderer_cfg.py @@ -14,7 +14,6 @@ from .output_contract import RenderBufferKind, RenderBufferSpec if TYPE_CHECKING: - from ..sensors.camera.camera_cfg import CameraCfg from .base_renderer import BaseRenderer @@ -45,14 +44,3 @@ def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec] | N renderer is selected dynamically. """ return None - - def apply_pre_reset_settings(self, camera_cfg: CameraCfg) -> None: - """Apply process-wide settings that a camera rendered by this renderer needs. - - Cameras call this when constructed, before ``sim.reset()``, because environments read some - of these settings before the renderer is created. The default applies no settings. - - Args: - camera_cfg: Configuration of the camera being constructed. - """ - pass diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index a69bcbf018af..61a7475a1f48 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -245,9 +245,6 @@ def __init__(self, cfg: CameraCfg): if sim_ctx is not None: sim_ctx.require_visual_shapes() - # Simulation setup and environments read some renderer settings before the renderer exists. - self.cfg.renderer_cfg.apply_pre_reset_settings(self.cfg) - # UsdGeom Camera prim for the sensor self._sensor_prims: list[UsdGeom.Camera] = [] # Allocated in :meth:`_create_buffers` once the renderer's output contract is known. @@ -258,6 +255,8 @@ def __init__(self, cfg: CameraCfg): self._renderer: BaseRenderer | None = None if sim_ctx is not None: self._renderer = sim_ctx.get_or_create_backend(self.cfg.renderer_cfg) + # The simulation and environments read some renderer settings during setup. + self._renderer.apply_camera_settings(self.cfg) with force_log_level(logging.INFO): logger.info("Using renderer: %s", type(self._renderer).__name__) # Render data — assigned in _initialize_impl. diff --git a/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst b/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst index 786857a47825..7c06df9da287 100644 --- a/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst +++ b/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst @@ -1,3 +1,2 @@ -* Changed :class:`~isaaclab_physx.renderers.IsaacRtxRendererCfg` to set the RTX sensor and HDR - routing settings for its cameras in - :meth:`~isaaclab_physx.renderers.IsaacRtxRendererCfg.apply_pre_reset_settings`. +* Changed :class:`~isaaclab_physx.renderers.IsaacRtxRenderer` to set the RTX sensor and HDR routing settings + for its cameras in :meth:`~isaaclab_physx.renderers.IsaacRtxRenderer.apply_camera_settings`. diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index 2108f66605ad..89987431e28e 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -47,6 +47,7 @@ from omni.replicator.core.scripts.utils.viewport_manager import HydraTexture + from isaaclab.sensors.camera import CameraCfg from isaaclab.sensors.camera.camera_data import CameraData from isaaclab.utils.warp import ProxyArray @@ -200,8 +201,19 @@ def __init__(self, cfg: IsaacRtxRendererCfg): if settings.get("/isaaclab/render/deterministic", False): apply_isaac_rtx_determinism_settings(settings) ensure_rtx_hydra_engine_attached() - # ``/isaaclab/render/rtx_sensors`` is set pre-``sim.reset()`` by - # ``IsaacRtxRendererCfg.apply_pre_reset_settings``. + + def apply_camera_settings(self, camera_cfg: CameraCfg) -> None: + """Mark the run as rendering RTX sensors and route HDR color for linear camera outputs. + + See :meth:`~isaaclab.renderers.base_renderer.BaseRenderer.apply_camera_settings`. + """ + settings = get_settings_manager() + settings.set("/isaaclab/render/rtx_sensors", True) + settings.set("/physics/fabricUpdateTransformations", True) + needs_hdr = camera_cfg.isp_cfg is not None or not {"rgb_hdr", "rgb_radiance"}.isdisjoint(camera_cfg.data_types) + if needs_hdr: + # Gaussian splats otherwise skip tonemapping and never reach the HdrColor AOV. + settings.set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) def initialize(self) -> None: """Bind shared Fabric destinations after scene creation.""" diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py index 7bb6ba1277a9..644f677e79d3 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py @@ -18,8 +18,6 @@ from isaaclab.utils.renderers import isaac_rtx_per_env_scene_partition_enabled if TYPE_CHECKING: - from isaaclab.sensors.camera import CameraCfg - from .isaac_rtx_renderer import IsaacRtxRenderer @@ -213,21 +211,6 @@ class IsaacRtxRendererCfg(RendererCfg): - ``"none"``: No clipping is applied. Values will be returned as ``inf``. """ - def apply_pre_reset_settings(self, camera_cfg: CameraCfg) -> None: - """Mark the run as rendering RTX sensors and route HDR color for linear camera outputs. - - See :meth:`~isaaclab.renderers.renderer_cfg.RendererCfg.apply_pre_reset_settings`. - """ - from isaaclab.app.settings_manager import get_settings_manager - - settings = get_settings_manager() - settings.set("/isaaclab/render/rtx_sensors", True) - settings.set("/physics/fabricUpdateTransformations", True) - needs_hdr = camera_cfg.isp_cfg is not None or not {"rgb_hdr", "rgb_radiance"}.isdisjoint(camera_cfg.data_types) - if needs_hdr: - # Gaussian splats otherwise skip tonemapping and never reach the HdrColor AOV. - settings.set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) - def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: """Return the per-output layouts supported by current Isaac RTX releases.""" diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py index 6bc93fbc0122..3efd53d25700 100644 --- a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py @@ -91,13 +91,16 @@ def test_native_fabric_geometry_needs_no_separate_publication(monkeypatch): ("data_types", "isp_cfg", "routes_hdr"), [(["rgb"], None, False), (["rgb_hdr"], None, True), (["rgb_radiance"], None, True), (["rgb"], object(), True)], ) -def test_pre_reset_settings_mark_rtx_sensors_and_route_hdr(data_types, isp_cfg, routes_hdr): +def test_camera_settings_mark_rtx_sensors_and_route_hdr(monkeypatch, data_types, isp_cfg, routes_hdr): """Isaac RTX cameras flag RTX sensors before reset and route Gaussian HDR only when linear color is used.""" - from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + _install_omni_stubs(monkeypatch) + import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer settings = MagicMock() - with patch("isaaclab.app.settings_manager.get_settings_manager", return_value=settings): - IsaacRtxRendererCfg().apply_pre_reset_settings(SimpleNamespace(data_types=data_types, isp_cfg=isp_cfg)) + with patch.object(rtx_renderer, "get_settings_manager", return_value=settings): + rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer).apply_camera_settings( + SimpleNamespace(data_types=data_types, isp_cfg=isp_cfg) + ) settings.set.assert_any_call("/isaaclab/render/rtx_sensors", True) settings.set.assert_any_call("/physics/fabricUpdateTransformations", True) From 08425dda595a0fa1890856e5b2aa7f6cd21d3ff2 Mon Sep 17 00:00:00 2001 From: Alexander Temirov Date: Wed, 7 Oct 2026 09:54:41 -0700 Subject: [PATCH 07/20] Keep the RTX exposure override internal The neutral-exposure override is a temporary workaround until RTX provides a native radiance output (NVBug 6858736), so it is not public API: rename it to _apply_rtx_exposure_overrides, document the module as internal, and drop it from the changelog. Signed-off-by: Alexander Temirov --- .../atemirov-rgb-radiance-groundwork.added.rst | 2 -- .../isaaclab/isaaclab/renderers/rtx_camera_overrides.py | 9 +++++---- .../test/renderers/test_camera_output_contract.py | 4 ++-- .../isaaclab/test/renderers/test_rtx_camera_overrides.py | 4 ++-- .../isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py | 4 ++-- .../isaaclab_physx/renderers/isaac_rtx_renderer.py | 4 ++-- .../atemirov-rgb-radiance-groundwork.removed.rst | 4 ++-- 7 files changed, 15 insertions(+), 16 deletions(-) diff --git a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst index 9a84f5317c78..d5643fae2cee 100644 --- a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst +++ b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst @@ -1,7 +1,5 @@ * Added the ``rgb_radiance`` camera output: scene-linear RGB before camera exposure and response, in renderer-relative intensity units. When ``rgb_hdr`` and ``rgb_radiance`` are both requested, they share one buffer. -* Added :func:`isaaclab.renderers.rtx_camera_overrides.apply_rtx_exposure_overrides`, shared by the - RTX renderers to author neutral camera exposure. * Added :meth:`~isaaclab.renderers.BaseRenderer.apply_camera_settings` so renderers apply the process-wide settings their cameras need before ``sim.reset()``. diff --git a/source/isaaclab/isaaclab/renderers/rtx_camera_overrides.py b/source/isaaclab/isaaclab/renderers/rtx_camera_overrides.py index f3937ab13663..632f07adc8ee 100644 --- a/source/isaaclab/isaaclab/renderers/rtx_camera_overrides.py +++ b/source/isaaclab/isaaclab/renderers/rtx_camera_overrides.py @@ -3,11 +3,12 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Shared RTX camera exposure overrides for scene-linear sensor inputs. +"""Internal RTX camera exposure overrides for scene-linear sensor inputs. RTX has no native pre-exposure radiance output, so ``rgb_radiance`` is derived by authoring neutral -exposure on the camera prim, which affects every output rendered from it. These overrides can be -removed once RTX provides a native radiance output (NVBug 6858736). +exposure on the camera prim, which affects every output rendered from it. This is a temporary +workaround shared by the Isaac RTX and OVRTX renderers, not public API. Remove it once RTX provides +a native radiance output (NVBug 6858736). """ from __future__ import annotations @@ -26,7 +27,7 @@ ) -def apply_rtx_exposure_overrides(stage: Any, prim_paths: list[str]) -> None: +def _apply_rtx_exposure_overrides(stage: Any, prim_paths: list[str]) -> None: """Disable RTX camera exposure so downstream processors receive scene-linear HDR. Existing API schemas are preserved. Schema metadata is authored directly so the RTX schema diff --git a/source/isaaclab/test/renderers/test_camera_output_contract.py b/source/isaaclab/test/renderers/test_camera_output_contract.py index 22dabde3a2b9..2e07fd64a9ac 100644 --- a/source/isaaclab/test/renderers/test_camera_output_contract.py +++ b/source/isaaclab/test/renderers/test_camera_output_contract.py @@ -275,7 +275,7 @@ def test_every_camera_prepares_before_the_shared_stage_export(monkeypatch): from pxr import Sdf, Usd, UsdGeom from isaaclab.physics import PhysicsEvent, PhysicsManager - from isaaclab.renderers.rtx_camera_overrides import apply_rtx_exposure_overrides + from isaaclab.renderers.rtx_camera_overrides import _apply_rtx_exposure_overrides from isaaclab.sensors.camera import Camera from isaaclab.sensors.camera import camera as camera_module from isaaclab.sensors.sensor_base import SensorBase @@ -296,7 +296,7 @@ class CameraPhysicsManager(PhysicsManager): def prepare_cameras(stage, spec): if "rgb_radiance" in spec.cfg.data_types: - apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) + _apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) def export_stage(stage, num_envs): if not exported_iso: diff --git a/source/isaaclab/test/renderers/test_rtx_camera_overrides.py b/source/isaaclab/test/renderers/test_rtx_camera_overrides.py index 8cc5b09e8f49..73fab8a08d23 100644 --- a/source/isaaclab/test/renderers/test_rtx_camera_overrides.py +++ b/source/isaaclab/test/renderers/test_rtx_camera_overrides.py @@ -7,7 +7,7 @@ from pxr import Sdf, Usd, UsdGeom -from isaaclab.renderers.rtx_camera_overrides import apply_rtx_exposure_overrides +from isaaclab.renderers.rtx_camera_overrides import _apply_rtx_exposure_overrides def test_apply_rtx_exposure_overrides_resets_authored_exposure_values(): @@ -23,7 +23,7 @@ def test_apply_rtx_exposure_overrides_resets_authored_exposure_values(): prim.CreateAttribute("exposure:time", Sdf.ValueTypeNames.Float).Set(1.0 / 60.0) prim.CreateAttribute("omni:rtx:autoExposure:enabled", Sdf.ValueTypeNames.Bool).Set(True) - apply_rtx_exposure_overrides(stage, ["/World/Camera"]) + _apply_rtx_exposure_overrides(stage, ["/World/Camera"]) assert prim.GetAttribute("exposure").Get() == 0.0 assert prim.GetAttribute("exposure:fStop").Get() == 1.0 diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index ba7abbae427d..0e469a8c5819 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -74,7 +74,7 @@ from isaaclab.cloner import ClonePlan from isaaclab.cloner import path as cloner_path from isaaclab.renderers import BaseRenderer, RenderBufferKind, RenderBufferSpec -from isaaclab.renderers.rtx_camera_overrides import apply_rtx_exposure_overrides +from isaaclab.renderers.rtx_camera_overrides import _apply_rtx_exposure_overrides from isaaclab.scene_data import SceneDataFormat from isaaclab.sim import SimulationContext from isaaclab.utils.warp import ProxyArray @@ -468,7 +468,7 @@ def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) needs_neutral_exposure = spec.cfg.isp_cfg is not None or "rgb_radiance" in spec.cfg.data_types if needs_neutral_exposure and spec.camera_prim_paths: - apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) + _apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) def prepare_stage(self, stage: Any, num_envs: int) -> None: """Prepare the USD stage for OVRTX before :meth:`create_render_data`. diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index 89987431e28e..a1a8a5a79ccf 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -24,7 +24,7 @@ from isaaclab.app.settings_manager import get_settings_manager from isaaclab.renderers import BaseRenderer, RenderBufferKind, RenderBufferSpec from isaaclab.renderers.camera_render_spec import CameraRenderSpec -from isaaclab.renderers.rtx_camera_overrides import apply_rtx_exposure_overrides +from isaaclab.renderers.rtx_camera_overrides import _apply_rtx_exposure_overrides from isaaclab.sim import SimulationContext from isaaclab.sim.utils import enable_extension from isaaclab.utils.version import get_isaac_sim_version @@ -250,7 +250,7 @@ def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) needs_neutral_exposure = spec.cfg.isp_cfg is not None or "rgb_radiance" in spec.cfg.data_types if needs_neutral_exposure and spec.camera_prim_paths: - apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) + _apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: """Publish the per-output Replicator layout this RTX backend writes. diff --git a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst index 69a0756ffc2c..db306a43dada 100644 --- a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst +++ b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst @@ -1,2 +1,2 @@ -* Removed ``isaaclab_ppisp.apply_rtx_exposure_overrides``. Import - :func:`isaaclab.renderers.rtx_camera_overrides.apply_rtx_exposure_overrides` instead. +* Removed ``isaaclab_ppisp.apply_rtx_exposure_overrides``. The RTX renderers now apply the exposure + override themselves when a camera requests ``rgb_radiance``. From 21c5b16f0888a2f26a76932f9a1e86f7a8284bf8 Mon Sep 17 00:00:00 2001 From: Alexander Temirov Date: Wed, 7 Oct 2026 10:31:50 -0700 Subject: [PATCH 08/20] Apply Isaac RTX camera settings in the renderer, without a new base method Isaac RTX marks RTX sensor rendering when it is created, which cameras do during scene construction, and enables Gaussian HDR routing in prepare_cameras for cameras that read HDR color. BaseRenderer.apply_camera_settings() and the Camera call to it are removed. Signed-off-by: Alexander Temirov --- ...atemirov-rgb-radiance-groundwork.added.rst | 2 -- ...emirov-rgb-radiance-groundwork.changed.rst | 4 +-- .../isaaclab/renderers/base_renderer.py | 12 ------- .../isaaclab/sensors/camera/camera.py | 2 -- ...emirov-rgb-radiance-groundwork.changed.rst | 5 +-- .../renderers/isaac_rtx_renderer.py | 19 ++++------ .../test_isaac_rtx_renderer_contract.py | 36 ++++++++++++++----- 7 files changed, 39 insertions(+), 41 deletions(-) diff --git a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst index d5643fae2cee..96bf99b197ab 100644 --- a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst +++ b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst @@ -1,5 +1,3 @@ * Added the ``rgb_radiance`` camera output: scene-linear RGB before camera exposure and response, in renderer-relative intensity units. When ``rgb_hdr`` and ``rgb_radiance`` are both requested, they share one buffer. -* Added :meth:`~isaaclab.renderers.BaseRenderer.apply_camera_settings` so renderers apply the - process-wide settings their cameras need before ``sim.reset()``. diff --git a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst index d4c2cb2f8dd1..29ca2a5cd245 100644 --- a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst +++ b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst @@ -1,2 +1,2 @@ -* Changed :class:`~isaaclab.sensors.camera.Camera` to apply renderer-specific settings through - :meth:`~isaaclab.renderers.BaseRenderer.apply_camera_settings` instead of checking the renderer type. +* Changed :class:`~isaaclab.sensors.camera.Camera` to no longer set renderer-specific settings; the + renderer applies them. diff --git a/source/isaaclab/isaaclab/renderers/base_renderer.py b/source/isaaclab/isaaclab/renderers/base_renderer.py index d42828eae7b6..3cea8c3f1db2 100644 --- a/source/isaaclab/isaaclab/renderers/base_renderer.py +++ b/source/isaaclab/isaaclab/renderers/base_renderer.py @@ -20,7 +20,6 @@ import torch import warp as wp - from ..sensors.camera.camera_cfg import CameraCfg from ..sensors.camera.camera_data import CameraData from ..utils.warp import ProxyArray @@ -52,17 +51,6 @@ def visual_material_writer(self) -> Callable[[tuple[VisualMaterialBatch, ...]], """ return None - def apply_camera_settings(self, camera_cfg: CameraCfg) -> None: - """Apply process-wide settings that a camera rendered by this renderer needs. - - Cameras call this when constructed, before ``sim.reset()``, because the simulation and - environments read some of these settings during setup. The default applies no settings. - - Args: - camera_cfg: Configuration of the camera being constructed. - """ - return - def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: """Pre-render per-camera setup the backend needs. diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 61a7475a1f48..a062fe33973a 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -255,8 +255,6 @@ def __init__(self, cfg: CameraCfg): self._renderer: BaseRenderer | None = None if sim_ctx is not None: self._renderer = sim_ctx.get_or_create_backend(self.cfg.renderer_cfg) - # The simulation and environments read some renderer settings during setup. - self._renderer.apply_camera_settings(self.cfg) with force_log_level(logging.INFO): logger.info("Using renderer: %s", type(self._renderer).__name__) # Render data — assigned in _initialize_impl. diff --git a/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst b/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst index 7c06df9da287..809a540dfddf 100644 --- a/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst +++ b/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst @@ -1,2 +1,3 @@ -* Changed :class:`~isaaclab_physx.renderers.IsaacRtxRenderer` to set the RTX sensor and HDR routing settings - for its cameras in :meth:`~isaaclab_physx.renderers.IsaacRtxRenderer.apply_camera_settings`. +* Changed :class:`~isaaclab_physx.renderers.IsaacRtxRenderer` to mark RTX sensor rendering when it is created + and to route Gaussian HDR in :meth:`~isaaclab_physx.renderers.IsaacRtxRenderer.prepare_cameras` for cameras + that request HDR color, instead of :class:`~isaaclab.sensors.camera.Camera` setting them. diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index a1a8a5a79ccf..3507061486cf 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -47,7 +47,6 @@ from omni.replicator.core.scripts.utils.viewport_manager import HydraTexture - from isaaclab.sensors.camera import CameraCfg from isaaclab.sensors.camera.camera_data import CameraData from isaaclab.utils.warp import ProxyArray @@ -200,20 +199,11 @@ def __init__(self, cfg: IsaacRtxRendererCfg): apply_isaac_rtx_global_settings(self.cfg.global_settings, settings) if settings.get("/isaaclab/render/deterministic", False): apply_isaac_rtx_determinism_settings(settings) - ensure_rtx_hydra_engine_attached() - - def apply_camera_settings(self, camera_cfg: CameraCfg) -> None: - """Mark the run as rendering RTX sensors and route HDR color for linear camera outputs. - - See :meth:`~isaaclab.renderers.base_renderer.BaseRenderer.apply_camera_settings`. - """ - settings = get_settings_manager() + # Cameras create this renderer during scene construction, before the simulation and + # environments read whether RTX sensors are rendered. settings.set("/isaaclab/render/rtx_sensors", True) settings.set("/physics/fabricUpdateTransformations", True) - needs_hdr = camera_cfg.isp_cfg is not None or not {"rgb_hdr", "rgb_radiance"}.isdisjoint(camera_cfg.data_types) - if needs_hdr: - # Gaussian splats otherwise skip tonemapping and never reach the HdrColor AOV. - settings.set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) + ensure_rtx_hydra_engine_attached() def initialize(self) -> None: """Bind shared Fabric destinations after scene creation.""" @@ -248,6 +238,9 @@ def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: camera_prim_path = spec.camera_prim_paths[0] if spec.camera_prim_paths else None spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) + if spec.cfg.isp_cfg is not None or not _HDR_DATA_TYPES.isdisjoint(spec.cfg.data_types): + # Gaussian splats otherwise skip tonemapping and never reach the HdrColor AOV. + get_settings_manager().set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) needs_neutral_exposure = spec.cfg.isp_cfg is not None or "rgb_radiance" in spec.cfg.data_types if needs_neutral_exposure and spec.camera_prim_paths: _apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py index 3efd53d25700..156da3c2a1b9 100644 --- a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py @@ -87,23 +87,43 @@ def test_native_fabric_geometry_needs_no_separate_publication(monkeypatch): provider.get_geometry_points.assert_not_called() +def test_renderer_marks_rtx_sensor_rendering_when_created(monkeypatch): + """Creating the renderer, which cameras do during scene construction, marks RTX sensor rendering.""" + _install_omni_stubs(monkeypatch) + import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + settings = MagicMock() + settings.get.return_value = False + with ( + patch.object(rtx_renderer, "get_settings_manager", return_value=settings), + patch.object(rtx_renderer, "enable_extension"), + patch.object(rtx_renderer, "apply_isaac_rtx_global_settings"), + patch.object(rtx_renderer, "ensure_rtx_hydra_engine_attached"), + ): + rtx_renderer.IsaacRtxRenderer(IsaacRtxRendererCfg()) + + settings.set.assert_any_call("/isaaclab/render/rtx_sensors", True) + settings.set.assert_any_call("/physics/fabricUpdateTransformations", True) + + @pytest.mark.parametrize( ("data_types", "isp_cfg", "routes_hdr"), [(["rgb"], None, False), (["rgb_hdr"], None, True), (["rgb_radiance"], None, True), (["rgb"], object(), True)], ) -def test_camera_settings_mark_rtx_sensors_and_route_hdr(monkeypatch, data_types, isp_cfg, routes_hdr): - """Isaac RTX cameras flag RTX sensors before reset and route Gaussian HDR only when linear color is used.""" +def test_prepare_cameras_routes_gaussian_hdr_for_hdr_cameras(monkeypatch, data_types, isp_cfg, routes_hdr): + """Gaussian HDR routing is enabled only for cameras that read HDR color.""" _install_omni_stubs(monkeypatch) import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer settings = MagicMock() - with patch.object(rtx_renderer, "get_settings_manager", return_value=settings): - rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer).apply_camera_settings( - SimpleNamespace(data_types=data_types, isp_cfg=isp_cfg) - ) + spec = SimpleNamespace(cfg=SimpleNamespace(data_types=data_types, isp_cfg=isp_cfg), camera_prim_paths=()) + with ( + patch.object(rtx_renderer, "get_settings_manager", return_value=settings), + patch("isaaclab_ppisp.resolve_and_normalize", side_effect=lambda cfg, *args: cfg), + ): + rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer).prepare_cameras(None, spec) - settings.set.assert_any_call("/isaaclab/render/rtx_sensors", True) - settings.set.assert_any_call("/physics/fabricUpdateTransformations", True) hdr_call = call("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) assert (hdr_call in settings.set.call_args_list) is routes_hdr From 17ddfca6b334911f0a045c16a4f9167c1ef67afd Mon Sep 17 00:00:00 2001 From: Alexander Temirov Date: Fri, 9 Oct 2026 12:35:13 -0700 Subject: [PATCH 09/20] Keep renderer preparation in the camera lifecycle --- .../cosmos-camera-review.changed.rst | 4 ++ .../isaaclab/sensors/camera/camera.py | 68 ++++++++++++++----- .../isaaclab/isaaclab/sensors/sensor_base.py | 27 +------- .../test/sensors/test_opencv_distortion.py | 2 +- .../test/sensors/test_sensor_destructors.py | 1 + 5 files changed, 57 insertions(+), 45 deletions(-) create mode 100644 source/isaaclab/changelog.d/cosmos-camera-review.changed.rst diff --git a/source/isaaclab/changelog.d/cosmos-camera-review.changed.rst b/source/isaaclab/changelog.d/cosmos-camera-review.changed.rst new file mode 100644 index 000000000000..accfee135856 --- /dev/null +++ b/source/isaaclab/changelog.d/cosmos-camera-review.changed.rst @@ -0,0 +1,4 @@ +* Inlined camera render specifications at initialization so renderer inputs use the existing simulation context + and remain visible at their call sites. +* Kept renderer stage preparation in the camera lifecycle instead of registering an empty preparation hook + for every sensor. All cameras still apply USD overrides before the shared stage is exported. diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index a062fe33973a..3ef2c6ce0f81 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -7,6 +7,7 @@ import logging import sys +import weakref from collections.abc import Sequence from typing import TYPE_CHECKING, Any, Literal, cast @@ -18,6 +19,7 @@ from ... import sim as sim_utils from ...app.logging_utils import force_log_level +from ...physics import PhysicsEvent, PhysicsManager from ...renderers import BaseRenderer, CameraRenderSpec from ...sim.views import FrameView from ...utils.math import ( @@ -206,6 +208,7 @@ def __init__(self, cfg: CameraCfg): RuntimeError: If no camera prim is found at the given path. ValueError: If the provided data types are not supported by the camera or active renderer. """ + self._prepare_renderer_handle = None # perform check on supported data types self._check_supported_data_types(cfg) # initialize base class @@ -257,9 +260,9 @@ def __init__(self, cfg: CameraCfg): self._renderer = sim_ctx.get_or_create_backend(self.cfg.renderer_cfg) with force_log_level(logging.INFO): logger.info("Using renderer: %s", type(self._renderer).__name__) - # Render data — assigned in _initialize_impl. + # Render data — assigned in _initialize_impl. self._render_data = None - # Frame view — assigned in _initialize_impl. + # Frame view — assigned in _initialize_impl. self._view: FrameView | None = None def __del__(self, _sys=sys): @@ -622,10 +625,16 @@ def _initialize_impl(self): # Per-camera USD setup must run **before** ``ensure_prepare_stage`` so renderers that snapshot # the stage (ovrtx's ``stage.Export``) capture the overrides in their exported USD. Every camera - # already did this in :meth:`_prepare_initialize_impl`; repeating it is harmless and covers + # already did this in :meth:`_prepare_renderer_callback`; repeating it is harmless and covers # cameras initialized without that callback. - render_spec = self._make_render_spec(self._num_envs) - cam_paths = render_spec.camera_prim_paths + cam_paths = tuple(str(p.GetPath()) for p in sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) + render_spec = CameraRenderSpec( + cfg=self.cfg, + device=str(sim_ctx.device), + num_instances=self._num_envs, + camera_prim_paths=cam_paths, + view_count=self._num_envs, + ) self._renderer.prepare_cameras(self.stage, render_spec) # Stage preprocessing must happen before creating the view because the view keeps @@ -664,17 +673,6 @@ def _initialize_impl(self): # Create internal buffers (includes intrinsic matrix and pose init) self._create_buffers() - def _make_render_spec(self, num_views: int) -> CameraRenderSpec: - """Describe this camera's prims and requested outputs to the renderer.""" - cam_paths = tuple(str(p.GetPath()) for p in sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) - return CameraRenderSpec( - cfg=self.cfg, - device=str(sim_utils.SimulationContext.instance().device), - num_instances=num_views, - camera_prim_paths=cam_paths, - view_count=num_views, - ) - def _prepare_camera(self, env_mask: wp.array) -> None: """Advance capture frames and refresh requested poses before rendering.""" if self.cfg.update_latest_camera_pose: @@ -1007,12 +1005,38 @@ def _env_mask_has_any(self, env_mask: wp.array) -> bool: Internal simulation callbacks. """ - def _prepare_initialize_impl(self): + def _register_callbacks(self): + """Register camera stage preparation before the ordinary sensor initialization callbacks.""" + super()._register_callbacks() + physics_mgr_cls = sim_utils.SimulationContext.instance().physics_manager + obj_ref = weakref.proxy(self) + + def _prepare(event): + obj_ref._prepare_renderer_callback(event) + + # All cameras must author their USD overrides before the first camera exports the shared stage. + # Sensor initialization runs at order 10. Preserve weak ownership and backend exception forwarding. + self._prepare_renderer_handle = physics_mgr_cls.register_callback( + lambda payload: PhysicsManager.safe_callback_invoke(_prepare, payload, physics_manager=physics_mgr_cls), + PhysicsEvent.PHYSICS_READY, + order=9, + ) + + def _clear_callbacks(self) -> None: + """Release the camera preparation subscription along with the ordinary sensor subscriptions.""" + if self._prepare_renderer_handle is not None: + self._prepare_renderer_handle.deregister() + self._prepare_renderer_handle = None + super()._clear_callbacks() + + def _prepare_renderer_callback(self, event): """Apply this camera's renderer USD overrides before any camera initializes. A renderer that exports the stage when the first camera initializes, such as OVRTX, then sees the overrides of every camera. """ + if self._is_initialized: + return sim_ctx = sim_utils.SimulationContext.instance() if self._renderer is None: self._renderer = sim_ctx.get_or_create_backend(self.cfg.renderer_cfg) @@ -1022,7 +1046,15 @@ def _prepare_initialize_impl(self): num_views = len(clone_plan.topology.world_prototype_layout) else: num_views = len(sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) - self._renderer.prepare_cameras(self.stage, self._make_render_spec(num_views)) + cam_paths = tuple(str(p.GetPath()) for p in sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) + render_spec = CameraRenderSpec( + cfg=self.cfg, + device=str(sim_ctx.device), + num_instances=num_views, + camera_prim_paths=cam_paths, + view_count=num_views, + ) + self._renderer.prepare_cameras(self.stage, render_spec) def _invalidate_initialize_callback(self, event): """Invalidates the scene elements.""" diff --git a/source/isaaclab/isaaclab/sensors/sensor_base.py b/source/isaaclab/isaaclab/sensors/sensor_base.py index b683515f28f7..b23b1fc43036 100644 --- a/source/isaaclab/isaaclab/sensors/sensor_base.py +++ b/source/isaaclab/isaaclab/sensors/sensor_base.py @@ -55,7 +55,6 @@ def __init__(self, cfg: SensorBaseCfg): cfg: The configuration parameters for the sensor. """ # start with unset callback handles so cleanup is safe if construction fails part way - self._prepare_initialize_handle = None self._initialize_handle = None self._invalidate_initialize_handle = None self._prim_deletion_handle = None @@ -344,15 +343,7 @@ def _register_callbacks(self): def _invoke(callback_name, event): getattr(obj_ref, callback_name)(event) - # Backend-agnostic: PHYSICS_READY (prepare, then init) and STOP (invalidate). Every sensor prepares - # before any sensor initializes, so sensors can set up shared state, such as a renderer's stage, together. - self._prepare_initialize_handle = physics_mgr_cls.register_callback( - lambda payload: PhysicsManager.safe_callback_invoke( - _invoke, "_prepare_initialize_callback", payload, physics_manager=physics_mgr_cls - ), - PhysicsEvent.PHYSICS_READY, - order=9, - ) + # Backend-agnostic: PHYSICS_READY initializes sensors; STOP invalidates them. self._initialize_handle = physics_mgr_cls.register_callback( lambda payload: PhysicsManager.safe_callback_invoke( _invoke, "_initialize_callback", payload, physics_manager=physics_mgr_cls @@ -381,19 +372,6 @@ def _invoke(callback_name, event): IsaacEvents.PRIM_DELETION, ) - def _prepare_initialize_callback(self, event): - """Prepares the sensor before any sensor initializes.""" - if not self._is_initialized: - self._prepare_initialize_impl() - - def _prepare_initialize_impl(self): - """Prepare state that the initialization of other sensors depends on. - - Runs for every sensor on :attr:`PhysicsEvent.PHYSICS_READY`, before any sensor's - :meth:`_initialize_impl`. The default does nothing. - """ - pass - def _initialize_callback(self, event): """Initializes the scene elements. @@ -425,9 +403,6 @@ def _on_prim_deletion(self, event) -> None: def _clear_callbacks(self) -> None: """Clears the callbacks.""" - if self._prepare_initialize_handle is not None: - self._prepare_initialize_handle.deregister() - self._prepare_initialize_handle = None if self._initialize_handle is not None: self._initialize_handle.deregister() self._initialize_handle = None diff --git a/source/isaaclab/test/sensors/test_opencv_distortion.py b/source/isaaclab/test/sensors/test_opencv_distortion.py index ef53a991fde3..6eb473d085f6 100644 --- a/source/isaaclab/test/sensors/test_opencv_distortion.py +++ b/source/isaaclab/test/sensors/test_opencv_distortion.py @@ -192,7 +192,7 @@ def _camera_for_prims(prims, width=640, height=480, device="cpu"): fake._ALL_INDICES = wp.array(np.arange(len(prims)), dtype=wp.int32, device=device) fake._ALL_ENV_MASK = wp.ones(len(prims), dtype=wp.bool, device=device) # attributes touched by ``__del__``/``_clear_callbacks`` when the fake object is garbage collected - fake._prepare_initialize_handle = None + fake._prepare_renderer_handle = None fake._initialize_handle = None fake._invalidate_initialize_handle = None fake._prim_deletion_handle = None diff --git a/source/isaaclab/test/sensors/test_sensor_destructors.py b/source/isaaclab/test/sensors/test_sensor_destructors.py index a5d057d9b8f3..99f910f60927 100644 --- a/source/isaaclab/test/sensors/test_sensor_destructors.py +++ b/source/isaaclab/test/sensors/test_sensor_destructors.py @@ -77,6 +77,7 @@ def cleanup(self, _render_data): view = _View() renderer = _Renderer() camera = object.__new__(Camera) + camera._prepare_renderer_handle = None camera._view = view camera._renderer = renderer camera._render_data = None From 16c623f1ee8e0aa8f4679f849fd9260efd6207c3 Mon Sep 17 00:00:00 2001 From: Alexander Temirov Date: Sun, 11 Oct 2026 19:48:11 -0700 Subject: [PATCH 10/20] Fix the camera comment encoding and the rgb_radiance changelog Restore the em dash in two Camera comments, mark the exposure-override removal as breaking, and drop a fragment that described review iterations rather than a change against develop. --- source/isaaclab/changelog.d/cosmos-camera-review.changed.rst | 4 ---- source/isaaclab/isaaclab/sensors/camera/camera.py | 4 ++-- .../changelog.d/atemirov-rgb-radiance-groundwork.removed.rst | 4 ++-- 3 files changed, 4 insertions(+), 8 deletions(-) delete mode 100644 source/isaaclab/changelog.d/cosmos-camera-review.changed.rst diff --git a/source/isaaclab/changelog.d/cosmos-camera-review.changed.rst b/source/isaaclab/changelog.d/cosmos-camera-review.changed.rst deleted file mode 100644 index accfee135856..000000000000 --- a/source/isaaclab/changelog.d/cosmos-camera-review.changed.rst +++ /dev/null @@ -1,4 +0,0 @@ -* Inlined camera render specifications at initialization so renderer inputs use the existing simulation context - and remain visible at their call sites. -* Kept renderer stage preparation in the camera lifecycle instead of registering an empty preparation hook - for every sensor. All cameras still apply USD overrides before the shared stage is exported. diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 9a7c4aa141b3..1861aa096152 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -260,9 +260,9 @@ def __init__(self, cfg: CameraCfg): self._renderer = sim_ctx.get_or_create_backend(self.cfg.renderer_cfg) with force_log_level(logging.INFO): logger.info("Using renderer: %s", type(self._renderer).__name__) - # Render data — assigned in _initialize_impl. + # Render data — assigned in _initialize_impl. self._render_data = None - # Frame view — assigned in _initialize_impl. + # Frame view — assigned in _initialize_impl. self._view: FrameView | None = None def __del__(self, _sys=sys): diff --git a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst index db306a43dada..e1794508e8b2 100644 --- a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst +++ b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst @@ -1,2 +1,2 @@ -* Removed ``isaaclab_ppisp.apply_rtx_exposure_overrides``. The RTX renderers now apply the exposure - override themselves when a camera requests ``rgb_radiance``. +* **Breaking:** Removed ``isaaclab_ppisp.apply_rtx_exposure_overrides``. The RTX renderers now apply the + exposure override themselves when a camera requests ``rgb_radiance``. From bcb22ec9ead8b3db3ba18978a9226b32c8108826 Mon Sep 17 00:00:00 2001 From: Alexander Temirov Date: Mon, 12 Oct 2026 13:57:15 -0700 Subject: [PATCH 11/20] Deprecate the PPISP RTX exposure helper instead of removing it isaaclab_ppisp.apply_rtx_exposure_overrides shipped as public API. Keep it, and its submodule path, as a wrapper that raises a DeprecationWarning and applies the renderers' override, which now runs automatically for cameras with PPISP or rgb_radiance. --- ...rov-rgb-radiance-groundwork.deprecated.rst | 3 ++ ...=> atemirov-rgb-radiance-groundwork.minor} | 0 ...emirov-rgb-radiance-groundwork.removed.rst | 2 -- .../isaaclab_ppisp/__init__.pyi | 2 ++ .../isaaclab_ppisp/rtx_camera_overrides.py | 33 +++++++++++++++++++ .../test_deprecated_rtx_exposure_overrides.py | 27 +++++++++++++++ 6 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.deprecated.rst rename source/isaaclab_ppisp/changelog.d/{atemirov-rgb-radiance-groundwork.major => atemirov-rgb-radiance-groundwork.minor} (100%) delete mode 100644 source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst create mode 100644 source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py create mode 100644 source/isaaclab_ppisp/test/test_deprecated_rtx_exposure_overrides.py diff --git a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.deprecated.rst b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.deprecated.rst new file mode 100644 index 000000000000..8e57f7b4d6c9 --- /dev/null +++ b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.deprecated.rst @@ -0,0 +1,3 @@ +* Deprecated ``isaaclab_ppisp.apply_rtx_exposure_overrides``. The Isaac RTX and OVRTX renderers now apply the + exposure override themselves to cameras with PPISP or an ``rgb_radiance`` output; remove calls to it. It raises a + ``DeprecationWarning`` and will be removed in a future release. diff --git a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.major b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.minor similarity index 100% rename from source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.major rename to source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.minor diff --git a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst deleted file mode 100644 index e1794508e8b2..000000000000 --- a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst +++ /dev/null @@ -1,2 +0,0 @@ -* **Breaking:** Removed ``isaaclab_ppisp.apply_rtx_exposure_overrides``. The RTX renderers now apply the - exposure override themselves when a camera requests ``rgb_radiance``. diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi index 476c537b9420..2d69e60325e9 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi +++ b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi @@ -10,6 +10,7 @@ __all__ = [ "PpispPipeline", "apply_ppisp_to_rgba", "apply_ppisp_to_rgba_with_controller_params", + "apply_rtx_exposure_overrides", "auto_any_ppisp_cfg", "auto_camera_ppisp_cfg", "compute_ppisp_controller_params", @@ -36,3 +37,4 @@ from .cfg import ( ) from .kernels import apply_ppisp_to_rgba, apply_ppisp_to_rgba_with_controller_params, compute_ppisp_controller_params from .pipeline import PpispPipeline +from .rtx_camera_overrides import apply_rtx_exposure_overrides diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py b/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py new file mode 100644 index 000000000000..9e70959d11ef --- /dev/null +++ b/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py @@ -0,0 +1,33 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Deprecated RTX camera exposure overrides; the RTX renderers now apply them themselves.""" + +from __future__ import annotations + +import warnings +from typing import Any + + +def apply_rtx_exposure_overrides(stage: Any, prim_paths: list[str]) -> None: + """Pin RTX-side exposure to neutral and apply the RTX exposure API schemas on each camera prim. + + .. deprecated:: 3.0 + The Isaac RTX and OVRTX renderers apply these overrides themselves to cameras with PPISP or an + ``rgb_radiance`` output. Remove the call. This function will be removed in a future release. + + Args: + stage: USD stage containing the camera prims. + prim_paths: Resolved camera prim paths. + """ + warnings.warn( + "isaaclab_ppisp.apply_rtx_exposure_overrides is deprecated: the Isaac RTX and OVRTX renderers apply it " + "themselves to cameras with PPISP or an rgb_radiance output. Remove the call.", + DeprecationWarning, + stacklevel=2, + ) + from isaaclab.renderers.rtx_camera_overrides import _apply_rtx_exposure_overrides + + _apply_rtx_exposure_overrides(stage, prim_paths) diff --git a/source/isaaclab_ppisp/test/test_deprecated_rtx_exposure_overrides.py b/source/isaaclab_ppisp/test/test_deprecated_rtx_exposure_overrides.py new file mode 100644 index 000000000000..82a3808839c6 --- /dev/null +++ b/source/isaaclab_ppisp/test/test_deprecated_rtx_exposure_overrides.py @@ -0,0 +1,27 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""The deprecated PPISP entry point to the RTX camera exposure overrides.""" + +import importlib + +import pytest + +from pxr import Sdf, Usd, UsdGeom + + +@pytest.mark.parametrize("module", ["isaaclab_ppisp", "isaaclab_ppisp.rtx_camera_overrides"]) +def test_the_deprecated_exposure_override_warns_and_still_neutralizes_exposure(module): + """Callers of the former public helper keep working from both import paths, with a warning to remove the call.""" + apply = importlib.import_module(module).apply_rtx_exposure_overrides + stage = Usd.Stage.CreateInMemory() + prim = UsdGeom.Camera.Define(stage, "/World/Camera").GetPrim() + prim.CreateAttribute("exposure:iso", Sdf.ValueTypeNames.Float).Set(800.0) + + with pytest.warns(DeprecationWarning, match="renderers apply it themselves"): + apply(stage, ["/World/Camera"]) + + assert prim.GetAttribute("exposure:iso").Get() == 0.0 + assert prim.GetAttribute("omni:rtx:autoExposure:enabled").Get() is False From 3fac60313c13ba9edfe31521f1ad3d329c4d386f Mon Sep 17 00:00:00 2001 From: Alexander Temirov Date: Mon, 12 Oct 2026 14:04:37 -0700 Subject: [PATCH 12/20] Keep the PPISP RTX exposure helper removal as a clean break This reverts commit bcb22ec9e. Reviewers asked for a clean break without compatibility shims; the changelog marks the removal as breaking with its migration. --- ...rov-rgb-radiance-groundwork.deprecated.rst | 3 -- ...=> atemirov-rgb-radiance-groundwork.major} | 0 ...emirov-rgb-radiance-groundwork.removed.rst | 2 ++ .../isaaclab_ppisp/__init__.pyi | 2 -- .../isaaclab_ppisp/rtx_camera_overrides.py | 33 ------------------- .../test_deprecated_rtx_exposure_overrides.py | 27 --------------- 6 files changed, 2 insertions(+), 65 deletions(-) delete mode 100644 source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.deprecated.rst rename source/isaaclab_ppisp/changelog.d/{atemirov-rgb-radiance-groundwork.minor => atemirov-rgb-radiance-groundwork.major} (100%) create mode 100644 source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst delete mode 100644 source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py delete mode 100644 source/isaaclab_ppisp/test/test_deprecated_rtx_exposure_overrides.py diff --git a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.deprecated.rst b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.deprecated.rst deleted file mode 100644 index 8e57f7b4d6c9..000000000000 --- a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.deprecated.rst +++ /dev/null @@ -1,3 +0,0 @@ -* Deprecated ``isaaclab_ppisp.apply_rtx_exposure_overrides``. The Isaac RTX and OVRTX renderers now apply the - exposure override themselves to cameras with PPISP or an ``rgb_radiance`` output; remove calls to it. It raises a - ``DeprecationWarning`` and will be removed in a future release. diff --git a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.minor b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.major similarity index 100% rename from source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.minor rename to source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.major diff --git a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst new file mode 100644 index 000000000000..e1794508e8b2 --- /dev/null +++ b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst @@ -0,0 +1,2 @@ +* **Breaking:** Removed ``isaaclab_ppisp.apply_rtx_exposure_overrides``. The RTX renderers now apply the + exposure override themselves when a camera requests ``rgb_radiance``. diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi index 2d69e60325e9..476c537b9420 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi +++ b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi @@ -10,7 +10,6 @@ __all__ = [ "PpispPipeline", "apply_ppisp_to_rgba", "apply_ppisp_to_rgba_with_controller_params", - "apply_rtx_exposure_overrides", "auto_any_ppisp_cfg", "auto_camera_ppisp_cfg", "compute_ppisp_controller_params", @@ -37,4 +36,3 @@ from .cfg import ( ) from .kernels import apply_ppisp_to_rgba, apply_ppisp_to_rgba_with_controller_params, compute_ppisp_controller_params from .pipeline import PpispPipeline -from .rtx_camera_overrides import apply_rtx_exposure_overrides diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py b/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py deleted file mode 100644 index 9e70959d11ef..000000000000 --- a/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Deprecated RTX camera exposure overrides; the RTX renderers now apply them themselves.""" - -from __future__ import annotations - -import warnings -from typing import Any - - -def apply_rtx_exposure_overrides(stage: Any, prim_paths: list[str]) -> None: - """Pin RTX-side exposure to neutral and apply the RTX exposure API schemas on each camera prim. - - .. deprecated:: 3.0 - The Isaac RTX and OVRTX renderers apply these overrides themselves to cameras with PPISP or an - ``rgb_radiance`` output. Remove the call. This function will be removed in a future release. - - Args: - stage: USD stage containing the camera prims. - prim_paths: Resolved camera prim paths. - """ - warnings.warn( - "isaaclab_ppisp.apply_rtx_exposure_overrides is deprecated: the Isaac RTX and OVRTX renderers apply it " - "themselves to cameras with PPISP or an rgb_radiance output. Remove the call.", - DeprecationWarning, - stacklevel=2, - ) - from isaaclab.renderers.rtx_camera_overrides import _apply_rtx_exposure_overrides - - _apply_rtx_exposure_overrides(stage, prim_paths) diff --git a/source/isaaclab_ppisp/test/test_deprecated_rtx_exposure_overrides.py b/source/isaaclab_ppisp/test/test_deprecated_rtx_exposure_overrides.py deleted file mode 100644 index 82a3808839c6..000000000000 --- a/source/isaaclab_ppisp/test/test_deprecated_rtx_exposure_overrides.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""The deprecated PPISP entry point to the RTX camera exposure overrides.""" - -import importlib - -import pytest - -from pxr import Sdf, Usd, UsdGeom - - -@pytest.mark.parametrize("module", ["isaaclab_ppisp", "isaaclab_ppisp.rtx_camera_overrides"]) -def test_the_deprecated_exposure_override_warns_and_still_neutralizes_exposure(module): - """Callers of the former public helper keep working from both import paths, with a warning to remove the call.""" - apply = importlib.import_module(module).apply_rtx_exposure_overrides - stage = Usd.Stage.CreateInMemory() - prim = UsdGeom.Camera.Define(stage, "/World/Camera").GetPrim() - prim.CreateAttribute("exposure:iso", Sdf.ValueTypeNames.Float).Set(800.0) - - with pytest.warns(DeprecationWarning, match="renderers apply it themselves"): - apply(stage, ["/World/Camera"]) - - assert prim.GetAttribute("exposure:iso").Get() == 0.0 - assert prim.GetAttribute("omni:rtx:autoExposure:enabled").Get() is False From d5de923ea9f8f2c03663168ed3feb5d44e0b9904 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Wed, 14 Oct 2026 14:50:20 -0400 Subject: [PATCH 13/20] Add shared render buffer pipes for cameras and observations --- docs/source/api/lab/isaaclab.envs.mdp.rst | 3 + docs/source/api/lab/isaaclab.utils.rst | 14 + docs/source/concepts/sensors/camera.rst | 164 +++++++++--- examples/sensors/ppisp_camera.py | 4 +- .../changelog.d/render-buffer-pipe.added.rst | 6 + .../changelog.d/render-buffer-pipe.major | 0 .../render-buffer-pipe.removed.rst | 3 + .../isaaclab/envs/manager_based_env.py | 10 + .../isaaclab/isaaclab/envs/mdp/__init__.pyi | 2 + .../isaaclab/envs/mdp/visual_observations.py | 72 ++++++ .../isaaclab/managers/manager_base.py | 13 + .../isaaclab/managers/observation_manager.py | 73 +++++- .../isaaclab/sensors/camera/camera.py | 217 +++++++++++++--- .../isaaclab/sensors/camera/camera_cfg.py | 52 ++-- .../isaaclab/sensors/camera/camera_data.py | 6 + .../isaaclab/sensors/camera/camera_isp.py | 17 +- source/isaaclab/isaaclab/utils/images.py | 243 +++++++++++++++++- .../isaaclab/utils/modifiers/__init__.pyi | 5 +- .../isaaclab/utils/modifiers/modifier_base.py | 4 + .../isaaclab/utils/modifiers/modifier_cfg.py | 35 ++- .../utils/modifiers/render_buffer_pipe.py | 50 ++++ .../envs/test_render_buffer_observations.py | 228 ++++++++++++++++ .../renderers/test_camera_output_contract.py | 163 +++++++++++- .../generate_synthetic_gaussian_asset.py | 25 +- .../test_camera_ppisp_gaussian_ovrtx.py | 45 ++-- .../test/utils/test_render_buffer_pipe.py | 215 ++++++++++++++++ .../changelog.d/feat-render-buffer-pipe.major | 0 .../feat-render-buffer-pipe.removed.rst | 2 + .../renderers/newton_warp_renderer.py | 99 +------ .../test_newton_manager_abstraction.py | 2 +- .../changelog.d/feat-render-buffer-pipe.major | 0 .../feat-render-buffer-pipe.removed.rst | 2 + .../isaaclab_ov/renderers/ovrtx_renderer.py | 101 +------- .../isaaclab_ov/renderers/ovrtx_usd.py | 5 +- .../test/test_ovrtx_renderer_contract.py | 99 +++---- source/isaaclab_ov/test/test_ovrtx_usd.py | 22 +- .../test/test_ovrtx_visual_material_sync.py | 7 +- .../changelog.d/feat-render-buffer-pipe.major | 0 .../feat-render-buffer-pipe.removed.rst | 2 + .../renderers/isaac_rtx_renderer.py | 112 +------- .../test_isaac_rtx_renderer_contract.py | 26 +- .../changelog.d/render-buffer-pipe.added.rst | 3 + .../render-buffer-pipe.changed.rst | 3 + .../changelog.d/render-buffer-pipe.fixed.rst | 2 + .../changelog.d/render-buffer-pipe.minor | 0 source/isaaclab_ppisp/docs/README.md | 33 ++- .../isaaclab_ppisp/isaaclab_ppisp/__init__.py | 10 +- .../isaaclab_ppisp/__init__.pyi | 8 +- source/isaaclab_ppisp/isaaclab_ppisp/cfg.py | 47 ++-- .../isaaclab_ppisp/isaaclab_ppisp/kernels.py | 46 ++-- .../isaaclab_ppisp/isaaclab_ppisp/pipeline.py | 147 ++++++----- .../isaaclab_ppisp/processor.py | 91 +++++++ source/isaaclab_ppisp/test/test_ppisp.py | 5 +- .../isaaclab_ppisp/test/test_ppisp_kernels.py | 78 +++++- 54 files changed, 1948 insertions(+), 673 deletions(-) create mode 100644 source/isaaclab/changelog.d/render-buffer-pipe.added.rst create mode 100644 source/isaaclab/changelog.d/render-buffer-pipe.major create mode 100644 source/isaaclab/changelog.d/render-buffer-pipe.removed.rst create mode 100644 source/isaaclab/isaaclab/envs/mdp/visual_observations.py create mode 100644 source/isaaclab/isaaclab/utils/modifiers/render_buffer_pipe.py create mode 100644 source/isaaclab/test/envs/test_render_buffer_observations.py create mode 100644 source/isaaclab/test/utils/test_render_buffer_pipe.py create mode 100644 source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.major create mode 100644 source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.removed.rst create mode 100644 source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.major create mode 100644 source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.removed.rst create mode 100644 source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.major create mode 100644 source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.removed.rst create mode 100644 source/isaaclab_ppisp/changelog.d/render-buffer-pipe.added.rst create mode 100644 source/isaaclab_ppisp/changelog.d/render-buffer-pipe.changed.rst create mode 100644 source/isaaclab_ppisp/changelog.d/render-buffer-pipe.fixed.rst create mode 100644 source/isaaclab_ppisp/changelog.d/render-buffer-pipe.minor create mode 100644 source/isaaclab_ppisp/isaaclab_ppisp/processor.py diff --git a/docs/source/api/lab/isaaclab.envs.mdp.rst b/docs/source/api/lab/isaaclab.envs.mdp.rst index 9775f0593305..ab5db418b7b4 100644 --- a/docs/source/api/lab/isaaclab.envs.mdp.rst +++ b/docs/source/api/lab/isaaclab.envs.mdp.rst @@ -9,6 +9,9 @@ Observations .. automodule:: isaaclab.envs.mdp.observations :members: +.. automodule:: isaaclab.envs.mdp.visual_observations + :members: + Actions ------- diff --git a/docs/source/api/lab/isaaclab.utils.rst b/docs/source/api/lab/isaaclab.utils.rst index d9cdd0e5cfa2..91005ca604d1 100644 --- a/docs/source/api/lab/isaaclab.utils.rst +++ b/docs/source/api/lab/isaaclab.utils.rst @@ -171,6 +171,20 @@ Image operations .. autofunction:: isaaclab.utils.images.compose_image +.. autoclass:: isaaclab.utils.images.ImageBufferSpec + :members: + +.. autoclass:: isaaclab.utils.images.PreparedBufferOperation + :members: + +.. autoclass:: isaaclab.utils.images.RenderBufferStepCfg + :members: + :exclude-members: __init__ + +.. autoclass:: isaaclab.utils.images.RenderBufferPipe + :members: + :special-members: __call__ + Timer operations ~~~~~~~~~~~~~~~~ diff --git a/docs/source/concepts/sensors/camera.rst b/docs/source/concepts/sensors/camera.rst index db1bc0df8b90..208619caafbb 100644 --- a/docs/source/concepts/sensors/camera.rst +++ b/docs/source/concepts/sensors/camera.rst @@ -153,8 +153,9 @@ updating it adds frame-query overhead. Output types ------------ -The active renderer validates ``data_types`` and allocates the channel count and data type declared -by its :class:`~isaaclab.renderers.RenderBufferSpec`. +For raw outputs, the active renderer validates the requested names and allocates the channel count +and data type declared by its :class:`~isaaclab.renderers.RenderBufferSpec`. A camera pipeline may +produce additional output names; its operation factories declare their output contracts. .. list-table:: Common output contracts :header-rows: 1 @@ -326,47 +327,148 @@ background. Set a normalized RGB tuple to use a solid color for pixels that miss The setting is per camera. Cameras with renderer-default and solid backgrounds can coexist in one scene. -Post-render image signal processing ------------------------------------ +Post-render image processing +---------------------------- -:attr:`~sensors.CameraCfg.isp_cfg` optionally applies an image signal processing (ISP) pass to the -renderer's scene-linear HDR output. The shipped implementation is PPISP (Physically Plausible Image -Signal Processing), which applies responsivity, exposure, vignetting, color correction, and a camera -response function before writing ``rgb`` or ``rgba``. +:attr:`~sensors.CameraCfg.pipeline` applies an ordered sequence of operations to named image buffers. +Each :class:`~isaaclab.utils.images.RenderBufferStepCfg` maps operation input and output roles to buffer +names. A step can read several channels, such as radiance and depth, and later steps read the preceding +step's results. The same :class:`~isaaclab.utils.images.RenderBufferPipe` can run independently or serve +an observation consumer; operations receive explicit buffers and resources without discovering a camera, +environment, or simulation singleton. -The field accepts: - -* ``None`` to disable post-render ISP. -* :class:`~isaaclab_ppisp.PpispCfg` for explicit coefficients or coefficients imported from a USD - camera. -* :class:`~sensors.CameraISPMode` to discover ``ppisp:*`` attributes on a camera prim. +The camera resolves external inputs before renderer initialization, including inputs needed only by the +pipeline. It publishes the names in ``data_types`` after processing. For example, this camera requests +private ``rgb_radiance`` from its renderer and publishes PPISP's processed ``rgb``: .. code-block:: python - from isaaclab.sensors.camera import CameraCfg, CameraISPMode from isaaclab.utils import replace - from isaaclab_ppisp import PpispCfg + from isaaclab_ppisp import PpispCfg, PpispStepCfg - explicit_isp = replace(front_camera, data_types=["rgb"], isp_cfg=PpispCfg(inputs={"exposureOffset": 1.5})) + processed_camera = replace( + front_camera, + data_types=["rgb"], + pipeline=[PpispStepCfg(params={"cfg": PpispCfg(inputs={"exposureOffset": 1.5})})], + ) - discovered_isp = replace(front_camera, data_types=["rgb"], isp_cfg=CameraISPMode.AUTO_CAMERA) +PPISP (Physically Plausible Image Signal Processing) applies responsivity, exposure, vignetting, color +correction, and a camera response function. It consumes float32 scene-linear radiance and produces +uint8 color. Set ``outputs={"color": "rgba"}`` and ``params={"cfg": ppisp_cfg, "channels": 4}`` on the +step for RGBA output. A pipeline-produced RGBA buffer also supplies a zero-copy RGB view when ``rgb`` +is requested. A pipeline producing only RGB cannot satisfy an explicit RGBA request. + +For a custom operation, provide a factory ``prepare(input_specs, **params)`` returning +:class:`~isaaclab.utils.images.PreparedBufferOperation`. Input and output mappings inside the operation +use role names, while the step configuration owns camera-buffer names. Declare output shape, dtype, +device, layout, and encoding with :class:`~isaaclab.utils.images.ImageBufferSpec`; then write each +supplied output destination in ``apply(inputs, outputs)``. The pipe allocates these destinations once. +Encoding is declared by the producer rather than inferred from dtype or pixel values. Camera radiance +uses ``scene_linear``, RGB/RGBA uses ``camera_response``, and depth uses ``meters``. +Published camera outputs must keep the camera's batch size, height, width, and NHWC layout so pose +and calibration metadata remain valid. Intermediate buffers may change shape. Use ``processed_image`` +below when the final observation should have a different image shape or layout. + +``camera.render_outputs`` exposes borrowed raw renderer buffers, including private inputs, and +``camera.render_output_specs`` exposes their contracts after initialization. Reading raw outputs updates +the camera when due. Neither an operation nor a caller may mutate borrowed inputs. Public processed +buffers and raw buffers retain their storage until camera reinitialization, but later captures replace +their contents; clone values that must survive another capture. Each camera owns its pipe and operation +workspace. Operations may share explicitly supplied immutable model resources. +Configuration copying must preserve the identity of borrowed resources: PPISP provides +``PpispControllerWeights`` for device weights. Custom factories can capture a resource in a closure +or use a wrapper with identity-preserving ``__deepcopy__``; raw Warp allocations are not configuration values. + +Pipelines run after scalar and batched captures. When an asynchronous renderer republishes a capture, +the camera retains its processed result and matching capture metadata. Camera reset forwards only the +selected environment indices to operation reset callbacks; teardown closes the operations. For an +observation-only pipeline, declare raw dependencies with ``camera.request_render_inputs(names)`` before +simulation initialization. Additional requests after initialization fail because renderer setup is +already complete. Observation consumers use their own pipe and snapshot results needed across captures. + +Processing observation images +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use :class:`~isaaclab.envs.mdp.processed_image` for observation-only processing, including operations +that combine several camera outputs or change image dimensions: -``AUTO_CAMERA`` checks the first matched camera prim. ``AUTO_ANY`` falls back to the first PPISP -camera anywhere on the stage. Discovery happens once during camera construction. +.. code-block:: python + + from isaaclab.envs import mdp + from isaaclab.managers import ObservationTermCfg, SceneEntityCfg + from isaaclab_ppisp import PpispStepCfg + + policy_image = ObservationTermCfg( + func=mdp.processed_image, + params={ + "sensor_cfg": SceneEntityCfg("front_camera"), + "pipeline": [PpispStepCfg()], + "output_name": "rgb", + }, + ) + +Manager-based environments collect the term's private renderer inputs before simulation reset. A +standalone observation manager must call ``ObservationManager.prepare_scene(cfg, env)`` before reset. +The term processes raw renderer outputs even when the camera has its own pipeline. To read the +camera's published processed output, use an existing image term such as ``mdp.image_rgb`` instead. +The observation pipe executes on every term evaluation and returns a snapshot; its state belongs to +that term rather than the camera's capture cadence. + +For a single incoming tensor, :class:`~isaaclab.utils.modifiers.RenderBufferPipeModifierCfg` uses the +same operations while preserving preceding observation transformations. This form must preserve +shape; use ``processed_image`` for multiple camera inputs or shape changes. The declared dtype, +encoding, and layout describe the tensor reaching the modifier: + +.. code-block:: python -PPISP is composed by Isaac RTX, OVRTX, and Newton Warp. It requires ``rgb`` or ``rgba`` output. A -static configuration is shared by all cloned views in one camera batch; controller weights may -predict per-view exposure and color parameters, while the remaining coefficients stay shared. ISP -configuration and discovered USD attributes are fixed for the camera lifetime. + import warp as wp + from isaaclab.utils import modifiers + + # This camera publishes radiance for the ordinary image term to read. + raw_camera = replace(front_camera, data_types=["rgb_radiance"], pipeline=[]) + policy_image = ObservationTermCfg( + func=mdp.image_rgb, + params={"sensor_cfg": SceneEntityCfg("front_camera"), "data_type": "rgb_radiance", "normalize": False}, + modifiers=[ + modifiers.RenderBufferPipeModifierCfg( + pipeline=[PpispStepCfg()], + input_name="rgb_radiance", + output_name="rgb", + dtype=wp.float32, + encoding="scene_linear", + layout="NHWC", + ) + ], + ) + +Migrating renderer-owned ISP configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``CameraCfg.isp_cfg`` no longer executes image processing. A non-None value raises a migration error; +replace it with ``CameraCfg.pipeline`` as above. ``CameraISPMode`` remains importable for legacy +configuration values, but new code should resolve USD attributes explicitly before constructing the +operation: + +.. code-block:: python + + from isaaclab_ppisp import PpispDiscoveryMode, PpispStepCfg, resolve_and_normalize + + ppisp_cfg = resolve_and_normalize(PpispDiscoveryMode.AUTO_CAMERA, stage, "/World/Camera") + discovered_camera = replace( + front_camera, + data_types=["rgb"], + pipeline=[PpispStepCfg(params={"cfg": ppisp_cfg})] if ppisp_cfg is not None else [], + ) -.. important:: +``AUTO_CAMERA`` checks the supplied camera prim; ``AUTO_ANY`` can fall back to the first PPISP camera +on the supplied stage. Alternatively, pass an already resolved camera prim to +``ppisp_cfg_from_usd_camera(camera_prim)``. Discovery has no implicit simulation or stage lookup inside +the operation. Static coefficients apply to the whole camera batch; controller weights may predict +per-view exposure and color parameters. - With Isaac RTX and OVRTX, enabling ``isp_cfg`` makes PPISP the ISP authority. The renderer - disables RTX auto-exposure, authors neutral ``exposure:*`` values, and applies the - ``OmniRtxCameraAutoExposureAPI_1`` and ``OmniRtxCameraExposureAPI_1`` schemas on every matched - camera prim so RTX does not process the image a second time. Do not combine ``isp_cfg`` with - separately authored RTX exposure or tonemapping settings. When ``isp_cfg`` is ``None``, the - renderer leaves authored camera exposure unchanged. +Requesting ``rgb_radiance``, including a private pipeline request, still neutralizes camera exposure on +Isaac RTX and OVRTX as described in :ref:`camera-supported-annotators`. Processing configuration itself +does not change renderer settings. Newton Warp has no exposure model. Run the ``ppisp-camera`` example for a complete PPISP workflow. It supports ``--renderer newton_renderer`` (Newton Warp, the default) and ``--renderer isaac_rtx`` (Isaac RTX). diff --git a/examples/sensors/ppisp_camera.py b/examples/sensors/ppisp_camera.py index 79e95688eb37..c2fcbf1b0c6d 100644 --- a/examples/sensors/ppisp_camera.py +++ b/examples/sensors/ppisp_camera.py @@ -120,7 +120,7 @@ import matplotlib.pyplot as plt import numpy as np import torch -from isaaclab_ppisp.cfg import PpispCfg, ppisp_cfg_from_usd_camera +from isaaclab_ppisp import PpispCfg, PpispStepCfg, ppisp_cfg_from_usd_camera import isaaclab.sim as sim_utils from isaaclab.assets import AssetBaseCfg, RigidObjectCfg @@ -374,7 +374,7 @@ def make_camera(camera_prim_path: str, *, ppisp_cfg: PpispCfg | None, width: int width=width, data_types=["rgb"], spawn=None, - isp_cfg=ppisp_cfg, + pipeline=[PpispStepCfg(params={"cfg": ppisp_cfg})] if ppisp_cfg is not None else [], renderer_cfg=make_renderer_cfg(), ) return instantiate(camera_cfg) diff --git a/source/isaaclab/changelog.d/render-buffer-pipe.added.rst b/source/isaaclab/changelog.d/render-buffer-pipe.added.rst new file mode 100644 index 000000000000..9a0b38138ef2 --- /dev/null +++ b/source/isaaclab/changelog.d/render-buffer-pipe.added.rst @@ -0,0 +1,6 @@ +* Added ``RenderBufferPipe`` and prepared operations over named image buffers, with explicit + shape, dtype, device, layout and encoding contracts, persistent destinations, and operation lifecycle hooks. +* Added ``CameraCfg.pipeline``, the ``processed_image`` observation term, and + ``RenderBufferPipeModifierCfg`` so cameras and observations use the same processing operations. + Observation dependencies are collected before renderer initialization; private inputs do not need + to be exposed as public camera outputs. diff --git a/source/isaaclab/changelog.d/render-buffer-pipe.major b/source/isaaclab/changelog.d/render-buffer-pipe.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab/changelog.d/render-buffer-pipe.removed.rst b/source/isaaclab/changelog.d/render-buffer-pipe.removed.rst new file mode 100644 index 000000000000..77f723ffa1d4 --- /dev/null +++ b/source/isaaclab/changelog.d/render-buffer-pipe.removed.rst @@ -0,0 +1,3 @@ +* Retired execution through ``CameraCfg.isp_cfg``. Configure a ``PpispStepCfg`` in + ``CameraCfg.pipeline`` or an observation pipe instead. Resolve USD-authored PPISP parameters + explicitly before configuring the step. Non-None legacy ``isp_cfg`` values raise a migration error. diff --git a/source/isaaclab/isaaclab/envs/manager_based_env.py b/source/isaaclab/isaaclab/envs/manager_based_env.py index f4026af3bd43..51fdbbb4fd8c 100644 --- a/source/isaaclab/isaaclab/envs/manager_based_env.py +++ b/source/isaaclab/isaaclab/envs/manager_based_env.py @@ -187,6 +187,8 @@ def _init_sim(self): if "prestartup" in self.event_manager.available_modes: self.event_manager.apply(mode="prestartup") + ObservationManager.prepare_scene(self.cfg.observations, self) + # play the simulator to activate physics handles # note: this activates the physics simulation view that exposes TensorAPIs # note: when started in extension mode, first call sim.reset_async() and then initialize the managers @@ -609,6 +611,12 @@ def close(self): # destructor is order-sensitive del self.action_manager + observation_error = None + try: + self.observation_manager.close() + except Exception as error: + # A user operation must not prevent scene and simulation teardown. + observation_error = error del self.observation_manager del self.event_manager del self.recorder_manager @@ -622,6 +630,8 @@ def close(self): self._window = None # update closing status self._is_closed = True + if observation_error is not None: + raise observation_error """ Helper functions. diff --git a/source/isaaclab/isaaclab/envs/mdp/__init__.pyi b/source/isaaclab/isaaclab/envs/mdp/__init__.pyi index 4bcfd3c0d824..226011e391b9 100644 --- a/source/isaaclab/isaaclab/envs/mdp/__init__.pyi +++ b/source/isaaclab/isaaclab/envs/mdp/__init__.pyi @@ -89,6 +89,7 @@ __all__ = [ "image_normals", "image_rgb", "image_segmentation", + "processed_image", "imu_ang_vel", "imu_lin_acc", "pva_orientation", @@ -270,6 +271,7 @@ from .observations import ( root_pos_w, root_quat_w, ) +from .visual_observations import processed_image from .recorders import ( ActionStateRecorderManagerCfg, InitialStateRecorder, diff --git a/source/isaaclab/isaaclab/envs/mdp/visual_observations.py b/source/isaaclab/isaaclab/envs/mdp/visual_observations.py new file mode 100644 index 000000000000..dc664dcaaf9e --- /dev/null +++ b/source/isaaclab/isaaclab/envs/mdp/visual_observations.py @@ -0,0 +1,72 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Observation adapters for renderer-independent image processing.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING + +import torch + +from isaaclab.managers import ManagerTermBase, ObservationTermCfg, SceneEntityCfg +from isaaclab.utils.images import RenderBufferPipe, RenderBufferStepCfg + +if TYPE_CHECKING: + from isaaclab.envs import ManagerBasedEnv + + +class processed_image(ManagerTermBase): + """Process named raw camera outputs with the shared render buffer pipe. + + Dependencies are requested before simulation initialization through + :meth:`~isaaclab.managers.ObservationManager.prepare_scene`. They remain private to the + renderer buffer bundle and need not appear in ``CameraCfg.data_types``. Each term owns + its own operation state and output storage. The pipe runs on every observation evaluation; + returned tensors are snapshots, so later evaluations cannot overwrite retained history. + + Use this term for multiple inputs or changed image dimensions. To process the result of + earlier observation modifiers instead, use :class:`~isaaclab.utils.modifiers.RenderBufferPipeModifier`. + To read a camera's already processed public output, use the existing image observation terms. + """ + + @classmethod + def prepare_scene(cls, cfg: ObservationTermCfg, env: ManagerBasedEnv) -> None: + sensor_cfg = cfg.params.get("sensor_cfg", SceneEntityCfg("camera")) + output_name = cfg.params.get("output_name", "rgb") + requirements = RenderBufferPipe.required_source_names(cfg.params["pipeline"], [output_name]) + env.scene.sensors[sensor_cfg.name].request_render_inputs(requirements) + + def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedEnv): + super().__init__(cfg, env) + self._camera = env.scene.sensors[cfg.params["sensor_cfg"].name] + self._output_name = cfg.params["output_name"] + self._pipe = RenderBufferPipe(self._camera.render_output_specs, cfg.params["pipeline"]) + if self._output_name not in self._pipe.output_specs: + self._pipe.close() + raise ValueError(f"Image processing does not provide output {self._output_name!r}.") + + def __call__( + self, + env: ManagerBasedEnv, + pipeline: Sequence[RenderBufferStepCfg], + sensor_cfg: SceneEntityCfg = SceneEntityCfg("camera"), + output_name: str = "rgb", + *, + out: torch.Tensor | None = None, + ) -> torch.Tensor: + """Return a processed image in the layout, dtype and shape declared by its final operation. + + The observation manager supplies ``out`` to receive the snapshot without another clone. + Direct callers may omit it to allocate a new snapshot. + """ + return self._pipe.as_tensor(self._camera.render_outputs, output_name=self._output_name, out=out) + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + self._pipe.reset(env_ids) + + def close(self) -> None: + self._pipe.close() diff --git a/source/isaaclab/isaaclab/managers/manager_base.py b/source/isaaclab/isaaclab/managers/manager_base.py index 1ca1435f6c76..1411374f2d32 100644 --- a/source/isaaclab/isaaclab/managers/manager_base.py +++ b/source/isaaclab/isaaclab/managers/manager_base.py @@ -90,6 +90,19 @@ def __name__(self) -> str: Operations. """ + @classmethod + def prepare_scene(cls, cfg: ManagerTermBaseCfg, env: ManagerBasedEnv) -> None: + """Declare scene requirements before simulation initialization. + + This hook may request sensor outputs but must not read runtime data or allocate + processing buffers. Observation terms are visited by :meth:`ObservationManager.prepare_scene`. + """ + pass + + def close(self) -> None: + """Release resources owned by the term before the scene closes.""" + pass + def reset(self, env_ids: Sequence[int] | None = None) -> None: """Resets the manager term. diff --git a/source/isaaclab/isaaclab/managers/observation_manager.py b/source/isaaclab/isaaclab/managers/observation_manager.py index a4ab4fefd3f4..4f520715232e 100644 --- a/source/isaaclab/isaaclab/managers/observation_manager.py +++ b/source/isaaclab/isaaclab/managers/observation_manager.py @@ -8,7 +8,9 @@ from __future__ import annotations import inspect -from collections.abc import Sequence +from builtins import ExceptionGroup +from collections.abc import Iterator, Sequence +from dataclasses import fields from typing import TYPE_CHECKING import numpy as np @@ -16,10 +18,10 @@ from prettytable import PrettyTable from ..envs.utils.io_descriptors import _warn_io_descriptors_deprecated -from ..utils import instantiate, modifiers, noise, to_dict +from ..utils import instantiate, modifiers, noise, string_to_callable, to_dict from ..utils.buffers import CircularBuffer, DelayBuffer from .manager_base import ManagerBase, ManagerTermBase -from .manager_term_cfg import ObservationGroupCfg, ObservationTermCfg +from .manager_term_cfg import ManagerTermBaseCfg, ObservationGroupCfg, ObservationTermCfg if TYPE_CHECKING: from ..envs import ManagerBasedEnv @@ -85,6 +87,7 @@ def __init__(self, cfg: object, env: ManagerBasedEnv): if cfg is None: raise ValueError("Observation manager configuration is None. Please provide a valid configuration.") + self._is_closed = False # call the base class constructor (this will parse the terms config) super().__init__(cfg, env) @@ -120,6 +123,54 @@ def __init__(self, cfg: object, env: ManagerBasedEnv): # Stores the latest observations. self._obs_buffer: dict[str, torch.Tensor | dict[str, torch.Tensor]] | None = None + @classmethod + def prepare_scene(cls, cfg: object, env: ManagerBasedEnv) -> None: + """Collect observation input requirements before renderer initialization. + + Manager-based environments call this after creating the scene and before resetting + the simulation. Standalone users must call it at the same point when terms require + private sensor outputs. Term instances and processing storage are created later. + """ + if cfg is None: + return + groups = cfg.values() if isinstance(cfg, dict) else vars(cfg).values() + for group in groups: + if not isinstance(group, ObservationGroupCfg): + continue + for term in _iter_term_configs(group): + if not isinstance(term, ManagerTermBaseCfg): + continue + func = string_to_callable(term.func) if isinstance(term.func, str) else term.func + if inspect.isclass(func) and issubclass(func, ManagerTermBase): + func.prepare_scene(term, env) + + def close(self) -> None: + """Close every owned term and modifier once, including nested configurations. + + All resources are closed before any callback errors are raised. Repeated calls are safe. + """ + if self._is_closed: + return + self._is_closed = True + resources = [] + seen = set() + for cfg in _iter_term_configs(self._group_obs_term_cfgs): + resource = cfg.func + if isinstance(resource, (ManagerTermBase, modifiers.ModifierBase)) and id(resource) not in seen: + seen.add(id(resource)) + resources.append(resource) + errors = [] + for resource in reversed(resources): + try: + resource.close() + except Exception as error: + errors.append(error) + self._obs_buffer = None + if len(errors) == 1: + raise errors[0] + if errors: + raise ExceptionGroup("Failed to close observation resources.", errors) + def __str__(self) -> str: """Returns: A string representation for the observation manager.""" msg = f" contains {len(self._group_obs_term_names)} groups.\n" @@ -770,3 +821,19 @@ def _prepare_terms(self): # add history buffers for each group self._group_obs_term_delay_buffer[group_name] = group_entry_delay_buffer self._group_obs_term_history_buffer[group_name] = group_entry_history_buffer + + +def _iter_term_configs(value: object) -> Iterator[ManagerTermBaseCfg | modifiers.ModifierCfg]: + """Visit manager-owned configurations without following term instances back into their configs.""" + if isinstance(value, (ManagerTermBaseCfg, modifiers.ModifierCfg, ObservationGroupCfg)): + if isinstance(value, (ManagerTermBaseCfg, modifiers.ModifierCfg)): + yield value + for field in fields(value): + if field.name != "func": + yield from _iter_term_configs(getattr(value, field.name)) + elif isinstance(value, dict): + for child in value.values(): + yield from _iter_term_configs(child) + elif isinstance(value, (list, tuple)): + for child in value: + yield from _iter_term_configs(child) diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 1861aa096152..cc46b2ec9a54 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -8,7 +8,10 @@ import logging import sys import weakref -from collections.abc import Sequence +from collections.abc import Mapping, Sequence +from copy import copy +from dataclasses import replace +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Literal, cast import numpy as np @@ -22,6 +25,7 @@ from ...physics import PhysicsEvent, PhysicsManager from ...renderers import BaseRenderer, CameraRenderSpec from ...sim.views import FrameView +from ...utils.images import ImageBufferSpec, RenderBufferPipe from ...utils.math import ( convert_camera_frame_orientation_convention, create_rotation_matrix_from_view, @@ -248,6 +252,13 @@ def __init__(self, cfg: CameraCfg): if sim_ctx is not None: sim_ctx.require_visual_shapes() + # Renderer requirements include private pipeline inputs and exclude produced public outputs. + self._render_cfg = replace(self.cfg, data_types=self.cfg.render_data_types(), pipeline=[]) + self._buffer_pipe: RenderBufferPipe | None = None + self._pipeline_capture: object | None = None + self._render_camera_data: CameraData | None = None + self._render_outputs: dict[str, ProxyArray] = {} + self._render_output_specs: dict[str, ImageBufferSpec] = {} # UsdGeom Camera prim for the sensor self._sensor_prims: list[UsdGeom.Camera] = [] # Allocated in :meth:`_create_buffers` once the renderer's output contract is known. @@ -273,16 +284,19 @@ def __del__(self, _sys=sys): """ if _sys.is_finalizing() or _sys.meta_path is None: return - # unsubscribe callbacks - super().__del__() - - # release the frame view's backend state, getattr covers partial initialization - if getattr(self, "_view", None) is not None: - self._view.close() - self._view = None - # cleanup render resources (renderer may be None if never initialized) - if getattr(self, "_renderer", None) is not None: - self._renderer.cleanup(getattr(self, "_render_data", None)) + try: + # unsubscribe callbacks + super().__del__() + + # release the frame view's backend state, getattr covers partial initialization + if getattr(self, "_view", None) is not None: + self._view.close() + self._view = None + # cleanup render resources (renderer may be None if never initialized) + if getattr(self, "_renderer", None) is not None: + self._renderer.cleanup(getattr(self, "_render_data", None)) + finally: + self._close_pipeline() def __str__(self) -> str: """Returns: A string containing information about the instance.""" @@ -322,6 +336,42 @@ def data(self) -> CameraData: # return the data return self._data + @property + def render_outputs(self) -> Mapping[str, ProxyArray]: + """Borrowed renderer buffers before the camera pipeline, including private pipeline inputs. + + Access updates the camera when due. Arrays must not be modified by callers, and their contents + are replaced by later captures. Copy any values that must outlive the next camera update. + """ + self._update_outdated_buffers() + return MappingProxyType(self._render_outputs) + + @property + def render_output_specs(self) -> Mapping[str, ImageBufferSpec]: + """Resolved raw image contracts, available after camera initialization and treated as read-only.""" + return MappingProxyType(self._render_output_specs) + + def request_render_inputs(self, names: Sequence[str]) -> None: + """Request private renderer buffers before camera initialization. + + Observation pipelines use this during scene preparation. Inputs already requested are a no-op; + adding outputs after initialization raises because renderer setup and USD export are complete. + These requests do not change the camera's public :attr:`CameraData.output` selection. + """ + additional = [ + name for name in names if name not in self._render_cfg.data_types and name not in self._render_outputs + ] + if not additional: + return + if self._is_initialized: + raise RuntimeError("Camera renderer inputs must be requested before initialization.") + render_cfg = replace( + self._render_cfg, data_types=list(dict.fromkeys([*self._render_cfg.data_types, *additional])) + ) + self._check_supported_data_types(render_cfg) + render_cfg.validate_config() + self._render_cfg = render_cfg + @property def frame(self) -> ProxyArray: """Frame number when the measurement took place.""" @@ -609,6 +659,9 @@ def reset(self, env_ids: Sequence[int] | slice | None = None, env_mask: wp.array self._renderer.reset(self._render_data, env_ids) # reset the timestamps super().reset(env_ids, env_mask) + if self._buffer_pipe is not None: + pipeline_env_ids = wp.to_torch(env_mask).nonzero().squeeze(-1) if env_mask is not None else env_ids + self._buffer_pipe.reset(pipeline_env_ids) # reset the data # note: this recomputation is useful if one performs events such as randomizations on the camera poses. if env_mask is not None: @@ -651,7 +704,7 @@ def _initialize_impl(self): # cameras initialized without that callback. cam_paths = tuple(str(p.GetPath()) for p in sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) render_spec = CameraRenderSpec( - cfg=self.cfg, + cfg=self._render_cfg, device=str(sim_ctx.device), num_instances=self._num_envs, camera_prim_paths=cam_paths, @@ -701,7 +754,7 @@ def _prepare_camera(self, env_mask: wp.array) -> None: self._update_poses(env_mask=env_mask, frame_op=1) else: self._update_camera_state(env_mask=env_mask, frame_op=1) - self._renderer.prepare_capture(self._render_data, self._data, self._frame) + self._renderer.prepare_capture(self._render_data, self._render_camera_data, self._frame) def _update_buffers_impl(self, env_mask: wp.array): if not self._env_mask_has_any(env_mask): @@ -715,12 +768,13 @@ def _update_buffers_impl(self, env_mask: wp.array): sim_ctx.render_context.render_into_camera( renderer, self._render_data, - self._data, + self._render_camera_data, sim_ctx.get_physics_step_count(), ) else: renderer.render(self._render_data) - renderer.read_output(self._render_data, self._data) + renderer.read_output(self._render_data, self._render_camera_data) + self._apply_pipeline() @staticmethod def _update_buffers_batch_impl(sensors: Sequence[SensorBase]) -> None: @@ -742,9 +796,41 @@ def _update_buffers_batch_impl(sensors: Sequence[SensorBase]) -> None: return sim_ctx.render_context.render_into_cameras( - [(camera._renderer, camera._render_data, camera._data) for camera in ready], + [(camera._renderer, camera._render_data, camera._render_camera_data) for camera in ready], sim_ctx.get_physics_step_count(), ) + for camera in ready: + camera._apply_pipeline() + + def _close_pipeline(self) -> None: + """Release pipeline resources, including partially initialized cameras.""" + pipe = getattr(self, "_buffer_pipe", None) + self._buffer_pipe = None + if pipe is not None: + pipe.close() + + def _apply_pipeline(self) -> None: + """Process a new capture once and publish metadata matching its processed pixels.""" + if self._buffer_pipe is None: + if self._data is not self._render_camera_data: + for name in self._data.output: + self._data.info[name] = self._render_camera_data.info.get(name) + return + info = self._render_camera_data.info + capture = next( + (value["capture"] for value in info.values() if isinstance(value, dict) and "capture" in value), None + ) + if capture is not None and capture is self._pipeline_capture: + return + self._buffer_pipe(self._render_outputs) + self._pipeline_capture = capture + for name in self._data.output: + if name in self._buffer_pipe.produced_outputs or ( + name == "rgb" and "rgba" in self._buffer_pipe.produced_outputs + ): + self._data.info[name] = {"capture": capture} if capture is not None else None + else: + self._data.info[name] = info.get(name) """ Private Helpers @@ -760,7 +846,7 @@ def _check_supported_data_types(self, cfg: CameraCfg): ) # check if there is any intersection in unsupported types # reason: these use np structured data types which are not compatible with the camera buffer contract - common_elements = set(cfg.data_types) & Camera.UNSUPPORTED_TYPES + common_elements = set(cfg.render_data_types()) & Camera.UNSUPPORTED_TYPES if common_elements: # provide alternative fast counterparts fast_common_elements = [] @@ -783,7 +869,7 @@ def _create_buffers(self): known: list[str] = [] unknown: list[str] = [] unsupported: list[str] = [] - for name in self.cfg.data_types: + for name in self._render_cfg.data_types: try: if RenderBufferKind(name) in specs: known.append(name) @@ -815,7 +901,75 @@ def _create_buffers(self): self._data.create_buffers(self._view.count, str(self._device)) self._initialize_intrinsics() self._update_poses() - self._renderer.set_outputs(self._render_data, self._data.output) + self._render_camera_data = self._data + self._render_outputs = self._render_camera_data.output + self._renderer.set_outputs(self._render_data, self._render_outputs) + encodings = { + "rgb_radiance": "scene_linear", + "rgb": "camera_response", + "rgba": "camera_response", + "depth": "meters", + "distance_to_camera": "meters", + "distance_to_image_plane": "meters", + } + self._render_output_specs = { + name: ImageBufferSpec( + shape=buffer.shape, + dtype=buffer.dtype, + device=str(buffer.device), + encoding=encodings.get(name, "unknown"), + ) + for name, buffer in self._render_outputs.items() + } + self._close_pipeline() + self._pipeline_capture = None + if self.cfg.pipeline: + self._buffer_pipe = RenderBufferPipe(self._render_output_specs, self.cfg.pipeline) + try: + outputs = {**self._render_outputs, **self._buffer_pipe.outputs} + alias_rgb = ( + "rgba" in self._buffer_pipe.produced_outputs and "rgb" not in self._buffer_pipe.produced_outputs + ) + expected_shape = (self._view.count, self.cfg.height, self.cfg.width) + for name in self.cfg.data_types: + spec = self._buffer_pipe.output_specs["rgba" if alias_rgb and name == "rgb" else name] + if spec.layout != "NHWC" or len(spec.shape) != 4 or spec.shape[:3] != expected_shape: + raise ValueError( + f"Camera pipeline output {name!r} must use NHWC with batch and spatial shape" + f" {expected_shape}; received {spec.layout} {spec.shape}. Use an observation pipe" + " for shape-changing results, because camera calibration describes the rendered image." + ) + # Preserve RGB's view semantics when an operation produces RGBA, without publishing stale raw RGB. + if alias_rgb: + rgba = outputs["rgba"].warp + if len(rgba.shape) != 4 or rgba.shape[-1] != 4: + raise ValueError("A pipeline output named rgba must have shape (N, H, W, 4).") + rgb = wp.array( + ptr=rgba.ptr, + shape=(*rgba.shape[:-1], 3), + strides=rgba.strides, + dtype=rgba.dtype, + device=rgba.device, + copy=False, + ) + outputs["rgb"] = ProxyArray(rgb) + self._data = copy(self._render_camera_data) + self._data._output = {name: outputs[name] for name in self.cfg.data_types} + self._data.info = dict.fromkeys(self._data.output) + except Exception as error: + try: + self._close_pipeline() + except Exception as cleanup_error: + error.add_note(f"Failed to close camera pipeline after initialization failed: {cleanup_error}") + raise + elif set(self._render_cfg.data_types) != set(self.cfg.data_types): + # Observation-only dependencies remain private even when this camera has no own pipeline. + public = set(self.cfg.data_types) + if not public.isdisjoint({"rgb", "rgba"}): + public.update({"rgb", "rgba"}) + self._data = copy(self._render_camera_data) + self._data._output = {name: buffer for name, buffer in self._render_outputs.items() if name in public} + self._data.info = dict.fromkeys(self._data.output) def _read_authored_opencv_intrinsics( self, prim: Usd.Prim, width: int, height: int, env_id: int @@ -1070,7 +1224,7 @@ def _prepare_renderer_callback(self, event): num_views = len(sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) cam_paths = tuple(str(p.GetPath()) for p in sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) render_spec = CameraRenderSpec( - cfg=self.cfg, + cfg=self._render_cfg, device=str(sim_ctx.device), num_instances=num_views, camera_prim_paths=cam_paths, @@ -1080,13 +1234,16 @@ def _prepare_renderer_callback(self, event): def _invalidate_initialize_callback(self, event): """Invalidates the scene elements.""" - if self._renderer is not None and self._render_data is not None: - self._renderer.cleanup(self._render_data) - self._render_data = None - self._renderer = None - # call parent - super()._invalidate_initialize_callback(event) - # release backend state deterministically, then invalidate the view - if self._view is not None: - self._view.close() - self._view = None + try: + self._close_pipeline() + finally: + if self._renderer is not None and self._render_data is not None: + self._renderer.cleanup(self._render_data) + self._render_data = None + self._renderer = None + # call parent + super()._invalidate_initialize_callback(event) + # release backend state deterministically, then invalidate the view + if self._view is not None: + self._view.close() + self._view = None diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py index e52396484d34..8984a805fd8c 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py @@ -13,9 +13,9 @@ from ...sim import FisheyeCameraCfg, PinholeCameraCfg from ...utils import configclass from ..sensor_base_cfg import SensorBaseCfg -from .camera_isp import CameraISPMode if TYPE_CHECKING: + from ...utils.images import RenderBufferStepCfg from .camera import Camera # Default values for the RTX-flavored fields kept on :class:`CameraCfg` for @@ -199,28 +199,37 @@ class OffsetCfg: renderer_cfg: RendererCfg = field(default_factory=RendererCfg) """Renderer configuration for camera sensor.""" - isp_cfg: Any | CameraISPMode | None = None - """Post-render ISP cfg applied by the renderer backend after it produces HDR output. + pipeline: list[RenderBufferStepCfg] = [] + """Ordered image operations applied after each newly published capture. - Defaults to ``None`` (ISP disabled). Auto-discovery is opt-in via a - :class:`CameraISPMode` sentinel — see below. + Steps bind operation roles to named buffers and may consume several render outputs or preceding + results. The renderer is configured with the pipeline's external dependencies before initialization. + Only :attr:`data_types` are published in :attr:`CameraData.output`; unprocessed inputs remain + available through :attr:`Camera.render_outputs`. Operations borrow their inputs read-only and + own persistent result buffers. Camera reset and teardown reset and close the operations. + """ - Accepted values: + isp_cfg: Any | None = None + """Retired renderer-owned ISP setting, retained to report a migration error. - * ``None`` — ISP disabled. No HDR AOV is requested and no RTX-side - tonemapping flags are flipped. - * A :class:`CameraISPMode` sentinel — the renderer backend walks the USD stage to - discover an ISP shader (e.g. via the :mod:`isaaclab_ppisp` package). - * A concrete ISP cfg dataclass (e.g. :class:`isaaclab_ppisp.PpispCfg`) — used directly. + Non-None values raise :class:`ValueError`. Configure :attr:`pipeline` with a PPISP step instead. + """ - The cfg applies once per Camera sensor batch. The PPISP Warp kernel takes - scalar coefficients, so every cloned view in a tiled batch shares the same - ISP configuration — there is no per-view ISP today. + def render_data_types(self) -> list[str]: + """Resolve raw renderer inputs, including requested outputs not produced by the pipeline.""" + if self.isp_cfg is not None: + raise ValueError( + "CameraCfg.isp_cfg has been retired. Configure CameraCfg.pipeline with a PPISP step instead." + ) + if not self.pipeline: + return list(self.data_types) + from ...utils.images import RenderBufferPipe - :mod:`isaaclab.sensors.camera` does not depend on any ISP implementation; the - annotation is intentionally loose (``Any``) so the sensor layer can carry the - cfg through to a renderer that knows what to do with it. - """ + produced = {name for step in self.pipeline for name in step.outputs.values()} + if "rgb" in produced and "rgba" not in produced and "rgba" in self.data_types: + raise ValueError("A pipeline producing rgb must also produce rgba when rgba is requested.") + published = [name for name in self.data_types if not (name == "rgb" and "rgba" in produced)] + return sorted(RenderBufferPipe.required_source_names(self.pipeline, published)) def __post_init__(self): """Forward deprecated RTX-flavored fields onto :attr:`renderer_cfg`. @@ -229,6 +238,10 @@ def __post_init__(self): :class:`DeprecationWarning` and is copied onto ``self.renderer_cfg`` when that cfg defines the same-named field. """ + if self.isp_cfg is not None: + raise ValueError( + "CameraCfg.isp_cfg has been retired. Configure CameraCfg.pipeline with a PPISP step instead." + ) renderer_type = getattr(self.renderer_cfg, "renderer_type", None) if renderer_type == "default": from ...utils.backend_utils import get_default_renderer_cfg @@ -253,13 +266,14 @@ def __post_init__(self): def validate_config(self) -> None: """Validate the requested data types against the selected renderer contract.""" + raw_names = self.render_data_types() if self.renderer_cfg is None: return supported_specs = self.renderer_cfg.supported_output_types() if supported_specs is None: return supported = {str(kind) for kind in supported_specs} - unsupported = sorted(set(self.data_types) - supported) + unsupported = sorted(set(raw_names) - supported) if unsupported: raise ValueError( f"Renderer {type(self.renderer_cfg).__name__} only supports data types {sorted(supported)}, " diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_data.py b/source/isaaclab/isaaclab/sensors/camera/camera_data.py index 92d6d613c02e..7f99f0a7ad58 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_data.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_data.py @@ -104,6 +104,12 @@ def output(self) -> dict[str, ProxyArray] | None: and C is the number of channels. Use ``.torch`` for a ``torch.Tensor`` view or ``.warp`` for the underlying ``wp.array``. + When :attr:`CameraCfg.pipeline` is configured, values are the pipeline results selected by + :attr:`CameraCfg.data_types`. Results preserve NHWC layout, batch size and image dimensions + so camera calibration remains applicable; channel count and dtype may change. Use an observation + pipe for results with different dimensions. Buffers are reused at the next capture. Unprocessed + inputs are available from :attr:`Camera.render_outputs`. + The format of the data is available in the `Replicator Documentation`_. For semantic-based data, this corresponds to the ``"data"`` key in the output of the sensor. diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_isp.py b/source/isaaclab/isaaclab/sensors/camera/camera_isp.py index 8934198bbcf8..17a2256f6ffe 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_isp.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_isp.py @@ -3,12 +3,10 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Sentinel enum for :attr:`~isaaclab.sensors.camera.CameraCfg.isp_cfg`. +"""Legacy ISP discovery names retained for migration of saved configurations. -The renderer backend (which depends on the appropriate ISP implementation -package) is responsible for resolving these sentinels to a concrete cfg and -applying the ISP pipeline. :mod:`isaaclab.sensors.camera` carries the sentinel -through to the renderer without knowing about any specific ISP implementation. +``CameraCfg.isp_cfg`` no longer executes processing. Use ``CameraCfg.pipeline`` and +the discovery configuration provided by the image-processing package instead. """ from __future__ import annotations @@ -17,14 +15,7 @@ class CameraISPMode(StrEnum): - """Sentinel modes for :attr:`~isaaclab.sensors.camera.CameraCfg.isp_cfg`. - - Selects how the renderer backend discovers the ISP configuration from the - USD stage at camera initialization time. Discovery resolves one cfg for - the whole Camera sensor batch, using the first matched camera prim as the - camera-bound lookup target. ``isp_cfg=None`` disables ISP entirely; this - enum carries only the *active* discovery strategies. - """ + """Retired camera ISP discovery modes; use the processing package's discovery configuration.""" AUTO_CAMERA = "auto_camera" """Discover an ISP cfg bound to the batch's first matched camera prim. diff --git a/source/isaaclab/isaaclab/utils/images.py b/source/isaaclab/isaaclab/utils/images.py index 58a419b74b4b..f326e4857737 100644 --- a/source/isaaclab/isaaclab/utils/images.py +++ b/source/isaaclab/isaaclab/utils/images.py @@ -19,13 +19,254 @@ from __future__ import annotations -from collections.abc import Callable, Sequence +from builtins import ExceptionGroup +from collections.abc import Callable, Mapping, Sequence +from dataclasses import MISSING, dataclass +from types import MappingProxyType from typing import Any import torch import warp as wp from .buffers import CircularBuffer +from .configclass import configclass +from .string import string_to_callable +from .warp import ProxyArray + + +@dataclass(frozen=True) +class ImageBufferSpec: + """Shape, storage and semantic encoding of one image-processing buffer. + + The producer declares the encoding: it cannot be inferred from pixel values or dtype. + ``shape`` includes the batch dimension. Borrowed arrays may be strided; operations must + either support their strides or explicitly copy to suitable storage. + """ + + shape: tuple[int, ...] + dtype: type + device: str + encoding: str = "unknown" + layout: str = "NHWC" + + def __post_init__(self): + if not self.shape or any(size <= 0 for size in self.shape): + raise ValueError("Image buffer dimensions must be positive.") + if self.layout == "NHWC" and len(self.shape) != 4: + raise ValueError("NHWC image buffers must have four dimensions.") + + def validate(self, name: str, value: ProxyArray) -> None: + """Validate the storage of a borrowed buffer before submitting work.""" + if ( + value.shape != self.shape + or value.dtype != self.dtype + or str(value.device) != str(wp.get_device(self.device)) + ): + raise ValueError(f"Buffer {name!r} must match shape, dtype and device {self}.") + + +@dataclass(frozen=True) +class PreparedBufferOperation: + """One operation bound to input contracts and resolved resources. + + ``apply`` receives input and destination dictionaries keyed by operation roles, not sensor + output names. Inputs are borrowed read-only by convention. The operation must fill each + supplied destination and enqueue GPU work on the current Warp stream. Output storage and + execution order belong to :class:`RenderBufferPipe`; private workspace belongs to the operation. + """ + + output_specs: Mapping[str, ImageBufferSpec] + apply: Callable[[Mapping[str, ProxyArray], Mapping[str, ProxyArray]], None] + reset: Callable[[Sequence[int] | None], None] | None = None + close: Callable[[], None] | None = None + + +@configclass +class RenderBufferStepCfg: + """Bind named buffers to the input/output roles of one processing operation. + + The factory ``prepare(input_specs, **params)`` runs once during pipe construction and returns + a :class:`PreparedBufferOperation`. ``input_specs`` is keyed by the roles in ``inputs``. + Factories receive concrete array contracts and explicitly supplied resources; they should not + discover cameras, environments, renderers or simulation singletons. + """ + + prepare: Callable[..., PreparedBufferOperation] | str = MISSING + """Operation factory or an import string in ``module:attribute`` form.""" + + inputs: dict[str, str] = MISSING + """Map operation input roles to buffer names read from the preceding stage.""" + + outputs: dict[str, str] = MISSING + """Map operation output roles to names created or replaced in the bundle.""" + + params: dict[str, Any] = {} + """Resolved operation parameters passed to the factory. + + Configurations are deep-copied by ``configclass`` and managers. Borrowed device allocations + or models need a resource wrapper that preserves identity through ``__deepcopy__``, or a + factory closure retaining the resource. Do not put a raw Warp allocation in this dictionary. + """ + + +class RenderBufferPipe: + """Apply an ordered sequence of operations to named :class:`ProxyArray` buffers. + + A pipe owns persistent output storage and prepared operations. Create one instance per + consumer; instances are not reentrant. Borrowed inputs are never modified by the pipe, and + untouched channels pass through by reference. Returned buffers remain valid until the next + call, reset or close; consumers retaining an observation must take a snapshot. + + Buffer dependencies are resolved in order, so an operation reads preceding replacements. + GPU execution uses the caller's current Warp stream without a device-wide synchronization. + The caller owns producer readiness and keeps borrowed storage alive until consumption ends. + + Args: + source_specs: Contracts for available external buffers. + steps: Ordered operation configurations. + """ + + def __init__(self, source_specs: Mapping[str, ImageBufferSpec], steps: Sequence[RenderBufferStepCfg]): + self.source_specs = MappingProxyType(dict(source_specs)) + self.required_inputs = self.required_source_names(steps) + self._operations: list[ + tuple[PreparedBufferOperation, dict[str, str], dict[str, str], dict[str, ProxyArray]] + ] = [] + self._closed = False + self.outputs: Mapping[str, ProxyArray] = MappingProxyType({}) + available = dict(source_specs) + final_outputs: dict[str, ProxyArray] = {} + try: + for step in steps: + inputs, outputs = dict(step.inputs), dict(step.outputs) + if not outputs or len(set(outputs.values())) != len(outputs): + raise ValueError("A processing step must bind distinct output names to its output roles.") + missing = set(inputs.values()) - available.keys() + if missing: + raise ValueError(f"Missing image buffer declarations: {sorted(missing)}.") + prepare = string_to_callable(step.prepare) if isinstance(step.prepare, str) else step.prepare + input_specs = {role: available[name] for role, name in inputs.items()} + operation = prepare(MappingProxyType(input_specs), **step.params) + if not isinstance(operation, PreparedBufferOperation): + raise TypeError("A buffer operation factory must return PreparedBufferOperation.") + destinations: dict[str, ProxyArray] = {} + # Register ownership before allocation/validation, so failure closes the operation. + self._operations.append((operation, inputs, outputs, destinations)) + if operation.output_specs.keys() != outputs.keys(): + raise ValueError("Operation output roles must match the step's output bindings.") + devices = { + str(wp.get_device(spec.device)) + for spec in (*input_specs.values(), *operation.output_specs.values()) + } + if len(devices) > 1: + raise ValueError("A buffer operation must use one device for its inputs and outputs.") + for role, name in outputs.items(): + spec = operation.output_specs[role] + destinations[role] = ProxyArray(wp.empty(spec.shape, dtype=spec.dtype, device=spec.device)) + available[name] = spec + final_outputs[name] = destinations[role] + except Exception as error: + try: + self.close() + except Exception as cleanup_error: + raise error from cleanup_error + raise + self.output_specs = MappingProxyType(available) + self.outputs = MappingProxyType(final_outputs) + self.produced_outputs = frozenset(final_outputs) + + @staticmethod + def required_source_names( + steps: Sequence[RenderBufferStepCfg], published_names: Sequence[str] = () + ) -> frozenset[str]: + """Resolve external names without constructing operations or allocating pixel storage. + + Call this before renderer initialization to request private processing inputs. Public + pass-through outputs also require a source, even when no operation reads them. + """ + required: set[str] = set() + produced: set[str] = set() + for step in steps: + required.update(set(step.inputs.values()) - produced) + produced.update(step.outputs.values()) + required.update(set(published_names) - produced) + return frozenset(required) + + def __call__(self, buffers: Mapping[str, ProxyArray]) -> dict[str, ProxyArray]: + """Execute the pipe on ready source buffers and return the resulting bundle.""" + if self._closed: + raise RuntimeError("Cannot execute a closed render buffer pipe.") + missing = self.required_inputs - buffers.keys() + if missing: + raise ValueError(f"Missing required image buffers: {sorted(missing)}.") + for name in self.required_inputs: + self.source_specs[name].validate(name, buffers[name]) + current = dict(buffers) + for operation, inputs, outputs, destinations in self._operations: + operation.apply({role: current[name] for role, name in inputs.items()}, destinations) + current.update({name: destinations[role] for role, name in outputs.items()}) + return current + + def apply_to_tensor( + self, + data: torch.Tensor, + *, + input_name: str, + output_name: str, + auxiliary_buffers: Mapping[str, ProxyArray] | None = None, + ) -> torch.Tensor: + """Process an incoming observation tensor and return an independently owned snapshot. + + The supplied tensor is authoritative: earlier observation transformations are preserved. + Auxiliary inputs must already be ready on the current Warp/Torch stream. CUDA work is + ordered with the caller's Torch stream; no host/device transfer is performed. + """ + if auxiliary_buffers is not None and input_name in auxiliary_buffers: + raise ValueError("Auxiliary buffers must not replace the incoming observation tensor.") + buffers = dict(auxiliary_buffers or {}) + buffers[input_name] = ProxyArray(wp.from_torch(data)) + return self.as_tensor(buffers, output_name=output_name) + + def as_tensor( + self, buffers: Mapping[str, ProxyArray], *, output_name: str, out: torch.Tensor | None = None + ) -> torch.Tensor: + """Execute on the current Torch stream and snapshot one output for an observation. + + Source buffers must be ready on the current Warp or Torch stream. Stream ordering + is established without a device-wide synchronization; no host/device copy is made. + A supplied ``out`` must match the selected output contract and is overwritten in place. + """ + if out is not None: + self.output_specs[output_name].validate("out", ProxyArray(wp.from_torch(out))) + device = wp.get_device(self.output_specs[output_name].device) + stream = wp.stream_from_torch(torch.cuda.current_stream(str(device))) if device.is_cuda else None + with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): + result = self(buffers)[output_name].torch + return result.clone() if out is None else out.copy_(result) + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + """Reset operation-owned state for selected batch entries.""" + for operation, *_ in self._operations: + if operation.reset is not None: + operation.reset(env_ids) + + def close(self) -> None: + """Release operation resources; repeated calls have no effect.""" + if self._closed: + return + self._closed = True + operations, self._operations = self._operations, [] + self.outputs = MappingProxyType({}) + errors = [] + for operation, *_ in reversed(operations): + if operation.close is not None: + try: + operation.close() + except Exception as error: + errors.append(error) + if errors: + raise ExceptionGroup("Failed to close render buffer operations.", errors) + _RGB_LIKE_PREFIXES: tuple[str, ...] = ("rgb", "albedo", "simple_shading") _DEPTH_LIKE_PATTERNS: tuple[str, ...] = ("depth", "distance_to") diff --git a/source/isaaclab/isaaclab/utils/modifiers/__init__.pyi b/source/isaaclab/isaaclab/utils/modifiers/__init__.pyi index e15f1284085c..7d4707aac60e 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/__init__.pyi +++ b/source/isaaclab/isaaclab/utils/modifiers/__init__.pyi @@ -5,6 +5,8 @@ __all__ = [ "ModifierCfg", + "RenderBufferPipeModifierCfg", + "RenderBufferPipeModifier", "DigitalFilterCfg", "IntegratorCfg", "ModifierBase", @@ -15,6 +17,7 @@ __all__ = [ "scale", ] -from .modifier_cfg import ModifierCfg, DigitalFilterCfg, IntegratorCfg +from .modifier_cfg import ModifierCfg, DigitalFilterCfg, IntegratorCfg, RenderBufferPipeModifierCfg +from .render_buffer_pipe import RenderBufferPipeModifier from .modifier_base import ModifierBase from .modifier import DigitalFilter, Integrator, bias, clip, scale diff --git a/source/isaaclab/isaaclab/utils/modifiers/modifier_base.py b/source/isaaclab/isaaclab/utils/modifiers/modifier_base.py index 65a7fe0bb8a2..ea0594fe093f 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/modifier_base.py +++ b/source/isaaclab/isaaclab/utils/modifiers/modifier_base.py @@ -56,6 +56,10 @@ def __init__(self, cfg: ModifierCfg, data_dim: tuple[int, ...], device: str) -> self._data_dim = data_dim self._device = device + def close(self) -> None: + """Release resources owned by this modifier.""" + pass + @abstractmethod def reset(self, env_ids: Sequence[int] | None = None): """Resets the Modifier. diff --git a/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py b/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py index d279fc62f2ab..9f22c9734430 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py +++ b/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py @@ -6,16 +6,19 @@ from __future__ import annotations from collections.abc import Callable -from dataclasses import MISSING +from dataclasses import MISSING, field from typing import TYPE_CHECKING, Any import torch +import warp as wp from .. import configclass if TYPE_CHECKING: + from ..images import RenderBufferStepCfg from .modifier import DigitalFilter, Integrator from .modifier_base import ModifierBase + from .render_buffer_pipe import RenderBufferPipeModifier @configclass @@ -37,6 +40,36 @@ class ModifierCfg: """ +@configclass +class RenderBufferPipeModifierCfg(ModifierCfg): + """Bind an incoming observation tensor to a render buffer pipe. + + The declared dtype, layout and encoding describe the tensor *after* preceding modifiers. + The pipe validates storage and lets each operation validate the semantic encoding. + """ + + func: type[RenderBufferPipeModifier] | str = "{DIR}.render_buffer_pipe:RenderBufferPipeModifier" + """Modifier class consuming the incoming tensor.""" + + pipeline: list[RenderBufferStepCfg] = MISSING + """Ordered buffer processing operations.""" + + input_name: str = "rgb" + """Name assigned to the incoming tensor in the buffer bundle.""" + + output_name: str = "rgb" + """Name selected from the processed bundle.""" + + dtype: type = field(default_factory=lambda: wp.float32) + """Expected Warp element type of the incoming tensor.""" + + encoding: str = "unknown" + """Semantic encoding of the incoming tensor, such as ``scene_linear``.""" + + layout: str = "NHWC" + """Axis layout of the incoming tensor.""" + + @configclass class DigitalFilterCfg(ModifierCfg): """Configuration parameters for a digital filter modifier. diff --git a/source/isaaclab/isaaclab/utils/modifiers/render_buffer_pipe.py b/source/isaaclab/isaaclab/utils/modifiers/render_buffer_pipe.py new file mode 100644 index 000000000000..19f481ddb1fc --- /dev/null +++ b/source/isaaclab/isaaclab/utils/modifiers/render_buffer_pipe.py @@ -0,0 +1,50 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Render buffer processing for an incoming observation tensor.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING + +import torch + +from ..images import ImageBufferSpec, RenderBufferPipe +from .modifier_base import ModifierBase + +if TYPE_CHECKING: + from .modifier_cfg import RenderBufferPipeModifierCfg + + +class RenderBufferPipeModifier(ModifierBase): + """Apply a render buffer pipe to the tensor received from the preceding modifier. + + The modifier owns a pipe and returns a snapshot. Its output must preserve the input + shape, as required by :class:`ModifierBase`. Multiple camera inputs or shape changes + belong in :class:`~isaaclab.envs.mdp.processed_image` instead. + """ + + def __init__(self, cfg: RenderBufferPipeModifierCfg, data_dim: tuple[int, ...], device: str): + super().__init__(cfg, data_dim, device) + self._pipe = RenderBufferPipe( + {cfg.input_name: ImageBufferSpec(tuple(data_dim), cfg.dtype, device, cfg.encoding, cfg.layout)}, + cfg.pipeline, + ) + if cfg.output_name not in self._pipe.output_specs: + self._pipe.close() + raise ValueError(f"Image processing does not provide output {cfg.output_name!r}.") + if self._pipe.output_specs[cfg.output_name].shape != tuple(data_dim): + self._pipe.close() + raise ValueError("Observation modifiers must preserve shape; use mdp.processed_image for shape changes.") + + def __call__(self, data: torch.Tensor) -> torch.Tensor: + return self._pipe.apply_to_tensor(data, input_name=self._cfg.input_name, output_name=self._cfg.output_name) + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + self._pipe.reset(env_ids) + + def close(self) -> None: + self._pipe.close() diff --git a/source/isaaclab/test/envs/test_render_buffer_observations.py b/source/isaaclab/test/envs/test_render_buffer_observations.py new file mode 100644 index 000000000000..8ca9e50b27ec --- /dev/null +++ b/source/isaaclab/test/envs/test_render_buffer_observations.py @@ -0,0 +1,228 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared pipe behavior at the observation manager boundary.""" + +from builtins import ExceptionGroup +from types import SimpleNamespace + +import pytest +import torch +import warp as wp + +from isaaclab.envs import ManagerBasedEnv +from isaaclab.managers import ( + ManagerTermBase, + ManagerTermBaseCfg, + ObservationGroupCfg, + ObservationManager, + ObservationTermCfg, +) +from isaaclab.utils import configclass, modifiers +from isaaclab.utils.images import ImageBufferSpec, PreparedBufferOperation, RenderBufferStepCfg +from isaaclab.utils.warp import ProxyArray + +pytestmark = pytest.mark.unit + + +def test_environment_releases_simulation_when_observation_cleanup_fails(): + """An operation teardown error must not strand the environment or its simulation singleton.""" + events = [] + + def close_observations(): + events.append("observations") + raise RuntimeError("Operation cleanup failed") + + env = ManagerBasedEnv.__new__(ManagerBasedEnv) + env._is_closed = False + env._window = None + env.sim = SimpleNamespace(stop=lambda: events.append("stop"), clear_instance=lambda: events.append("clear")) + env.action_manager = object() + env.observation_manager = SimpleNamespace(close=close_observations) + env.event_manager = object() + env.recorder_manager = object() + env.scene = object() + env.obs_buf = {"policy": torch.ones(1)} + with pytest.raises(RuntimeError, match="Operation cleanup failed"): + env.close() + assert events == ["stop", "observations", "clear"] + assert env.obs_buf == {} + env.close() + assert events == ["stop", "observations", "clear"] + + +@wp.kernel +def _gain(source: wp.array4d(dtype=wp.float32), gain: float, result: wp.array4d(dtype=wp.float32)): + n, y, x, c = wp.tid() + result[n, y, x, c] = source[n, y, x, c] * gain + + +@wp.kernel +def _depth_weighted_half( + color: wp.array4d(dtype=wp.float32), + depth: wp.array4d(dtype=wp.float32), + result: wp.array4d(dtype=wp.float32), +): + n, y, x, c = wp.tid() + result[n, y, x, c] = color[n, y, 2 * x, c] * depth[n, y, 2 * x, 0] + + +def _prepare_gain(specs, gain): + def apply(inputs, outputs): + wp.launch( + _gain, + dim=outputs["color"].shape, + inputs=[inputs["color"].warp, gain, outputs["color"].warp], + device=specs["color"].device, + ) + + return PreparedBufferOperation({"color": specs["color"]}, apply) + + +def _read_image(env): + return env.image + + +def _env(**kwargs): + return SimpleNamespace(num_envs=2, device="cpu", sim=SimpleNamespace(is_playing=lambda: True), **kwargs) + + +class _NestedImage(ManagerTermBase): + """Compose an image term through the manager's supported nested configuration containers.""" + + def __call__(self, env, terms, *, out=None): + term = terms[0]["image"] + params = {name: value for name, value in term.params.items() if name != "out"} + return term.func(env, **params, out=out) + + def reset(self, env_ids=None): + self.cfg.params["terms"][0]["image"].func.reset(env_ids) + + def close(self): + self._env.lifecycle.append(("outer_close",)) + + +def test_pipe_modifier_uses_preceding_modifier_output(): + """A pipe must consume the current observation, not re-read the source camera.""" + image = torch.ones((2, 2, 4, 3)) + + @configclass + class Policy(ObservationGroupCfg): + concatenate_terms = False + image = ObservationTermCfg( + func=_read_image, + modifiers=[ + modifiers.ModifierCfg(func=modifiers.bias, params={"value": 3.0}), + modifiers.RenderBufferPipeModifierCfg( + pipeline=[ + RenderBufferStepCfg( + prepare=_prepare_gain, + inputs={"color": "rgb"}, + outputs={"color": "rgb"}, + params={"gain": 2.0}, + ) + ] + ), + ], + ) + + manager = ObservationManager({"policy": Policy()}, _env(image=image)) + first = manager.compute()["policy"]["image"] + torch.testing.assert_close(first, torch.full_like(image, 8.0)) + torch.testing.assert_close(image, torch.ones_like(image)) + image.fill_(2.0) + torch.testing.assert_close(manager.compute()["policy"]["image"], torch.full_like(image, 10.0)) + torch.testing.assert_close(first, torch.full_like(image, 8.0)) + manager.close() + + +def test_processed_image_declares_private_inputs_and_reports_transformed_shape(): + """Preparation resolves private AOVs before binding; manager dimensions describe the processed output.""" + lifecycle = [] + + def prepare(specs): + color = specs["color"] + n, h, w, c = color.shape + output_spec = ImageBufferSpec((n, h, w // 2, c), wp.float32, color.device) + + def apply(inputs, outputs): + wp.launch( + _depth_weighted_half, + dim=output_spec.shape, + inputs=[inputs["color"].warp, inputs["depth"].warp, outputs["color"].warp], + device=color.device, + ) + + def close(): + lifecycle.append(("close",)) + raise ValueError("operation close failed") + + return PreparedBufferOperation( + {"color": output_spec}, + apply, + reset=lambda env_ids: lifecycle.append(("reset", env_ids)), + close=close, + ) + + radiance = torch.full((2, 2, 4, 3), 2.0) + depth = torch.tensor([1.0, 9.0, 3.0, 9.0]).reshape(1, 1, 4, 1).expand(2, 2, 4, 1).contiguous() + requests = [] + camera = SimpleNamespace( + request_render_inputs=lambda names: requests.append(set(names)), + render_output_specs={ + "rgb_radiance": ImageBufferSpec(tuple(radiance.shape), wp.float32, "cpu", "scene_linear"), + "depth": ImageBufferSpec(tuple(depth.shape), wp.float32, "cpu", "distance"), + }, + render_outputs={"rgb_radiance": ProxyArray(wp.from_torch(radiance)), "depth": ProxyArray(wp.from_torch(depth))}, + ) + sensors = {"camera": camera} + env = _env(scene=SimpleNamespace(sensors=sensors, keys=sensors.keys), lifecycle=lifecycle) + + @configclass + class Policy(ObservationGroupCfg): + concatenate_terms = False + image = ObservationTermCfg( + func=_NestedImage, + params={ + "terms": [ + { + "image": ManagerTermBaseCfg( + func="isaaclab.envs.mdp:processed_image", + params={ + "pipeline": [ + RenderBufferStepCfg( + prepare=prepare, + inputs={"color": "rgb_radiance", "depth": "depth"}, + outputs={"color": "rgb"}, + ) + ] + }, + ) + } + ] + }, + ) + + @configclass + class Observations: + policy = Policy() + + cfg = Observations() + ObservationManager.prepare_scene(cfg, env) + assert requests == [{"rgb_radiance", "depth"}] + manager = ObservationManager(cfg, env) + assert manager.group_obs_dim["policy"] == [(2, 2, 3)] + result = manager.compute()["policy"]["image"] + expected = torch.tensor([2.0, 6.0]).reshape(1, 1, 2, 1).expand(2, 2, 2, 3) + torch.testing.assert_close(result, expected) + radiance.fill_(4.0) + torch.testing.assert_close(manager.compute()["policy"]["image"], expected * 2) + torch.testing.assert_close(result, expected) + manager.reset([1]) + with pytest.raises(ExceptionGroup, match="Failed to close render buffer operations") as error: + manager.close() + assert str(error.value.exceptions[0]) == "operation close failed" + manager.close() + assert lifecycle[-3:] == [("reset", [1]), ("close",), ("outer_close",)] diff --git a/source/isaaclab/test/renderers/test_camera_output_contract.py b/source/isaaclab/test/renderers/test_camera_output_contract.py index f36ff6daa8b0..60069d452bbc 100644 --- a/source/isaaclab/test/renderers/test_camera_output_contract.py +++ b/source/isaaclab/test/renderers/test_camera_output_contract.py @@ -6,10 +6,13 @@ """Tests for the renderer→camera output contract.""" import warnings +from builtins import ExceptionGroup +from dataclasses import replace from types import SimpleNamespace import numpy as np import pytest +import torch import warp as wp pytest.importorskip("isaaclab_physx") @@ -22,6 +25,7 @@ from isaaclab.sim import PinholeCameraCfg from isaaclab.test.utils import DeviceScope, test_devices from isaaclab.utils import clone, validate +from isaaclab.utils.images import PreparedBufferOperation, RenderBufferStepCfg pytestmark = [pytest.mark.integration, pytest.mark.rendering] @@ -376,9 +380,24 @@ def export_stage(stage, num_envs): lambda path, **kwargs: SimpleNamespace(count=2, prims=[stage.GetPrimAtPath(path)] * 2, close=lambda: None), ) cameras = [] - for prim, data_types in zip(prims, (["rgb"], ["rgb_radiance"])): + + def prepare_identity(input_specs): + return PreparedBufferOperation( + {"image": input_specs["image"]}, + lambda inputs, outputs: wp.copy(outputs["image"].warp, inputs["image"].warp), + ) + + for prim, data_types in zip(prims, (["rgb"], ["processed_radiance"])): camera = Camera.__new__(Camera) - camera.cfg = SimpleNamespace(prim_path=str(prim.GetPath()), data_types=data_types, height=2, width=3) + pipeline = ( + [RenderBufferStepCfg(prepare_identity, {"image": "rgb_radiance"}, {"image": "processed_radiance"})] + if data_types == ["processed_radiance"] + else [] + ) + camera.cfg = CameraCfg( + prim_path=str(prim.GetPath()), data_types=data_types, height=2, width=3, spawn=None, pipeline=pipeline + ) + camera._render_cfg = replace(camera.cfg, data_types=camera.cfg.render_data_types(), pipeline=[]) camera.stage = stage camera._device = "cpu" camera._num_envs = 2 @@ -399,3 +418,143 @@ def export_stage(stage, num_envs): for camera in cameras: camera.__del__() assert not CameraPhysicsManager._callbacks + + +def test_camera_pipeline_keeps_raw_inputs_and_processes_only_new_captures(monkeypatch): + """Camera publication preserves raw pixels, capture metadata and calibration-compatible geometry.""" + from isaaclab.sensors.camera import Camera + from isaaclab.sensors.sensor_base import SensorBase + from isaaclab.sim import SimulationContext + + events = [] + output_shape = (2, 1, 1, 4) + close_fails = False + + def prepare_color(input_specs): + assert input_specs["radiance"].encoding == "scene_linear" + assert input_specs["range"].encoding == "meters" + spec = replace(input_specs["radiance"], shape=output_shape, dtype=wp.uint8, encoding="camera_response") + + def apply(inputs, outputs): + events.append("apply") + output = outputs["color"].torch + output[..., :3].copy_(100 * (inputs["radiance"].torch + inputs["range"].torch)) + output[..., 3] = 255 + + def close(): + events.append("close") + if close_fails: + raise RuntimeError("processor close failed") + + return PreparedBufferOperation({"color": spec}, apply, reset=lambda ids: events.append(tuple(ids)), close=close) + + cfg = CameraCfg( + prim_path="/World/Camera", + spawn=None, + height=1, + width=1, + data_types=["rgb", "depth"], + pipeline=[ + RenderBufferStepCfg(prepare_color, {"radiance": "rgb_radiance", "range": "depth"}, {"color": "rgba"}) + ], + ) + specs = { + RenderBufferKind.RGB_RADIANCE: RenderBufferSpec(3, wp.float32), + RenderBufferKind.DEPTH: RenderBufferSpec(1, wp.float32), + } + capture = {"frame": 1} + radiance = 0.2 + + def read_output(render_data, camera_data): + camera_data.output["rgb_radiance"].torch.fill_(radiance) + camera_data.output["depth"].torch.fill_(0.5) + for name in camera_data.info: + camera_data.info[name] = {"capture": capture} + + camera = Camera.__new__(Camera) + camera.cfg = cfg + camera._render_cfg = replace(cfg, data_types=cfg.render_data_types(), pipeline=[]) + camera._view = SimpleNamespace(count=2, close=lambda: None) + camera._device = "cpu" + camera._num_envs = 2 + camera._is_initialized = True + camera._render_data = None + camera._clear_callbacks = lambda: None + camera._renderer = SimpleNamespace( + supported_output_types=lambda: specs, + set_outputs=lambda data, outputs: None, + render=lambda data: None, + read_output=read_output, + reset=lambda data, ids: None, + cleanup=lambda data: None, + ) + monkeypatch.setattr(SimulationContext, "instance", staticmethod(lambda: None)) + monkeypatch.setattr(SensorBase, "reset", lambda self, ids, mask: None) + monkeypatch.setattr(camera, "_initialize_intrinsics", lambda: None) + monkeypatch.setattr(camera, "_update_poses", lambda **kwargs: None) + monkeypatch.setattr(camera, "_prepare_camera", lambda mask: None) + monkeypatch.setattr(camera, "_update_outdated_buffers", lambda: None) + camera._create_buffers() + mask = wp.ones(2, dtype=wp.bool, device="cpu") + camera._update_buffers_impl(mask) + + assert set(camera.data.output) == {"rgb", "depth"} + assert set(camera.render_outputs) == {"rgb_radiance", "depth"} + torch.testing.assert_close(camera.data.output["rgb"].torch, torch.full((2, 1, 1, 3), 70, dtype=torch.uint8)) + torch.testing.assert_close(camera.render_outputs["rgb_radiance"].torch, torch.full((2, 1, 1, 3), 0.2)) + assert camera.data.output["depth"] is camera.render_outputs["depth"] + assert camera.data.info["rgb"]["capture"] is capture + + camera._update_buffers_impl(mask) + assert events == ["apply"] + capture = {"frame": 2} + radiance = 0.4 + camera._update_buffers_impl(mask) + torch.testing.assert_close(camera.data.output["rgb"].torch, torch.full((2, 1, 1, 3), 90, dtype=torch.uint8)) + assert events == ["apply", "apply"] + reset_mask = wp.array([False, True], dtype=wp.bool, device="cpu") + camera.reset(env_mask=reset_mask) + assert events[-1] == (1,) + # A rebuilt pipe cannot publish resized pixels against unchanged camera calibration. + output_shape = (2, 2, 1, 4) + with pytest.raises(ValueError, match="camera calibration"): + camera._create_buffers() + assert camera._buffer_pipe is None + assert events[-2:] == ["close", "close"] + output_shape = (2, 1, 1, 4) + camera._create_buffers() + camera._render_data = object() + camera._renderer.cleanup = lambda data: events.append("renderer-close") + camera._view.close = lambda: events.append("view-close") + close_fails = True + with pytest.raises(ExceptionGroup, match="close render buffer operations"): + camera._invalidate_initialize_callback(None) + assert events[-3:] == ["close", "renderer-close", "view-close"] + assert not camera.is_initialized + assert camera._renderer is camera._render_data is camera._view is camera._buffer_pipe is None + camera.__del__() + + +def test_camera_requests_private_observation_inputs_before_initialization(): + """An observation may request raw inputs before reset, while late additions fail clearly.""" + from isaaclab.sensors.camera import Camera + + camera = Camera.__new__(Camera) + camera.cfg = _make_camera_cfg(["rgb"]) + camera._render_cfg = replace(camera.cfg, pipeline=[]) + camera._render_outputs = {} + camera._is_initialized = False + camera._clear_callbacks = lambda: None + camera.request_render_inputs(["depth", "rgb_radiance"]) + assert camera.cfg.data_types == ["rgb"] + assert set(camera._render_cfg.data_types) == {"rgb", "depth", "rgb_radiance"} + camera._is_initialized = True + camera.request_render_inputs(["depth"]) + with pytest.raises(RuntimeError, match="before initialization"): + camera.request_render_inputs(["normals"]) + + +def test_camera_retired_isp_setting_reports_pipeline_migration(): + """Legacy camera ISP configuration fails before it can silently skip image processing.""" + with pytest.raises(ValueError, match="CameraCfg.pipeline"): + CameraCfg(isp_cfg="auto_any") diff --git a/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py b/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py index 0756f2b72d26..945492c850e0 100644 --- a/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py +++ b/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py @@ -24,7 +24,7 @@ from typing import TYPE_CHECKING import torch -from isaaclab_ppisp import PpispCfg, normalize_ppisp_cfg +from isaaclab_ppisp import PpispCfg, PpispStepCfg, normalize_ppisp_cfg, ppisp_cfg_from_usd_camera from pxr import Gf, Sdf, Usd, UsdGeom, Vt @@ -32,7 +32,6 @@ from isaaclab.assets import AssetBaseCfg, RigidObjectCfg from isaaclab.scene import InteractiveScene, InteractiveSceneCfg from isaaclab.sensors.camera import Camera, CameraCfg -from isaaclab.sensors.camera.camera_isp import CameraISPMode from isaaclab.terrains import TerrainImporterCfg from isaaclab.utils import configclass @@ -728,7 +727,7 @@ def render_synthetic_gaussian_scene( width=width, data_types=data_types, spawn=None, - isp_cfg=isp_cfg, + pipeline=[PpispStepCfg(params={"cfg": isp_cfg})], renderer_cfg=renderer_cfg, ) camera = Camera(cfg) @@ -736,7 +735,9 @@ def render_synthetic_gaussian_scene( for _ in range(stabilisation_steps): sim.step() camera.update(sim_dt) - outputs = {name: tensor.clone().detach().cpu().to(torch.float32) for name, tensor in camera.data.output.items()} + outputs = { + name: tensor.torch.clone().detach().cpu().to(torch.float32) for name, tensor in camera.data.output.items() + } del camera return outputs @@ -756,9 +757,7 @@ def render_synthetic_gaussian_scene_with_static_ppisp_attrs( ) -> dict[str, torch.Tensor]: """Render the synthesised gaussian asset through authored static PPISP camera attributes. - The camera uses :class:`CameraISPMode.AUTO_CAMERA`; renderer backends must - discover the camera-authored PPISP attributes and route them through their - PPISP workflow. + Resolve the camera-authored attributes before constructing the shared PPISP image operation. """ with fresh_synthetic_gaussian_interactive_scene(usd_path, sim_cfg, num_envs=num_envs) as sim: author_static_ppisp_camera_attrs(sim.stage, ppisp_cfg=ppisp_cfg) @@ -769,7 +768,7 @@ def render_synthetic_gaussian_scene_with_static_ppisp_attrs( width=width, sim_dt=sim_dt, stabilisation_steps=stabilisation_steps, - isp_cfg=CameraISPMode.AUTO_CAMERA, + ppisp_cfg=ppisp_cfg_from_usd_camera(sim.stage.GetPrimAtPath(_camera_path_for_env(0))), sim=sim, ) @@ -797,7 +796,7 @@ def render_synthetic_gaussian_scene_with_controller_ppisp_attrs( width=width, sim_dt=sim_dt, stabilisation_steps=stabilisation_steps, - isp_cfg=CameraISPMode.AUTO_CAMERA, + ppisp_cfg=ppisp_cfg_from_usd_camera(sim.stage.GetPrimAtPath(_camera_path_for_env(0))), sim=sim, ) @@ -810,7 +809,7 @@ def _render_synthetic_gaussian_camera( width: int, sim_dt: float, stabilisation_steps: int, - isp_cfg: PpispCfg | CameraISPMode | None, + ppisp_cfg: PpispCfg, sim: SimulationContext, ) -> dict[str, torch.Tensor]: cfg = CameraCfg( @@ -820,7 +819,7 @@ def _render_synthetic_gaussian_camera( width=width, data_types=data_types, spawn=None, - isp_cfg=isp_cfg, + pipeline=[PpispStepCfg(params={"cfg": ppisp_cfg})], renderer_cfg=renderer_cfg, ) camera = Camera(cfg) @@ -828,6 +827,8 @@ def _render_synthetic_gaussian_camera( for _ in range(stabilisation_steps): sim.step() camera.update(sim_dt) - outputs = {name: tensor.clone().detach().cpu().to(torch.float32) for name, tensor in camera.data.output.items()} + outputs = { + name: tensor.torch.clone().detach().cpu().to(torch.float32) for name, tensor in camera.data.output.items() + } del camera return outputs diff --git a/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py b/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py index 1371bebe5b0e..f58b8eb9e93c 100644 --- a/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py +++ b/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py @@ -3,12 +3,12 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Validate the camera PPISP wrapper applied to a 3D Gaussian (NuRec / +"""Validate the camera PPISP pipeline applied to a 3D Gaussian (NuRec / ParticleField) scene through the ``ovrtx`` renderer. The asset is synthesised at test time by :mod:`generate_synthetic_gaussian_asset` -and rendered via :func:`generate_synthetic_gaussian_asset.render_synthetic_gaussian_scene`. -The aggressive wrapper PPISP cfg +and rendered through the camera's shared render-buffer pipeline. +The aggressive PPISP cfg (:func:`generate_synthetic_gaussian_asset.make_aggressive_ppisp_cfg`) intentionally engages every feature past its subtle-correction defaults so each can be asserted independently: @@ -38,8 +38,8 @@ * Uses Newton physics because ``ovrtx`` is incompatible with Kit/Isaac Sim and the PhysX backend requires Kit (``carb``) to bootstrap. * Requests ``"rgb_hdr"`` in ``data_types`` because the test asserts the raw - HDR source with :func:`assert_ppisp_lifts_exposure`. The PPISP render path - itself also allocates an internal HDR buffer when ``isp_cfg`` is set, so + HDR source with :func:`assert_ppisp_lifts_exposure`. The PPISP camera pipeline + requests its private ``"rgb_radiance"`` input automatically, so ``"rgb_hdr"`` is not required just to enable PPISP. """ @@ -57,10 +57,10 @@ make_aggressive_ppisp_cfg, make_neutral_ppisp_cfg, make_synthetic_gaussian_usd, - render_synthetic_gaussian_scene, render_synthetic_gaussian_scene_with_controller_ppisp_attrs, render_synthetic_gaussian_scene_with_static_ppisp_attrs, ) +from isaaclab_ppisp import PpispCfg from isaaclab.sim import SimulationCfg @@ -87,17 +87,14 @@ SIM_DT = 1.0 / 60.0 MULTI_TILE_COUNT = 4 -# Mark the gaussian-on-OVRTX tests xfail by default: the wrapper-side -# ``carb.settings.set_bool`` call that disables RTX-side tonemapping is a -# no-op for the kit-less ovrtx backend, and the equivalent must be applied -# externally before launching pytest. -_XFAIL_OVRTX_GAUSSIAN_PPISP = pytest.mark.xfail( - reason=( - "kit-less ovrtx cannot toggle RTX-side tonemapping at runtime; the equivalent must be " - "applied externally before launching pytest" - ), - strict=False, -) + +def _ovrtx_ppisp_cfg() -> PpispCfg: + """Expose the unexposed scene-linear Gaussian signal into the CRF's useful range.""" + cfg = make_aggressive_ppisp_cfg() + # rgb_radiance has a center near 0.12; +2 EV places it near 0.48 before the + # response curve. The shared fixture's -5 EV targets legacy exposed HDR. + cfg.inputs["exposureOffset"] = 2.0 + return cfg def _ovrtx_sim_cfg(device: str) -> SimulationCfg: @@ -110,12 +107,11 @@ def _ovrtx_sim_cfg(device: str) -> SimulationCfg: @pytest.mark.parametrize("device", ["cuda:0"]) @_SKIP_MISSING_OVRTX -@_XFAIL_OVRTX_GAUSSIAN_PPISP def test_camera_ppisp_authored_static_attrs_are_applied_on_synthetic_gaussians_ovrtx(device): """OVRTX must apply camera-authored static PPISP attributes.""" with tempfile.TemporaryDirectory(prefix="isaaclab-synth-gauss-") as tmpdir: asset_path = make_synthetic_gaussian_usd(f"{tmpdir}/synthetic_gaussians.usda") - aggressive_cfg = make_aggressive_ppisp_cfg() + aggressive_cfg = _ovrtx_ppisp_cfg() neutral = render_synthetic_gaussian_scene_with_static_ppisp_attrs( asset_path, @@ -141,12 +137,11 @@ def test_camera_ppisp_authored_static_attrs_are_applied_on_synthetic_gaussians_o @pytest.mark.parametrize("device", ["cuda:0"]) @_SKIP_MISSING_OVRTX -@_XFAIL_OVRTX_GAUSSIAN_PPISP def test_camera_ppisp_controller_matches_static_attrs_on_synthetic_gaussians_ovrtx(device): """OVRTX controller output must match the equivalent static PPISP cfg.""" with tempfile.TemporaryDirectory(prefix="isaaclab-synth-gauss-") as tmpdir: asset_path = make_synthetic_gaussian_usd(f"{tmpdir}/synthetic_gaussians.usda") - ppisp_cfg = make_aggressive_ppisp_cfg() + ppisp_cfg = _ovrtx_ppisp_cfg() static = render_synthetic_gaussian_scene_with_static_ppisp_attrs( asset_path, @@ -171,7 +166,10 @@ def test_camera_ppisp_controller_matches_static_attrs_on_synthetic_gaussians_ovr @pytest.mark.parametrize("device", ["cuda:0"]) @_SKIP_MISSING_OVRTX -@_XFAIL_OVRTX_GAUSSIAN_PPISP +@pytest.mark.xfail( + reason="OVRTX renders cloned Gaussian tiles black in native RGB and radiance, including without a camera pipeline", + strict=False, +) def test_camera_ppisp_wrapper_signatures_on_synthetic_gaussians_ovrtx_multitile(device): """Multi-tile wrapper PPISP via ``ovrtx`` must hold the same invariants independently for every tile. @@ -184,10 +182,11 @@ def test_camera_ppisp_wrapper_signatures_on_synthetic_gaussians_ovrtx_multitile( """ with tempfile.TemporaryDirectory(prefix="isaaclab-synth-gauss-") as tmpdir: asset_path = make_synthetic_gaussian_usd(f"{tmpdir}/synthetic_gaussians.usda") - output = render_synthetic_gaussian_scene( + output = render_synthetic_gaussian_scene_with_static_ppisp_attrs( asset_path, sim_cfg=_ovrtx_sim_cfg(device), renderer_cfg=OVRTXRendererCfg(), + ppisp_cfg=_ovrtx_ppisp_cfg(), data_types=["rgb", "rgb_hdr"], num_envs=MULTI_TILE_COUNT, sim_dt=SIM_DT, diff --git a/source/isaaclab/test/utils/test_render_buffer_pipe.py b/source/isaaclab/test/utils/test_render_buffer_pipe.py new file mode 100644 index 000000000000..f01f0bfffda4 --- /dev/null +++ b/source/isaaclab/test/utils/test_render_buffer_pipe.py @@ -0,0 +1,215 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Contracts for buffer routing, ownership and lifecycle independent of camera rendering.""" + +from builtins import ExceptionGroup +from dataclasses import replace + +import pytest +import torch +import warp as wp + +from isaaclab.utils.images import ImageBufferSpec, PreparedBufferOperation, RenderBufferPipe, RenderBufferStepCfg +from isaaclab.utils.warp import ProxyArray + +pytestmark = pytest.mark.unit + + +def _prepare_scale(specs, *, factor=2.0): + def apply(inputs, outputs): + torch.mul(inputs["value"].torch, factor, out=outputs["value"].torch) + + return PreparedBufferOperation({"value": specs["value"]}, apply) + + +def _prepare_composite(specs): + """Combine color and depth, retaining only one spatial sample per batch entry.""" + assert specs["color"].encoding == "scene_linear" + assert specs["depth"].encoding == "meters" + shape = (specs["color"].shape[0], 1, 1, 3) + + def apply(inputs, outputs): + torch.sub( + inputs["color"].torch[:, :1, :1], + inputs["depth"].torch[:, :1, :1], + out=outputs["value"].torch, + ) + + return PreparedBufferOperation({"value": replace(specs["color"], shape=shape)}, apply) + + +@pytest.fixture +def composite_pipe(): + specs = { + "raw_color": ImageBufferSpec((2, 2, 2, 3), wp.float32, "cpu", "scene_linear"), + "range": ImageBufferSpec((2, 2, 2, 1), wp.float32, "cpu", "meters"), + } + steps = [ + RenderBufferStepCfg(prepare=_prepare_scale, inputs={"value": "raw_color"}, outputs={"value": "color"}), + RenderBufferStepCfg( + prepare=_prepare_composite, + inputs={"color": "color", "depth": "range"}, + outputs={"value": "color"}, + ), + ] + assert RenderBufferPipe.required_source_names(steps, ["color", "labels"]) == {"raw_color", "range", "labels"} + pipe = RenderBufferPipe(specs, steps) + yield pipe + pipe.close() + + +def test_ordered_multi_input_operations_preserve_sources_and_reuse_output_storage(composite_pipe): + """Bindings consume earlier replacements; shape changes and strided inputs preserve ownership.""" + backing = torch.full((2, 2, 2, 4), 3.0) + color = ProxyArray(wp.from_torch(backing[..., :3])) + depth = ProxyArray(wp.full((2, 2, 2, 1), 2.0, dtype=wp.float32, device="cpu")) + labels = ProxyArray(wp.ones((2, 2, 2, 1), dtype=wp.int32, device="cpu")) + sources = {"raw_color": color, "range": depth, "labels": labels} + output = composite_pipe(sources) + snapshot = output["color"].torch.clone() + destination = output["color"] + + torch.testing.assert_close(snapshot, torch.full((2, 1, 1, 3), 4.0)) + torch.testing.assert_close(backing, torch.full_like(backing, 3.0)) + torch.testing.assert_close(depth.torch, torch.full_like(depth.torch, 2.0)) + assert output["labels"] is labels + assert output["raw_color"] is color + assert set(sources) == {"raw_color", "range", "labels"} + assert composite_pipe.output_specs["color"].shape == (2, 1, 1, 3) + + backing.fill_(5.0) + updated = composite_pipe(sources) + assert updated["color"] is destination + torch.testing.assert_close(destination.torch, torch.full_like(snapshot, 8.0)) + torch.testing.assert_close(snapshot, torch.full_like(snapshot, 4.0)) + + +def test_tensor_adapter_preserves_incoming_transformations_and_returns_snapshots(composite_pipe): + """The current observation supplies color, auxiliary depth stays read-only, and outputs are independent.""" + original = torch.full((2, 2, 2, 3), 3.0) + incoming = original * 2.0 + depth = ProxyArray(wp.full((2, 2, 2, 1), 2.0, dtype=wp.float32, device="cpu")) + kwargs = {"input_name": "raw_color", "output_name": "color", "auxiliary_buffers": {"range": depth}} + first = composite_pipe.apply_to_tensor(incoming, **kwargs) + second = composite_pipe.apply_to_tensor(torch.full_like(incoming, 9.0), **kwargs) + torch.testing.assert_close(first, torch.full((2, 1, 1, 3), 10.0)) + torch.testing.assert_close(second, torch.full_like(first, 16.0)) + torch.testing.assert_close(incoming, torch.full_like(incoming, 6.0)) + torch.testing.assert_close(depth.torch, torch.full_like(depth.torch, 2.0)) + assert first.data_ptr() != second.data_ptr() + + with pytest.raises(ValueError, match="must not replace"): + composite_pipe.apply_to_tensor( + incoming, + input_name="raw_color", + output_name="color", + auxiliary_buffers={"raw_color": ProxyArray(wp.from_torch(original)), "range": depth}, + ) + + +@pytest.mark.parametrize("invalid", ["missing", "shape", "dtype"]) +def test_invalid_source_buffers_are_rejected_before_processing(invalid): + """Missing or incompatible external storage must fail before a callback reads it.""" + calls = [] + spec = ImageBufferSpec((2, 1, 1, 1), wp.float32, "cpu") + + def prepare(specs): + return PreparedBufferOperation({"value": specs["value"]}, lambda *_: calls.append(True)) + + pipe = RenderBufferPipe( + {"source": spec}, + [RenderBufferStepCfg(prepare=prepare, inputs={"value": "source"}, outputs={"value": "result"})], + ) + try: + shape = (1, 1, 1, 1) if invalid == "shape" else spec.shape + dtype = wp.uint8 if invalid == "dtype" else wp.float32 + inputs = {} if invalid == "missing" else {"source": ProxyArray(wp.zeros(shape, dtype=dtype, device="cpu"))} + with pytest.raises(ValueError, match="Missing required|shape, dtype and device"): + pipe(inputs) + assert not calls + finally: + pipe.close() + + +@pytest.mark.parametrize("invalid", ["missing_dependency", "output_roles", "duplicate_outputs"]) +def test_invalid_bindings_release_prepared_operations(invalid): + """Construction failure closes owned operations, including one whose outputs could not be bound.""" + closed = [] + spec = ImageBufferSpec((2, 1, 1, 1), wp.float32, "cpu") + + def prepare(specs, *, name): + return PreparedBufferOperation({"value": specs["value"]}, lambda *_: None, close=lambda: closed.append(name)) + + first = RenderBufferStepCfg( + prepare=prepare, inputs={"value": "source"}, outputs={"value": "middle"}, params={"name": "first"} + ) + second = RenderBufferStepCfg( + prepare=prepare, inputs={"value": "middle"}, outputs={"value": "result"}, params={"name": "second"} + ) + if invalid == "missing_dependency": + second.inputs = {"value": "absent"} + message = "Missing image buffer declarations" + elif invalid == "output_roles": + second.outputs = {"wrong": "result"} + message = "output roles" + else: + second.outputs = {"value": "result", "other": "result"} + message = "distinct output names" + with pytest.raises(ValueError, match=message): + RenderBufferPipe({"source": spec}, [first, second]) + assert closed == (["second", "first"] if invalid == "output_roles" else ["first"]) + + +@pytest.mark.parametrize("close_fails", [False, True], ids=["normal_close", "failing_close"]) +def test_reset_selects_batch_entries_and_close_releases_operations_in_reverse_order(close_fails): + """State belongs to each operation; selected resets and idempotent teardown preserve its lifecycle.""" + closed = [] + spec = ImageBufferSpec((2, 1, 1, 1), wp.float32, "cpu") + + def prepare(specs, *, name): + history = torch.zeros(specs["value"].shape) + + def apply(inputs, outputs): + history.add_(inputs["value"].torch) + outputs["value"].torch.copy_(history) + + def reset(env_ids): + history[slice(None) if env_ids is None else env_ids] = 0 + + def close(): + closed.append(name) + if close_fails and name == "second": + raise RuntimeError("Operation cleanup failed") + + return PreparedBufferOperation({"value": specs["value"]}, apply, reset=reset, close=close) + + pipe = RenderBufferPipe( + {"value": spec}, + [ + RenderBufferStepCfg( + prepare=prepare, inputs={"value": "value"}, outputs={"value": "value"}, params={"name": name} + ) + for name in ("first", "second") + ], + ) + source = {"value": ProxyArray(wp.ones(spec.shape, dtype=wp.float32, device="cpu"))} + try: + torch.testing.assert_close(pipe(source)["value"].torch.flatten(), torch.tensor([1.0, 1.0])) + torch.testing.assert_close(pipe(source)["value"].torch.flatten(), torch.tensor([3.0, 3.0])) + pipe.reset([0]) + torch.testing.assert_close(pipe(source)["value"].torch.flatten(), torch.tensor([1.0, 6.0])) + pipe.reset() + torch.testing.assert_close(pipe(source)["value"].torch.flatten(), torch.tensor([1.0, 1.0])) + finally: + if close_fails: + with pytest.raises(ExceptionGroup, match="Failed to close"): + pipe.close() + else: + pipe.close() + pipe.close() + assert closed == ["second", "first"] + with pytest.raises(RuntimeError, match="closed"): + pipe(source) diff --git a/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.major b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.removed.rst b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.removed.rst new file mode 100644 index 000000000000..2dfc3e3f54d0 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.removed.rst @@ -0,0 +1,2 @@ +* **Breaking:** Removed renderer-owned PPISP processing. Request ``rgb_radiance`` and apply PPISP + through the render buffer pipe instead of configuring ``CameraCfg.isp_cfg``. diff --git a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py index 193631c877f0..609beeb276f8 100644 --- a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py +++ b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py @@ -11,7 +11,7 @@ import warnings from dataclasses import dataclass from functools import partial -from typing import TYPE_CHECKING, Any, NoReturn +from typing import TYPE_CHECKING, Any import newton import warp as wp @@ -27,8 +27,6 @@ from .segmentation import NewtonSegmentationMapper, NewtonSegmentationMapping if TYPE_CHECKING: - from isaaclab_ppisp import PpispPipeline - from isaaclab.sensors.camera.camera_data import CameraData from isaaclab.utils.warp import ProxyArray @@ -68,21 +66,6 @@ def _update_camera_rays(intrinsics: wp.array(dtype=wp.mat33f), rays: wp.array4d( rays[0, y, x, 1] = wp.normalize(direction) -_PPISP_IMPORT_ERROR_MESSAGE = ( - "isaaclab_ppisp is required when CameraCfg.isp_cfg is set. " - "Run `uv sync` from the Isaac Lab source checkout, or install the Isaac Lab wheel " - "with `uv pip install isaaclab`." -) - - -def _raise_missing_ppisp_error(exc: ModuleNotFoundError) -> NoReturn: - # Only translate missing isaaclab_ppisp imports into the optional-dependency hint; - # unrelated missing modules should surface unchanged for easier debugging. - if exc.name != "isaaclab_ppisp" and not (exc.name and exc.name.startswith("isaaclab_ppisp.")): - raise exc - raise ModuleNotFoundError(_PPISP_IMPORT_ERROR_MESSAGE, name="isaaclab_ppisp") from exc - - class RenderData: # Back-compat alias for callers of ``RenderData.OutputNames``. OutputNames = RenderBufferKind @@ -185,27 +168,6 @@ def __init__( # OpenCV lens-distortion model (``spawn.distortion``), consumed by :meth:`_build_distortion_rays` # to trace distorted per-pixel rays instead of the centered, square-pixel pinhole field. self._distortion = spawn.distortion if spawn is not None else None - # Post-render PPISP pipeline composed when ``spec.cfg.isp_cfg`` is set. - # ``isp_cfg`` is already fully normalized by ``prepare_cameras`` by the time it reaches here. - self.ppisp_pipeline: PpispPipeline | None = None - if spec.cfg.isp_cfg is not None: - try: - from isaaclab_ppisp import PpispPipeline - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - self.ppisp_pipeline = PpispPipeline(spec.cfg.isp_cfg) - self._hdr_scratch_wp: wp.array | None = None - """Internal HDR scratch buffer allocated when PPISP is composed but the - user did not request ``"rgb_hdr"`` in ``data_types``. Also exposed to - the Newton sensor through :attr:`CameraOutputs.hdr_color_image` as a - vec3f reinterpretation of this same backing storage.""" - self._ppisp_hdr_source: wp.array | None = None - """PPISP HDR source bound once in :meth:`set_outputs` from the caller's - ``rgb_hdr`` output or :attr:`_hdr_scratch_wp`.""" - self._ppisp_rgba_dest: wp.array | None = None - """PPISP LDR destination bound once in :meth:`set_outputs` from the - caller's ``rgba`` output.""" def _view(self, proxy: ProxyArray, dtype: type, shape: tuple[int, ...]) -> wp.array: """Alias the caller's output buffer as a ``(world_count, 1, H, W)`` warp array of ``dtype``. @@ -264,24 +226,6 @@ def set_outputs(self, output_data: dict[str, ProxyArray]): # requested segmentation outputs are remapped from this single buffer in :meth:`_convert_segmentation`. if self._seg_dests: self.outputs.shape_index_image = wp.zeros(shape, dtype=wp.uint32, device=self.newton_sensor.model.device) - # When PPISP is composed but the user did not request the raw HDR AOV, - # allocate an internal HDR scratch buffer and route a vec3f-shaped view - # of it as the Newton sensor's ``hdr_color_image`` so the renderer - # fills it directly. - if self.ppisp_pipeline is not None and self.outputs.hdr_color_image is None: - ref_proxy = next(iter(output_data.values())) - self._hdr_scratch_wp = wp.zeros( - (self.newton_sensor.model.world_count, self.height, self.width, 3), - dtype=wp.float32, - device=ref_proxy.device, - ) - self.outputs.hdr_color_image = wp.array( - ptr=self._hdr_scratch_wp.ptr, - dtype=wp.vec3f, - shape=shape, - device=self._hdr_scratch_wp.device, - copy=False, - ) # Newton clears HDR to black regardless of clear_color. A hit mask preserves dark geometry. if ( self.outputs.hdr_color_image is not None @@ -289,19 +233,6 @@ def set_outputs(self, output_data: dict[str, ProxyArray]): and self.outputs.shape_index_image is None ): self.outputs.shape_index_image = wp.zeros(shape, dtype=wp.uint32, device=self.newton_sensor.model.device) - # Bind the two warp arrays the per-frame PPISP dispatch needs. - if self.ppisp_pipeline is not None: - if str(RenderBufferKind.RGBA) not in output_data: - raise ValueError( - "Newton renderer ISP requires 'rgba' (or 'rgb', which aliases into rgba) as the" - " LDR output destination, but neither was provided. Add 'rgb' or 'rgba' to" - " Camera.cfg.data_types when isp_cfg is set." - ) - hdr_proxy = output_data.get(str(RenderBufferKind.RGB_HDR)) - if hdr_proxy is None: - hdr_proxy = output_data.get(str(RenderBufferKind.RGB_RADIANCE)) - self._ppisp_hdr_source = hdr_proxy.warp if hdr_proxy is not None else self._hdr_scratch_wp - self._ppisp_rgba_dest = output_data[str(RenderBufferKind.RGBA)].warp def get_output(self, output_name: str) -> wp.array: if output_name in self._depth_dests: @@ -516,15 +447,10 @@ def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: return self.cfg.supported_output_types() def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: - """Resolve the camera's PPISP cfg before rendering. - - :mod:`isaaclab.sensors.camera` does not depend on PPISP; the renderer - owns the sentinel-resolution + cfg-normalization step. Newton has no - USD-side overrides to author beyond this. - - Also captures the USD ``stage`` so the segmentation mapper can read the scene's - :class:`UsdSemantics.LabelsAPI` labels when a segmentation output is requested. + """Capture the USD stage for the segmentation mapper. + The mapper reads the scene's :class:`UsdSemantics.LabelsAPI` labels when a segmentation + output is requested. Newton authors no camera overrides. """ self._stage = stage if spec.cfg.spawn is None or spec.cfg.spawn.distortion is None: @@ -543,15 +469,6 @@ def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: stacklevel=2, ) break - if spec.cfg.isp_cfg is None: - return - try: - from isaaclab_ppisp import resolve_and_normalize - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - camera_prim_path = spec.camera_prim_paths[0] if spec.camera_prim_paths else None - spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) def prepare_stage(self, stage: Any, num_envs: int) -> None: """No-op for Newton Warp - uses Newton scene directly without stage export. @@ -646,14 +563,6 @@ def render(self, render_data: RenderData): use_cuda_graph=self.cfg.use_cuda_graph and backend.model.tri_count == 0, ) - # Post-render PPISP: HDR scene-linear → LDR RGBA. Source/destination - # tensors were bound once in ``set_outputs``. - if render_data.ppisp_pipeline is not None: - render_data.ppisp_pipeline.apply( - render_data._ppisp_hdr_source, - render_data._ppisp_rgba_dest, - ) - def _launch_render(self, render_data: RenderData) -> None: """Launch the tiled-camera render kernels for sensor graph capture.""" # default_render_config is shared state across all Newton sensors, so set max_distance diff --git a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py index 201689809193..584ceddcd84a 100644 --- a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py +++ b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py @@ -312,7 +312,7 @@ def read(_data): monkeypatch.setattr(renderer, "_launch_render", read) for _ in range(2): - render_data = SimpleNamespace(graph=None, ppisp_pipeline=None) + render_data = SimpleNamespace(graph=None) renderer.render(render_data) assert render_data.graph is None assert len(refits) == 1 diff --git a/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.major b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.removed.rst b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.removed.rst new file mode 100644 index 000000000000..2dfc3e3f54d0 --- /dev/null +++ b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.removed.rst @@ -0,0 +1,2 @@ +* **Breaking:** Removed renderer-owned PPISP processing. Request ``rgb_radiance`` and apply PPISP + through the render buffer pipe instead of configuring ``CameraCfg.isp_cfg``. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index 88e8904ba945..e1d2d896d27f 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -32,7 +32,7 @@ from collections import deque from collections.abc import Iterable, Iterator, Sequence from pathlib import Path -from typing import TYPE_CHECKING, Any, NoReturn, cast +from typing import TYPE_CHECKING, Any, cast logger = logging.getLogger(__name__) @@ -113,7 +113,6 @@ ) if TYPE_CHECKING: - from isaaclab_ppisp import PpispPipeline from ovrtx import AttributeBinding, Operation, PendingFetch, RenderProductSetOutputs from isaaclab.renderers.base_renderer import VisualMaterialBatch @@ -151,11 +150,6 @@ } -_PPISP_IMPORT_ERROR_MESSAGE = ( - "isaaclab_ppisp is required when CameraCfg.isp_cfg is set. " - "Run `uv sync` from the Isaac Lab source checkout, or install the Isaac Lab wheel " - "with `uv pip install isaaclab`." -) _READ_GPU_TRANSFORMS_ENV = "ISAAC_LAB_OVRTX_READ_GPU_TRANSFORMS" @@ -183,14 +177,6 @@ def ovrtx_use_ovstage_enabled() -> bool: return value == "1" -def _raise_missing_ppisp_error(exc: ModuleNotFoundError) -> NoReturn: - # Only translate missing isaaclab_ppisp imports into the optional-dependency hint; - # unrelated missing modules should surface unchanged for easier debugging. - if exc.name != "isaaclab_ppisp" and not (exc.name and exc.name.startswith("isaaclab_ppisp.")): - raise exc - raise ModuleNotFoundError(_PPISP_IMPORT_ERROR_MESSAGE, name="isaaclab_ppisp") from exc - - def _read_gpu_transforms_enabled() -> bool: """Return whether OVRTX should read GPU transforms from its internal transform cache.""" value = os.environ.get(_READ_GPU_TRANSFORMS_ENV, "1").strip() @@ -341,16 +327,6 @@ def __init__(self, spec: CameraRenderSpec, device, render_scope_name: str): # Populated for "semantic_segmentation" (with an "idToLabels" mapping) and # "instance_segmentation" (with "idToLabels" and "idToSemantics" mappings). self.renderer_info: dict[str, Any] = {} - # Post-render PPISP pipeline composed when ``spec.cfg.isp_cfg`` is set. - # ``isp_cfg`` is already fully normalized by ``prepare_cameras`` by the time it reaches here. - self.ppisp_pipeline: PpispPipeline | None = None - if spec.cfg.isp_cfg is not None: - try: - from isaaclab_ppisp import PpispPipeline - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - self.ppisp_pipeline = PpispPipeline(spec.cfg.isp_cfg) def cleanup(self) -> None: """Release this camera's native resources and buffers. Safe to call repeatedly. @@ -369,7 +345,6 @@ def cleanup(self) -> None: self.intrinsic_bindings.clear() self.warp_buffers.clear() self.renderer_info.clear() - self.ppisp_pipeline = None class OVRTXRenderer(BaseRenderer): @@ -442,26 +417,13 @@ def visual_material_writer(self): return self._create_visual_material_writer if self._initialized_scene else None def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: - """Resolve the camera's PPISP cfg and apply OVRTX-specific USD overrides. - - When ``spec.cfg.isp_cfg`` is set, resolves it (sentinel discovery + - normalization) via :func:`isaaclab_ppisp.resolve_and_normalize` so - :mod:`isaaclab` does not need to know about PPISP. Then pins - ``exposure:*`` to neutral and applies ``OmniRtxCameraExposureAPI_1`` so - the RTX exposure model OVRTX embeds does not compound on top of the - ISP. Exposure is also neutralized when ``rgb_radiance`` is requested. Otherwise, the + """Neutralize camera exposure when ``rgb_radiance`` is requested. + + The RTX core OVRTX embeds has no pre-exposure radiance output, so neutral ``exposure:*`` + values and ``OmniRtxCameraExposureAPI_1`` are authored on the camera prims. Otherwise, the camera prim's authored exposure is left alone. """ - if spec.cfg.isp_cfg is not None: - try: - from isaaclab_ppisp import resolve_and_normalize - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - camera_prim_path = spec.camera_prim_paths[0] if spec.camera_prim_paths else None - spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) - needs_neutral_exposure = spec.cfg.isp_cfg is not None or "rgb_radiance" in spec.cfg.data_types - if needs_neutral_exposure and spec.camera_prim_paths: + if "rgb_radiance" in spec.cfg.data_types and spec.camera_prim_paths: _apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) def prepare_stage(self, stage: Any, num_envs: int) -> None: @@ -905,23 +867,6 @@ def set_outputs(self, render_data: OVRTXCameraRenderData, output_data: dict[str, radiance = render_data.warp_buffers.get(str(RenderBufferKind.RGB_RADIANCE)) if radiance is not None: render_data.warp_buffers.setdefault(str(RenderBufferKind.RGB_HDR), radiance) - # When PPISP is composed but the user did not request the raw HDR AOV, - # allocate an internal HDR scratch buffer under "rgb_hdr" so both the - # HdrColor extractor and PPISP dispatch can use the same buffer map. - if render_data.ppisp_pipeline is not None and str(RenderBufferKind.RGB_HDR) not in render_data.warp_buffers: - ref_proxy = next(iter(output_data.values())) - render_data.warp_buffers[str(RenderBufferKind.RGB_HDR)] = wp.zeros( - (render_data.num_envs, render_data.height, render_data.width, 3), - dtype=wp.float32, - device=ref_proxy.device, - ) - if render_data.ppisp_pipeline is not None: - if str(RenderBufferKind.RGBA) not in render_data.warp_buffers: - raise ValueError( - "OVRTX renderer ISP requires 'rgba' (or 'rgb', which aliases into rgba) as the" - " LDR output destination, but neither was provided. Add 'rgb' or 'rgba' to" - " Camera.cfg.data_types when isp_cfg is set." - ) def _update_camera_legacy( self, @@ -1243,24 +1188,12 @@ def _extract_hdr_color_tiles( return if tiled_data.dtype not in (wp.float16, wp.float32): raise TypeError(f"Unsupported OVRTX HdrColor dtype: {tiled_data.dtype}.") - self._launch_extract_all_tiles(render_data, tiled_data, output_buffers["rgb_hdr"]) - - def _prepare_ppisp_hdr_source( - self, render_data: OVRTXCameraRenderData, tiled_data: wp.array, output_buffers: dict - ) -> wp.array: - """Return the PPISP HdrColor source on the output buffer device.""" - if render_data.ppisp_pipeline is None: - return tiled_data - - output_device = str(output_buffers[str(RenderBufferKind.RGB_HDR)].device) - if str(tiled_data.device) == output_device: - return tiled_data - - # The render product pins ``deviceIds`` to this renderer's CUDA device, so the mapping - # normally lands on the output device already. This stays as a fallback for the case OVRTX - # reports as "deviceIds ... not in the active device set" and falls back to automatic - # assignment. - return wp.clone(tiled_data, device=output_device) + output = output_buffers["rgb_hdr"] + if tiled_data.device != output.device: + # OVRTX may fall back to automatic device assignment when the requested deviceIds + # is outside its active device set. Extract on the destination's device in that case. + tiled_data = wp.clone(tiled_data, device=output.device) + self._launch_extract_all_tiles(render_data, tiled_data, output) def _process_render_frame(self, render_data: OVRTXCameraRenderData, frame, output_buffers: dict) -> None: """Extract RGB, depth, albedo, and semantic from a single render frame into output_buffers.""" @@ -1273,7 +1206,7 @@ def _process_render_frame(self, render_data: OVRTXCameraRenderData, frame, outpu if ldr_color is not None: buffer_key = None - if render_data.ppisp_pipeline is None and "rgba" in output_buffers: + if "rgba" in output_buffers: buffer_key = "rgba" else: # The output buffers must contain only one simple shading data type at most after resolution of the data @@ -1308,7 +1241,6 @@ def _process_render_frame(self, render_data: OVRTXCameraRenderData, frame, outpu hdr_color = frame.render_vars.get(render_data.render_var_keys[_HDR_COLOR_VAR]) if hdr_color is not None and "rgb_hdr" in output_buffers: with self._map_render_var_to_dlpack(hdr_color) as tiled_hdr_data: - tiled_hdr_data = self._prepare_ppisp_hdr_source(render_data, tiled_hdr_data, output_buffers) self._extract_hdr_color_tiles(render_data, tiled_hdr_data, output_buffers) self._process_id_segmentation_render_var( @@ -1424,13 +1356,6 @@ def _process_render_products( data.warp_buffers, ) - # Post-render PPISP uses each camera's own HDR source and RGBA destination. - if data.ppisp_pipeline is not None: - data.ppisp_pipeline.apply( - data.warp_buffers[str(RenderBufferKind.RGB_HDR)], - data.warp_buffers[str(RenderBufferKind.RGBA)], - ) - def _close_legacy(self) -> None: """Release the renderer's tensor bindings. See :meth:`close`.""" diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py index 65d691a49739..f74190e45f0e 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py @@ -156,8 +156,7 @@ def build_render_product_as_string( resolved camera path after runtime cloning. Args: - spec: Camera configuration, environment count, and camera paths. ISP configurations - automatically request the HDR render variable in addition to the configured outputs. + spec: Camera configuration, environment count, and camera paths. render_data: Camera's render scope and product identity. device_id: CUDA device index the render product is pinned to, so its render var buffers are allocated on the same device as the Warp kernels that read them. @@ -168,8 +167,6 @@ def build_render_product_as_string( Render product USD layer, including the USDA header and default prim metadata. """ data_types = list(spec.cfg.data_types) - if spec.cfg.isp_cfg is not None and "rgb_hdr" not in data_types: - data_types.append("rgb_hdr") tiled_width, tiled_height = render_data.num_cols * render_data.width, render_data.num_rows * render_data.height render_var_configs = get_render_var_configs(data_types, render_data.render_scope_name) minimal_mode = next( diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index 96153d1799ad..1f8ad68e193c 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -196,13 +196,11 @@ def test_ovrtx_render_submits_every_product_and_routes_requested_outputs( cameras = [_make_ovrtx_camera_render_data() for _ in range(2 if batch else 1)] products = {} processed = [] - postprocessed = [] submissions = [] published_ordinals = [] for index, camera in enumerate(cameras): camera.render_product_path = f"/Render/Camera{index}" camera.warp_buffers = {str(RenderBufferKind.RGB_HDR): object(), str(RenderBufferKind.RGBA): object()} - camera.ppisp_pipeline = types.SimpleNamespace(apply=lambda *buffers: postprocessed.append(buffers)) products[camera.render_product_path] = types.SimpleNamespace(frames=[object()]) renderer._render_product_paths = [*products, "/Render/UnrequestedCamera"] if missing_output == "product": @@ -229,15 +227,10 @@ def advance_write_floor(*, ordinal): assert processed == [ (camera, products[camera.render_product_path].frames[0], camera.warp_buffers) for camera in cameras ] - assert postprocessed == [ - (camera.warp_buffers[str(RenderBufferKind.RGB_HDR)], camera.warp_buffers[str(RenderBufferKind.RGBA)]) - for camera in cameras - ] else: with pytest.raises(RuntimeError, match=cameras[-1].render_product_path): render(request) assert not processed - assert not postprocessed assert len(submissions) == 1 # Unrequested products are submitted too, and only the requested cameras are read back. @@ -578,9 +571,7 @@ def test_ovrtx_prepare_cameras_neutralizes_exposure_for_radiance(data_types): stage = Usd.Stage.CreateInMemory() camera = UsdGeom.Camera.Define(stage, "/World/Camera").GetPrim() camera.CreateAttribute("exposure:iso", Sdf.ValueTypeNames.Float).Set(100.0) - spec = types.SimpleNamespace( - cfg=types.SimpleNamespace(data_types=data_types, isp_cfg=None), camera_prim_paths=("/World/Camera",) - ) + spec = types.SimpleNamespace(cfg=types.SimpleNamespace(data_types=data_types), camera_prim_paths=("/World/Camera",)) _make_ovrtx_renderer_without_backend().prepare_cameras(stage, spec) assert camera.GetAttribute("exposure:iso").Get() == (0.0 if "rgb_radiance" in data_types else 100.0) @@ -619,47 +610,6 @@ def test_ovrtx_set_outputs_wraps_caller_torch_zero_copy(): assert "rgb" not in render_data.warp_buffers -def test_ovrtx_set_outputs_routes_ppisp_buffers_through_warp_buffers(): - """OVRTXRenderer.set_outputs stores PPISP source/destination in warp_buffers.""" - renderer = _make_ovrtx_renderer_without_backend() - - cfg = _make_camera_cfg(["rgb"]) - data = CameraData.allocate( - data_types=cfg.data_types, - height=8, - width=16, - num_views=2, - device="cpu", - supported_specs=renderer.supported_output_types(), - ) - render_data = _make_ovrtx_camera_render_data() - render_data.ppisp_pipeline = object() - renderer.set_outputs(render_data, data.output) - - assert render_data.warp_buffers["rgba"].ptr == data.output["rgba"].warp.ptr - assert "rgb_hdr" in render_data.warp_buffers - assert render_data.warp_buffers["rgb_hdr"].shape == (2, 8, 16, 3) - assert render_data.warp_buffers["rgb_hdr"].dtype is wp.float32 - - -def test_ovrtx_process_frame_skips_ldr_rgba_when_ppisp_is_active(): - """PPISP owns RGBA output, so OVRTX LdrColor should not pre-fill it.""" - - class FailingRenderVar: - def map(self, *args, **kwargs): - raise AssertionError("PPISP RGBA output must not read OVRTX LdrColor") - - renderer = _make_ovrtx_renderer_without_backend() - render_data = _make_ovrtx_camera_render_data() - render_data.ppisp_pipeline = object() - key = "LdrColor" - if ovrtx_renderer_module.uses_prim_path_render_vars(ovrtx_renderer_module.OVRTX_VERSION): - key = f"/{render_data.render_scope_name}/Vars/{key}" - frame = types.SimpleNamespace(render_vars={key: FailingRenderVar()}) - - renderer._process_render_frame(render_data, frame, {"rgba": object()}) - - # Render-var keys depend only on the OVRTX version; ``use_ovstage`` only selects the registration path. @pytest.mark.parametrize(("use_ovstage", "version"), [(False, "0.4"), (True, "0.5")]) def test_ovrtx_process_frame_reads_authored_camera_render_vars(monkeypatch, use_ovstage, version): @@ -749,31 +699,42 @@ def fake_map(self, render_var): render_data.cleanup() -def test_ovrtx_ppisp_hdr_source_is_cloned_to_output_device(monkeypatch): - """PPISP HdrColor source is moved to the HDR output buffer device.""" - - class FakeArray: - device = "cuda:1" - - class OutputArray: - device = "cuda:0" - - cloned = object() +def test_ovrtx_radiance_is_extracted_on_the_output_device(monkeypatch): + """A mapped HDR buffer on another device is copied before radiance tile extraction.""" + renderer = _make_ovrtx_renderer_without_backend() + renderer._device = "cpu" + render_data = _make_ovrtx_camera_render_data() + data = CameraData.allocate( + data_types=["rgb_radiance"], + height=8, + width=16, + num_views=2, + device="cpu", + supported_specs=renderer.supported_output_types(), + ) + renderer.set_outputs(render_data, data.output) + # Only the cross-device transport is mocked; the destination is filled by the real extractor. + source = types.SimpleNamespace(device="cuda:1", dtype=wp.float32) + cloned = wp.full((8, 32, 4), 2.0, dtype=wp.float32, device="cpu") clone_calls = [] def fake_clone(src, *, device): clone_calls.append((src, device)) return cloned - monkeypatch.setattr(wp, "clone", fake_clone) + extract = renderer._launch_extract_all_tiles - renderer = _make_ovrtx_renderer_without_backend() - render_data = _make_ovrtx_camera_render_data() - render_data.ppisp_pipeline = object() - source = FakeArray() + def extract_on_output_device(render_data, source_buffer, output_buffer): + assert clone_calls == [(source, output_buffer.device)], "Copy the mapped HDR buffer before extracting it." + extract(render_data, source_buffer, output_buffer) + + monkeypatch.setattr(wp, "clone", fake_clone) + monkeypatch.setattr(renderer, "_launch_extract_all_tiles", extract_on_output_device) + monkeypatch.setattr(renderer, "_map_render_var_to_dlpack", lambda source: contextlib.nullcontext(source)) + frame = types.SimpleNamespace(render_vars={render_data.render_var_keys["HdrColor"]: source}) + renderer._process_render_frame(render_data, frame, render_data.warp_buffers) - assert renderer._prepare_ppisp_hdr_source(render_data, source, {"rgb_hdr": OutputArray()}) is cloned - assert clone_calls == [(source, "cuda:0")] + np.testing.assert_array_equal(data.output["rgb_radiance"].warp.numpy(), 2.0) def test_launch_extract_all_tiles_rejects_wider_output_channels(): @@ -918,7 +879,6 @@ def test_ovrtx_cleanup_releases_only_the_given_render_data(monkeypatch, cleanup_ renderer._camera_render_data.append(render_data) render_data.warp_buffers = {"rgba": wp.zeros((8, 16, 4), dtype=wp.uint8, device="cpu")} render_data.renderer_info = {"semantic_segmentation": {"idToLabels": {}}} - render_data.ppisp_pipeline = object() if use_ovstage: render_data.camera_xform_query = "to_remove" render_data.resources.callback(renderer.backend.paths.destroy_path_list, "to_remove") @@ -965,7 +925,6 @@ def test_ovrtx_cleanup_releases_only_the_given_render_data(monkeypatch, cleanup_ assert render_data.intrinsic_bindings == [] assert render_data.warp_buffers == {} assert render_data.renderer_info == {} - assert render_data.ppisp_pipeline is None assert render_data.pending is render_data.ready is None assert renderer._render_product_paths == ["/RenderCamera_0/RenderProduct_camera"] assert renderer._initialized_scene is True diff --git a/source/isaaclab_ov/test/test_ovrtx_usd.py b/source/isaaclab_ov/test/test_ovrtx_usd.py index f56d3cf9bf85..cb5989c0b6b0 100644 --- a/source/isaaclab_ov/test/test_ovrtx_usd.py +++ b/source/isaaclab_ov/test/test_ovrtx_usd.py @@ -311,30 +311,18 @@ def test_render_product_pins_device_ids_to_the_requested_cuda_device(camera_spec @pytest.mark.parametrize( - ("data_types", "use_isp", "routes_hdr"), - [ - (["rgb"], True, True), - (["rgb", "rgb_hdr"], False, True), - (["rgb", "rgb_radiance"], False, True), - ([], True, True), - (["rgb"], False, False), - ], + ("data_types", "routes_hdr"), + [(["rgb"], False), (["rgb", "rgb_hdr"], True), (["rgb", "rgb_radiance"], True)], ) -def test_render_product_isp_requests_hdr_without_mutating_camera_outputs( - camera_spec, render_data, data_types, use_isp, routes_hdr -): - """Route Gaussian HDR for explicit or ISP inputs without changing public outputs.""" +def test_render_product_routes_gaussian_hdr_only_for_hdr_outputs(camera_spec, render_data, data_types, routes_hdr): + """Route Gaussian HDR when an HDR output is requested, without changing the camera outputs.""" camera_spec.cfg.data_types = data_types.copy() - camera_spec.cfg.isp_cfg = object() if use_isp else None render_product = build_render_product_as_string(camera_spec, render_data, device_id=0) layer = Sdf.Layer.CreateAnonymous(".usda") assert layer.ImportFromString(render_product) product_path = "/RenderCamera_0/RenderProduct" ordered_vars = layer.GetRelationshipAtPath(product_path + ".orderedVars") - # An empty output list requests no LDR color; ISP still adds its HDR source. - expected_vars = ["LdrColor"] if data_types else [] - if routes_hdr: - expected_vars.append("HdrColor") + expected_vars = ["LdrColor", "HdrColor"] if routes_hdr else ["LdrColor"] assert list(ordered_vars.targetPathList.explicitItems) == [ Sdf.Path(f"/RenderCamera_0/Vars/{name}") for name in expected_vars ] diff --git a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py index 83e4f140cfb9..02eeb87b6512 100644 --- a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py +++ b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py @@ -222,10 +222,7 @@ def drain(self): writer = Writer() renderer._visual_material_writer_ref = lambda: writer renderer.render_batch( - [ - SimpleNamespace(render_product_path=path, ppisp_pipeline=None, warp_buffers={}) - for path in renderer._render_product_paths - ] + [SimpleNamespace(render_product_path=path, warp_buffers={}) for path in renderer._render_product_paths] ) assert events == expected_events @@ -258,7 +255,7 @@ def advance_write_floor(**_kwargs): renderer.backend.stage.advance_write_floor = advance_write_floor with pytest.raises(ValueError, match=failure): - renderer.render(SimpleNamespace(render_product_path="/RenderCamera_0/Product", ppisp_pipeline=None)) + renderer.render(SimpleNamespace(render_product_path="/RenderCamera_0/Product")) assert events == expected_events diff --git a/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.major b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.removed.rst b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.removed.rst new file mode 100644 index 000000000000..2dfc3e3f54d0 --- /dev/null +++ b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.removed.rst @@ -0,0 +1,2 @@ +* **Breaking:** Removed renderer-owned PPISP processing. Request ``rgb_radiance`` and apply PPISP + through the render buffer pipe instead of configuring ``CameraCfg.isp_cfg``. diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index 3507061486cf..51899fe2341e 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -13,7 +13,7 @@ import uuid from collections.abc import Sequence from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, NoReturn +from typing import TYPE_CHECKING, Any import numpy as np import warp as wp @@ -43,8 +43,6 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: - from isaaclab_ppisp import PpispPipeline - from omni.replicator.core.scripts.utils.viewport_manager import HydraTexture from isaaclab.sensors.camera.camera_data import CameraData @@ -52,21 +50,6 @@ from .isaac_rtx_renderer_cfg import SIMPLE_SHADING_MODES, IsaacRtxRendererCfg -_PPISP_IMPORT_ERROR_MESSAGE = ( - "isaaclab_ppisp is required when CameraCfg.isp_cfg is set. " - "Run `uv sync` from the Isaac Lab source checkout, or install the Isaac Lab wheel " - "with `uv pip install isaaclab`." -) - - -def _raise_missing_ppisp_error(exc: ModuleNotFoundError) -> NoReturn: - # Only translate missing isaaclab_ppisp imports into the optional-dependency hint; - # unrelated missing modules should surface unchanged for easier debugging. - if exc.name != "isaaclab_ppisp" and not (exc.name and exc.name.startswith("isaaclab_ppisp.")): - raise exc - raise ModuleNotFoundError(_PPISP_IMPORT_ERROR_MESSAGE, name="isaaclab_ppisp") from exc - - # Outputs read from the HdrColor AOV. Radiance differs only by the neutral camera exposure. _HDR_DATA_TYPES = frozenset({str(RenderBufferKind.RGB_HDR), str(RenderBufferKind.RGB_RADIANCE)}) @@ -140,12 +123,6 @@ class IsaacRtxRenderData: intrinsic_selection: Any = None intrinsic_row_attribute: str = "" intrinsic_stage: Any = None - ppisp_pipeline: PpispPipeline | None = None - """Post-render PPISP pipeline composed when ``spec.cfg.isp_cfg`` is set.""" - _hdr_scratch_wp: wp.array | None = None - """Internal HDR scratch buffer allocated when the user did not request - ``"rgb_hdr"`` in ``data_types`` but the PPISP pipeline still needs - somewhere to receive the HDR AOV before LDR conversion.""" def __post_init__(self): """Compile the stage-bound calibration columns owned by this render product.""" @@ -217,32 +194,19 @@ def visual_material_writer(self): return FabricVisualMaterialWriter def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: - """Resolve the camera's PPISP cfg and apply RTX-specific USD overrides. - - When ``spec.cfg.isp_cfg`` is set, resolves it (sentinel discovery + - normalization) via :func:`isaaclab_ppisp.resolve_and_normalize` so - :mod:`isaaclab` does not need to know about PPISP. Then pins - ``exposure:*`` to neutral and applies ``OmniRtxCameraExposureAPI_1`` so - RTX's physical-camera exposure model does not compound on top of the - ISP. Exposure is also neutralized when ``rgb_radiance`` is requested. Otherwise, the + """Neutralize camera exposure when ``rgb_radiance`` is requested. + + RTX has no pre-exposure radiance output, so neutral ``exposure:*`` values and + ``OmniRtxCameraExposureAPI_1`` are authored on the camera prims. Otherwise, the camera prim's authored exposure is left alone. :attr:`~isaaclab.sensors.camera.CameraCfg.background_color` is applied per-render-product in :meth:`create_render_data` via USD attributes. """ - if spec.cfg.isp_cfg is not None: - try: - from isaaclab_ppisp import resolve_and_normalize - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - camera_prim_path = spec.camera_prim_paths[0] if spec.camera_prim_paths else None - spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) - if spec.cfg.isp_cfg is not None or not _HDR_DATA_TYPES.isdisjoint(spec.cfg.data_types): + if not _HDR_DATA_TYPES.isdisjoint(spec.cfg.data_types): # Gaussian splats otherwise skip tonemapping and never reach the HdrColor AOV. get_settings_manager().set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) - needs_neutral_exposure = spec.cfg.isp_cfg is not None or "rgb_radiance" in spec.cfg.data_types - if needs_neutral_exposure and spec.camera_prim_paths: + if "rgb_radiance" in spec.cfg.data_types and spec.camera_prim_paths: _apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: @@ -410,10 +374,7 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: aov=SIMPLE_SHADING_AOV, output_data_type=np.uint8, output_channels=4 ) - needs_hdr_color = not _HDR_DATA_TYPES.isdisjoint(spec.cfg.data_types) or ( - spec.cfg.isp_cfg is not None and any(data_type in ("rgb", "rgba") for data_type in spec.cfg.data_types) - ) - if needs_hdr_color: + if not _HDR_DATA_TYPES.isdisjoint(spec.cfg.data_types): rep.AnnotatorRegistry.register_annotator_from_aov( aov="HdrColor", output_data_type=np.float32, output_channels=4 ) @@ -422,15 +383,8 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: annotators = {} for annotator_type in spec.cfg.data_types: if annotator_type == "rgba" or annotator_type == "rgb": - if spec.cfg.isp_cfg is not None: - if str(RenderBufferKind.RGB_HDR) not in annotators: - annotator = rep.AnnotatorRegistry.get_annotator( - "HdrColor", device=spec.device, do_array_copy=False - ) - annotators[str(RenderBufferKind.RGB_HDR)] = annotator - else: - annotator = rep.AnnotatorRegistry.get_annotator("rgb", device=spec.device, do_array_copy=False) - annotators["rgba"] = annotator + annotator = rep.AnnotatorRegistry.get_annotator("rgb", device=spec.device, do_array_copy=False) + annotators["rgba"] = annotator elif annotator_type in _HDR_DATA_TYPES: # Radiance reads the same HdrColor AOV through the ``rgb_hdr`` alias set in # :meth:`set_outputs`; exposure is neutralized in :meth:`prepare_cameras`. @@ -496,20 +450,10 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: enable_minimal_render_mode=not needs_color_render, ) - ppisp_pipeline = None - if spec.cfg.isp_cfg is not None: - try: - from isaaclab_ppisp import PpispPipeline - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - ppisp_pipeline = PpispPipeline(spec.cfg.isp_cfg) - return IsaacRtxRenderData( annotators=annotators, render_product=rp, spec=spec, - ppisp_pipeline=ppisp_pipeline, ) def _apply_simple_shading_settings( @@ -570,30 +514,10 @@ def _resolve_simple_shading_mode(self, spec: CameraRenderSpec) -> int | None: def set_outputs(self, render_data: IsaacRtxRenderData, output_data: dict[str, ProxyArray]): """Store reference to output buffers for writing during render. See :meth:`~isaaclab.renderers.base_renderer.BaseRenderer.set_outputs`.""" - if render_data.ppisp_pipeline is not None and str(RenderBufferKind.RGBA) not in output_data: - raise ValueError( - "Isaac RTX renderer ISP requires 'rgba' (or 'rgb', which aliases into rgba) as the" - " LDR output destination, but neither was provided. Add 'rgb' or 'rgba' to" - " Camera.cfg.data_types when isp_cfg is set." - ) radiance = output_data.get(str(RenderBufferKind.RGB_RADIANCE)) if radiance is not None and str(RenderBufferKind.RGB_HDR) not in output_data: output_data = {**output_data, str(RenderBufferKind.RGB_HDR): radiance} render_data.output_data = output_data - # Allocate an internal HDR scratch buffer when PPISP is composed but - # the user did not request the raw HDR AOV in ``data_types`` — the - # PPISP kernel still needs somewhere to receive the HDR annotator - # output before LDR conversion. - if render_data.ppisp_pipeline is not None and str(RenderBufferKind.RGB_HDR) not in output_data: - spec = render_data.spec - assert spec is not None - hdr_spec = self.supported_output_types()[RenderBufferKind.RGB_HDR] - assert hdr_spec.dtype is wp.float32 - render_data._hdr_scratch_wp = wp.zeros( - (spec.num_instances, spec.cfg.height, spec.cfg.width, hdr_spec.channels), - dtype=wp.float32, - device=spec.device, - ) def update_transforms(self) -> None: """Update shared Fabric transforms and propagate the visual hierarchy.""" @@ -682,11 +606,7 @@ def tiling_grid_shape(): # The RTX annotator may return an empty frame while its render product is warming up. # Clear the destination so callers do not observe stale data, then wait for the next frame. - if data_type == str(RenderBufferKind.RGB_HDR) and data_type not in output_data: - assert render_data._hdr_scratch_wp is not None - buf_wp = render_data._hdr_scratch_wp - else: - buf_wp = output_data[data_type].warp + buf_wp = output_data[data_type].warp if tiled_data_buffer.size == 0: buf_wp.zero_() continue @@ -768,14 +688,6 @@ def tiling_grid_shape(): replacement = 0.0 if self.cfg.depth_clipping_behavior == "zero" else cfg.spawn.clipping_range[1] replace_inf_depth_wp(buf_wp, replacement, device=device) - # Post-render PPISP: HDR scene-linear → LDR RGBA. The camera enforces - # that ``rgba`` (or ``rgb`` aliasing into it) is present when an ISP is - # configured, so writing to ``output_data["rgba"]`` is safe. - if render_data.ppisp_pipeline is not None: - hdr_proxy = output_data.get(str(RenderBufferKind.RGB_HDR)) - hdr_source = hdr_proxy.warp if hdr_proxy is not None else render_data._hdr_scratch_wp - render_data.ppisp_pipeline.apply(hdr_source, output_data[str(RenderBufferKind.RGBA)].warp) - def read_output(self, render_data: IsaacRtxRenderData, camera_data: CameraData) -> None: """Populate per-output metadata collected during render(). Pixel data already written in render(). @@ -814,5 +726,3 @@ def cleanup(self, render_data: IsaacRtxRenderData | None): render_data.output_data = None render_data.spec = None render_data.renderer_info.clear() - render_data.ppisp_pipeline = None - render_data._hdr_scratch_wp = None diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py index 156da3c2a1b9..d58f7f2e64a1 100644 --- a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py @@ -108,20 +108,16 @@ def test_renderer_marks_rtx_sensor_rendering_when_created(monkeypatch): @pytest.mark.parametrize( - ("data_types", "isp_cfg", "routes_hdr"), - [(["rgb"], None, False), (["rgb_hdr"], None, True), (["rgb_radiance"], None, True), (["rgb"], object(), True)], + ("data_types", "routes_hdr"), [(["rgb"], False), (["rgb_hdr"], True), (["rgb_radiance"], True)] ) -def test_prepare_cameras_routes_gaussian_hdr_for_hdr_cameras(monkeypatch, data_types, isp_cfg, routes_hdr): +def test_prepare_cameras_routes_gaussian_hdr_for_hdr_cameras(monkeypatch, data_types, routes_hdr): """Gaussian HDR routing is enabled only for cameras that read HDR color.""" _install_omni_stubs(monkeypatch) import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer settings = MagicMock() - spec = SimpleNamespace(cfg=SimpleNamespace(data_types=data_types, isp_cfg=isp_cfg), camera_prim_paths=()) - with ( - patch.object(rtx_renderer, "get_settings_manager", return_value=settings), - patch("isaaclab_ppisp.resolve_and_normalize", side_effect=lambda cfg, *args: cfg), - ): + spec = SimpleNamespace(cfg=SimpleNamespace(data_types=data_types), camera_prim_paths=()) + with patch.object(rtx_renderer, "get_settings_manager", return_value=settings): rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer).prepare_cameras(None, spec) hdr_call = call("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) @@ -139,9 +135,7 @@ def test_prepare_cameras_neutralizes_exposure_for_radiance(monkeypatch, data_typ stage = Usd.Stage.CreateInMemory() camera = UsdGeom.Camera.Define(stage, "/World/Camera").GetPrim() camera.CreateAttribute("exposure:iso", Sdf.ValueTypeNames.Float).Set(100.0) - spec = SimpleNamespace( - cfg=SimpleNamespace(data_types=data_types, isp_cfg=None), camera_prim_paths=("/World/Camera",) - ) + spec = SimpleNamespace(cfg=SimpleNamespace(data_types=data_types), camera_prim_paths=("/World/Camera",)) IsaacRtxRenderer.__new__(IsaacRtxRenderer).prepare_cameras(stage, spec) assert camera.GetAttribute("exposure:iso").Get() == (0.0 if "rgb_radiance" in data_types else 100.0) @@ -176,7 +170,7 @@ def test_radiance_reads_the_hdr_annotator_into_its_output(monkeypatch, data_type camera_prim_paths=("/World/Camera",), device="cpu", view_count=1, - cfg=SimpleNamespace(data_types=data_types, width=3, height=2, isp_cfg=None), + cfg=SimpleNamespace(data_types=data_types, width=3, height=2), ) renderer = rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer) renderer.cfg = IsaacRtxRendererCfg() @@ -245,7 +239,6 @@ def test_create_render_data_uses_unique_sdf_safe_render_product_name(monkeypatch data_types=["rgb"], width=64, height=64, - isp_cfg=None, colorize_semantic_segmentation=False, colorize_instance_segmentation=False, colorize_instance_id_segmentation=False, @@ -362,7 +355,6 @@ def _create_attribute(name, value_type): data_types=data_types, width=64, height=64, - isp_cfg=None, colorize_semantic_segmentation=False, colorize_instance_segmentation=False, colorize_instance_id_segmentation=False, @@ -580,8 +572,6 @@ def test_render_treats_empty_annotator_frame_as_not_ready(monkeypatch, data_type cfg=SimpleNamespace(width=64, height=64), ), renderer_info={}, - ppisp_pipeline=None, - _hdr_scratch_wp=None, ) renderer = rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer) renderer.cfg = IsaacRtxRendererCfg() @@ -621,8 +611,6 @@ def test_render_keeps_leading_channels_of_padded_annotator_tiles(monkeypatch, da output_data={data_type: SimpleNamespace(warp=output_buffer)}, spec=SimpleNamespace(view_count=view_count, device="cpu", cfg=SimpleNamespace(width=width, height=height)), renderer_info={}, - ppisp_pipeline=None, - _hdr_scratch_wp=None, ) renderer = rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer) renderer.cfg = IsaacRtxRendererCfg() @@ -674,8 +662,6 @@ def test_render_batch_updates_once_before_extracting_ready_cameras(monkeypatch, else SimpleNamespace(view_count=1, device="cpu", cfg=SimpleNamespace(width=1, height=1)) ), renderer_info={}, - ppisp_pipeline=None, - _hdr_scratch_wp=None, ) ) if state == "ready": diff --git a/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.added.rst b/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.added.rst new file mode 100644 index 000000000000..630e96c62874 --- /dev/null +++ b/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.added.rst @@ -0,0 +1,3 @@ +* Added ``PpispStepCfg`` and ``prepare_ppisp`` for shared camera and observation render-buffer + pipelines, including RGB destinations and optional read-only shared controller weights. +* Added ``PpispDiscoveryMode`` for explicit USD configuration discovery before pipeline preparation. diff --git a/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.changed.rst b/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..68372f67d027 --- /dev/null +++ b/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.changed.rst @@ -0,0 +1,3 @@ +* Separated PPISP numerical execution from camera and renderer discovery. Configured processing with + ``CameraCfg.pipeline=[PpispStepCfg(params={"cfg": resolved_cfg})]`` instead of ``CameraCfg.isp_cfg``; + resolved USD settings with ``ppisp_cfg_from_usd_camera`` or ``resolve_and_normalize`` first. diff --git a/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.fixed.rst b/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.fixed.rst new file mode 100644 index 000000000000..daf552f71a68 --- /dev/null +++ b/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.fixed.rst @@ -0,0 +1,2 @@ +* Fixed static PPISP execution on CPU by compiling image-transform kernels separately from + CUDA-only controller code. diff --git a/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.minor b/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ppisp/docs/README.md b/source/isaaclab_ppisp/docs/README.md index 351571fa30db..985bc3635bf9 100644 --- a/source/isaaclab_ppisp/docs/README.md +++ b/source/isaaclab_ppisp/docs/README.md @@ -1,8 +1,31 @@ # Isaac Lab PPISP -This extension provides a renderer-backend-agnostic PPISP (Physically Plausible -Image Signal Processing) pipeline for Isaac Lab camera outputs. +This extension provides PPISP (Physically Plausible Image Signal Processing) as +an operation in the shared render-buffer pipeline used by cameras and observations. -PPISP consumes HDR scene-linear color from renderer backends and writes LDR -`rgb` / `rgba` camera output after each render tick when `CameraCfg.isp_cfg` is -set. +`PpispStepCfg` reads float32 NHWC scene-linear `rgb_radiance` and writes uint8 +`rgb` with `camera_response` encoding. For camera processing, configure +`CameraCfg.pipeline=[PpispStepCfg(params={"cfg": ppisp_cfg})]`. The camera requests +required radiance internally; only the requested public outputs are published. + +Resolve authored settings explicitly before constructing the pipeline: + +```python +from isaaclab_ppisp import PpispStepCfg, ppisp_cfg_from_usd_camera + +ppisp_cfg = ppisp_cfg_from_usd_camera(source_camera_prim) +step = PpispStepCfg(params={"cfg": ppisp_cfg}) +``` + +The same step can run in `isaaclab.utils.images.RenderBufferPipe` outside a camera. +Its input/output bindings can be changed independently of PPISP math. For RGBA, +use `outputs={"color": "rgba"}` and `params={"cfg": ppisp_cfg, "channels": 4}`. +PPISP consumes the preceding operation's radiance, so an earlier exposure or scale +operation changes its result. Display-encoded RGB is rejected as PPISP input. + +Each operation owns its controller scratch and output buffers. Optional +`params["controller_weights"]=PpispControllerWeights(device_weights)` can supply +shared, read-only device weights; configuration copies preserve this resource's identity. The +caller owns upload synchronization, and mutable controller workspace stays private. +The controller requires CUDA; static PPISP also runs on CPU. `PpispPipeline` remains +available for direct numerical use with explicitly supplied Warp buffers. diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py index 416f7d9e134e..fb6d434387d7 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py @@ -5,13 +5,9 @@ """Post-render PPISP (Physically Plausible Image Signal Processing) for IsaacLab. -Provides the renderer-backend-agnostic ISP pipeline that converts the renderer's -HDR scene-linear AOV to LDR RGBA at the end of a render tick. Renderer backends -(:class:`~isaaclab_physx.renderers.IsaacRtxRenderer`, -:class:`~isaaclab_ov.renderers.OVRTXRenderer`, -:class:`~isaaclab_newton.renderers.NewtonWarpRenderer`) compose -:class:`PpispPipeline` internally when the camera's -:attr:`~isaaclab.sensors.camera.CameraCfg.isp_cfg` is set. +Provides a shared render-buffer operation that converts scene-linear radiance to RGB or RGBA. +Camera and observation pipelines use :class:`PpispStepCfg` with concrete settings. USD discovery +helpers remain explicit, and renderer backends do not own PPISP execution or controller workspace. """ import importlib.metadata diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi index 476c537b9420..7b7b417455eb 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi +++ b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi @@ -7,7 +7,10 @@ __all__ = [ "PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN", "PPISP_DEFAULT_INPUTS", "PpispCfg", + "PpispControllerWeights", + "PpispDiscoveryMode", "PpispPipeline", + "PpispStepCfg", "apply_ppisp_to_rgba", "apply_ppisp_to_rgba_with_controller_params", "auto_any_ppisp_cfg", @@ -18,6 +21,7 @@ __all__ = [ "normalize_ppisp_cfg", "ppisp_cfg_from_usd_camera", "ppisp_cfg_from_usd_stage", + "prepare_ppisp", "resolve_and_normalize", ] @@ -25,6 +29,7 @@ from .cfg import ( PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN, PPISP_DEFAULT_INPUTS, PpispCfg, + PpispDiscoveryMode, auto_any_ppisp_cfg, auto_camera_ppisp_cfg, default_ppisp_inputs, @@ -35,4 +40,5 @@ from .cfg import ( resolve_and_normalize, ) from .kernels import apply_ppisp_to_rgba, apply_ppisp_to_rgba_with_controller_params, compute_ppisp_controller_params -from .pipeline import PpispPipeline +from .pipeline import PpispControllerWeights, PpispPipeline +from .processor import PpispStepCfg, prepare_ppisp diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py b/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py index 37859bda6d86..d5c9079b6b45 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py @@ -12,6 +12,7 @@ from __future__ import annotations from dataclasses import field +from enum import StrEnum from typing import Any from isaaclab.utils import configclass @@ -79,6 +80,16 @@ def default_ppisp_inputs() -> dict[str, float | tuple[float, float]]: return dict(PPISP_DEFAULT_INPUTS) +class PpispDiscoveryMode(StrEnum): + """Explicit USD configuration discovery strategies used before preparing a PPISP stage.""" + + AUTO_CAMERA = "auto_camera" + """Read the specified camera, or the first PPISP camera when no path is supplied.""" + + AUTO_ANY = "auto_any" + """Read the specified camera, then fall back to the first PPISP camera on the stage.""" + + @configclass class PpispCfg: """Configuration for PPISP post-processing. @@ -321,29 +332,17 @@ def has_ppisp_camera_attrs(camera_prim: Any | None) -> bool: return _has_ppisp_camera_attrs(camera_prim) -def resolve_and_normalize(isp_cfg: Any, stage: Any, camera_prim_path: str | None = None) -> PpispCfg | None: - """Resolve a Camera sensor batch's ``isp_cfg`` to a normalised cfg or ``None``. - - Handles all three legal forms of :attr:`~isaaclab.sensors.camera.CameraCfg.isp_cfg`: - - * ``None`` → returns ``None``. - * :class:`~isaaclab.sensors.camera.CameraISPMode` sentinel — checks the - target camera via :func:`auto_camera_ppisp_cfg` (and uses - :func:`auto_any_ppisp_cfg` for ``AUTO_ANY`` or when no camera path is - supplied) to discover a PPISP camera. Returns the parsed + normalised - :class:`PpispCfg`, or ``None`` if no PPISP camera matched. - * Concrete :class:`PpispCfg` — normalises in place (validates input keys, - fills defaults, and merges camera-authored USD values when - ``camera_prim_path`` is set). +def resolve_and_normalize( + isp_cfg: PpispCfg | PpispDiscoveryMode | None, stage: Any, camera_prim_path: str | None = None +) -> PpispCfg | None: + """Resolve explicit settings or a discovery mode using the caller's USD stage. - This is the single entry point renderer backends call inside their - ``prepare_cameras`` hook so :mod:`isaaclab.sensors.camera` does not need - to know about PPISP types at all. The returned cfg applies to the whole - Camera sensor batch; callers pass the first matched camera prim path for - the camera-local discovery phase. + Call this before constructing a PPISP stage. The processing operation receives only concrete + settings and never looks up a camera or stage. Legacy ``CameraISPMode`` string-enum values + remain accepted for migration without importing the camera package. Args: - isp_cfg: The Camera sensor's :attr:`isp_cfg` value (``None``, ``CameraISPMode``, or :class:`PpispCfg`). + isp_cfg: Explicit settings, a discovery mode, or ``None`` to disable processing. stage: USD stage used for sentinel discovery and camera-path resolution. camera_prim_path: Optional absolute path of the first matched camera prim in the Camera sensor batch. When omitted, discovery uses the @@ -352,14 +351,12 @@ def resolve_and_normalize(isp_cfg: Any, stage: Any, camera_prim_path: str | None Returns: A fully-normalised :class:`PpispCfg`, or ``None`` if the batch has no ISP. """ - # Local import avoids a top-of-module dep on isaaclab.sensors. - from isaaclab.sensors.camera.camera_isp import CameraISPMode - if isp_cfg is None: return None - if isinstance(isp_cfg, CameraISPMode): + if isinstance(isp_cfg, str): + mode = PpispDiscoveryMode(isp_cfg) resolved = auto_camera_ppisp_cfg(stage, camera_prim_path) if camera_prim_path else None - if resolved is None and (isp_cfg == CameraISPMode.AUTO_ANY or not camera_prim_path): + if resolved is None and (mode == PpispDiscoveryMode.AUTO_ANY or not camera_prim_path): resolved = auto_any_ppisp_cfg(stage) if resolved is None: return None diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py b/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py index 02a3ed36cd7c..631a3e5f6e87 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py @@ -508,7 +508,8 @@ def _ppisp_controller_mlp_native_kernel( _ppisp_controller_mlp_native(features, weights, controller_params, prior_exposure, camera_id, lane) -@wp.kernel(enable_backward=False) +# Keep image transforms compilable on CPU without the CUDA-only native controller snippets. +@wp.kernel(enable_backward=False, module="unique") def _apply_ppisp_kernel( hdr_color: wp.array4d(dtype=wp.float32), out_rgba: wp.array4d(dtype=wp.uint8), @@ -591,10 +592,11 @@ def _apply_ppisp_kernel( out_rgba[camera_id, height_id, width_id, 0] = wp.uint8(wp.clamp(out_rgb[0], 0.0, 1.0) * 255.0) out_rgba[camera_id, height_id, width_id, 1] = wp.uint8(wp.clamp(out_rgb[1], 0.0, 1.0) * 255.0) out_rgba[camera_id, height_id, width_id, 2] = wp.uint8(wp.clamp(out_rgb[2], 0.0, 1.0) * 255.0) - out_rgba[camera_id, height_id, width_id, 3] = wp.uint8(255) + if out_rgba.shape[3] == 4: + out_rgba[camera_id, height_id, width_id, 3] = wp.uint8(255) -@wp.kernel(enable_backward=False) +@wp.kernel(enable_backward=False, module="unique") def _apply_ppisp_controller_kernel( hdr_color: wp.array4d(dtype=wp.float32), out_rgba: wp.array4d(dtype=wp.uint8), @@ -670,23 +672,38 @@ def _apply_ppisp_controller_kernel( out_rgba[camera_id, height_id, width_id, 0] = wp.uint8(wp.clamp(out_rgb[0], 0.0, 1.0) * 255.0) out_rgba[camera_id, height_id, width_id, 1] = wp.uint8(wp.clamp(out_rgb[1], 0.0, 1.0) * 255.0) out_rgba[camera_id, height_id, width_id, 2] = wp.uint8(wp.clamp(out_rgb[2], 0.0, 1.0) * 255.0) - out_rgba[camera_id, height_id, width_id, 3] = wp.uint8(255) + if out_rgba.shape[3] == 4: + out_rgba[camera_id, height_id, width_id, 3] = wp.uint8(255) + + +def _validate_ppisp_image_buffers(hdr_color: wp.array, out_rgba: wp.array) -> None: + """Check the shapes, scalar types, and device required by both image kernels.""" + if hdr_color.dtype is not wp.float32: + raise ValueError(f"Camera PPISP HDR input must be wp.float32, got {hdr_color.dtype}.") + if out_rgba.dtype is not wp.uint8: + raise ValueError(f"Camera PPISP RGB/RGBA output must be wp.uint8, got {out_rgba.dtype}.") + if len(hdr_color.shape) != 4 or hdr_color.shape[-1] != 3: + raise ValueError("PPISP input must have shape (N, H, W, 3).") + if ( + len(out_rgba.shape) != 4 + or out_rgba.shape[-1] not in (3, 4) + or out_rgba.shape[:3] != hdr_color.shape[:3] + or out_rgba.device != hdr_color.device + ): + raise ValueError("PPISP output must match the input batch, image dimensions, and device, with 3 or 4 channels.") def apply_ppisp_to_rgba(hdr_color: wp.array, out_rgba: wp.array, cfg: PpispCfg) -> None: - """Apply PPISP to ``hdr_color`` and write LDR RGBA into ``out_rgba``. + """Apply PPISP to ``hdr_color`` and write LDR RGB or RGBA into ``out_rgba``. Args: hdr_color: HDR scene-linear input. ``wp.array4d(dtype=wp.float32)`` of shape ``(N, H, W, 3)``. - out_rgba: LDR RGBA output. ``wp.array4d(dtype=wp.uint8)`` of shape - ``(N, H, W, 4)``. Written in place. + out_rgba: LDR output. ``wp.array4d(dtype=wp.uint8)`` of shape + ``(N, H, W, 3)`` or ``(N, H, W, 4)``. Written in place; alpha is 255 when present. cfg: PPISP configuration. """ - if hdr_color.dtype is not wp.float32: - raise ValueError(f"Camera PPISP HDR input must be wp.float32, got {hdr_color.dtype}.") - if out_rgba.dtype is not wp.uint8: - raise ValueError(f"Camera PPISP RGBA output must be wp.uint8, got {out_rgba.dtype}.") + _validate_ppisp_image_buffers(hdr_color, out_rgba) inputs = cfg.inputs wp.launch( @@ -812,17 +829,14 @@ def apply_ppisp_to_rgba_with_controller_params( Args: hdr_color: HDR scene-linear input, shape ``(N, H, W, 3)``. - out_rgba: LDR RGBA output, shape ``(N, H, W, 4)``. + out_rgba: LDR RGB/RGBA output, shape ``(N, H, W, 3)`` or ``(N, H, W, 4)``. cfg: PPISP configuration. Static inputs provide responsivity, vignetting and CRF. controller_params: ``wp.array2d(float32)`` of shape ``(N, 9)`` holding ``[exposureOffset, colorLatentBlue.xy, colorLatentRed.xy, colorLatentGreen.xy, colorLatentNeutral.xy]``. """ - if hdr_color.dtype is not wp.float32: - raise ValueError(f"Camera PPISP HDR input must be wp.float32, got {hdr_color.dtype}.") - if out_rgba.dtype is not wp.uint8: - raise ValueError(f"Camera PPISP RGBA output must be wp.uint8, got {out_rgba.dtype}.") + _validate_ppisp_image_buffers(hdr_color, out_rgba) if controller_params.dtype is not wp.float32: raise ValueError(f"Camera PPISP controller params must be wp.float32, got {controller_params.dtype}.") if controller_params.shape[0] != out_rgba.shape[0] or controller_params.shape[1] != 9: diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py b/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py index 1fe7a3228e25..3f6d51d8cd96 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py @@ -3,20 +3,15 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Post-render PPISP pipeline composed into renderer backends. - -:class:`PpispPipeline` runs *after* a renderer fills its HDR scene-linear AOV, -converting HDR → LDR via a single Warp kernel. Renderer backends instantiate -this class when their cfg/spec carries a :class:`PpispCfg` and dispatch -:meth:`apply` once per render tick. The HDR scratch buffer is owned by the -renderer backend, not by this class. -""" +"""PPISP numerical execution with owned controller workspace and optional shared weights.""" from __future__ import annotations +from dataclasses import dataclass + import warp as wp -from .cfg import PpispCfg, normalize_ppisp_cfg +from .cfg import PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN, PpispCfg, normalize_ppisp_cfg from .kernels import ( PPISP_CONTROLLER_FEATURE_LEN, PPISP_CONTROLLER_PARAM_COUNT, @@ -26,89 +21,105 @@ ) -class PpispPipeline: - """Post-render PPISP kernel applier. +@dataclass(frozen=True, eq=False, slots=True) +class PpispControllerWeights: + """Shared, read-only device weights with identity preserved when configurations are copied. - Constructed by renderer backends when ``spec.cfg.isp_cfg`` is set. Owns the - normalised :class:`PpispCfg` and dispatches the PPISP Warp kernel once per - render tick via :meth:`apply`. + The caller supplies the allocation and orders its upload before processing. The wrapper retains + that allocation for every consumer; neither processors nor configuration copies mutate it. + """ - One pipeline instance applies to the whole Camera sensor batch. The PPISP - Warp kernel takes scalar coefficients, so every cloned view in a tiled - batch shares the same ISP configuration — there is no per-view ISP today. - Per-view support would require packing the cfg into GPU arrays and indexing - by ``camera_id`` inside the kernel. + values: wp.array - Today only :class:`PpispCfg` is accepted; future ISP implementations can - either subclass or be selected by cfg type without changes to the backend - renderers. - """ + def __post_init__(self) -> None: + if self.values.shape != (PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN,) or self.values.dtype != wp.float32: + raise ValueError("PPISP controller weights require the expected one-dimensional float32 buffer.") - def __init__(self, cfg: PpispCfg): - """Initialize the PPISP pipeline. + def __deepcopy__(self, memo: dict) -> PpispControllerWeights: + # Config cloning must share model resources, while each operation creates its own workspace. + return self - Normalises ``cfg`` on construction (validates input keys, fills - defaults). - :class:`~isaaclab.sensors.camera.Camera` already normalises ``isp_cfg`` - before passing the :class:`~isaaclab.renderers.CameraRenderSpec` to - the backend, so renderer backends pass a concrete, resolved config here. - Args: - cfg: The PPISP configuration. - """ +class PpispPipeline: + """Apply concrete PPISP settings to explicitly supplied image buffers. + + This numerical object performs no camera, scene, or renderer discovery. Resolve USD settings + with :func:`~isaaclab_ppisp.ppisp_cfg_from_usd_camera` before construction. Each instance owns + its controller workspace and is not reentrant; coefficients apply to its whole image batch. + :func:`~isaaclab_ppisp.prepare_ppisp` creates an independent instance for each buffer-pipe stage. + + Args: + cfg: Concrete PPISP settings, normalized in place for direct numerical use. + controller_weights: Optional device-resident controller weights to borrow read-only. + Callers may share these across instances, but must keep them unchanged and order their + upload before execution. Workspace and outputs are never shared automatically. + """ + + def __init__(self, cfg: PpispCfg, *, controller_weights: PpispControllerWeights | None = None): normalized_cfg = normalize_ppisp_cfg(cfg) if normalized_cfg is None: raise ValueError("PpispPipeline requires a concrete PpispCfg.") self.cfg = normalized_cfg + self._shared_controller_weights = controller_weights self._controller_weights_by_device: dict[str, wp.array] = {} self._controller_buffers_by_shape: dict[tuple[str, int, int, int], tuple[wp.array, ...]] = {} + self._closed = False + + def initialize(self, shape: tuple[int, ...], device: str | wp.Device) -> None: + """Prepare controller weights and scratch before the first image is processed.""" + if self._closed: + raise RuntimeError("Cannot initialize a closed PPISP pipeline.") + if self.cfg.controller_weights is None and self._shared_controller_weights is None: + return + if not wp.get_device(device).is_cuda: + raise ValueError("PPISP controller requires a CUDA device.") + device_name = str(wp.get_device(device)) + if device_name not in self._controller_weights_by_device: + if self._shared_controller_weights is None: + if len(self.cfg.controller_weights) != PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN: + raise ValueError(f"PPISP controller requires {PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN} weights.") + weights = wp.array(self.cfg.controller_weights, dtype=wp.float32, device=device_name) + else: + weights = self._shared_controller_weights.values + if weights.device != wp.get_device(device_name): + raise ValueError("Shared PPISP controller weights must match the image device.") + self._controller_weights_by_device[device_name] = weights + self._controller_buffers(shape, device_name) def apply(self, hdr: wp.array, rgba: wp.array) -> None: - """Run the PPISP kernel: HDR scene-linear → LDR RGBA, in place on ``rgba``.""" - if self.cfg.controller_weights is None: + """Write RGB or RGBA to the supplied destination on the current Warp stream.""" + if self._closed: + raise RuntimeError("Cannot apply a closed PPISP pipeline.") + if self.cfg.controller_weights is None and self._shared_controller_weights is None: apply_ppisp_to_rgba(hdr, rgba, self.cfg) return - controller_params = self._compute_controller_params(hdr) - apply_ppisp_to_rgba_with_controller_params(hdr, rgba, self.cfg, controller_params) - - def _compute_controller_params(self, hdr: wp.array) -> wp.array: - """Run the exported PPISP controller and return a Warp view of ``(N, 9)`` params.""" - controller_weights = self.cfg.controller_weights - assert controller_weights is not None - + self.initialize(hdr.shape, hdr.device) device = str(hdr.device) - weights = self._controller_weights_by_device.get(device) - if weights is None: - weights = wp.array(controller_weights, dtype=wp.float32, device=device) - self._controller_weights_by_device[device] = weights - - features, controller_params = self._controller_buffers(hdr) + features, controller_params = self._controller_buffers(hdr.shape, device) compute_ppisp_controller_params( hdr, - weights, + self._controller_weights_by_device[device], features, controller_params, self.cfg.controller_prior_exposure, float(self.cfg.controller_responsivity), ) - return controller_params + apply_ppisp_to_rgba_with_controller_params(hdr, rgba, self.cfg, controller_params) - def _controller_buffers(self, hdr: wp.array) -> tuple[wp.array, ...]: - """Return cached controller scratch buffers matching ``hdr`` shape/device.""" - num_cameras = int(hdr.shape[0]) - image_height = int(hdr.shape[1]) - image_width = int(hdr.shape[2]) - device = str(hdr.device) - key = (device, num_cameras, image_height, image_width) + def close(self) -> None: + """Release references to owned workspace and borrowed weights. Repeated calls are safe.""" + self._controller_weights_by_device.clear() + self._controller_buffers_by_shape.clear() + self._shared_controller_weights = None + self._closed = True + + def _controller_buffers(self, shape: tuple[int, ...], device: str) -> tuple[wp.array, ...]: + key = (device, *shape[:3]) buffers = self._controller_buffers_by_shape.get(key) - if buffers is not None: - return buffers - - # Camera output shape is fixed for a pipeline lifetime; cache buffers so - # controller execution does not allocate every frame. - buffers = ( - wp.empty((num_cameras, PPISP_CONTROLLER_FEATURE_LEN), dtype=wp.float32, device=device), - wp.empty((num_cameras, PPISP_CONTROLLER_PARAM_COUNT), dtype=wp.float32, device=device), - ) - self._controller_buffers_by_shape[key] = buffers + if buffers is None: + buffers = ( + wp.empty((shape[0], PPISP_CONTROLLER_FEATURE_LEN), dtype=wp.float32, device=device), + wp.empty((shape[0], PPISP_CONTROLLER_PARAM_COUNT), dtype=wp.float32, device=device), + ) + self._controller_buffers_by_shape[key] = buffers return buffers diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/processor.py b/source/isaaclab_ppisp/isaaclab_ppisp/processor.py new file mode 100644 index 000000000000..67e550c91ac7 --- /dev/null +++ b/source/isaaclab_ppisp/isaaclab_ppisp/processor.py @@ -0,0 +1,91 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""PPISP operation for camera and observation render-buffer pipes.""" + +from __future__ import annotations + +from collections.abc import Mapping +from copy import deepcopy +from typing import Any + +import warp as wp + +from isaaclab.utils import configclass +from isaaclab.utils.images import ImageBufferSpec, PreparedBufferOperation, RenderBufferStepCfg +from isaaclab.utils.warp import ProxyArray + +from .cfg import PpispCfg +from .pipeline import PpispControllerWeights, PpispPipeline + + +def prepare_ppisp( + input_specs: Mapping[str, ImageBufferSpec], + *, + cfg: PpispCfg, + channels: int = 3, + controller_weights: PpispControllerWeights | None = None, +) -> PreparedBufferOperation: + """Prepare a PPISP stage from concrete settings and a scene-linear radiance contract. + + Args: + input_specs: One ``radiance`` input: NHWC float32, three scene-linear channels. + cfg: Concrete configuration. Resolve USD camera attributes before preparing the pipe. + channels: Number of output channels, three for RGB or four for opaque RGBA. + controller_weights: Optional read-only device weights shared across operations. Each operation + retains its own mutable controller workspace; weights must be ready on the calling stream. + + Returns: + A prepared operation producing uint8 ``color`` with ``camera_response`` encoding. + """ + if set(input_specs) != {"radiance"}: + raise ValueError("PPISP requires exactly one input role named 'radiance'.") + source = input_specs["radiance"] + if ( + source.layout != "NHWC" + or len(source.shape) != 4 + or source.shape[-1] != 3 + or source.dtype != wp.float32 + or source.encoding != "scene_linear" + ): + raise ValueError("PPISP radiance must be NHWC float32 with three channels and scene_linear encoding.") + if channels not in (3, 4): + raise ValueError("PPISP color output requires three RGB or four RGBA channels.") + if not isinstance(cfg, PpispCfg) or cfg.camera_prim_path is not None: + raise ValueError("PPISP stages require a concrete PpispCfg; resolve USD camera attributes before preparation.") + pipeline = PpispPipeline(deepcopy(cfg), controller_weights=controller_weights) + try: + pipeline.initialize(source.shape, source.device) + except Exception: + pipeline.close() + raise + + def apply(inputs: Mapping[str, ProxyArray], outputs: Mapping[str, ProxyArray]) -> None: + pipeline.apply(inputs["radiance"].warp, outputs["color"].warp) + + return PreparedBufferOperation( + output_specs={ + "color": ImageBufferSpec( + shape=(*source.shape[:3], channels), dtype=wp.uint8, device=source.device, encoding="camera_response" + ), + }, + apply=apply, + close=pipeline.close, + ) + + +@configclass +class PpispStepCfg(RenderBufferStepCfg): + """Route scene-linear radiance through PPISP into a public RGB buffer. + + Supply concrete settings with ``params={"cfg": ppisp_cfg}``. To produce RGBA, set + ``outputs={"color": "rgba"}`` and ``params={"cfg": ppisp_cfg, "channels": 4}``. + Both camera and observation pipes use the same operation implementation. + """ + + prepare = "isaaclab_ppisp:prepare_ppisp" + inputs: dict[str, str] = {"radiance": "rgb_radiance"} + outputs: dict[str, str] = {"color": "rgb"} + params: dict[str, Any] = {"cfg": PpispCfg()} diff --git a/source/isaaclab_ppisp/test/test_ppisp.py b/source/isaaclab_ppisp/test/test_ppisp.py index 453595d849d0..c60841320a31 100644 --- a/source/isaaclab_ppisp/test/test_ppisp.py +++ b/source/isaaclab_ppisp/test/test_ppisp.py @@ -10,6 +10,7 @@ import pytest from isaaclab_ppisp import ( PpispCfg, + PpispDiscoveryMode, auto_any_ppisp_cfg, auto_camera_ppisp_cfg, default_ppisp_inputs, @@ -226,13 +227,11 @@ def test_auto_any_ppisp_cfg_reads_first_camera_with_ppisp_attrs(): def test_resolve_and_normalize_without_camera_uses_first_ppisp_camera(): - from isaaclab.sensors.camera.camera_isp import CameraISPMode - stage = Usd.Stage.CreateInMemory() _author_camera(stage, "/World/CameraWithoutPpisp") _author_ppisp_camera(stage, "/World/Camera_ppisp", inherits=None, attrs={"exposureOffset": 2.0}) - cfg = resolve_and_normalize(CameraISPMode.AUTO_CAMERA, stage) + cfg = resolve_and_normalize(PpispDiscoveryMode.AUTO_CAMERA, stage) assert cfg is not None assert cfg.camera_prim_path is None diff --git a/source/isaaclab_ppisp/test/test_ppisp_kernels.py b/source/isaaclab_ppisp/test/test_ppisp_kernels.py index 6b3ac48d7ea7..7a295c6d9c42 100644 --- a/source/isaaclab_ppisp/test/test_ppisp_kernels.py +++ b/source/isaaclab_ppisp/test/test_ppisp_kernels.py @@ -4,9 +4,12 @@ # SPDX-License-Identifier: BSD-3-Clause import pytest +import torch import warp as wp from isaaclab_ppisp import ( PpispCfg, + PpispControllerWeights, + PpispStepCfg, apply_ppisp_to_rgba, apply_ppisp_to_rgba_with_controller_params, compute_ppisp_controller_params, @@ -37,6 +40,10 @@ PPISP_CONTROLLER_PARAM_COUNT, ) +from isaaclab.test.utils import DeviceScope, test_devices +from isaaclab.utils.images import ImageBufferSpec, RenderBufferPipe +from isaaclab.utils.warp import ProxyArray + wp.init() @@ -282,15 +289,14 @@ def bounded_softplus(raw: float, min_value: float) -> float: return np.maximum(y, 0.0) ** gamma -def test_ppisp_warp_crf_extreme_centers_match_reference(): - """Saturated CRF centers produce the reference curve instead of NaN-cast bytes.""" +def test_ppisp_pipe_rgb_crf_extreme_centers_match_reference(): + """A CPU RGB pipe preserves the reference curve, including saturated CRF centers.""" import numpy as np # a gray ramp passes the default exposure, vignetting, and color stages unchanged ramp = np.linspace(0.0, 1.0, 16, dtype=np.float32) hdr_data = np.repeat(ramp[None, None, :, None], 3, axis=3) - hdr_color = wp.from_numpy(hdr_data, dtype=wp.float32) - rgba = _rgba((1, 1, 16, 4)) + hdr_color = wp.from_numpy(hdr_data, dtype=wp.float32, device="cpu") cfg = normalize_ppisp_cfg( PpispCfg( inputs={ @@ -307,10 +313,12 @@ def test_ppisp_warp_crf_extreme_centers_match_reference(): ) ) - apply_ppisp_to_rgba(hdr_color, rgba, cfg) - - rgba_np = rgba.numpy()[0, 0] - assert (rgba_np[:, 3] == 255).all() + pipe = RenderBufferPipe( + {"rgb_radiance": ImageBufferSpec(hdr_color.shape, wp.float32, "cpu", encoding="scene_linear")}, + [PpispStepCfg(params={"cfg": cfg})], + ) + rgb = pipe({"rgb_radiance": ProxyArray(hdr_color)})["rgb"] + rgb_np = rgb.warp.numpy()[0, 0] for channel, suffix in enumerate("RGB"): expected = _crf_reference( ramp, @@ -321,4 +329,56 @@ def test_ppisp_warp_crf_extreme_centers_match_reference(): ) expected_bytes = np.floor(np.clip(expected, 0.0, 1.0) * 255.0) # float32 kernel math may land one byte away from the float64 reference at a floor boundary - np.testing.assert_allclose(rgba_np[:, channel].astype(float), expected_bytes, atol=1.0, err_msg=suffix) + np.testing.assert_allclose(rgb_np[:, channel].astype(float), expected_bytes, atol=1.0, err_msg=suffix) + np.testing.assert_array_equal(hdr_color.numpy(), hdr_data) + pipe.close() + + +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) +def test_ppisp_controller_pipes_share_weights_for_rgb_and_rgba(device): + """Both controller output layouts match a known response; closing one consumer preserves another.""" + import numpy as np + + weights_np = np.zeros(PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN, dtype=np.float32) + weights_np[PPISP_CONTROLLER_OFF_EXP_B] = 1.0 # A constant one-stop controller exposure. + weights = wp.from_numpy(weights_np, dtype=wp.float32, device=device) + source = ImageBufferSpec((1, 1, 4, 3), wp.float32, device, encoding="scene_linear") + cfg = PpispCfg() + shared_weights = PpispControllerWeights(weights) + pipes = [ + RenderBufferPipe( + {"rgb_radiance": source}, + [PpispStepCfg(params={"cfg": cfg, "channels": channels, "controller_weights": shared_weights})], + ) + for channels in (3, 4) + ] + ramp = np.array([0.0, 0.125, 0.25, 0.5], dtype=np.float32) + hdr_data = np.repeat(ramp[None, None, :, None], 3, axis=3) + hdr = wp.from_numpy(hdr_data, dtype=wp.float32, device=device) + # The observation path must honor work already queued on a non-default Torch stream. + with torch.cuda.stream(torch.cuda.Stream(device=device)): + incoming = torch.from_numpy(hdr_data * 0.5).to(device) * 2.0 + observation = pipes[0].apply_to_tensor(incoming, input_name="rgb_radiance", output_name="rgb") + results = [observation.cpu().numpy()] + results.append(pipes[1]({"rgb_radiance": ProxyArray(hdr)})["rgb"].warp.numpy()) + expected = np.floor(_crf_reference(ramp * 2.0, 0.013659, 0.013659, 0.378165, 0.0) * 255.0) + for result in results: + np.testing.assert_allclose(result[0, 0, :, :3].astype(float), np.repeat(expected[:, None], 3, axis=1), atol=1.0) + assert (results[1][..., 3] == 255).all() + pipes[0].close() + hdr.zero_() + np.testing.assert_array_equal(pipes[1]({"rgb_radiance": ProxyArray(hdr)})["rgb"].warp.numpy()[..., :3], 0) + np.testing.assert_array_equal(weights.numpy(), weights_np) + pipes[1].close() + + +def test_ppisp_pipe_requires_scene_linear_input_and_resolved_configuration(): + """Reject semantically wrong radiance and unresolved scene dependencies before execution.""" + source = ImageBufferSpec((1, 2, 2, 3), wp.float32, "cpu", encoding="srgb") + with pytest.raises(ValueError, match="scene_linear encoding"): + RenderBufferPipe({"rgb_radiance": source}, [PpispStepCfg()]) + source = ImageBufferSpec((1, 2, 2, 3), wp.float32, "cpu", encoding="scene_linear") + with pytest.raises(ValueError, match="resolve USD camera attributes"): + RenderBufferPipe( + {"rgb_radiance": source}, [PpispStepCfg(params={"cfg": PpispCfg(camera_prim_path="/World/Camera")})] + ) From 37d95f315d8a9686e1b0d7bcffd3551eab27d09f Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Wed, 14 Oct 2026 15:08:22 -0400 Subject: [PATCH 14/20] Move camera preparation into the render context --- docs/source/concepts/sensors/camera.rst | 12 ++- ...atemirov-rgb-radiance-groundwork.added.rst | 3 - ...emirov-rgb-radiance-groundwork.changed.rst | 2 - ...atemirov-rgb-radiance-groundwork.fixed.rst | 2 - .../feat-render-buffer-pipe.changed.rst | 3 + ...rk.minor => feat-render-buffer-pipe.major} | 0 .../changelog.d/render-buffer-pipe.added.rst | 6 -- .../render-buffer-pipe.removed.rst | 3 - .../isaaclab/renderers/render_context.py | 23 ++++- .../isaaclab/sensors/camera/camera.py | 38 +++----- .../isaaclab/sim/simulation_context.py | 9 ++ .../renderers/test_camera_output_contract.py | 93 ++++++++++++------- .../test_simulation_render_context.py | 2 + .../test/sensors/test_opencv_distortion.py | 2 +- .../test/sensors/test_sensor_destructors.py | 2 +- ...atemirov-rgb-radiance-groundwork.added.rst | 2 - .../atemirov-rgb-radiance-groundwork.minor | 0 .../feat-render-buffer-pipe.changed.rst | 3 + .../feat-render-buffer-pipe.removed.rst | 2 - ...atemirov-rgb-radiance-groundwork.added.rst | 2 - ...atemirov-rgb-radiance-groundwork.fixed.rst | 2 - .../atemirov-rgb-radiance-groundwork.minor | 0 .../feat-render-buffer-pipe.changed.rst | 3 + .../feat-render-buffer-pipe.removed.rst | 2 - ...atemirov-rgb-radiance-groundwork.added.rst | 2 - ...emirov-rgb-radiance-groundwork.changed.rst | 3 - .../atemirov-rgb-radiance-groundwork.minor | 0 .../feat-render-buffer-pipe.changed.rst | 3 + .../feat-render-buffer-pipe.removed.rst | 2 - .../atemirov-rgb-radiance-groundwork.major | 0 ...emirov-rgb-radiance-groundwork.removed.rst | 2 - .../feat-render-buffer-pipe.changed.rst | 4 + .../feat-render-buffer-pipe.major} | 0 .../changelog.d/render-buffer-pipe.added.rst | 3 - .../render-buffer-pipe.changed.rst | 3 - .../changelog.d/render-buffer-pipe.fixed.rst | 2 - .../changelog.d/render-buffer-pipe.minor | 0 37 files changed, 132 insertions(+), 108 deletions(-) delete mode 100644 source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst delete mode 100644 source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst delete mode 100644 source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst create mode 100644 source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst rename source/isaaclab/changelog.d/{atemirov-rgb-radiance-groundwork.minor => feat-render-buffer-pipe.major} (100%) delete mode 100644 source/isaaclab/changelog.d/render-buffer-pipe.added.rst delete mode 100644 source/isaaclab/changelog.d/render-buffer-pipe.removed.rst delete mode 100644 source/isaaclab_newton/changelog.d/atemirov-rgb-radiance-groundwork.added.rst delete mode 100644 source/isaaclab_newton/changelog.d/atemirov-rgb-radiance-groundwork.minor create mode 100644 source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst delete mode 100644 source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.removed.rst delete mode 100644 source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.added.rst delete mode 100644 source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst delete mode 100644 source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.minor create mode 100644 source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst delete mode 100644 source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.removed.rst delete mode 100644 source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.added.rst delete mode 100644 source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst delete mode 100644 source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.minor create mode 100644 source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst delete mode 100644 source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.removed.rst delete mode 100644 source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.major delete mode 100644 source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst create mode 100644 source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst rename source/{isaaclab/changelog.d/render-buffer-pipe.major => isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.major} (100%) delete mode 100644 source/isaaclab_ppisp/changelog.d/render-buffer-pipe.added.rst delete mode 100644 source/isaaclab_ppisp/changelog.d/render-buffer-pipe.changed.rst delete mode 100644 source/isaaclab_ppisp/changelog.d/render-buffer-pipe.fixed.rst delete mode 100644 source/isaaclab_ppisp/changelog.d/render-buffer-pipe.minor diff --git a/docs/source/concepts/sensors/camera.rst b/docs/source/concepts/sensors/camera.rst index 208619caafbb..c6b1664ec6d6 100644 --- a/docs/source/concepts/sensors/camera.rst +++ b/docs/source/concepts/sensors/camera.rst @@ -202,9 +202,15 @@ produce additional output names; its operation factories declare their output co non-colorized segmentation uses one ``int32`` ID channel. Label and prim-path mappings are stored in ``camera_data.info[output_name]``. -Requesting ``rgb_hdr`` alone preserves the renderer's existing camera settings. ``rgb_radiance`` is -the input for image processing that applies its own exposure and camera response, such as PPISP. -If both are requested, they share one buffer. +``rgb_hdr`` does not request exposure neutralization. ``rgb_radiance`` is the input for image +processing that applies its own exposure and camera response, such as PPISP. +A PPISP pipeline requests ``rgb_radiance`` as a private input automatically; add it to +``CameraCfg.data_types`` only when it should also be published in ``camera.data.output``. + +If both ``rgb_hdr`` and ``rgb_radiance`` are requested, they share the same scene-linear buffer. +On Isaac RTX and OVRTX, requesting radiance neutralizes exposure for the entire camera prim, so +``rgb_hdr`` also contains neutralized color. Use separate camera prims to obtain radiance and color +with the authored exposure; see the limitation in :ref:`camera-supported-annotators`. .. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/camera-renderer-isaac-rtx.webp :align: center diff --git a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst deleted file mode 100644 index 96bf99b197ab..000000000000 --- a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.added.rst +++ /dev/null @@ -1,3 +0,0 @@ -* Added the ``rgb_radiance`` camera output: scene-linear RGB before camera exposure and response, - in renderer-relative intensity units. When ``rgb_hdr`` and ``rgb_radiance`` are both requested, - they share one buffer. diff --git a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst deleted file mode 100644 index 29ca2a5cd245..000000000000 --- a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst +++ /dev/null @@ -1,2 +0,0 @@ -* Changed :class:`~isaaclab.sensors.camera.Camera` to no longer set renderer-specific settings; the - renderer applies them. diff --git a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst b/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst deleted file mode 100644 index 54a0fe5ec933..000000000000 --- a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst +++ /dev/null @@ -1,2 +0,0 @@ -* Fixed cameras after the first one on a renderer that exports the stage once, such as OVRTX, missing their - renderer camera overrides. Every camera now applies them before any camera initializes. diff --git a/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..063c1e1b8540 --- /dev/null +++ b/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst @@ -0,0 +1,3 @@ +* **Breaking:** Replaced ``CameraCfg.isp_cfg`` with reusable image pipelines. Configure camera + processing with ``CameraCfg.pipeline=[PpispStepCfg(...)]``; use ``mdp.processed_image`` or + ``RenderBufferPipeModifierCfg`` for observation processing with the same operations. diff --git a/source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.minor b/source/isaaclab/changelog.d/feat-render-buffer-pipe.major similarity index 100% rename from source/isaaclab/changelog.d/atemirov-rgb-radiance-groundwork.minor rename to source/isaaclab/changelog.d/feat-render-buffer-pipe.major diff --git a/source/isaaclab/changelog.d/render-buffer-pipe.added.rst b/source/isaaclab/changelog.d/render-buffer-pipe.added.rst deleted file mode 100644 index 9a0b38138ef2..000000000000 --- a/source/isaaclab/changelog.d/render-buffer-pipe.added.rst +++ /dev/null @@ -1,6 +0,0 @@ -* Added ``RenderBufferPipe`` and prepared operations over named image buffers, with explicit - shape, dtype, device, layout and encoding contracts, persistent destinations, and operation lifecycle hooks. -* Added ``CameraCfg.pipeline``, the ``processed_image`` observation term, and - ``RenderBufferPipeModifierCfg`` so cameras and observations use the same processing operations. - Observation dependencies are collected before renderer initialization; private inputs do not need - to be exposed as public camera outputs. diff --git a/source/isaaclab/changelog.d/render-buffer-pipe.removed.rst b/source/isaaclab/changelog.d/render-buffer-pipe.removed.rst deleted file mode 100644 index 77f723ffa1d4..000000000000 --- a/source/isaaclab/changelog.d/render-buffer-pipe.removed.rst +++ /dev/null @@ -1,3 +0,0 @@ -* Retired execution through ``CameraCfg.isp_cfg``. Configure a ``PpispStepCfg`` in - ``CameraCfg.pipeline`` or an observation pipe instead. Resolve USD-authored PPISP parameters - explicitly before configuring the step. Non-None legacy ``isp_cfg`` values raise a migration error. diff --git a/source/isaaclab/isaaclab/renderers/render_context.py b/source/isaaclab/isaaclab/renderers/render_context.py index cdd1515532b6..2ae854ae9bfa 100644 --- a/source/isaaclab/isaaclab/renderers/render_context.py +++ b/source/isaaclab/isaaclab/renderers/render_context.py @@ -9,8 +9,9 @@ import logging import warnings +import weakref from collections.abc import Sequence -from typing import Any +from typing import TYPE_CHECKING, Any import torch import warp as wp @@ -19,6 +20,9 @@ from .base_renderer import BaseRenderer, VisualMaterialBatch from .renderer_cfg import RendererCfg +if TYPE_CHECKING: + from ..sensors.camera.camera import Camera + logger = logging.getLogger(__name__) @@ -65,6 +69,7 @@ class RenderContext: __slots__ = ( "clone_contexts", "_backend_registry", + "_cameras", "_physics_initialized", "_prepared_renderer_ids", "_prepared_num_envs", @@ -82,6 +87,7 @@ def __init__(self, backend_registry: list[tuple[Any, Any]]) -> None: self.clone_contexts: set[type | str] = set() """Scene representations declared by camera renderers and visualizers before cloning.""" self._backend_registry = backend_registry + self._cameras: weakref.WeakValueDictionary[int, Camera] = weakref.WeakValueDictionary() self._physics_initialized: bool = False # Set to True after the first PHYSICS_READY callback fires. self._prepared_renderer_ids: set[int] = set() self._prepared_num_envs: int | None = None @@ -133,6 +139,19 @@ def ensure_initialize(self) -> None: for _cfg, renderer in self._renderer_entries: renderer.initialize() + def register_camera(self, camera: Camera) -> None: + """Include a camera in stage preparation without extending its lifetime.""" + self._cameras[id(camera)] = camera + + def unregister_camera(self, camera: Camera) -> None: + """Stop preparing a camera whose sensor callbacks have been cleared.""" + self._cameras.pop(id(camera), None) + + def prepare_cameras(self) -> None: + """Prepare every live camera before any sensor can export the shared renderer stage.""" + for camera in tuple(self._cameras.values()): + camera._prepare_renderer() + def register_visual_material(self, material: Any) -> None: """Register one initialized material asset for flat channel composition.""" if any(registered is material for registered in self._visual_materials): @@ -370,6 +389,8 @@ def reset_stage_prepare_flag(self) -> None: def close(self) -> None: """Release material writers and lifecycle bookkeeping, not registry-owned renderers. + Live camera registrations survive a stop so their stage preparation runs again on reset. + Raises: RuntimeError: If a material writer failed to close. """ diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index cc46b2ec9a54..37d3f7302eef 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -7,7 +7,6 @@ import logging import sys -import weakref from collections.abc import Mapping, Sequence from copy import copy from dataclasses import replace @@ -22,7 +21,6 @@ from ... import sim as sim_utils from ...app.logging_utils import force_log_level -from ...physics import PhysicsEvent, PhysicsManager from ...renderers import BaseRenderer, CameraRenderSpec from ...sim.views import FrameView from ...utils.images import ImageBufferSpec, RenderBufferPipe @@ -36,6 +34,7 @@ from .camera_data import CameraData, RenderBufferKind if TYPE_CHECKING: + from ...renderers.render_context import RenderContext from .camera_cfg import CameraCfg logger = logging.getLogger(__name__) @@ -212,7 +211,7 @@ def __init__(self, cfg: CameraCfg): RuntimeError: If no camera prim is found at the given path. ValueError: If the provided data types are not supported by the camera or active renderer. """ - self._prepare_renderer_handle = None + self._render_context: RenderContext | None = None # perform check on supported data types self._check_supported_data_types(cfg) # initialize base class @@ -264,7 +263,7 @@ def __init__(self, cfg: CameraCfg): # Allocated in :meth:`_create_buffers` once the renderer's output contract is known. self._data: CameraData | None = None # The backend's ``__init__`` is its pre-physics phase, so it has to exist before - # ``sim.reset()``; sensor initialization only runs on ``PhysicsEvent.PHYSICS_READY``, which is + # ``sim.reset()``; sensor initialization only runs once physics is ready, which is # too late. Backends are shared per renderer config, so this stays cheap for many cameras. self._renderer: BaseRenderer | None = None if sim_ctx is not None: @@ -700,8 +699,8 @@ def _initialize_impl(self): # Per-camera USD setup must run **before** ``ensure_prepare_stage`` so renderers that snapshot # the stage (ovrtx's ``stage.Export``) capture the overrides in their exported USD. Every camera - # already did this in :meth:`_prepare_renderer_callback`; repeating it is harmless and covers - # cameras initialized without that callback. + # already did this in :meth:`_prepare_renderer`; repeating it is harmless and covers + # cameras initialized outside that preparation pass. cam_paths = tuple(str(p.GetPath()) for p in sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) render_spec = CameraRenderSpec( cfg=self._render_cfg, @@ -1182,30 +1181,19 @@ def _env_mask_has_any(self, env_mask: wp.array) -> bool: """ def _register_callbacks(self): - """Register camera stage preparation before the ordinary sensor initialization callbacks.""" + """Register the sensor lifecycle and shared renderer stage preparation.""" super()._register_callbacks() - physics_mgr_cls = sim_utils.SimulationContext.instance().physics_manager - obj_ref = weakref.proxy(self) - - def _prepare(event): - obj_ref._prepare_renderer_callback(event) - - # All cameras must author their USD overrides before the first camera exports the shared stage. - # Sensor initialization runs at order 10. Preserve weak ownership and backend exception forwarding. - self._prepare_renderer_handle = physics_mgr_cls.register_callback( - lambda payload: PhysicsManager.safe_callback_invoke(_prepare, payload, physics_manager=physics_mgr_cls), - PhysicsEvent.PHYSICS_READY, - order=9, - ) + self._render_context = sim_utils.SimulationContext.instance().render_context + self._render_context.register_camera(self) def _clear_callbacks(self) -> None: - """Release the camera preparation subscription along with the ordinary sensor subscriptions.""" - if self._prepare_renderer_handle is not None: - self._prepare_renderer_handle.deregister() - self._prepare_renderer_handle = None + """Release stage preparation along with the ordinary sensor subscriptions.""" + if self._render_context is not None: + self._render_context.unregister_camera(self) + self._render_context = None super()._clear_callbacks() - def _prepare_renderer_callback(self, event): + def _prepare_renderer(self): """Apply this camera's renderer USD overrides before any camera initializes. A renderer that exports the stage when the first camera initializes, such as OVRTX, then sees diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index a35f849a9925..b3ba99a5b94e 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -263,6 +263,15 @@ def __init__(self, cfg: SimulationCfg | None = None): PhysicsEvent.PHYSICS_READY, order=5, ) + # Prepare every camera before SensorBase initializes any sensor at order 10. Renderers + # that export the stage during the first camera's initialization then see every override. + self.physics_manager.register_callback( + lambda _payload: PhysicsManager.safe_callback_invoke( + self._render_context.prepare_cameras, physics_manager=self.physics_manager + ), + PhysicsEvent.PHYSICS_READY, + order=9, + ) self.physics_manager.register_callback( lambda _payload: self._render_context.close(), PhysicsEvent.STOP, order=100 ) diff --git a/source/isaaclab/test/renderers/test_camera_output_contract.py b/source/isaaclab/test/renderers/test_camera_output_contract.py index 60069d452bbc..aa3348aff66f 100644 --- a/source/isaaclab/test/renderers/test_camera_output_contract.py +++ b/source/isaaclab/test/renderers/test_camera_output_contract.py @@ -323,54 +323,50 @@ def test_camera_data_allocate_raises_on_unknown_name(): def test_every_camera_prepares_before_the_shared_stage_export(monkeypatch): - """A renderer that snapshots the stage at the first camera's initialization sees every camera's overrides.""" - import numpy as np + """Every live camera prepares before stage export, including after stop and with callback error forwarding.""" + import weakref - from pxr import Sdf, Usd, UsdGeom + pytest.importorskip("isaaclab_newton") + pytest.importorskip("newton") + from isaaclab_newton.physics import NewtonCfg, NewtonManager + + from pxr import Sdf, UsdGeom - from isaaclab.physics import PhysicsEvent, PhysicsManager + from isaaclab.physics import PhysicsEvent + from isaaclab.renderers import RendererCfg from isaaclab.renderers.rtx_camera_overrides import _apply_rtx_exposure_overrides from isaaclab.sensors.camera import Camera from isaaclab.sensors.camera import camera as camera_module from isaaclab.sensors.sensor_base import SensorBase - from isaaclab.sim import SimulationContext - - class CameraPhysicsManager(PhysicsManager): - _callbacks = {} - stage = Usd.Stage.CreateInMemory() - prims = [UsdGeom.Camera.Define(stage, path).GetPrim() for path in ("/World/First", "/World/Second")] - prims[1].CreateAttribute("exposure:iso", Sdf.ValueTypeNames.Float).Set(100.0) specs = { RenderBufferKind.RGB: RenderBufferSpec(3, wp.uint8), RenderBufferKind.RGBA: RenderBufferSpec(4, wp.uint8), RenderBufferKind.RGB_RADIANCE: RenderBufferSpec(3, wp.float32), } exported_iso = [] + prepare_error = None def prepare_cameras(stage, spec): + if prepare_error is not None: + raise prepare_error if "rgb_radiance" in spec.cfg.data_types: _apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) def export_stage(stage, num_envs): - if not exported_iso: - exported_iso.append(prims[1].GetAttribute("exposure:iso").Get()) + exported_iso.append(prims[1].GetAttribute("exposure:iso").Get()) renderer = SimpleNamespace( + initialize=lambda: None, supported_output_types=lambda: specs, prepare_cameras=prepare_cameras, + prepare_stage=export_stage, create_render_data=lambda spec: SimpleNamespace(spec=spec), set_outputs=lambda data, outputs: None, cleanup=lambda data: None, + close=lambda: None, ) - sim = SimpleNamespace( - device="cpu", - physics_manager=CameraPhysicsManager, - get_clone_plan=lambda: SimpleNamespace(topology=SimpleNamespace(world_prototype_layout=np.zeros(2))), - render_context=SimpleNamespace(ensure_prepare_stage=export_stage), - vis_marker_registry=SimpleNamespace(clear_debug_vis_callback=lambda sensor: None), - ) - monkeypatch.setattr(SimulationContext, "instance", staticmethod(lambda: sim)) + renderer_cfg = RendererCfg(class_type=lambda cfg: renderer, renderer_type="contract") monkeypatch.setattr(SensorBase, "_initialize_impl", lambda self: None) monkeypatch.setattr(Camera, "_initialize_intrinsics", lambda self: None) monkeypatch.setattr(Camera, "_update_poses", lambda self: None) @@ -379,7 +375,6 @@ def export_stage(stage, num_envs): "FrameView", lambda path, **kwargs: SimpleNamespace(count=2, prims=[stage.GetPrimAtPath(path)] * 2, close=lambda: None), ) - cameras = [] def prepare_identity(input_specs): return PreparedBufferOperation( @@ -387,7 +382,7 @@ def prepare_identity(input_specs): lambda inputs, outputs: wp.copy(outputs["image"].warp, inputs["image"].warp), ) - for prim, data_types in zip(prims, (["rgb"], ["processed_radiance"])): + def make_camera(prim, data_types): camera = Camera.__new__(Camera) pipeline = ( [RenderBufferStepCfg(prepare_identity, {"image": "rgb_radiance"}, {"image": "processed_radiance"})] @@ -395,7 +390,13 @@ def prepare_identity(input_specs): else [] ) camera.cfg = CameraCfg( - prim_path=str(prim.GetPath()), data_types=data_types, height=2, width=3, spawn=None, pipeline=pipeline + prim_path=str(prim.GetPath()), + data_types=data_types, + height=2, + width=3, + spawn=None, + pipeline=pipeline, + renderer_cfg=renderer_cfg, ) camera._render_cfg = replace(camera.cfg, data_types=camera.cfg.render_data_types(), pipeline=[]) camera.stage = stage @@ -407,17 +408,41 @@ def prepare_identity(input_specs): camera._render_data = None camera._view = None camera._register_callbacks() - cameras.append(camera) - - try: - CameraPhysicsManager.dispatch_event(PhysicsEvent.PHYSICS_READY) + return camera - assert all(camera.is_initialized for camera in cameras) - assert exported_iso == [0.0] - finally: - for camera in cameras: - camera.__del__() - assert not CameraPhysicsManager._callbacks + sim_cfg = sim_utils.SimulationCfg(device="cpu", physics=NewtonCfg()) + with sim_utils.build_simulation_context(sim_cfg=sim_cfg) as sim: + stage = sim.stage + prims = [UsdGeom.Camera.Define(stage, path).GetPrim() for path in ("/World/First", "/World/Second")] + iso = prims[1].CreateAttribute("exposure:iso", Sdf.ValueTypeNames.Float) + monkeypatch.setattr( + sim, "get_clone_plan", lambda: SimpleNamespace(topology=SimpleNamespace(world_prototype_layout=np.zeros(2))) + ) + sim.get_or_create_backend(renderer_cfg) + cameras = [make_camera(prim, names) for prim, names in zip(prims, (["rgb"], ["processed_radiance"]))] + discarded = make_camera(prims[0], ["rgb"]) + discarded_ref = weakref.ref(discarded) + del discarded + assert discarded_ref() is None + + try: + for _ in range(2): + iso.Set(100.0) + sim.physics_manager.dispatch_event(PhysicsEvent.PHYSICS_READY) + assert all(camera.is_initialized for camera in cameras) + sim.physics_manager.dispatch_event(PhysicsEvent.STOP) + assert exported_iso == [0.0, 0.0] + + # External event buses store callback failures instead of propagating them directly. + errors = [] + monkeypatch.setattr(NewtonManager, "store_callback_exception", errors.append, raising=False) + prepare_error = ValueError("camera preparation failed") + sim.physics_manager.dispatch_event(PhysicsEvent.PHYSICS_READY) + assert errors and errors[0] is prepare_error + assert exported_iso == [0.0, 0.0] + finally: + for camera in cameras: + camera.__del__() def test_camera_pipeline_keeps_raw_inputs_and_processes_only_new_captures(monkeypatch): diff --git a/source/isaaclab/test/renderers/test_simulation_render_context.py b/source/isaaclab/test/renderers/test_simulation_render_context.py index 7256e7e7a015..ab6beb2bf63c 100644 --- a/source/isaaclab/test/renderers/test_simulation_render_context.py +++ b/source/isaaclab/test/renderers/test_simulation_render_context.py @@ -294,6 +294,8 @@ def __init__(self, renderer, name, update_period=0.0): self._data = CameraData() self._data.create_buffers(2, "cpu") self._data.info = {} + self._render_camera_data = self._data + self._buffer_pipe = None self._frame = ProxyArray(wp.zeros(2, dtype=wp.int64, device="cpu")) self._ALL_INDICES = wp.array([0, 1], dtype=wp.int32, device="cpu") self._ALL_ENV_MASK = wp.ones(2, dtype=wp.bool, device="cpu") diff --git a/source/isaaclab/test/sensors/test_opencv_distortion.py b/source/isaaclab/test/sensors/test_opencv_distortion.py index 6eb473d085f6..bb7e6dc8caf3 100644 --- a/source/isaaclab/test/sensors/test_opencv_distortion.py +++ b/source/isaaclab/test/sensors/test_opencv_distortion.py @@ -192,7 +192,7 @@ def _camera_for_prims(prims, width=640, height=480, device="cpu"): fake._ALL_INDICES = wp.array(np.arange(len(prims)), dtype=wp.int32, device=device) fake._ALL_ENV_MASK = wp.ones(len(prims), dtype=wp.bool, device=device) # attributes touched by ``__del__``/``_clear_callbacks`` when the fake object is garbage collected - fake._prepare_renderer_handle = None + fake._render_context = None fake._initialize_handle = None fake._invalidate_initialize_handle = None fake._prim_deletion_handle = None diff --git a/source/isaaclab/test/sensors/test_sensor_destructors.py b/source/isaaclab/test/sensors/test_sensor_destructors.py index 99f910f60927..b4bbb7245d64 100644 --- a/source/isaaclab/test/sensors/test_sensor_destructors.py +++ b/source/isaaclab/test/sensors/test_sensor_destructors.py @@ -77,7 +77,7 @@ def cleanup(self, _render_data): view = _View() renderer = _Renderer() camera = object.__new__(Camera) - camera._prepare_renderer_handle = None + camera._render_context = None camera._view = view camera._renderer = renderer camera._render_data = None diff --git a/source/isaaclab_newton/changelog.d/atemirov-rgb-radiance-groundwork.added.rst b/source/isaaclab_newton/changelog.d/atemirov-rgb-radiance-groundwork.added.rst deleted file mode 100644 index 73cdcc547da4..000000000000 --- a/source/isaaclab_newton/changelog.d/atemirov-rgb-radiance-groundwork.added.rst +++ /dev/null @@ -1,2 +0,0 @@ -* Added ``rgb_radiance`` support to :class:`~isaaclab_newton.renderers.NewtonWarpRenderer`. Newton - applies no camera exposure, so it matches ``rgb_hdr``. diff --git a/source/isaaclab_newton/changelog.d/atemirov-rgb-radiance-groundwork.minor b/source/isaaclab_newton/changelog.d/atemirov-rgb-radiance-groundwork.minor deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..6e9cc7742b12 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst @@ -0,0 +1,3 @@ +* **Breaking:** Moved PPISP processing from the renderer to ``CameraCfg.pipeline``. + Replace ``CameraCfg.isp_cfg`` with ``PpispStepCfg``; use ``rgb_radiance`` for its scene-linear + input, equivalent to ``rgb_hdr`` on Newton. diff --git a/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.removed.rst b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.removed.rst deleted file mode 100644 index 2dfc3e3f54d0..000000000000 --- a/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.removed.rst +++ /dev/null @@ -1,2 +0,0 @@ -* **Breaking:** Removed renderer-owned PPISP processing. Request ``rgb_radiance`` and apply PPISP - through the render buffer pipe instead of configuring ``CameraCfg.isp_cfg``. diff --git a/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.added.rst b/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.added.rst deleted file mode 100644 index 7e5255120d85..000000000000 --- a/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.added.rst +++ /dev/null @@ -1,2 +0,0 @@ -* Added ``rgb_radiance`` support to :class:`~isaaclab_ov.renderers.OVRTXRenderer`. Requesting it - neutralizes exposure on the camera prim, which also affects the prim's other color outputs. diff --git a/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst b/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst deleted file mode 100644 index 99dc1553081d..000000000000 --- a/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.fixed.rst +++ /dev/null @@ -1,2 +0,0 @@ -* Fixed black Gaussian HDR inputs by disabling Gaussian skip-tonemapping on HDR-consuming - OVRTX render products, including HDR inputs requested by camera ISP. diff --git a/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.minor b/source/isaaclab_ov/changelog.d/atemirov-rgb-radiance-groundwork.minor deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..6cbb5cc3bc3c --- /dev/null +++ b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst @@ -0,0 +1,3 @@ +* **Breaking:** Moved PPISP processing from the renderer to ``CameraCfg.pipeline``. + Replace ``CameraCfg.isp_cfg`` with ``PpispStepCfg``; its ``rgb_radiance`` input neutralizes + exposure for all color outputs sharing the camera prim. diff --git a/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.removed.rst b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.removed.rst deleted file mode 100644 index 2dfc3e3f54d0..000000000000 --- a/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.removed.rst +++ /dev/null @@ -1,2 +0,0 @@ -* **Breaking:** Removed renderer-owned PPISP processing. Request ``rgb_radiance`` and apply PPISP - through the render buffer pipe instead of configuring ``CameraCfg.isp_cfg``. diff --git a/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.added.rst b/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.added.rst deleted file mode 100644 index 79a4627d1afb..000000000000 --- a/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.added.rst +++ /dev/null @@ -1,2 +0,0 @@ -* Added ``rgb_radiance`` support to :class:`~isaaclab_physx.renderers.IsaacRtxRenderer`. Requesting - it neutralizes exposure on the camera prim, which also affects the prim's other color outputs. diff --git a/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst b/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst deleted file mode 100644 index 809a540dfddf..000000000000 --- a/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.changed.rst +++ /dev/null @@ -1,3 +0,0 @@ -* Changed :class:`~isaaclab_physx.renderers.IsaacRtxRenderer` to mark RTX sensor rendering when it is created - and to route Gaussian HDR in :meth:`~isaaclab_physx.renderers.IsaacRtxRenderer.prepare_cameras` for cameras - that request HDR color, instead of :class:`~isaaclab.sensors.camera.Camera` setting them. diff --git a/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.minor b/source/isaaclab_physx/changelog.d/atemirov-rgb-radiance-groundwork.minor deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..6cbb5cc3bc3c --- /dev/null +++ b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst @@ -0,0 +1,3 @@ +* **Breaking:** Moved PPISP processing from the renderer to ``CameraCfg.pipeline``. + Replace ``CameraCfg.isp_cfg`` with ``PpispStepCfg``; its ``rgb_radiance`` input neutralizes + exposure for all color outputs sharing the camera prim. diff --git a/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.removed.rst b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.removed.rst deleted file mode 100644 index 2dfc3e3f54d0..000000000000 --- a/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.removed.rst +++ /dev/null @@ -1,2 +0,0 @@ -* **Breaking:** Removed renderer-owned PPISP processing. Request ``rgb_radiance`` and apply PPISP - through the render buffer pipe instead of configuring ``CameraCfg.isp_cfg``. diff --git a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.major b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.major deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst b/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst deleted file mode 100644 index e1794508e8b2..000000000000 --- a/source/isaaclab_ppisp/changelog.d/atemirov-rgb-radiance-groundwork.removed.rst +++ /dev/null @@ -1,2 +0,0 @@ -* **Breaking:** Removed ``isaaclab_ppisp.apply_rtx_exposure_overrides``. The RTX renderers now apply the - exposure override themselves when a camera requests ``rgb_radiance``. diff --git a/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..5edaa8601455 --- /dev/null +++ b/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst @@ -0,0 +1,4 @@ +* **Breaking:** Moved PPISP into camera and observation pipelines through ``PpispStepCfg``. + Resolve USD settings explicitly with ``ppisp_cfg_from_usd_camera`` or ``resolve_and_normalize`` + before configuring the step. Replaced ``apply_rtx_exposure_overrides`` with renderer-managed + exposure when requesting ``rgb_radiance``. diff --git a/source/isaaclab/changelog.d/render-buffer-pipe.major b/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.major similarity index 100% rename from source/isaaclab/changelog.d/render-buffer-pipe.major rename to source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.major diff --git a/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.added.rst b/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.added.rst deleted file mode 100644 index 630e96c62874..000000000000 --- a/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.added.rst +++ /dev/null @@ -1,3 +0,0 @@ -* Added ``PpispStepCfg`` and ``prepare_ppisp`` for shared camera and observation render-buffer - pipelines, including RGB destinations and optional read-only shared controller weights. -* Added ``PpispDiscoveryMode`` for explicit USD configuration discovery before pipeline preparation. diff --git a/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.changed.rst b/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.changed.rst deleted file mode 100644 index 68372f67d027..000000000000 --- a/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.changed.rst +++ /dev/null @@ -1,3 +0,0 @@ -* Separated PPISP numerical execution from camera and renderer discovery. Configured processing with - ``CameraCfg.pipeline=[PpispStepCfg(params={"cfg": resolved_cfg})]`` instead of ``CameraCfg.isp_cfg``; - resolved USD settings with ``ppisp_cfg_from_usd_camera`` or ``resolve_and_normalize`` first. diff --git a/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.fixed.rst b/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.fixed.rst deleted file mode 100644 index daf552f71a68..000000000000 --- a/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.fixed.rst +++ /dev/null @@ -1,2 +0,0 @@ -* Fixed static PPISP execution on CPU by compiling image-transform kernels separately from - CUDA-only controller code. diff --git a/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.minor b/source/isaaclab_ppisp/changelog.d/render-buffer-pipe.minor deleted file mode 100644 index e69de29bb2d1..000000000000 From 4cef5c31529da8fd9fda20e87db450c136668823 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Wed, 14 Oct 2026 15:46:29 -0400 Subject: [PATCH 15/20] Use ProxyArray buffers directly in render pipelines --- docs/source/api/lab/isaaclab.utils.rst | 3 - docs/source/concepts/sensors/camera.rst | 33 +++-- .../isaaclab/envs/mdp/visual_observations.py | 11 +- .../isaaclab/sensors/camera/camera.py | 35 +---- source/isaaclab/isaaclab/utils/images.py | 126 ++++++++---------- .../isaaclab/utils/modifiers/modifier_cfg.py | 16 +-- .../utils/modifiers/render_buffer_pipe.py | 51 ++++--- .../envs/test_render_buffer_observations.py | 83 +++++++++--- .../renderers/test_camera_output_contract.py | 16 ++- .../test/utils/test_render_buffer_pipe.py | 93 ++++++++----- source/isaaclab_ppisp/docs/README.md | 15 ++- .../isaaclab_ppisp/processor.py | 50 +++---- .../isaaclab_ppisp/test/test_ppisp_kernels.py | 39 +++--- 13 files changed, 315 insertions(+), 256 deletions(-) diff --git a/docs/source/api/lab/isaaclab.utils.rst b/docs/source/api/lab/isaaclab.utils.rst index 91005ca604d1..d58f30ddd65e 100644 --- a/docs/source/api/lab/isaaclab.utils.rst +++ b/docs/source/api/lab/isaaclab.utils.rst @@ -171,9 +171,6 @@ Image operations .. autofunction:: isaaclab.utils.images.compose_image -.. autoclass:: isaaclab.utils.images.ImageBufferSpec - :members: - .. autoclass:: isaaclab.utils.images.PreparedBufferOperation :members: diff --git a/docs/source/concepts/sensors/camera.rst b/docs/source/concepts/sensors/camera.rst index c6b1664ec6d6..4a0445949bed 100644 --- a/docs/source/concepts/sensors/camera.rst +++ b/docs/source/concepts/sensors/camera.rst @@ -155,7 +155,7 @@ Output types For raw outputs, the active renderer validates the requested names and allocates the channel count and data type declared by its :class:`~isaaclab.renderers.RenderBufferSpec`. A camera pipeline may -produce additional output names; its operation factories declare their output contracts. +produce additional output names; its operation factories allocate those buffers. .. list-table:: Common output contracts :header-rows: 1 @@ -364,20 +364,22 @@ uint8 color. Set ``outputs={"color": "rgba"}`` and ``params={"cfg": ppisp_cfg, " step for RGBA output. A pipeline-produced RGBA buffer also supplies a zero-copy RGB view when ``rgb`` is requested. A pipeline producing only RGB cannot satisfy an explicit RGBA request. -For a custom operation, provide a factory ``prepare(input_specs, **params)`` returning -:class:`~isaaclab.utils.images.PreparedBufferOperation`. Input and output mappings inside the operation -use role names, while the step configuration owns camera-buffer names. Declare output shape, dtype, -device, layout, and encoding with :class:`~isaaclab.utils.images.ImageBufferSpec`; then write each -supplied output destination in ``apply(inputs, outputs)``. The pipe allocates these destinations once. -Encoding is declared by the producer rather than inferred from dtype or pixel values. Camera radiance -uses ``scene_linear``, RGB/RGBA uses ``camera_response``, and depth uses ``meters``. +For a custom operation, provide a factory ``prepare(inputs, **params)`` returning +:class:`~isaaclab.utils.images.PreparedBufferOperation`. The factory receives actual +:class:`~isaaclab.utils.warp.ProxyArray` inputs, inspects their shape, dtype, and device, and allocates +its output buffers and workspace once. Return the output arrays in the prepared operation, then write +each supplied destination in ``apply(inputs, outputs)``. Both mappings use operation role names; the +step configuration maps those roles to camera-buffer names. Preparation must not read input pixels or +retain their addresses for execution: each ``apply`` call receives the current inputs. +Operations document their expected image layout and pixel meaning. PPISP requires NHWC float32 +scene-linear radiance and produces NHWC uint8 color; a float32 array alone does not establish radiance. Published camera outputs must keep the camera's batch size, height, width, and NHWC layout so pose and calibration metadata remain valid. Intermediate buffers may change shape. Use ``processed_image`` below when the final observation should have a different image shape or layout. -``camera.render_outputs`` exposes borrowed raw renderer buffers, including private inputs, and -``camera.render_output_specs`` exposes their contracts after initialization. Reading raw outputs updates -the camera when due. Neither an operation nor a caller may mutate borrowed inputs. Public processed +``camera.render_outputs`` exposes borrowed raw renderer buffers, including private inputs. Their +``ProxyArray`` properties provide shape, dtype, and device. Reading raw outputs updates the camera +when due. Neither an operation nor a caller may mutate borrowed inputs. Public processed buffers and raw buffers retain their storage until camera reinitialization, but later captures replace their contents; clone values that must survive another capture. Each camera owns its pipe and operation workspace. Operations may share explicitly supplied immutable model resources. @@ -422,12 +424,12 @@ that term rather than the camera's capture cadence. For a single incoming tensor, :class:`~isaaclab.utils.modifiers.RenderBufferPipeModifierCfg` uses the same operations while preserving preceding observation transformations. This form must preserve -shape; use ``processed_image`` for multiple camera inputs or shape changes. The declared dtype, -encoding, and layout describe the tensor reaching the modifier: +shape; use ``processed_image`` for multiple camera inputs or shape changes. The first incoming tensor +prepares the operations; later calls reuse their output storage. Its values must meet the operation's +documented input requirements, including any transformations from preceding modifiers: .. code-block:: python - import warp as wp from isaaclab.utils import modifiers # This camera publishes radiance for the ordinary image term to read. @@ -440,9 +442,6 @@ encoding, and layout describe the tensor reaching the modifier: pipeline=[PpispStepCfg()], input_name="rgb_radiance", output_name="rgb", - dtype=wp.float32, - encoding="scene_linear", - layout="NHWC", ) ], ) diff --git a/source/isaaclab/isaaclab/envs/mdp/visual_observations.py b/source/isaaclab/isaaclab/envs/mdp/visual_observations.py index dc664dcaaf9e..fb5496cadb02 100644 --- a/source/isaaclab/isaaclab/envs/mdp/visual_observations.py +++ b/source/isaaclab/isaaclab/envs/mdp/visual_observations.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING import torch +import warp as wp from isaaclab.managers import ManagerTermBase, ObservationTermCfg, SceneEntityCfg from isaaclab.utils.images import RenderBufferPipe, RenderBufferStepCfg @@ -44,8 +45,12 @@ def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedEnv): super().__init__(cfg, env) self._camera = env.scene.sensors[cfg.params["sensor_cfg"].name] self._output_name = cfg.params["output_name"] - self._pipe = RenderBufferPipe(self._camera.render_output_specs, cfg.params["pipeline"]) - if self._output_name not in self._pipe.output_specs: + buffers = self._camera.render_outputs + device = wp.get_device(self.device) + stream = wp.stream_from_torch(torch.cuda.current_stream(str(device))) if device.is_cuda else None + with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): + self._pipe = RenderBufferPipe(buffers, cfg.params["pipeline"]) + if self._output_name not in self._pipe.outputs: self._pipe.close() raise ValueError(f"Image processing does not provide output {self._output_name!r}.") @@ -58,7 +63,7 @@ def __call__( *, out: torch.Tensor | None = None, ) -> torch.Tensor: - """Return a processed image in the layout, dtype and shape declared by its final operation. + """Return a snapshot of the selected operation output. The observation manager supplies ``out`` to receive the snapshot without another clone. Direct callers may omit it to allocate a new snapshot. diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 37d3f7302eef..33e1d3429a4d 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -23,7 +23,7 @@ from ...app.logging_utils import force_log_level from ...renderers import BaseRenderer, CameraRenderSpec from ...sim.views import FrameView -from ...utils.images import ImageBufferSpec, RenderBufferPipe +from ...utils.images import RenderBufferPipe from ...utils.math import ( convert_camera_frame_orientation_convention, create_rotation_matrix_from_view, @@ -257,7 +257,6 @@ def __init__(self, cfg: CameraCfg): self._pipeline_capture: object | None = None self._render_camera_data: CameraData | None = None self._render_outputs: dict[str, ProxyArray] = {} - self._render_output_specs: dict[str, ImageBufferSpec] = {} # UsdGeom Camera prim for the sensor self._sensor_prims: list[UsdGeom.Camera] = [] # Allocated in :meth:`_create_buffers` once the renderer's output contract is known. @@ -345,11 +344,6 @@ def render_outputs(self) -> Mapping[str, ProxyArray]: self._update_outdated_buffers() return MappingProxyType(self._render_outputs) - @property - def render_output_specs(self) -> Mapping[str, ImageBufferSpec]: - """Resolved raw image contracts, available after camera initialization and treated as read-only.""" - return MappingProxyType(self._render_output_specs) - def request_render_inputs(self, names: Sequence[str]) -> None: """Request private renderer buffers before camera initialization. @@ -903,39 +897,22 @@ def _create_buffers(self): self._render_camera_data = self._data self._render_outputs = self._render_camera_data.output self._renderer.set_outputs(self._render_data, self._render_outputs) - encodings = { - "rgb_radiance": "scene_linear", - "rgb": "camera_response", - "rgba": "camera_response", - "depth": "meters", - "distance_to_camera": "meters", - "distance_to_image_plane": "meters", - } - self._render_output_specs = { - name: ImageBufferSpec( - shape=buffer.shape, - dtype=buffer.dtype, - device=str(buffer.device), - encoding=encodings.get(name, "unknown"), - ) - for name, buffer in self._render_outputs.items() - } self._close_pipeline() self._pipeline_capture = None if self.cfg.pipeline: - self._buffer_pipe = RenderBufferPipe(self._render_output_specs, self.cfg.pipeline) + self._buffer_pipe = RenderBufferPipe(self._render_outputs, self.cfg.pipeline) try: - outputs = {**self._render_outputs, **self._buffer_pipe.outputs} + outputs = dict(self._buffer_pipe.outputs) alias_rgb = ( "rgba" in self._buffer_pipe.produced_outputs and "rgb" not in self._buffer_pipe.produced_outputs ) expected_shape = (self._view.count, self.cfg.height, self.cfg.width) for name in self.cfg.data_types: - spec = self._buffer_pipe.output_specs["rgba" if alias_rgb and name == "rgb" else name] - if spec.layout != "NHWC" or len(spec.shape) != 4 or spec.shape[:3] != expected_shape: + buffer = outputs["rgba" if alias_rgb and name == "rgb" else name] + if len(buffer.shape) != 4 or buffer.shape[:3] != expected_shape: raise ValueError( f"Camera pipeline output {name!r} must use NHWC with batch and spatial shape" - f" {expected_shape}; received {spec.layout} {spec.shape}. Use an observation pipe" + f" {expected_shape}; received {buffer.shape}. Use an observation pipe" " for shape-changing results, because camera calibration describes the rendered image." ) # Preserve RGB's view semantics when an operation produces RGBA, without publishing stale raw RGB. diff --git a/source/isaaclab/isaaclab/utils/images.py b/source/isaaclab/isaaclab/utils/images.py index f326e4857737..51a548d6671e 100644 --- a/source/isaaclab/isaaclab/utils/images.py +++ b/source/isaaclab/isaaclab/utils/images.py @@ -34,48 +34,18 @@ from .warp import ProxyArray -@dataclass(frozen=True) -class ImageBufferSpec: - """Shape, storage and semantic encoding of one image-processing buffer. - - The producer declares the encoding: it cannot be inferred from pixel values or dtype. - ``shape`` includes the batch dimension. Borrowed arrays may be strided; operations must - either support their strides or explicitly copy to suitable storage. - """ - - shape: tuple[int, ...] - dtype: type - device: str - encoding: str = "unknown" - layout: str = "NHWC" - - def __post_init__(self): - if not self.shape or any(size <= 0 for size in self.shape): - raise ValueError("Image buffer dimensions must be positive.") - if self.layout == "NHWC" and len(self.shape) != 4: - raise ValueError("NHWC image buffers must have four dimensions.") - - def validate(self, name: str, value: ProxyArray) -> None: - """Validate the storage of a borrowed buffer before submitting work.""" - if ( - value.shape != self.shape - or value.dtype != self.dtype - or str(value.device) != str(wp.get_device(self.device)) - ): - raise ValueError(f"Buffer {name!r} must match shape, dtype and device {self}.") - - @dataclass(frozen=True) class PreparedBufferOperation: - """One operation bound to input contracts and resolved resources. + """One operation with persistent output buffers and resolved resources. ``apply`` receives input and destination dictionaries keyed by operation roles, not sensor - output names. Inputs are borrowed read-only by convention. The operation must fill each - supplied destination and enqueue GPU work on the current Warp stream. Output storage and - execution order belong to :class:`RenderBufferPipe`; private workspace belongs to the operation. + output names. Preparation allocates the outputs once. Inputs are borrowed read-only by + convention; outputs must use independent storage. The operation must fill each supplied + destination and enqueue GPU work on the current Warp stream. The pipe retains the outputs + and controls execution order; private workspace belongs to the operation. """ - output_specs: Mapping[str, ImageBufferSpec] + outputs: Mapping[str, ProxyArray] apply: Callable[[Mapping[str, ProxyArray], Mapping[str, ProxyArray]], None] reset: Callable[[Sequence[int] | None], None] | None = None close: Callable[[], None] | None = None @@ -85,10 +55,12 @@ class PreparedBufferOperation: class RenderBufferStepCfg: """Bind named buffers to the input/output roles of one processing operation. - The factory ``prepare(input_specs, **params)`` runs once during pipe construction and returns - a :class:`PreparedBufferOperation`. ``input_specs`` is keyed by the roles in ``inputs``. - Factories receive concrete array contracts and explicitly supplied resources; they should not - discover cameras, environments, renderers or simulation singletons. + The factory ``prepare(inputs, **params)`` runs once during pipe construction and returns + a :class:`PreparedBufferOperation` with allocated output buffers. ``inputs`` contains actual + :class:`ProxyArray` objects keyed by operation roles. Factories inspect array shape, dtype + and device directly; each operation documents its expected layout and pixel meaning. + Factories should not read input pixels during preparation or retain input pointers for execution: + ``apply`` receives the current inputs on every call. Scene resources must be supplied explicitly. """ prepare: Callable[..., PreparedBufferOperation] | str = MISSING @@ -112,7 +84,7 @@ class RenderBufferStepCfg: class RenderBufferPipe: """Apply an ordered sequence of operations to named :class:`ProxyArray` buffers. - A pipe owns persistent output storage and prepared operations. Create one instance per + A pipe retains persistent output storage allocated by its prepared operations. Create one instance per consumer; instances are not reentrant. Borrowed inputs are never modified by the pipe, and untouched channels pass through by reference. Returned buffers remain valid until the next call, reset or close; consumers retaining an observation must take a snapshot. @@ -122,20 +94,20 @@ class RenderBufferPipe: The caller owns producer readiness and keeps borrowed storage alive until consumption ends. Args: - source_specs: Contracts for available external buffers. + source_buffers: Available external buffers, used to prepare operations and validate later inputs. steps: Ordered operation configurations. """ - def __init__(self, source_specs: Mapping[str, ImageBufferSpec], steps: Sequence[RenderBufferStepCfg]): - self.source_specs = MappingProxyType(dict(source_specs)) + def __init__(self, source_buffers: Mapping[str, ProxyArray], steps: Sequence[RenderBufferStepCfg]): + self._source_buffers = dict(source_buffers) self.required_inputs = self.required_source_names(steps) self._operations: list[ - tuple[PreparedBufferOperation, dict[str, str], dict[str, str], dict[str, ProxyArray]] + tuple[PreparedBufferOperation, dict[str, str], dict[str, str], Mapping[str, ProxyArray]] ] = [] self._closed = False self.outputs: Mapping[str, ProxyArray] = MappingProxyType({}) - available = dict(source_specs) - final_outputs: dict[str, ProxyArray] = {} + available = dict(source_buffers) + produced: set[str] = set() try: for step in steps: inputs, outputs = dict(step.inputs), dict(step.outputs) @@ -143,37 +115,46 @@ def __init__(self, source_specs: Mapping[str, ImageBufferSpec], steps: Sequence[ raise ValueError("A processing step must bind distinct output names to its output roles.") missing = set(inputs.values()) - available.keys() if missing: - raise ValueError(f"Missing image buffer declarations: {sorted(missing)}.") + raise ValueError(f"Missing input buffers for preparation: {sorted(missing)}.") prepare = string_to_callable(step.prepare) if isinstance(step.prepare, str) else step.prepare - input_specs = {role: available[name] for role, name in inputs.items()} - operation = prepare(MappingProxyType(input_specs), **step.params) + input_buffers = {role: available[name] for role, name in inputs.items()} + operation = prepare(MappingProxyType(input_buffers), **step.params) if not isinstance(operation, PreparedBufferOperation): raise TypeError("A buffer operation factory must return PreparedBufferOperation.") - destinations: dict[str, ProxyArray] = {} - # Register ownership before allocation/validation, so failure closes the operation. + # Own the operation before checking its outputs so failed validation releases its resources. + output_buffers = {} + destinations = MappingProxyType(output_buffers) self._operations.append((operation, inputs, outputs, destinations)) - if operation.output_specs.keys() != outputs.keys(): + output_buffers.update(operation.outputs) + if destinations.keys() != outputs.keys(): raise ValueError("Operation output roles must match the step's output bindings.") - devices = { - str(wp.get_device(spec.device)) - for spec in (*input_specs.values(), *operation.output_specs.values()) - } + if any(not isinstance(buffer, ProxyArray) for buffer in destinations.values()): + raise TypeError("Prepared operation outputs must be ProxyArray objects.") + devices = {str(buffer.device) for buffer in (*input_buffers.values(), *destinations.values())} if len(devices) > 1: raise ValueError("A buffer operation must use one device for its inputs and outputs.") for role, name in outputs.items(): - spec = operation.output_specs[role] - destinations[role] = ProxyArray(wp.empty(spec.shape, dtype=spec.dtype, device=spec.device)) - available[name] = spec - final_outputs[name] = destinations[role] + available[name] = destinations[role] + produced.update(outputs.values()) except Exception as error: try: self.close() except Exception as cleanup_error: raise error from cleanup_error raise - self.output_specs = MappingProxyType(available) - self.outputs = MappingProxyType(final_outputs) - self.produced_outputs = frozenset(final_outputs) + self.outputs = MappingProxyType(available) + self.produced_outputs = frozenset(produced) + + @staticmethod + def _validate_buffer(name: str, value: ProxyArray, reference: ProxyArray) -> None: + """Check replacement storage against the actual buffer used to prepare the pipe.""" + if not isinstance(value, ProxyArray): + raise TypeError(f"Buffer {name!r} must be a ProxyArray.") + if value.shape != reference.shape or value.dtype != reference.dtype or value.device != reference.device: + raise ValueError( + f"Buffer {name!r} must match shape, dtype and device" + f" {reference.shape}, {reference.dtype}, {reference.device}." + ) @staticmethod def required_source_names( @@ -199,12 +180,15 @@ def __call__(self, buffers: Mapping[str, ProxyArray]) -> dict[str, ProxyArray]: missing = self.required_inputs - buffers.keys() if missing: raise ValueError(f"Missing required image buffers: {sorted(missing)}.") - for name in self.required_inputs: - self.source_specs[name].validate(name, buffers[name]) + for name in self._source_buffers.keys() & buffers.keys(): + self._validate_buffer(name, buffers[name], self._source_buffers[name]) current = dict(buffers) for operation, inputs, outputs, destinations in self._operations: operation.apply({role: current[name] for role, name in inputs.items()}, destinations) current.update({name: destinations[role] for role, name in outputs.items()}) + # Release replaced source references, such as the preceding observation's tensor. + self._source_buffers.update({name: buffers[name] for name in self._source_buffers.keys() & buffers.keys()}) + self.outputs = MappingProxyType(dict(current)) return current def apply_to_tensor( @@ -234,11 +218,14 @@ def as_tensor( Source buffers must be ready on the current Warp or Torch stream. Stream ordering is established without a device-wide synchronization; no host/device copy is made. - A supplied ``out`` must match the selected output contract and is overwritten in place. + A supplied ``out`` must match the prepared output's shape, dtype and device and is overwritten in place. """ + if self._closed: + raise RuntimeError("Cannot execute a closed render buffer pipe.") + reference = self.outputs[output_name] if output_name in self.produced_outputs else buffers[output_name] if out is not None: - self.output_specs[output_name].validate("out", ProxyArray(wp.from_torch(out))) - device = wp.get_device(self.output_specs[output_name].device) + self._validate_buffer("out", ProxyArray(wp.from_torch(out)), reference) + device = wp.get_device(reference.device) stream = wp.stream_from_torch(torch.cuda.current_stream(str(device))) if device.is_cuda else None with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): result = self(buffers)[output_name].torch @@ -256,6 +243,7 @@ def close(self) -> None: return self._closed = True operations, self._operations = self._operations, [] + self._source_buffers.clear() self.outputs = MappingProxyType({}) errors = [] for operation, *_ in reversed(operations): diff --git a/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py b/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py index 9f22c9734430..7527877bd9c0 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py +++ b/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py @@ -6,11 +6,10 @@ from __future__ import annotations from collections.abc import Callable -from dataclasses import MISSING, field +from dataclasses import MISSING from typing import TYPE_CHECKING, Any import torch -import warp as wp from .. import configclass @@ -44,8 +43,8 @@ class ModifierCfg: class RenderBufferPipeModifierCfg(ModifierCfg): """Bind an incoming observation tensor to a render buffer pipe. - The declared dtype, layout and encoding describe the tensor *after* preceding modifiers. - The pipe validates storage and lets each operation validate the semantic encoding. + Operations are prepared from the first tensor received after preceding modifiers. + Their documented input requirements define the expected image layout and value semantics. """ func: type[RenderBufferPipeModifier] | str = "{DIR}.render_buffer_pipe:RenderBufferPipeModifier" @@ -60,15 +59,6 @@ class RenderBufferPipeModifierCfg(ModifierCfg): output_name: str = "rgb" """Name selected from the processed bundle.""" - dtype: type = field(default_factory=lambda: wp.float32) - """Expected Warp element type of the incoming tensor.""" - - encoding: str = "unknown" - """Semantic encoding of the incoming tensor, such as ``scene_linear``.""" - - layout: str = "NHWC" - """Axis layout of the incoming tensor.""" - @configclass class DigitalFilterCfg(ModifierCfg): diff --git a/source/isaaclab/isaaclab/utils/modifiers/render_buffer_pipe.py b/source/isaaclab/isaaclab/utils/modifiers/render_buffer_pipe.py index 19f481ddb1fc..8777bdb0c4f3 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/render_buffer_pipe.py +++ b/source/isaaclab/isaaclab/utils/modifiers/render_buffer_pipe.py @@ -11,8 +11,10 @@ from typing import TYPE_CHECKING import torch +import warp as wp -from ..images import ImageBufferSpec, RenderBufferPipe +from ..images import RenderBufferPipe +from ..warp import ProxyArray from .modifier_base import ModifierBase if TYPE_CHECKING: @@ -23,28 +25,47 @@ class RenderBufferPipeModifier(ModifierBase): """Apply a render buffer pipe to the tensor received from the preceding modifier. The modifier owns a pipe and returns a snapshot. Its output must preserve the input - shape, as required by :class:`ModifierBase`. Multiple camera inputs or shape changes - belong in :class:`~isaaclab.envs.mdp.processed_image` instead. + shape, as required by :class:`ModifierBase`. The first incoming tensor prepares the pipe; + subsequent calls reuse its output storage. Multiple camera inputs or shape changes belong + in :class:`~isaaclab.envs.mdp.processed_image` instead. """ def __init__(self, cfg: RenderBufferPipeModifierCfg, data_dim: tuple[int, ...], device: str): super().__init__(cfg, data_dim, device) - self._pipe = RenderBufferPipe( - {cfg.input_name: ImageBufferSpec(tuple(data_dim), cfg.dtype, device, cfg.encoding, cfg.layout)}, - cfg.pipeline, - ) - if cfg.output_name not in self._pipe.output_specs: - self._pipe.close() - raise ValueError(f"Image processing does not provide output {cfg.output_name!r}.") - if self._pipe.output_specs[cfg.output_name].shape != tuple(data_dim): - self._pipe.close() - raise ValueError("Observation modifiers must preserve shape; use mdp.processed_image for shape changes.") + self._pipe: RenderBufferPipe | None = None + self._closed = False def __call__(self, data: torch.Tensor) -> torch.Tensor: + if self._closed: + raise RuntimeError("The render buffer pipe modifier is closed.") + if tuple(data.shape) != tuple(self._data_dim): + raise ValueError( + f"Observation modifier input shape must be {tuple(self._data_dim)}, got {tuple(data.shape)}." + ) + if self._pipe is None: + stream = wp.stream_from_torch(torch.cuda.current_stream(data.device)) if data.is_cuda else None + with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): + pipe = RenderBufferPipe({self._cfg.input_name: ProxyArray(wp.from_torch(data))}, self._cfg.pipeline) + try: + if self._cfg.output_name not in pipe.outputs: + raise ValueError(f"Image processing does not provide output {self._cfg.output_name!r}.") + if pipe.outputs[self._cfg.output_name].shape != tuple(self._data_dim): + raise ValueError( + "Observation modifiers must preserve shape; use mdp.processed_image for shape changes." + ) + except Exception: + pipe.close() + raise + self._pipe = pipe return self._pipe.apply_to_tensor(data, input_name=self._cfg.input_name, output_name=self._cfg.output_name) def reset(self, env_ids: Sequence[int] | None = None) -> None: - self._pipe.reset(env_ids) + if self._pipe is not None: + self._pipe.reset(env_ids) def close(self) -> None: - self._pipe.close() + if self._closed: + return + self._closed = True + if self._pipe is not None: + self._pipe.close() diff --git a/source/isaaclab/test/envs/test_render_buffer_observations.py b/source/isaaclab/test/envs/test_render_buffer_observations.py index 8ca9e50b27ec..6a757aaf1c98 100644 --- a/source/isaaclab/test/envs/test_render_buffer_observations.py +++ b/source/isaaclab/test/envs/test_render_buffer_observations.py @@ -21,7 +21,7 @@ ObservationTermCfg, ) from isaaclab.utils import configclass, modifiers -from isaaclab.utils.images import ImageBufferSpec, PreparedBufferOperation, RenderBufferStepCfg +from isaaclab.utils.images import PreparedBufferOperation, RenderBufferStepCfg from isaaclab.utils.warp import ProxyArray pytestmark = pytest.mark.unit @@ -69,16 +69,18 @@ def _depth_weighted_half( result[n, y, x, c] = color[n, y, 2 * x, c] * depth[n, y, 2 * x, 0] -def _prepare_gain(specs, gain): +def _prepare_gain(inputs, gain): + result = ProxyArray(wp.empty_like(inputs["color"].warp)) + def apply(inputs, outputs): wp.launch( _gain, dim=outputs["color"].shape, inputs=[inputs["color"].warp, gain, outputs["color"].warp], - device=specs["color"].device, + device=result.device, ) - return PreparedBufferOperation({"color": specs["color"]}, apply) + return PreparedBufferOperation({"color": result}, apply) def _read_image(env): @@ -107,6 +109,11 @@ def close(self): def test_pipe_modifier_uses_preceding_modifier_output(): """A pipe must consume the current observation, not re-read the source camera.""" image = torch.ones((2, 2, 4, 3)) + preparations = [] + + def prepare(inputs, gain): + preparations.append(inputs["color"].shape) + return _prepare_gain(inputs, gain) @configclass class Policy(ObservationGroupCfg): @@ -118,7 +125,7 @@ class Policy(ObservationGroupCfg): modifiers.RenderBufferPipeModifierCfg( pipeline=[ RenderBufferStepCfg( - prepare=_prepare_gain, + prepare=prepare, inputs={"color": "rgb"}, outputs={"color": "rgb"}, params={"gain": 2.0}, @@ -128,31 +135,77 @@ class Policy(ObservationGroupCfg): ], ) - manager = ObservationManager({"policy": Policy()}, _env(image=image)) + env = _env(image=image) + manager = ObservationManager({"policy": Policy()}, env) + manager.reset([1]) + assert preparations == [] first = manager.compute()["policy"]["image"] torch.testing.assert_close(first, torch.full_like(image, 8.0)) torch.testing.assert_close(image, torch.ones_like(image)) - image.fill_(2.0) + env.image = torch.full_like(image, 2.0) torch.testing.assert_close(manager.compute()["policy"]["image"], torch.full_like(image, 10.0)) torch.testing.assert_close(first, torch.full_like(image, 8.0)) + assert preparations == [tuple(image.shape)] manager.close() +def test_pipe_modifier_retries_failed_preparation_and_stays_closed(): + """A rejected output is released; a later valid preparation works only before close.""" + image = torch.ones((2, 2, 4, 3)) + lifecycle = [] + valid_shape = False + + def prepare(inputs): + lifecycle.append("prepare") + operation = _prepare_gain(inputs, 2.0) + output = ( + operation.outputs["color"] + if valid_shape + else ProxyArray(wp.empty((2, 2, 2, 3), dtype=wp.float32, device="cpu")) + ) + return PreparedBufferOperation({"color": output}, operation.apply, close=lambda: lifecycle.append("close")) + + cfg = modifiers.RenderBufferPipeModifierCfg( + pipeline=[RenderBufferStepCfg(prepare=prepare, inputs={"color": "rgb"}, outputs={"color": "rgb"})] + ) + modifier = modifiers.RenderBufferPipeModifier(cfg, tuple(image.shape), "cpu") + modifier.reset([1]) + with pytest.raises(ValueError, match="input shape"): + modifier(image[:, :, ::2]) + assert lifecycle == [] + with pytest.raises(ValueError, match="must preserve shape"): + modifier(image) + assert lifecycle == ["prepare", "close"] + valid_shape = True + torch.testing.assert_close(modifier(image), image * 2) + modifier.close() + modifier.close() + with pytest.raises(RuntimeError, match="closed"): + modifier(image) + assert lifecycle == ["prepare", "close", "prepare", "close"] + + unused_modifier = modifiers.RenderBufferPipeModifier(cfg, tuple(image.shape), "cpu") + unused_modifier.close() + with pytest.raises(RuntimeError, match="closed"): + unused_modifier(image) + assert lifecycle == ["prepare", "close", "prepare", "close"] + + def test_processed_image_declares_private_inputs_and_reports_transformed_shape(): """Preparation resolves private AOVs before binding; manager dimensions describe the processed output.""" lifecycle = [] - def prepare(specs): - color = specs["color"] + def prepare(inputs): + color = inputs["color"] n, h, w, c = color.shape - output_spec = ImageBufferSpec((n, h, w // 2, c), wp.float32, color.device) + output = ProxyArray(wp.empty((n, h, w // 2, c), dtype=wp.float32, device=color.device)) def apply(inputs, outputs): wp.launch( _depth_weighted_half, - dim=output_spec.shape, + dim=output.shape, inputs=[inputs["color"].warp, inputs["depth"].warp, outputs["color"].warp], - device=color.device, + device=output.device, ) def close(): @@ -160,7 +213,7 @@ def close(): raise ValueError("operation close failed") return PreparedBufferOperation( - {"color": output_spec}, + {"color": output}, apply, reset=lambda env_ids: lifecycle.append(("reset", env_ids)), close=close, @@ -171,10 +224,6 @@ def close(): requests = [] camera = SimpleNamespace( request_render_inputs=lambda names: requests.append(set(names)), - render_output_specs={ - "rgb_radiance": ImageBufferSpec(tuple(radiance.shape), wp.float32, "cpu", "scene_linear"), - "depth": ImageBufferSpec(tuple(depth.shape), wp.float32, "cpu", "distance"), - }, render_outputs={"rgb_radiance": ProxyArray(wp.from_torch(radiance)), "depth": ProxyArray(wp.from_torch(depth))}, ) sensors = {"camera": camera} diff --git a/source/isaaclab/test/renderers/test_camera_output_contract.py b/source/isaaclab/test/renderers/test_camera_output_contract.py index aa3348aff66f..4e3997634b33 100644 --- a/source/isaaclab/test/renderers/test_camera_output_contract.py +++ b/source/isaaclab/test/renderers/test_camera_output_contract.py @@ -26,6 +26,7 @@ from isaaclab.test.utils import DeviceScope, test_devices from isaaclab.utils import clone, validate from isaaclab.utils.images import PreparedBufferOperation, RenderBufferStepCfg +from isaaclab.utils.warp import ProxyArray pytestmark = [pytest.mark.integration, pytest.mark.rendering] @@ -376,9 +377,10 @@ def export_stage(stage, num_envs): lambda path, **kwargs: SimpleNamespace(count=2, prims=[stage.GetPrimAtPath(path)] * 2, close=lambda: None), ) - def prepare_identity(input_specs): + def prepare_identity(inputs): + output = ProxyArray(wp.empty_like(inputs["image"].warp)) return PreparedBufferOperation( - {"image": input_specs["image"]}, + {"image": output}, lambda inputs, outputs: wp.copy(outputs["image"].warp, inputs["image"].warp), ) @@ -455,10 +457,8 @@ def test_camera_pipeline_keeps_raw_inputs_and_processes_only_new_captures(monkey output_shape = (2, 1, 1, 4) close_fails = False - def prepare_color(input_specs): - assert input_specs["radiance"].encoding == "scene_linear" - assert input_specs["range"].encoding == "meters" - spec = replace(input_specs["radiance"], shape=output_shape, dtype=wp.uint8, encoding="camera_response") + def prepare_color(inputs): + output = ProxyArray(wp.empty(output_shape, dtype=wp.uint8, device=inputs["radiance"].device)) def apply(inputs, outputs): events.append("apply") @@ -471,7 +471,9 @@ def close(): if close_fails: raise RuntimeError("processor close failed") - return PreparedBufferOperation({"color": spec}, apply, reset=lambda ids: events.append(tuple(ids)), close=close) + return PreparedBufferOperation( + {"color": output}, apply, reset=lambda ids: events.append(tuple(ids)), close=close + ) cfg = CameraCfg( prim_path="/World/Camera", diff --git a/source/isaaclab/test/utils/test_render_buffer_pipe.py b/source/isaaclab/test/utils/test_render_buffer_pipe.py index f01f0bfffda4..7a05df2d1d50 100644 --- a/source/isaaclab/test/utils/test_render_buffer_pipe.py +++ b/source/isaaclab/test/utils/test_render_buffer_pipe.py @@ -6,30 +6,30 @@ """Contracts for buffer routing, ownership and lifecycle independent of camera rendering.""" from builtins import ExceptionGroup -from dataclasses import replace import pytest import torch import warp as wp -from isaaclab.utils.images import ImageBufferSpec, PreparedBufferOperation, RenderBufferPipe, RenderBufferStepCfg +from isaaclab.utils.images import PreparedBufferOperation, RenderBufferPipe, RenderBufferStepCfg from isaaclab.utils.warp import ProxyArray pytestmark = pytest.mark.unit -def _prepare_scale(specs, *, factor=2.0): +def _prepare_scale(inputs, *, factor=2.0): + destination = ProxyArray(wp.empty_like(inputs["value"].warp)) + def apply(inputs, outputs): torch.mul(inputs["value"].torch, factor, out=outputs["value"].torch) - return PreparedBufferOperation({"value": specs["value"]}, apply) + return PreparedBufferOperation({"value": destination}, apply) -def _prepare_composite(specs): +def _prepare_composite(inputs): """Combine color and depth, retaining only one spatial sample per batch entry.""" - assert specs["color"].encoding == "scene_linear" - assert specs["depth"].encoding == "meters" - shape = (specs["color"].shape[0], 1, 1, 3) + color = inputs["color"] + destination = ProxyArray(wp.empty((color.shape[0], 1, 1, 3), dtype=color.dtype, device=color.device)) def apply(inputs, outputs): torch.sub( @@ -38,14 +38,14 @@ def apply(inputs, outputs): out=outputs["value"].torch, ) - return PreparedBufferOperation({"value": replace(specs["color"], shape=shape)}, apply) + return PreparedBufferOperation({"value": destination}, apply) @pytest.fixture def composite_pipe(): - specs = { - "raw_color": ImageBufferSpec((2, 2, 2, 3), wp.float32, "cpu", "scene_linear"), - "range": ImageBufferSpec((2, 2, 2, 1), wp.float32, "cpu", "meters"), + sources = { + "raw_color": ProxyArray(wp.zeros((2, 2, 2, 3), dtype=wp.float32, device="cpu")), + "range": ProxyArray(wp.zeros((2, 2, 2, 1), dtype=wp.float32, device="cpu")), } steps = [ RenderBufferStepCfg(prepare=_prepare_scale, inputs={"value": "raw_color"}, outputs={"value": "color"}), @@ -56,7 +56,7 @@ def composite_pipe(): ), ] assert RenderBufferPipe.required_source_names(steps, ["color", "labels"]) == {"raw_color", "range", "labels"} - pipe = RenderBufferPipe(specs, steps) + pipe = RenderBufferPipe(sources, steps) yield pipe pipe.close() @@ -78,13 +78,15 @@ def test_ordered_multi_input_operations_preserve_sources_and_reuse_output_storag assert output["labels"] is labels assert output["raw_color"] is color assert set(sources) == {"raw_color", "range", "labels"} - assert composite_pipe.output_specs["color"].shape == (2, 1, 1, 3) + assert composite_pipe.outputs["color"].shape == (2, 1, 1, 3) backing.fill_(5.0) updated = composite_pipe(sources) assert updated["color"] is destination torch.testing.assert_close(destination.torch, torch.full_like(snapshot, 8.0)) torch.testing.assert_close(snapshot, torch.full_like(snapshot, 4.0)) + updated.pop("color") + assert composite_pipe.outputs["color"] is destination def test_tensor_adapter_preserves_incoming_transformations_and_returns_snapshots(composite_pipe): @@ -110,21 +112,36 @@ def test_tensor_adapter_preserves_incoming_transformations_and_returns_snapshots ) +def test_tensor_snapshot_accepts_passthrough_buffer_omitted_from_previous_call(): + """Optional pass-through buffers can return after a call that did not supply them.""" + original = ProxyArray(wp.zeros((2,), dtype=wp.float32, device="cpu")) + pipe = RenderBufferPipe({"value": original}, []) + try: + pipe({}) + current = ProxyArray(wp.full((2,), 3.0, dtype=wp.float32, device="cpu")) + snapshot = pipe.as_tensor({"value": current}, output_name="value") + current.torch.fill_(5.0) + torch.testing.assert_close(snapshot, torch.full((2,), 3.0)) + finally: + pipe.close() + + @pytest.mark.parametrize("invalid", ["missing", "shape", "dtype"]) def test_invalid_source_buffers_are_rejected_before_processing(invalid): """Missing or incompatible external storage must fail before a callback reads it.""" calls = [] - spec = ImageBufferSpec((2, 1, 1, 1), wp.float32, "cpu") + source = ProxyArray(wp.zeros((2, 1, 1, 1), dtype=wp.float32, device="cpu")) - def prepare(specs): - return PreparedBufferOperation({"value": specs["value"]}, lambda *_: calls.append(True)) + def prepare(inputs): + destination = ProxyArray(wp.empty_like(inputs["value"].warp)) + return PreparedBufferOperation({"value": destination}, lambda *_: calls.append(True)) pipe = RenderBufferPipe( - {"source": spec}, + {"source": source}, [RenderBufferStepCfg(prepare=prepare, inputs={"value": "source"}, outputs={"value": "result"})], ) try: - shape = (1, 1, 1, 1) if invalid == "shape" else spec.shape + shape = (1, 1, 1, 1) if invalid == "shape" else source.shape dtype = wp.uint8 if invalid == "dtype" else wp.float32 inputs = {} if invalid == "missing" else {"source": ProxyArray(wp.zeros(shape, dtype=dtype, device="cpu"))} with pytest.raises(ValueError, match="Missing required|shape, dtype and device"): @@ -134,14 +151,17 @@ def prepare(specs): pipe.close() -@pytest.mark.parametrize("invalid", ["missing_dependency", "output_roles", "duplicate_outputs"]) +@pytest.mark.parametrize("invalid", ["missing_dependency", "output_roles", "duplicate_outputs", "output_type"]) def test_invalid_bindings_release_prepared_operations(invalid): """Construction failure closes owned operations, including one whose outputs could not be bound.""" closed = [] - spec = ImageBufferSpec((2, 1, 1, 1), wp.float32, "cpu") + source = ProxyArray(wp.zeros((2, 1, 1, 1), dtype=wp.float32, device="cpu")) - def prepare(specs, *, name): - return PreparedBufferOperation({"value": specs["value"]}, lambda *_: None, close=lambda: closed.append(name)) + def prepare(inputs, *, name): + destination = ProxyArray(wp.empty_like(inputs["value"].warp)) + if invalid == "output_type" and name == "second": + destination = destination.warp + return PreparedBufferOperation({"value": destination}, lambda *_: None, close=lambda: closed.append(name)) first = RenderBufferStepCfg( prepare=prepare, inputs={"value": "source"}, outputs={"value": "middle"}, params={"name": "first"} @@ -151,26 +171,29 @@ def prepare(specs, *, name): ) if invalid == "missing_dependency": second.inputs = {"value": "absent"} - message = "Missing image buffer declarations" + message = "Missing input buffers for preparation" elif invalid == "output_roles": second.outputs = {"wrong": "result"} message = "output roles" - else: + elif invalid == "duplicate_outputs": second.outputs = {"value": "result", "other": "result"} message = "distinct output names" - with pytest.raises(ValueError, match=message): - RenderBufferPipe({"source": spec}, [first, second]) - assert closed == (["second", "first"] if invalid == "output_roles" else ["first"]) + else: + message = "outputs must be ProxyArray" + with pytest.raises(TypeError if invalid == "output_type" else ValueError, match=message): + RenderBufferPipe({"source": source}, [first, second]) + assert closed == (["second", "first"] if invalid in ("output_roles", "output_type") else ["first"]) @pytest.mark.parametrize("close_fails", [False, True], ids=["normal_close", "failing_close"]) def test_reset_selects_batch_entries_and_close_releases_operations_in_reverse_order(close_fails): """State belongs to each operation; selected resets and idempotent teardown preserve its lifecycle.""" closed = [] - spec = ImageBufferSpec((2, 1, 1, 1), wp.float32, "cpu") + source = ProxyArray(wp.zeros((2, 1, 1, 1), dtype=wp.float32, device="cpu")) - def prepare(specs, *, name): - history = torch.zeros(specs["value"].shape) + def prepare(inputs, *, name): + destination = ProxyArray(wp.empty_like(inputs["value"].warp)) + history = torch.zeros(inputs["value"].shape) def apply(inputs, outputs): history.add_(inputs["value"].torch) @@ -184,10 +207,10 @@ def close(): if close_fails and name == "second": raise RuntimeError("Operation cleanup failed") - return PreparedBufferOperation({"value": specs["value"]}, apply, reset=reset, close=close) + return PreparedBufferOperation({"value": destination}, apply, reset=reset, close=close) pipe = RenderBufferPipe( - {"value": spec}, + {"value": source}, [ RenderBufferStepCfg( prepare=prepare, inputs={"value": "value"}, outputs={"value": "value"}, params={"name": name} @@ -195,7 +218,7 @@ def close(): for name in ("first", "second") ], ) - source = {"value": ProxyArray(wp.ones(spec.shape, dtype=wp.float32, device="cpu"))} + source = {"value": ProxyArray(wp.ones(source.shape, dtype=wp.float32, device="cpu"))} try: torch.testing.assert_close(pipe(source)["value"].torch.flatten(), torch.tensor([1.0, 1.0])) torch.testing.assert_close(pipe(source)["value"].torch.flatten(), torch.tensor([3.0, 3.0])) @@ -213,3 +236,5 @@ def close(): assert closed == ["second", "first"] with pytest.raises(RuntimeError, match="closed"): pipe(source) + with pytest.raises(RuntimeError, match="closed"): + pipe.as_tensor(source, output_name="value") diff --git a/source/isaaclab_ppisp/docs/README.md b/source/isaaclab_ppisp/docs/README.md index 985bc3635bf9..8e9d0c269481 100644 --- a/source/isaaclab_ppisp/docs/README.md +++ b/source/isaaclab_ppisp/docs/README.md @@ -3,8 +3,8 @@ This extension provides PPISP (Physically Plausible Image Signal Processing) as an operation in the shared render-buffer pipeline used by cameras and observations. -`PpispStepCfg` reads float32 NHWC scene-linear `rgb_radiance` and writes uint8 -`rgb` with `camera_response` encoding. For camera processing, configure +`PpispStepCfg` reads scene-linear `rgb_radiance` from a float32 `ProxyArray` with +shape `(N, H, W, 3)` and writes uint8 `rgb`. For camera processing, configure `CameraCfg.pipeline=[PpispStepCfg(params={"cfg": ppisp_cfg})]`. The camera requests required radiance internally; only the requested public outputs are published. @@ -17,11 +17,16 @@ ppisp_cfg = ppisp_cfg_from_usd_camera(source_camera_prim) step = PpispStepCfg(params={"cfg": ppisp_cfg}) ``` -The same step can run in `isaaclab.utils.images.RenderBufferPipe` outside a camera. -Its input/output bindings can be changed independently of PPISP math. For RGBA, +The same step can run in `isaaclab.utils.images.RenderBufferPipe` outside a camera: +`RenderBufferPipe({"rgb_radiance": radiance_buffer}, [step])`, where `radiance_buffer` +is an allocated `ProxyArray`. Preparation validates its shape and dtype and allocates +the persistent output once on the same device. Input/output bindings can be changed +independently of PPISP math. For RGBA, use `outputs={"color": "rgba"}` and `params={"cfg": ppisp_cfg, "channels": 4}`. PPISP consumes the preceding operation's radiance, so an earlier exposure or scale -operation changes its result. Display-encoded RGB is rejected as PPISP input. +operation changes its result. The producer must supply scene-linear color before +camera exposure and response. Array storage cannot establish that semantic contract; +float32 display-encoded RGB must not be supplied as radiance. Each operation owns its controller scratch and output buffers. Optional `params["controller_weights"]=PpispControllerWeights(device_weights)` can supply diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/processor.py b/source/isaaclab_ppisp/isaaclab_ppisp/processor.py index 67e550c91ac7..db438e534f35 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/processor.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/processor.py @@ -14,7 +14,7 @@ import warp as wp from isaaclab.utils import configclass -from isaaclab.utils.images import ImageBufferSpec, PreparedBufferOperation, RenderBufferStepCfg +from isaaclab.utils.images import PreparedBufferOperation, RenderBufferStepCfg from isaaclab.utils.warp import ProxyArray from .cfg import PpispCfg @@ -22,35 +22,36 @@ def prepare_ppisp( - input_specs: Mapping[str, ImageBufferSpec], + inputs: Mapping[str, ProxyArray], *, cfg: PpispCfg, channels: int = 3, controller_weights: PpispControllerWeights | None = None, ) -> PreparedBufferOperation: - """Prepare a PPISP stage from concrete settings and a scene-linear radiance contract. + """Prepare PPISP and allocate its persistent output from an actual radiance buffer. + + The producer must supply scene-linear color before camera exposure and response. This semantic + requirement cannot be inferred from the array's dtype, shape, or pixel values. Args: - input_specs: One ``radiance`` input: NHWC float32, three scene-linear channels. + inputs: One ``radiance`` input: float32 with shape ``(N, H, W, 3)``. cfg: Concrete configuration. Resolve USD camera attributes before preparing the pipe. channels: Number of output channels, three for RGB or four for opaque RGBA. controller_weights: Optional read-only device weights shared across operations. Each operation retains its own mutable controller workspace; weights must be ready on the calling stream. Returns: - A prepared operation producing uint8 ``color`` with ``camera_response`` encoding. + A prepared operation owning a uint8 ``color`` output with three or four channels. """ - if set(input_specs) != {"radiance"}: + if set(inputs) != {"radiance"}: raise ValueError("PPISP requires exactly one input role named 'radiance'.") - source = input_specs["radiance"] - if ( - source.layout != "NHWC" - or len(source.shape) != 4 - or source.shape[-1] != 3 - or source.dtype != wp.float32 - or source.encoding != "scene_linear" - ): - raise ValueError("PPISP radiance must be NHWC float32 with three channels and scene_linear encoding.") + source = inputs["radiance"] + if not isinstance(source, ProxyArray): + raise TypeError("PPISP radiance must be a ProxyArray.") + if len(source.shape) != 4 or source.shape[-1] != 3 or any(size <= 0 for size in source.shape): + raise ValueError("PPISP radiance must have shape (N, H, W, 3) with positive dimensions.") + if source.dtype is not wp.float32: + raise ValueError("PPISP radiance must have dtype wp.float32.") if channels not in (3, 4): raise ValueError("PPISP color output requires three RGB or four RGBA channels.") if not isinstance(cfg, PpispCfg) or cfg.camera_prim_path is not None: @@ -58,23 +59,16 @@ def prepare_ppisp( pipeline = PpispPipeline(deepcopy(cfg), controller_weights=controller_weights) try: pipeline.initialize(source.shape, source.device) + color = ProxyArray(wp.empty((*source.shape[:3], channels), dtype=wp.uint8, device=source.device)) + + def apply(inputs: Mapping[str, ProxyArray], outputs: Mapping[str, ProxyArray]) -> None: + pipeline.apply(inputs["radiance"].warp, outputs["color"].warp) + + return PreparedBufferOperation(outputs={"color": color}, apply=apply, close=pipeline.close) except Exception: pipeline.close() raise - def apply(inputs: Mapping[str, ProxyArray], outputs: Mapping[str, ProxyArray]) -> None: - pipeline.apply(inputs["radiance"].warp, outputs["color"].warp) - - return PreparedBufferOperation( - output_specs={ - "color": ImageBufferSpec( - shape=(*source.shape[:3], channels), dtype=wp.uint8, device=source.device, encoding="camera_response" - ), - }, - apply=apply, - close=pipeline.close, - ) - @configclass class PpispStepCfg(RenderBufferStepCfg): diff --git a/source/isaaclab_ppisp/test/test_ppisp_kernels.py b/source/isaaclab_ppisp/test/test_ppisp_kernels.py index 7a295c6d9c42..0324423a6e7a 100644 --- a/source/isaaclab_ppisp/test/test_ppisp_kernels.py +++ b/source/isaaclab_ppisp/test/test_ppisp_kernels.py @@ -41,7 +41,7 @@ ) from isaaclab.test.utils import DeviceScope, test_devices -from isaaclab.utils.images import ImageBufferSpec, RenderBufferPipe +from isaaclab.utils.images import RenderBufferPipe from isaaclab.utils.warp import ProxyArray wp.init() @@ -314,7 +314,7 @@ def test_ppisp_pipe_rgb_crf_extreme_centers_match_reference(): ) pipe = RenderBufferPipe( - {"rgb_radiance": ImageBufferSpec(hdr_color.shape, wp.float32, "cpu", encoding="scene_linear")}, + {"rgb_radiance": ProxyArray(hdr_color)}, [PpispStepCfg(params={"cfg": cfg})], ) rgb = pipe({"rgb_radiance": ProxyArray(hdr_color)})["rgb"] @@ -342,27 +342,26 @@ def test_ppisp_controller_pipes_share_weights_for_rgb_and_rgba(device): weights_np = np.zeros(PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN, dtype=np.float32) weights_np[PPISP_CONTROLLER_OFF_EXP_B] = 1.0 # A constant one-stop controller exposure. weights = wp.from_numpy(weights_np, dtype=wp.float32, device=device) - source = ImageBufferSpec((1, 1, 4, 3), wp.float32, device, encoding="scene_linear") + ramp = np.array([0.0, 0.125, 0.25, 0.5], dtype=np.float32) + hdr_data = np.repeat(ramp[None, None, :, None], 3, axis=3) + hdr = wp.from_numpy(hdr_data, dtype=wp.float32, device=device) cfg = PpispCfg() shared_weights = PpispControllerWeights(weights) pipes = [ RenderBufferPipe( - {"rgb_radiance": source}, + {"rgb_radiance": ProxyArray(hdr)}, [PpispStepCfg(params={"cfg": cfg, "channels": channels, "controller_weights": shared_weights})], ) for channels in (3, 4) ] - ramp = np.array([0.0, 0.125, 0.25, 0.5], dtype=np.float32) - hdr_data = np.repeat(ramp[None, None, :, None], 3, axis=3) - hdr = wp.from_numpy(hdr_data, dtype=wp.float32, device=device) - # The observation path must honor work already queued on a non-default Torch stream. + # Use different pixels from preparation and honor prior work on a non-default Torch stream. with torch.cuda.stream(torch.cuda.Stream(device=device)): - incoming = torch.from_numpy(hdr_data * 0.5).to(device) * 2.0 + incoming = torch.from_numpy(hdr_data * 0.5).to(device) * 3.0 observation = pipes[0].apply_to_tensor(incoming, input_name="rgb_radiance", output_name="rgb") results = [observation.cpu().numpy()] results.append(pipes[1]({"rgb_radiance": ProxyArray(hdr)})["rgb"].warp.numpy()) - expected = np.floor(_crf_reference(ramp * 2.0, 0.013659, 0.013659, 0.378165, 0.0) * 255.0) - for result in results: + for result, input_scale in zip(results, (1.5, 1.0), strict=True): + expected = np.floor(_crf_reference(ramp * input_scale * 2.0, 0.013659, 0.013659, 0.378165, 0.0) * 255.0) np.testing.assert_allclose(result[0, 0, :, :3].astype(float), np.repeat(expected[:, None], 3, axis=1), atol=1.0) assert (results[1][..., 3] == 255).all() pipes[0].close() @@ -372,12 +371,20 @@ def test_ppisp_controller_pipes_share_weights_for_rgb_and_rgba(device): pipes[1].close() -def test_ppisp_pipe_requires_scene_linear_input_and_resolved_configuration(): - """Reject semantically wrong radiance and unresolved scene dependencies before execution.""" - source = ImageBufferSpec((1, 2, 2, 3), wp.float32, "cpu", encoding="srgb") - with pytest.raises(ValueError, match="scene_linear encoding"): +@pytest.mark.parametrize( + ("shape", "dtype", "message"), + [((1, 2, 2, 4), wp.float32, "must have shape"), ((1, 2, 2, 3), wp.uint8, "must have dtype")], +) +def test_ppisp_pipe_rejects_incompatible_input_storage(shape, dtype, message): + """Reject invalid image channels and scalar types before preparing numerical execution.""" + source = ProxyArray(wp.empty(shape, dtype=dtype, device="cpu")) + with pytest.raises(ValueError, match=message): RenderBufferPipe({"rgb_radiance": source}, [PpispStepCfg()]) - source = ImageBufferSpec((1, 2, 2, 3), wp.float32, "cpu", encoding="scene_linear") + + +def test_ppisp_pipe_requires_resolved_configuration(): + """An image operation must not discover its settings from a camera during preparation.""" + source = ProxyArray(wp.empty((1, 2, 2, 3), dtype=wp.float32, device="cpu")) with pytest.raises(ValueError, match="resolve USD camera attributes"): RenderBufferPipe( {"rgb_radiance": source}, [PpispStepCfg(params={"cfg": PpispCfg(camera_prim_path="/World/Camera")})] From bf53ccf9162c1969b36447ff1698d24627286a0e Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Wed, 14 Oct 2026 16:44:18 -0400 Subject: [PATCH 16/20] Skip stateless camera pipeline reset synchronization --- .../changelog.d/feat-render-buffer-pipe.fixed.rst | 2 ++ source/isaaclab/isaaclab/sensors/camera/camera.py | 2 +- source/isaaclab/isaaclab/utils/images.py | 5 +++++ .../test/renderers/test_camera_output_contract.py | 13 ++++++++++++- 4 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 source/isaaclab/changelog.d/feat-render-buffer-pipe.fixed.rst diff --git a/source/isaaclab/changelog.d/feat-render-buffer-pipe.fixed.rst b/source/isaaclab/changelog.d/feat-render-buffer-pipe.fixed.rst new file mode 100644 index 000000000000..db061e515934 --- /dev/null +++ b/source/isaaclab/changelog.d/feat-render-buffer-pipe.fixed.rst @@ -0,0 +1,2 @@ +* Avoided unnecessary CUDA synchronization during masked camera resets when image pipeline + operations had no reset callbacks. diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 33e1d3429a4d..61afef7af19e 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -652,7 +652,7 @@ def reset(self, env_ids: Sequence[int] | slice | None = None, env_mask: wp.array self._renderer.reset(self._render_data, env_ids) # reset the timestamps super().reset(env_ids, env_mask) - if self._buffer_pipe is not None: + if self._buffer_pipe is not None and self._buffer_pipe.has_reset_callbacks: pipeline_env_ids = wp.to_torch(env_mask).nonzero().squeeze(-1) if env_mask is not None else env_ids self._buffer_pipe.reset(pipeline_env_ids) # reset the data diff --git a/source/isaaclab/isaaclab/utils/images.py b/source/isaaclab/isaaclab/utils/images.py index 51a548d6671e..13a0bdfd7607 100644 --- a/source/isaaclab/isaaclab/utils/images.py +++ b/source/isaaclab/isaaclab/utils/images.py @@ -145,6 +145,11 @@ def __init__(self, source_buffers: Mapping[str, ProxyArray], steps: Sequence[Ren self.outputs = MappingProxyType(available) self.produced_outputs = frozenset(produced) + @property + def has_reset_callbacks(self) -> bool: + """Whether any operation needs selected batch entries reset.""" + return any(operation.reset is not None for operation, *_ in self._operations) + @staticmethod def _validate_buffer(name: str, value: ProxyArray, reference: ProxyArray) -> None: """Check replacement storage against the actual buffer used to prepare the pipe.""" diff --git a/source/isaaclab/test/renderers/test_camera_output_contract.py b/source/isaaclab/test/renderers/test_camera_output_contract.py index 4e3997634b33..281d402314d1 100644 --- a/source/isaaclab/test/renderers/test_camera_output_contract.py +++ b/source/isaaclab/test/renderers/test_camera_output_contract.py @@ -456,6 +456,7 @@ def test_camera_pipeline_keeps_raw_inputs_and_processes_only_new_captures(monkey events = [] output_shape = (2, 1, 1, 4) close_fails = False + reset_enabled = True def prepare_color(inputs): output = ProxyArray(wp.empty(output_shape, dtype=wp.uint8, device=inputs["radiance"].device)) @@ -472,7 +473,10 @@ def close(): raise RuntimeError("processor close failed") return PreparedBufferOperation( - {"color": output}, apply, reset=lambda ids: events.append(tuple(ids)), close=close + {"color": output}, + apply, + reset=(lambda ids: events.append(tuple(ids))) if reset_enabled else None, + close=close, ) cfg = CameraCfg( @@ -549,7 +553,14 @@ def read_output(render_data, camera_data): assert camera._buffer_pipe is None assert events[-2:] == ["close", "close"] output_shape = (2, 1, 1, 4) + reset_enabled = False camera._create_buffers() + # Stateless operations must not materialize indices: CUDA nonzero would synchronize the host. + with monkeypatch.context() as patch: + patch.setattr( + torch.Tensor, "nonzero", lambda self: pytest.fail("Stateless camera reset materialized mask indices") + ) + camera.reset(env_mask=reset_mask) camera._render_data = object() camera._renderer.cleanup = lambda data: events.append("renderer-close") camera._view.close = lambda: events.append("view-close") From ae84472c98d08449d260dc1684764e36ad48d5f8 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Wed, 14 Oct 2026 17:25:47 -0400 Subject: [PATCH 17/20] Rename shared buffer processing APIs --- docs/source/api/lab/isaaclab.envs.mdp.rst | 2 +- docs/source/api/lab/isaaclab.utils.rst | 31 ++- docs/source/concepts/sensors/camera.rst | 36 +-- .../feat-render-buffer-pipe.changed.rst | 8 +- .../isaaclab/isaaclab/envs/mdp/__init__.pyi | 4 +- ...observations.py => sensor_observations.py} | 31 ++- .../isaaclab/sensors/camera/camera.py | 6 +- .../isaaclab/sensors/camera/camera_cfg.py | 8 +- source/isaaclab/isaaclab/utils/images.py | 236 +--------------- .../isaaclab/utils/modifiers/__init__.pyi | 8 +- .../isaaclab/utils/modifiers/modifier_cfg.py | 14 +- ...ffer_pipe.py => processing_buffer_pipe.py} | 28 +- source/isaaclab/isaaclab/utils/processing.py | 251 ++++++++++++++++++ ...vations.py => test_buffer_observations.py} | 67 ++--- .../renderers/test_camera_output_contract.py | 8 +- ...pipe.py => test_processing_buffer_pipe.py} | 26 +- source/isaaclab_ppisp/docs/README.md | 6 +- .../isaaclab_ppisp/processor.py | 6 +- .../isaaclab_ppisp/test/test_ppisp_kernels.py | 10 +- 19 files changed, 416 insertions(+), 370 deletions(-) rename source/isaaclab/isaaclab/envs/mdp/{visual_observations.py => sensor_observations.py} (66%) rename source/isaaclab/isaaclab/utils/modifiers/{render_buffer_pipe.py => processing_buffer_pipe.py} (65%) create mode 100644 source/isaaclab/isaaclab/utils/processing.py rename source/isaaclab/test/envs/{test_render_buffer_observations.py => test_buffer_observations.py} (79%) rename source/isaaclab/test/utils/{test_render_buffer_pipe.py => test_processing_buffer_pipe.py} (92%) diff --git a/docs/source/api/lab/isaaclab.envs.mdp.rst b/docs/source/api/lab/isaaclab.envs.mdp.rst index ab5db418b7b4..4cffb72ae505 100644 --- a/docs/source/api/lab/isaaclab.envs.mdp.rst +++ b/docs/source/api/lab/isaaclab.envs.mdp.rst @@ -9,7 +9,7 @@ Observations .. automodule:: isaaclab.envs.mdp.observations :members: -.. automodule:: isaaclab.envs.mdp.visual_observations +.. automodule:: isaaclab.envs.mdp.sensor_observations :members: Actions diff --git a/docs/source/api/lab/isaaclab.utils.rst b/docs/source/api/lab/isaaclab.utils.rst index d58f30ddd65e..f81f3ad459d1 100644 --- a/docs/source/api/lab/isaaclab.utils.rst +++ b/docs/source/api/lab/isaaclab.utils.rst @@ -19,6 +19,7 @@ mesh modifiers noise + processing seed sensors string @@ -145,6 +146,25 @@ Noise operations :show-inheritance: :exclude-members: __init__, func +Processing operations +~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: isaaclab.utils.processing + +``ProcessingBufferPipe`` processes named ``ProxyArray`` buffers, including camera outputs and nonvisual +sensor data. Operations define their own input shape, dtype, and semantic requirements. + +.. autoclass:: isaaclab.utils.processing.PreparedBufferOperation + :members: + +.. autoclass:: isaaclab.utils.processing.ProcessingBufferStepCfg + :members: + :exclude-members: __init__ + +.. autoclass:: isaaclab.utils.processing.ProcessingBufferPipe + :members: + :special-members: __call__ + Seed operations ~~~~~~~~~~~~~~~ @@ -171,17 +191,6 @@ Image operations .. autofunction:: isaaclab.utils.images.compose_image -.. autoclass:: isaaclab.utils.images.PreparedBufferOperation - :members: - -.. autoclass:: isaaclab.utils.images.RenderBufferStepCfg - :members: - :exclude-members: __init__ - -.. autoclass:: isaaclab.utils.images.RenderBufferPipe - :members: - :special-members: __call__ - Timer operations ~~~~~~~~~~~~~~~~ diff --git a/docs/source/concepts/sensors/camera.rst b/docs/source/concepts/sensors/camera.rst index 4a0445949bed..f9b2b9015cc4 100644 --- a/docs/source/concepts/sensors/camera.rst +++ b/docs/source/concepts/sensors/camera.rst @@ -337,11 +337,12 @@ Post-render image processing ---------------------------- :attr:`~sensors.CameraCfg.pipeline` applies an ordered sequence of operations to named image buffers. -Each :class:`~isaaclab.utils.images.RenderBufferStepCfg` maps operation input and output roles to buffer +Each :class:`~isaaclab.utils.processing.ProcessingBufferStepCfg` maps operation input and output roles to buffer names. A step can read several channels, such as radiance and depth, and later steps read the preceding -step's results. The same :class:`~isaaclab.utils.images.RenderBufferPipe` can run independently or serve +step's results. The same :class:`~isaaclab.utils.processing.ProcessingBufferPipe` can run independently or serve an observation consumer; operations receive explicit buffers and resources without discovering a camera, -environment, or simulation singleton. +environment, or simulation singleton. The shared pipe accepts named ``ProxyArray`` buffers, including +nonvisual sensor data; image layout and pixel meaning are requirements of individual operations. The camera resolves external inputs before renderer initialization, including inputs needed only by the pipeline. It publishes the names in ``data_types`` after processing. For example, this camera requests @@ -365,7 +366,7 @@ step for RGBA output. A pipeline-produced RGBA buffer also supplies a zero-copy is requested. A pipeline producing only RGB cannot satisfy an explicit RGBA request. For a custom operation, provide a factory ``prepare(inputs, **params)`` returning -:class:`~isaaclab.utils.images.PreparedBufferOperation`. The factory receives actual +:class:`~isaaclab.utils.processing.PreparedBufferOperation`. The factory receives actual :class:`~isaaclab.utils.warp.ProxyArray` inputs, inspects their shape, dtype, and device, and allocates its output buffers and workspace once. Return the output arrays in the prepared operation, then write each supplied destination in ``apply(inputs, outputs)``. Both mappings use operation role names; the @@ -374,7 +375,7 @@ retain their addresses for execution: each ``apply`` call receives the current i Operations document their expected image layout and pixel meaning. PPISP requires NHWC float32 scene-linear radiance and produces NHWC uint8 color; a float32 array alone does not establish radiance. Published camera outputs must keep the camera's batch size, height, width, and NHWC layout so pose -and calibration metadata remain valid. Intermediate buffers may change shape. Use ``processed_image`` +and calibration metadata remain valid. Intermediate buffers may change shape. Use ``processed_sensor`` below when the final observation should have a different image shape or layout. ``camera.render_outputs`` exposes borrowed raw renderer buffers, including private inputs. Their @@ -394,11 +395,12 @@ observation-only pipeline, declare raw dependencies with ``camera.request_render simulation initialization. Additional requests after initialization fail because renderer setup is already complete. Observation consumers use their own pipe and snapshot results needed across captures. -Processing observation images -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Processing sensor observations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Use :class:`~isaaclab.envs.mdp.processed_image` for observation-only processing, including operations -that combine several camera outputs or change image dimensions: +Use :class:`~isaaclab.envs.mdp.processed_sensor` for observation-only processing, including operations +that combine several camera outputs or change image dimensions. The term currently binds sensors +through their ``request_render_inputs()`` and ``render_outputs`` interface: .. code-block:: python @@ -407,7 +409,7 @@ that combine several camera outputs or change image dimensions: from isaaclab_ppisp import PpispStepCfg policy_image = ObservationTermCfg( - func=mdp.processed_image, + func=mdp.processed_sensor, params={ "sensor_cfg": SceneEntityCfg("front_camera"), "pipeline": [PpispStepCfg()], @@ -422,11 +424,13 @@ camera's published processed output, use an existing image term such as ``mdp.im The observation pipe executes on every term evaluation and returns a snapshot; its state belongs to that term rather than the camera's capture cadence. -For a single incoming tensor, :class:`~isaaclab.utils.modifiers.RenderBufferPipeModifierCfg` uses the -same operations while preserving preceding observation transformations. This form must preserve -shape; use ``processed_image`` for multiple camera inputs or shape changes. The first incoming tensor -prepares the operations; later calls reuse their output storage. Its values must meet the operation's -documented input requirements, including any transformations from preceding modifiers: +For a single incoming tensor, :class:`~isaaclab.utils.modifiers.ProcessingBufferPipeModifierCfg` uses the +same operations while preserving preceding observation transformations. This includes nonvisual +observation tensors from sensors without the render-input interface. This form must preserve shape; +use ``processed_sensor`` for multiple render inputs or changes to render-output shapes, or construct +a ``ProcessingBufferPipe`` directly for other named buffers. The first incoming tensor prepares the +operations; later calls reuse their output storage. Its values must meet the operation's documented +input requirements, including any transformations from preceding modifiers: .. code-block:: python @@ -438,7 +442,7 @@ documented input requirements, including any transformations from preceding modi func=mdp.image_rgb, params={"sensor_cfg": SceneEntityCfg("front_camera"), "data_type": "rgb_radiance", "normalize": False}, modifiers=[ - modifiers.RenderBufferPipeModifierCfg( + modifiers.ProcessingBufferPipeModifierCfg( pipeline=[PpispStepCfg()], input_name="rgb_radiance", output_name="rgb", diff --git a/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst index 063c1e1b8540..e7515e13a8eb 100644 --- a/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst +++ b/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst @@ -1,3 +1,5 @@ -* **Breaking:** Replaced ``CameraCfg.isp_cfg`` with reusable image pipelines. Configure camera - processing with ``CameraCfg.pipeline=[PpispStepCfg(...)]``; use ``mdp.processed_image`` or - ``RenderBufferPipeModifierCfg`` for observation processing with the same operations. +* **Breaking:** Replaced ``CameraCfg.isp_cfg`` with reusable buffer pipelines. Configure camera + processing with ``CameraCfg.pipeline=[PpispStepCfg(...)]``; use ``mdp.processed_sensor`` or + ``ProcessingBufferPipeModifierCfg`` for observation processing with the same operations. The shared + ``isaaclab.utils.processing.ProcessingBufferPipe`` accepts named ``ProxyArray`` buffers for both image and + nonvisual sensor data. diff --git a/source/isaaclab/isaaclab/envs/mdp/__init__.pyi b/source/isaaclab/isaaclab/envs/mdp/__init__.pyi index 226011e391b9..500a12b82456 100644 --- a/source/isaaclab/isaaclab/envs/mdp/__init__.pyi +++ b/source/isaaclab/isaaclab/envs/mdp/__init__.pyi @@ -89,7 +89,7 @@ __all__ = [ "image_normals", "image_rgb", "image_segmentation", - "processed_image", + "processed_sensor", "imu_ang_vel", "imu_lin_acc", "pva_orientation", @@ -271,7 +271,7 @@ from .observations import ( root_pos_w, root_quat_w, ) -from .visual_observations import processed_image +from .sensor_observations import processed_sensor from .recorders import ( ActionStateRecorderManagerCfg, InitialStateRecorder, diff --git a/source/isaaclab/isaaclab/envs/mdp/visual_observations.py b/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py similarity index 66% rename from source/isaaclab/isaaclab/envs/mdp/visual_observations.py rename to source/isaaclab/isaaclab/envs/mdp/sensor_observations.py index fb5496cadb02..af390bee049c 100644 --- a/source/isaaclab/isaaclab/envs/mdp/visual_observations.py +++ b/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Observation adapters for renderer-independent image processing.""" +"""Observation adapters for processing named sensor buffers.""" from __future__ import annotations @@ -14,14 +14,18 @@ import warp as wp from isaaclab.managers import ManagerTermBase, ObservationTermCfg, SceneEntityCfg -from isaaclab.utils.images import RenderBufferPipe, RenderBufferStepCfg +from isaaclab.utils.processing import ProcessingBufferPipe, ProcessingBufferStepCfg if TYPE_CHECKING: from isaaclab.envs import ManagerBasedEnv -class processed_image(ManagerTermBase): - """Process named raw camera outputs with the shared render buffer pipe. +class processed_sensor(ManagerTermBase): + """Process named sensor render outputs with the shared buffer pipe. + + The sensor must expose ``request_render_inputs`` and ``render_outputs``, as + :class:`~isaaclab.sensors.Camera` does. Operations define the supported buffer layouts and + value semantics; outputs need not be visual images. Dependencies are requested before simulation initialization through :meth:`~isaaclab.managers.ObservationManager.prepare_scene`. They remain private to the @@ -29,8 +33,9 @@ class processed_image(ManagerTermBase): its own operation state and output storage. The pipe runs on every observation evaluation; returned tensors are snapshots, so later evaluations cannot overwrite retained history. - Use this term for multiple inputs or changed image dimensions. To process the result of - earlier observation modifiers instead, use :class:`~isaaclab.utils.modifiers.RenderBufferPipeModifier`. + Use this term for multiple render inputs or changed buffer dimensions. To process other + sensor data or the result of earlier observation modifiers, use + :class:`~isaaclab.utils.modifiers.ProcessingBufferPipeModifier` on an observation tensor. To read a camera's already processed public output, use the existing image observation terms. """ @@ -38,26 +43,26 @@ class processed_image(ManagerTermBase): def prepare_scene(cls, cfg: ObservationTermCfg, env: ManagerBasedEnv) -> None: sensor_cfg = cfg.params.get("sensor_cfg", SceneEntityCfg("camera")) output_name = cfg.params.get("output_name", "rgb") - requirements = RenderBufferPipe.required_source_names(cfg.params["pipeline"], [output_name]) + requirements = ProcessingBufferPipe.required_source_names(cfg.params["pipeline"], [output_name]) env.scene.sensors[sensor_cfg.name].request_render_inputs(requirements) def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedEnv): super().__init__(cfg, env) - self._camera = env.scene.sensors[cfg.params["sensor_cfg"].name] + self._sensor = env.scene.sensors[cfg.params["sensor_cfg"].name] self._output_name = cfg.params["output_name"] - buffers = self._camera.render_outputs + buffers = self._sensor.render_outputs device = wp.get_device(self.device) stream = wp.stream_from_torch(torch.cuda.current_stream(str(device))) if device.is_cuda else None with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): - self._pipe = RenderBufferPipe(buffers, cfg.params["pipeline"]) + self._pipe = ProcessingBufferPipe(buffers, cfg.params["pipeline"]) if self._output_name not in self._pipe.outputs: self._pipe.close() - raise ValueError(f"Image processing does not provide output {self._output_name!r}.") + raise ValueError(f"Buffer processing does not provide output {self._output_name!r}.") def __call__( self, env: ManagerBasedEnv, - pipeline: Sequence[RenderBufferStepCfg], + pipeline: Sequence[ProcessingBufferStepCfg], sensor_cfg: SceneEntityCfg = SceneEntityCfg("camera"), output_name: str = "rgb", *, @@ -68,7 +73,7 @@ def __call__( The observation manager supplies ``out`` to receive the snapshot without another clone. Direct callers may omit it to allocate a new snapshot. """ - return self._pipe.as_tensor(self._camera.render_outputs, output_name=self._output_name, out=out) + return self._pipe.as_tensor(self._sensor.render_outputs, output_name=self._output_name, out=out) def reset(self, env_ids: Sequence[int] | None = None) -> None: self._pipe.reset(env_ids) diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 61afef7af19e..c5ee6dc3be57 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -23,12 +23,12 @@ from ...app.logging_utils import force_log_level from ...renderers import BaseRenderer, CameraRenderSpec from ...sim.views import FrameView -from ...utils.images import RenderBufferPipe from ...utils.math import ( convert_camera_frame_orientation_convention, create_rotation_matrix_from_view, quat_from_matrix, ) +from ...utils.processing import ProcessingBufferPipe from ...utils.warp import ProxyArray from ..sensor_base import SensorBase from .camera_data import CameraData, RenderBufferKind @@ -253,7 +253,7 @@ def __init__(self, cfg: CameraCfg): # Renderer requirements include private pipeline inputs and exclude produced public outputs. self._render_cfg = replace(self.cfg, data_types=self.cfg.render_data_types(), pipeline=[]) - self._buffer_pipe: RenderBufferPipe | None = None + self._buffer_pipe: ProcessingBufferPipe | None = None self._pipeline_capture: object | None = None self._render_camera_data: CameraData | None = None self._render_outputs: dict[str, ProxyArray] = {} @@ -900,7 +900,7 @@ def _create_buffers(self): self._close_pipeline() self._pipeline_capture = None if self.cfg.pipeline: - self._buffer_pipe = RenderBufferPipe(self._render_outputs, self.cfg.pipeline) + self._buffer_pipe = ProcessingBufferPipe(self._render_outputs, self.cfg.pipeline) try: outputs = dict(self._buffer_pipe.outputs) alias_rgb = ( diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py index 8984a805fd8c..507d32319a20 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py @@ -15,7 +15,7 @@ from ..sensor_base_cfg import SensorBaseCfg if TYPE_CHECKING: - from ...utils.images import RenderBufferStepCfg + from ...utils.processing import ProcessingBufferStepCfg from .camera import Camera # Default values for the RTX-flavored fields kept on :class:`CameraCfg` for @@ -199,7 +199,7 @@ class OffsetCfg: renderer_cfg: RendererCfg = field(default_factory=RendererCfg) """Renderer configuration for camera sensor.""" - pipeline: list[RenderBufferStepCfg] = [] + pipeline: list[ProcessingBufferStepCfg] = [] """Ordered image operations applied after each newly published capture. Steps bind operation roles to named buffers and may consume several render outputs or preceding @@ -223,13 +223,13 @@ def render_data_types(self) -> list[str]: ) if not self.pipeline: return list(self.data_types) - from ...utils.images import RenderBufferPipe + from ...utils.processing import ProcessingBufferPipe produced = {name for step in self.pipeline for name in step.outputs.values()} if "rgb" in produced and "rgba" not in produced and "rgba" in self.data_types: raise ValueError("A pipeline producing rgb must also produce rgba when rgba is requested.") published = [name for name in self.data_types if not (name == "rgb" and "rgba" in produced)] - return sorted(RenderBufferPipe.required_source_names(self.pipeline, published)) + return sorted(ProcessingBufferPipe.required_source_names(self.pipeline, published)) def __post_init__(self): """Forward deprecated RTX-flavored fields onto :attr:`renderer_cfg`. diff --git a/source/isaaclab/isaaclab/utils/images.py b/source/isaaclab/isaaclab/utils/images.py index 13a0bdfd7607..58a419b74b4b 100644 --- a/source/isaaclab/isaaclab/utils/images.py +++ b/source/isaaclab/isaaclab/utils/images.py @@ -19,247 +19,13 @@ from __future__ import annotations -from builtins import ExceptionGroup -from collections.abc import Callable, Mapping, Sequence -from dataclasses import MISSING, dataclass -from types import MappingProxyType +from collections.abc import Callable, Sequence from typing import Any import torch import warp as wp from .buffers import CircularBuffer -from .configclass import configclass -from .string import string_to_callable -from .warp import ProxyArray - - -@dataclass(frozen=True) -class PreparedBufferOperation: - """One operation with persistent output buffers and resolved resources. - - ``apply`` receives input and destination dictionaries keyed by operation roles, not sensor - output names. Preparation allocates the outputs once. Inputs are borrowed read-only by - convention; outputs must use independent storage. The operation must fill each supplied - destination and enqueue GPU work on the current Warp stream. The pipe retains the outputs - and controls execution order; private workspace belongs to the operation. - """ - - outputs: Mapping[str, ProxyArray] - apply: Callable[[Mapping[str, ProxyArray], Mapping[str, ProxyArray]], None] - reset: Callable[[Sequence[int] | None], None] | None = None - close: Callable[[], None] | None = None - - -@configclass -class RenderBufferStepCfg: - """Bind named buffers to the input/output roles of one processing operation. - - The factory ``prepare(inputs, **params)`` runs once during pipe construction and returns - a :class:`PreparedBufferOperation` with allocated output buffers. ``inputs`` contains actual - :class:`ProxyArray` objects keyed by operation roles. Factories inspect array shape, dtype - and device directly; each operation documents its expected layout and pixel meaning. - Factories should not read input pixels during preparation or retain input pointers for execution: - ``apply`` receives the current inputs on every call. Scene resources must be supplied explicitly. - """ - - prepare: Callable[..., PreparedBufferOperation] | str = MISSING - """Operation factory or an import string in ``module:attribute`` form.""" - - inputs: dict[str, str] = MISSING - """Map operation input roles to buffer names read from the preceding stage.""" - - outputs: dict[str, str] = MISSING - """Map operation output roles to names created or replaced in the bundle.""" - - params: dict[str, Any] = {} - """Resolved operation parameters passed to the factory. - - Configurations are deep-copied by ``configclass`` and managers. Borrowed device allocations - or models need a resource wrapper that preserves identity through ``__deepcopy__``, or a - factory closure retaining the resource. Do not put a raw Warp allocation in this dictionary. - """ - - -class RenderBufferPipe: - """Apply an ordered sequence of operations to named :class:`ProxyArray` buffers. - - A pipe retains persistent output storage allocated by its prepared operations. Create one instance per - consumer; instances are not reentrant. Borrowed inputs are never modified by the pipe, and - untouched channels pass through by reference. Returned buffers remain valid until the next - call, reset or close; consumers retaining an observation must take a snapshot. - - Buffer dependencies are resolved in order, so an operation reads preceding replacements. - GPU execution uses the caller's current Warp stream without a device-wide synchronization. - The caller owns producer readiness and keeps borrowed storage alive until consumption ends. - - Args: - source_buffers: Available external buffers, used to prepare operations and validate later inputs. - steps: Ordered operation configurations. - """ - - def __init__(self, source_buffers: Mapping[str, ProxyArray], steps: Sequence[RenderBufferStepCfg]): - self._source_buffers = dict(source_buffers) - self.required_inputs = self.required_source_names(steps) - self._operations: list[ - tuple[PreparedBufferOperation, dict[str, str], dict[str, str], Mapping[str, ProxyArray]] - ] = [] - self._closed = False - self.outputs: Mapping[str, ProxyArray] = MappingProxyType({}) - available = dict(source_buffers) - produced: set[str] = set() - try: - for step in steps: - inputs, outputs = dict(step.inputs), dict(step.outputs) - if not outputs or len(set(outputs.values())) != len(outputs): - raise ValueError("A processing step must bind distinct output names to its output roles.") - missing = set(inputs.values()) - available.keys() - if missing: - raise ValueError(f"Missing input buffers for preparation: {sorted(missing)}.") - prepare = string_to_callable(step.prepare) if isinstance(step.prepare, str) else step.prepare - input_buffers = {role: available[name] for role, name in inputs.items()} - operation = prepare(MappingProxyType(input_buffers), **step.params) - if not isinstance(operation, PreparedBufferOperation): - raise TypeError("A buffer operation factory must return PreparedBufferOperation.") - # Own the operation before checking its outputs so failed validation releases its resources. - output_buffers = {} - destinations = MappingProxyType(output_buffers) - self._operations.append((operation, inputs, outputs, destinations)) - output_buffers.update(operation.outputs) - if destinations.keys() != outputs.keys(): - raise ValueError("Operation output roles must match the step's output bindings.") - if any(not isinstance(buffer, ProxyArray) for buffer in destinations.values()): - raise TypeError("Prepared operation outputs must be ProxyArray objects.") - devices = {str(buffer.device) for buffer in (*input_buffers.values(), *destinations.values())} - if len(devices) > 1: - raise ValueError("A buffer operation must use one device for its inputs and outputs.") - for role, name in outputs.items(): - available[name] = destinations[role] - produced.update(outputs.values()) - except Exception as error: - try: - self.close() - except Exception as cleanup_error: - raise error from cleanup_error - raise - self.outputs = MappingProxyType(available) - self.produced_outputs = frozenset(produced) - - @property - def has_reset_callbacks(self) -> bool: - """Whether any operation needs selected batch entries reset.""" - return any(operation.reset is not None for operation, *_ in self._operations) - - @staticmethod - def _validate_buffer(name: str, value: ProxyArray, reference: ProxyArray) -> None: - """Check replacement storage against the actual buffer used to prepare the pipe.""" - if not isinstance(value, ProxyArray): - raise TypeError(f"Buffer {name!r} must be a ProxyArray.") - if value.shape != reference.shape or value.dtype != reference.dtype or value.device != reference.device: - raise ValueError( - f"Buffer {name!r} must match shape, dtype and device" - f" {reference.shape}, {reference.dtype}, {reference.device}." - ) - - @staticmethod - def required_source_names( - steps: Sequence[RenderBufferStepCfg], published_names: Sequence[str] = () - ) -> frozenset[str]: - """Resolve external names without constructing operations or allocating pixel storage. - - Call this before renderer initialization to request private processing inputs. Public - pass-through outputs also require a source, even when no operation reads them. - """ - required: set[str] = set() - produced: set[str] = set() - for step in steps: - required.update(set(step.inputs.values()) - produced) - produced.update(step.outputs.values()) - required.update(set(published_names) - produced) - return frozenset(required) - - def __call__(self, buffers: Mapping[str, ProxyArray]) -> dict[str, ProxyArray]: - """Execute the pipe on ready source buffers and return the resulting bundle.""" - if self._closed: - raise RuntimeError("Cannot execute a closed render buffer pipe.") - missing = self.required_inputs - buffers.keys() - if missing: - raise ValueError(f"Missing required image buffers: {sorted(missing)}.") - for name in self._source_buffers.keys() & buffers.keys(): - self._validate_buffer(name, buffers[name], self._source_buffers[name]) - current = dict(buffers) - for operation, inputs, outputs, destinations in self._operations: - operation.apply({role: current[name] for role, name in inputs.items()}, destinations) - current.update({name: destinations[role] for role, name in outputs.items()}) - # Release replaced source references, such as the preceding observation's tensor. - self._source_buffers.update({name: buffers[name] for name in self._source_buffers.keys() & buffers.keys()}) - self.outputs = MappingProxyType(dict(current)) - return current - - def apply_to_tensor( - self, - data: torch.Tensor, - *, - input_name: str, - output_name: str, - auxiliary_buffers: Mapping[str, ProxyArray] | None = None, - ) -> torch.Tensor: - """Process an incoming observation tensor and return an independently owned snapshot. - - The supplied tensor is authoritative: earlier observation transformations are preserved. - Auxiliary inputs must already be ready on the current Warp/Torch stream. CUDA work is - ordered with the caller's Torch stream; no host/device transfer is performed. - """ - if auxiliary_buffers is not None and input_name in auxiliary_buffers: - raise ValueError("Auxiliary buffers must not replace the incoming observation tensor.") - buffers = dict(auxiliary_buffers or {}) - buffers[input_name] = ProxyArray(wp.from_torch(data)) - return self.as_tensor(buffers, output_name=output_name) - - def as_tensor( - self, buffers: Mapping[str, ProxyArray], *, output_name: str, out: torch.Tensor | None = None - ) -> torch.Tensor: - """Execute on the current Torch stream and snapshot one output for an observation. - - Source buffers must be ready on the current Warp or Torch stream. Stream ordering - is established without a device-wide synchronization; no host/device copy is made. - A supplied ``out`` must match the prepared output's shape, dtype and device and is overwritten in place. - """ - if self._closed: - raise RuntimeError("Cannot execute a closed render buffer pipe.") - reference = self.outputs[output_name] if output_name in self.produced_outputs else buffers[output_name] - if out is not None: - self._validate_buffer("out", ProxyArray(wp.from_torch(out)), reference) - device = wp.get_device(reference.device) - stream = wp.stream_from_torch(torch.cuda.current_stream(str(device))) if device.is_cuda else None - with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): - result = self(buffers)[output_name].torch - return result.clone() if out is None else out.copy_(result) - - def reset(self, env_ids: Sequence[int] | None = None) -> None: - """Reset operation-owned state for selected batch entries.""" - for operation, *_ in self._operations: - if operation.reset is not None: - operation.reset(env_ids) - - def close(self) -> None: - """Release operation resources; repeated calls have no effect.""" - if self._closed: - return - self._closed = True - operations, self._operations = self._operations, [] - self._source_buffers.clear() - self.outputs = MappingProxyType({}) - errors = [] - for operation, *_ in reversed(operations): - if operation.close is not None: - try: - operation.close() - except Exception as error: - errors.append(error) - if errors: - raise ExceptionGroup("Failed to close render buffer operations.", errors) - _RGB_LIKE_PREFIXES: tuple[str, ...] = ("rgb", "albedo", "simple_shading") _DEPTH_LIKE_PATTERNS: tuple[str, ...] = ("depth", "distance_to") diff --git a/source/isaaclab/isaaclab/utils/modifiers/__init__.pyi b/source/isaaclab/isaaclab/utils/modifiers/__init__.pyi index 7d4707aac60e..9b0fa3c886da 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/__init__.pyi +++ b/source/isaaclab/isaaclab/utils/modifiers/__init__.pyi @@ -5,8 +5,8 @@ __all__ = [ "ModifierCfg", - "RenderBufferPipeModifierCfg", - "RenderBufferPipeModifier", + "ProcessingBufferPipeModifierCfg", + "ProcessingBufferPipeModifier", "DigitalFilterCfg", "IntegratorCfg", "ModifierBase", @@ -17,7 +17,7 @@ __all__ = [ "scale", ] -from .modifier_cfg import ModifierCfg, DigitalFilterCfg, IntegratorCfg, RenderBufferPipeModifierCfg -from .render_buffer_pipe import RenderBufferPipeModifier +from .modifier_cfg import ModifierCfg, DigitalFilterCfg, IntegratorCfg, ProcessingBufferPipeModifierCfg +from .processing_buffer_pipe import ProcessingBufferPipeModifier from .modifier_base import ModifierBase from .modifier import DigitalFilter, Integrator, bias, clip, scale diff --git a/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py b/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py index 7527877bd9c0..b2f234e8217c 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py +++ b/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py @@ -14,10 +14,10 @@ from .. import configclass if TYPE_CHECKING: - from ..images import RenderBufferStepCfg + from ..processing import ProcessingBufferStepCfg from .modifier import DigitalFilter, Integrator from .modifier_base import ModifierBase - from .render_buffer_pipe import RenderBufferPipeModifier + from .processing_buffer_pipe import ProcessingBufferPipeModifier @configclass @@ -40,17 +40,17 @@ class ModifierCfg: @configclass -class RenderBufferPipeModifierCfg(ModifierCfg): - """Bind an incoming observation tensor to a render buffer pipe. +class ProcessingBufferPipeModifierCfg(ModifierCfg): + """Bind an incoming observation tensor to a buffer pipe. Operations are prepared from the first tensor received after preceding modifiers. - Their documented input requirements define the expected image layout and value semantics. + Their documented input requirements define the expected array layout and value semantics. """ - func: type[RenderBufferPipeModifier] | str = "{DIR}.render_buffer_pipe:RenderBufferPipeModifier" + func: type[ProcessingBufferPipeModifier] | str = "{DIR}.processing_buffer_pipe:ProcessingBufferPipeModifier" """Modifier class consuming the incoming tensor.""" - pipeline: list[RenderBufferStepCfg] = MISSING + pipeline: list[ProcessingBufferStepCfg] = MISSING """Ordered buffer processing operations.""" input_name: str = "rgb" diff --git a/source/isaaclab/isaaclab/utils/modifiers/render_buffer_pipe.py b/source/isaaclab/isaaclab/utils/modifiers/processing_buffer_pipe.py similarity index 65% rename from source/isaaclab/isaaclab/utils/modifiers/render_buffer_pipe.py rename to source/isaaclab/isaaclab/utils/modifiers/processing_buffer_pipe.py index 8777bdb0c4f3..79edd26d74bd 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/render_buffer_pipe.py +++ b/source/isaaclab/isaaclab/utils/modifiers/processing_buffer_pipe.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Render buffer processing for an incoming observation tensor.""" +"""Buffer processing for an incoming observation tensor.""" from __future__ import annotations @@ -13,31 +13,32 @@ import torch import warp as wp -from ..images import RenderBufferPipe +from ..processing import ProcessingBufferPipe from ..warp import ProxyArray from .modifier_base import ModifierBase if TYPE_CHECKING: - from .modifier_cfg import RenderBufferPipeModifierCfg + from .modifier_cfg import ProcessingBufferPipeModifierCfg -class RenderBufferPipeModifier(ModifierBase): - """Apply a render buffer pipe to the tensor received from the preceding modifier. +class ProcessingBufferPipeModifier(ModifierBase): + """Apply a buffer pipe to the tensor received from the preceding modifier. The modifier owns a pipe and returns a snapshot. Its output must preserve the input shape, as required by :class:`ModifierBase`. The first incoming tensor prepares the pipe; - subsequent calls reuse its output storage. Multiple camera inputs or shape changes belong - in :class:`~isaaclab.envs.mdp.processed_image` instead. + subsequent calls reuse its output storage. Inputs may represent images or nonvisual sensor + observations. For multiple render inputs or shape changes to render outputs, use + :class:`~isaaclab.envs.mdp.processed_sensor` instead. """ - def __init__(self, cfg: RenderBufferPipeModifierCfg, data_dim: tuple[int, ...], device: str): + def __init__(self, cfg: ProcessingBufferPipeModifierCfg, data_dim: tuple[int, ...], device: str): super().__init__(cfg, data_dim, device) - self._pipe: RenderBufferPipe | None = None + self._pipe: ProcessingBufferPipe | None = None self._closed = False def __call__(self, data: torch.Tensor) -> torch.Tensor: if self._closed: - raise RuntimeError("The render buffer pipe modifier is closed.") + raise RuntimeError("The buffer pipe modifier is closed.") if tuple(data.shape) != tuple(self._data_dim): raise ValueError( f"Observation modifier input shape must be {tuple(self._data_dim)}, got {tuple(data.shape)}." @@ -45,13 +46,14 @@ def __call__(self, data: torch.Tensor) -> torch.Tensor: if self._pipe is None: stream = wp.stream_from_torch(torch.cuda.current_stream(data.device)) if data.is_cuda else None with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): - pipe = RenderBufferPipe({self._cfg.input_name: ProxyArray(wp.from_torch(data))}, self._cfg.pipeline) + pipe = ProcessingBufferPipe({self._cfg.input_name: ProxyArray(wp.from_torch(data))}, self._cfg.pipeline) try: if self._cfg.output_name not in pipe.outputs: - raise ValueError(f"Image processing does not provide output {self._cfg.output_name!r}.") + raise ValueError(f"Buffer processing does not provide output {self._cfg.output_name!r}.") if pipe.outputs[self._cfg.output_name].shape != tuple(self._data_dim): raise ValueError( - "Observation modifiers must preserve shape; use mdp.processed_image for shape changes." + "Observation modifiers must preserve shape;" + " use a processing observation term for shape changes." ) except Exception: pipe.close() diff --git a/source/isaaclab/isaaclab/utils/processing.py b/source/isaaclab/isaaclab/utils/processing.py new file mode 100644 index 000000000000..a6864042e27d --- /dev/null +++ b/source/isaaclab/isaaclab/utils/processing.py @@ -0,0 +1,251 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Named buffer processing shared by sensors and observations.""" + +from __future__ import annotations + +from builtins import ExceptionGroup +from collections.abc import Callable, Mapping, Sequence +from dataclasses import MISSING, dataclass +from types import MappingProxyType +from typing import Any + +import torch +import warp as wp + +from .configclass import configclass +from .string import string_to_callable +from .warp import ProxyArray + + +@dataclass(frozen=True) +class PreparedBufferOperation: + """One operation with persistent output buffers and resolved resources. + + ``apply`` receives input and destination dictionaries keyed by operation roles, not sensor + output names. Preparation allocates the outputs once. Inputs are borrowed read-only by + convention; outputs must use independent storage. The operation must fill each supplied + destination and enqueue GPU work on the current Warp stream. The pipe retains the outputs + and controls execution order; private workspace belongs to the operation. + """ + + outputs: Mapping[str, ProxyArray] + apply: Callable[[Mapping[str, ProxyArray], Mapping[str, ProxyArray]], None] + reset: Callable[[Sequence[int] | None], None] | None = None + close: Callable[[], None] | None = None + + +@configclass +class ProcessingBufferStepCfg: + """Bind named buffers to the input/output roles of one processing operation. + + The factory ``prepare(inputs, **params)`` runs once during pipe construction and returns + a :class:`PreparedBufferOperation` with allocated output buffers. ``inputs`` contains actual + :class:`ProxyArray` objects keyed by operation roles. Factories inspect array shape, dtype + and device directly; each operation documents its expected layout and value semantics. + Factories should not read input values during preparation or retain input pointers for execution: + ``apply`` receives the current inputs on every call. Scene resources must be supplied explicitly. + """ + + prepare: Callable[..., PreparedBufferOperation] | str = MISSING + """Operation factory or an import string in ``module:attribute`` form.""" + + inputs: dict[str, str] = MISSING + """Map operation input roles to buffer names read from the preceding stage.""" + + outputs: dict[str, str] = MISSING + """Map operation output roles to names created or replaced in the bundle.""" + + params: dict[str, Any] = {} + """Resolved operation parameters passed to the factory. + + Configurations are deep-copied by ``configclass`` and managers. Borrowed device allocations + or models need a resource wrapper that preserves identity through ``__deepcopy__``, or a + factory closure retaining the resource. Do not put a raw Warp allocation in this dictionary. + """ + + +class ProcessingBufferPipe: + """Apply an ordered sequence of operations to named :class:`ProxyArray` buffers. + + Buffers may hold images, LiDAR points, or other sensor and observation data. The pipe imposes + no image layout; each operation defines its input requirements and allocates its outputs. + + A pipe retains persistent output storage allocated by its prepared operations. Create one instance per + consumer; instances are not reentrant. Borrowed inputs are never modified by the pipe, and + untouched channels pass through by reference. Returned buffers remain valid until the next + call, reset or close; consumers retaining an observation must take a snapshot. + + Buffer dependencies are resolved in order, so an operation reads preceding replacements. + GPU execution uses the caller's current Warp stream without a device-wide synchronization. + The caller owns producer readiness and keeps borrowed storage alive until consumption ends. + + Args: + source_buffers: Available external buffers, used to prepare operations and validate later inputs. + steps: Ordered operation configurations. + """ + + def __init__(self, source_buffers: Mapping[str, ProxyArray], steps: Sequence[ProcessingBufferStepCfg]): + self._source_buffers = dict(source_buffers) + self.required_inputs = self.required_source_names(steps) + self._operations: list[ + tuple[PreparedBufferOperation, dict[str, str], dict[str, str], Mapping[str, ProxyArray]] + ] = [] + self._closed = False + self.outputs: Mapping[str, ProxyArray] = MappingProxyType({}) + available = dict(source_buffers) + produced: set[str] = set() + try: + for step in steps: + inputs, outputs = dict(step.inputs), dict(step.outputs) + if not outputs or len(set(outputs.values())) != len(outputs): + raise ValueError("A processing step must bind distinct output names to its output roles.") + missing = set(inputs.values()) - available.keys() + if missing: + raise ValueError(f"Missing input buffers for preparation: {sorted(missing)}.") + prepare = string_to_callable(step.prepare) if isinstance(step.prepare, str) else step.prepare + input_buffers = {role: available[name] for role, name in inputs.items()} + operation = prepare(MappingProxyType(input_buffers), **step.params) + if not isinstance(operation, PreparedBufferOperation): + raise TypeError("A buffer operation factory must return PreparedBufferOperation.") + # Own the operation before checking its outputs so failed validation releases its resources. + output_buffers = {} + destinations = MappingProxyType(output_buffers) + self._operations.append((operation, inputs, outputs, destinations)) + output_buffers.update(operation.outputs) + if destinations.keys() != outputs.keys(): + raise ValueError("Operation output roles must match the step's output bindings.") + if any(not isinstance(buffer, ProxyArray) for buffer in destinations.values()): + raise TypeError("Prepared operation outputs must be ProxyArray objects.") + devices = {str(buffer.device) for buffer in (*input_buffers.values(), *destinations.values())} + if len(devices) > 1: + raise ValueError("A buffer operation must use one device for its inputs and outputs.") + for role, name in outputs.items(): + available[name] = destinations[role] + produced.update(outputs.values()) + except Exception as error: + try: + self.close() + except Exception as cleanup_error: + raise error from cleanup_error + raise + self.outputs = MappingProxyType(available) + self.produced_outputs = frozenset(produced) + + @property + def has_reset_callbacks(self) -> bool: + """Whether any operation needs selected batch entries reset.""" + return any(operation.reset is not None for operation, *_ in self._operations) + + @staticmethod + def _validate_buffer(name: str, value: ProxyArray, reference: ProxyArray) -> None: + """Check replacement storage against the actual buffer used to prepare the pipe.""" + if not isinstance(value, ProxyArray): + raise TypeError(f"Buffer {name!r} must be a ProxyArray.") + if value.shape != reference.shape or value.dtype != reference.dtype or value.device != reference.device: + raise ValueError( + f"Buffer {name!r} must match shape, dtype and device" + f" {reference.shape}, {reference.dtype}, {reference.device}." + ) + + @staticmethod + def required_source_names( + steps: Sequence[ProcessingBufferStepCfg], published_names: Sequence[str] = () + ) -> frozenset[str]: + """Resolve external names without constructing operations or allocating buffer storage. + + Call this before producer initialization to request private processing inputs. Public + pass-through outputs also require a source, even when no operation reads them. + """ + required: set[str] = set() + produced: set[str] = set() + for step in steps: + required.update(set(step.inputs.values()) - produced) + produced.update(step.outputs.values()) + required.update(set(published_names) - produced) + return frozenset(required) + + def __call__(self, buffers: Mapping[str, ProxyArray]) -> dict[str, ProxyArray]: + """Execute the pipe on ready source buffers and return the resulting bundle.""" + if self._closed: + raise RuntimeError("Cannot execute a closed buffer pipe.") + missing = self.required_inputs - buffers.keys() + if missing: + raise ValueError(f"Missing required buffers: {sorted(missing)}.") + for name in self._source_buffers.keys() & buffers.keys(): + self._validate_buffer(name, buffers[name], self._source_buffers[name]) + current = dict(buffers) + for operation, inputs, outputs, destinations in self._operations: + operation.apply({role: current[name] for role, name in inputs.items()}, destinations) + current.update({name: destinations[role] for role, name in outputs.items()}) + # Release replaced source references, such as the preceding observation's tensor. + self._source_buffers.update({name: buffers[name] for name in self._source_buffers.keys() & buffers.keys()}) + self.outputs = MappingProxyType(dict(current)) + return current + + def apply_to_tensor( + self, + data: torch.Tensor, + *, + input_name: str, + output_name: str, + auxiliary_buffers: Mapping[str, ProxyArray] | None = None, + ) -> torch.Tensor: + """Process an incoming observation tensor and return an independently owned snapshot. + + The supplied tensor is authoritative: earlier observation transformations are preserved. + Auxiliary inputs must already be ready on the current Warp/Torch stream. CUDA work is + ordered with the caller's Torch stream; no host/device transfer is performed. + """ + if auxiliary_buffers is not None and input_name in auxiliary_buffers: + raise ValueError("Auxiliary buffers must not replace the incoming observation tensor.") + buffers = dict(auxiliary_buffers or {}) + buffers[input_name] = ProxyArray(wp.from_torch(data)) + return self.as_tensor(buffers, output_name=output_name) + + def as_tensor( + self, buffers: Mapping[str, ProxyArray], *, output_name: str, out: torch.Tensor | None = None + ) -> torch.Tensor: + """Execute on the current Torch stream and snapshot one output for an observation. + + Source buffers must be ready on the current Warp or Torch stream. Stream ordering + is established without a device-wide synchronization; no host/device copy is made. + A supplied ``out`` must match the prepared output's shape, dtype and device and is overwritten in place. + """ + if self._closed: + raise RuntimeError("Cannot execute a closed buffer pipe.") + reference = self.outputs[output_name] if output_name in self.produced_outputs else buffers[output_name] + if out is not None: + self._validate_buffer("out", ProxyArray(wp.from_torch(out)), reference) + device = wp.get_device(reference.device) + stream = wp.stream_from_torch(torch.cuda.current_stream(str(device))) if device.is_cuda else None + with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): + result = self(buffers)[output_name].torch + return result.clone() if out is None else out.copy_(result) + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + """Reset operation-owned state for selected batch entries.""" + for operation, *_ in self._operations: + if operation.reset is not None: + operation.reset(env_ids) + + def close(self) -> None: + """Release operation resources; repeated calls have no effect.""" + if self._closed: + return + self._closed = True + operations, self._operations = self._operations, [] + self._source_buffers.clear() + self.outputs = MappingProxyType({}) + errors = [] + for operation, *_ in reversed(operations): + if operation.close is not None: + try: + operation.close() + except Exception as error: + errors.append(error) + if errors: + raise ExceptionGroup("Failed to close buffer operations.", errors) diff --git a/source/isaaclab/test/envs/test_render_buffer_observations.py b/source/isaaclab/test/envs/test_buffer_observations.py similarity index 79% rename from source/isaaclab/test/envs/test_render_buffer_observations.py rename to source/isaaclab/test/envs/test_buffer_observations.py index 6a757aaf1c98..fb2d03909696 100644 --- a/source/isaaclab/test/envs/test_render_buffer_observations.py +++ b/source/isaaclab/test/envs/test_buffer_observations.py @@ -21,7 +21,7 @@ ObservationTermCfg, ) from isaaclab.utils import configclass, modifiers -from isaaclab.utils.images import PreparedBufferOperation, RenderBufferStepCfg +from isaaclab.utils.processing import PreparedBufferOperation, ProcessingBufferStepCfg from isaaclab.utils.warp import ProxyArray pytestmark = pytest.mark.unit @@ -83,8 +83,8 @@ def apply(inputs, outputs): return PreparedBufferOperation({"color": result}, apply) -def _read_image(env): - return env.image +def _read_ranges(env): + return env.ranges def _env(**kwargs): @@ -107,45 +107,52 @@ def close(self): def test_pipe_modifier_uses_preceding_modifier_output(): - """A pipe must consume the current observation, not re-read the source camera.""" - image = torch.ones((2, 2, 4, 3)) + """A pipe consumes the current non-image observation after preceding modifiers.""" + ranges = torch.ones((2, 4)) preparations = [] def prepare(inputs, gain): - preparations.append(inputs["color"].shape) - return _prepare_gain(inputs, gain) + preparations.append(inputs["ranges"].shape) + result = ProxyArray(wp.empty_like(inputs["ranges"].warp)) + + def apply(inputs, outputs): + torch.mul(inputs["ranges"].torch, gain, out=outputs["ranges"].torch) + + return PreparedBufferOperation({"ranges": result}, apply) @configclass class Policy(ObservationGroupCfg): concatenate_terms = False - image = ObservationTermCfg( - func=_read_image, + ranges = ObservationTermCfg( + func=_read_ranges, modifiers=[ modifiers.ModifierCfg(func=modifiers.bias, params={"value": 3.0}), - modifiers.RenderBufferPipeModifierCfg( + modifiers.ProcessingBufferPipeModifierCfg( pipeline=[ - RenderBufferStepCfg( + ProcessingBufferStepCfg( prepare=prepare, - inputs={"color": "rgb"}, - outputs={"color": "rgb"}, + inputs={"ranges": "ranges"}, + outputs={"ranges": "ranges"}, params={"gain": 2.0}, ) - ] + ], + input_name="ranges", + output_name="ranges", ), ], ) - env = _env(image=image) + env = _env(ranges=ranges) manager = ObservationManager({"policy": Policy()}, env) manager.reset([1]) assert preparations == [] - first = manager.compute()["policy"]["image"] - torch.testing.assert_close(first, torch.full_like(image, 8.0)) - torch.testing.assert_close(image, torch.ones_like(image)) - env.image = torch.full_like(image, 2.0) - torch.testing.assert_close(manager.compute()["policy"]["image"], torch.full_like(image, 10.0)) - torch.testing.assert_close(first, torch.full_like(image, 8.0)) - assert preparations == [tuple(image.shape)] + first = manager.compute()["policy"]["ranges"] + torch.testing.assert_close(first, torch.full_like(ranges, 8.0)) + torch.testing.assert_close(ranges, torch.ones_like(ranges)) + env.ranges = torch.full_like(ranges, 2.0) + torch.testing.assert_close(manager.compute()["policy"]["ranges"], torch.full_like(ranges, 10.0)) + torch.testing.assert_close(first, torch.full_like(ranges, 8.0)) + assert preparations == [tuple(ranges.shape)] manager.close() @@ -165,10 +172,10 @@ def prepare(inputs): ) return PreparedBufferOperation({"color": output}, operation.apply, close=lambda: lifecycle.append("close")) - cfg = modifiers.RenderBufferPipeModifierCfg( - pipeline=[RenderBufferStepCfg(prepare=prepare, inputs={"color": "rgb"}, outputs={"color": "rgb"})] + cfg = modifiers.ProcessingBufferPipeModifierCfg( + pipeline=[ProcessingBufferStepCfg(prepare=prepare, inputs={"color": "rgb"}, outputs={"color": "rgb"})] ) - modifier = modifiers.RenderBufferPipeModifier(cfg, tuple(image.shape), "cpu") + modifier = modifiers.ProcessingBufferPipeModifier(cfg, tuple(image.shape), "cpu") modifier.reset([1]) with pytest.raises(ValueError, match="input shape"): modifier(image[:, :, ::2]) @@ -184,14 +191,14 @@ def prepare(inputs): modifier(image) assert lifecycle == ["prepare", "close", "prepare", "close"] - unused_modifier = modifiers.RenderBufferPipeModifier(cfg, tuple(image.shape), "cpu") + unused_modifier = modifiers.ProcessingBufferPipeModifier(cfg, tuple(image.shape), "cpu") unused_modifier.close() with pytest.raises(RuntimeError, match="closed"): unused_modifier(image) assert lifecycle == ["prepare", "close", "prepare", "close"] -def test_processed_image_declares_private_inputs_and_reports_transformed_shape(): +def test_processed_sensor_declares_private_inputs_and_reports_transformed_shape(): """Preparation resolves private AOVs before binding; manager dimensions describe the processed output.""" lifecycle = [] @@ -238,10 +245,10 @@ class Policy(ObservationGroupCfg): "terms": [ { "image": ManagerTermBaseCfg( - func="isaaclab.envs.mdp:processed_image", + func="isaaclab.envs.mdp:processed_sensor", params={ "pipeline": [ - RenderBufferStepCfg( + ProcessingBufferStepCfg( prepare=prepare, inputs={"color": "rgb_radiance", "depth": "depth"}, outputs={"color": "rgb"}, @@ -270,7 +277,7 @@ class Observations: torch.testing.assert_close(manager.compute()["policy"]["image"], expected * 2) torch.testing.assert_close(result, expected) manager.reset([1]) - with pytest.raises(ExceptionGroup, match="Failed to close render buffer operations") as error: + with pytest.raises(ExceptionGroup, match="Failed to close buffer operations") as error: manager.close() assert str(error.value.exceptions[0]) == "operation close failed" manager.close() diff --git a/source/isaaclab/test/renderers/test_camera_output_contract.py b/source/isaaclab/test/renderers/test_camera_output_contract.py index 281d402314d1..3a4129bb47aa 100644 --- a/source/isaaclab/test/renderers/test_camera_output_contract.py +++ b/source/isaaclab/test/renderers/test_camera_output_contract.py @@ -25,7 +25,7 @@ from isaaclab.sim import PinholeCameraCfg from isaaclab.test.utils import DeviceScope, test_devices from isaaclab.utils import clone, validate -from isaaclab.utils.images import PreparedBufferOperation, RenderBufferStepCfg +from isaaclab.utils.processing import PreparedBufferOperation, ProcessingBufferStepCfg from isaaclab.utils.warp import ProxyArray pytestmark = [pytest.mark.integration, pytest.mark.rendering] @@ -387,7 +387,7 @@ def prepare_identity(inputs): def make_camera(prim, data_types): camera = Camera.__new__(Camera) pipeline = ( - [RenderBufferStepCfg(prepare_identity, {"image": "rgb_radiance"}, {"image": "processed_radiance"})] + [ProcessingBufferStepCfg(prepare_identity, {"image": "rgb_radiance"}, {"image": "processed_radiance"})] if data_types == ["processed_radiance"] else [] ) @@ -486,7 +486,7 @@ def close(): width=1, data_types=["rgb", "depth"], pipeline=[ - RenderBufferStepCfg(prepare_color, {"radiance": "rgb_radiance", "range": "depth"}, {"color": "rgba"}) + ProcessingBufferStepCfg(prepare_color, {"radiance": "rgb_radiance", "range": "depth"}, {"color": "rgba"}) ], ) specs = { @@ -565,7 +565,7 @@ def read_output(render_data, camera_data): camera._renderer.cleanup = lambda data: events.append("renderer-close") camera._view.close = lambda: events.append("view-close") close_fails = True - with pytest.raises(ExceptionGroup, match="close render buffer operations"): + with pytest.raises(ExceptionGroup, match="close buffer operations"): camera._invalidate_initialize_callback(None) assert events[-3:] == ["close", "renderer-close", "view-close"] assert not camera.is_initialized diff --git a/source/isaaclab/test/utils/test_render_buffer_pipe.py b/source/isaaclab/test/utils/test_processing_buffer_pipe.py similarity index 92% rename from source/isaaclab/test/utils/test_render_buffer_pipe.py rename to source/isaaclab/test/utils/test_processing_buffer_pipe.py index 7a05df2d1d50..5278aec38c9d 100644 --- a/source/isaaclab/test/utils/test_render_buffer_pipe.py +++ b/source/isaaclab/test/utils/test_processing_buffer_pipe.py @@ -11,7 +11,7 @@ import torch import warp as wp -from isaaclab.utils.images import PreparedBufferOperation, RenderBufferPipe, RenderBufferStepCfg +from isaaclab.utils.processing import PreparedBufferOperation, ProcessingBufferPipe, ProcessingBufferStepCfg from isaaclab.utils.warp import ProxyArray pytestmark = pytest.mark.unit @@ -48,15 +48,15 @@ def composite_pipe(): "range": ProxyArray(wp.zeros((2, 2, 2, 1), dtype=wp.float32, device="cpu")), } steps = [ - RenderBufferStepCfg(prepare=_prepare_scale, inputs={"value": "raw_color"}, outputs={"value": "color"}), - RenderBufferStepCfg( + ProcessingBufferStepCfg(prepare=_prepare_scale, inputs={"value": "raw_color"}, outputs={"value": "color"}), + ProcessingBufferStepCfg( prepare=_prepare_composite, inputs={"color": "color", "depth": "range"}, outputs={"value": "color"}, ), ] - assert RenderBufferPipe.required_source_names(steps, ["color", "labels"]) == {"raw_color", "range", "labels"} - pipe = RenderBufferPipe(sources, steps) + assert ProcessingBufferPipe.required_source_names(steps, ["color", "labels"]) == {"raw_color", "range", "labels"} + pipe = ProcessingBufferPipe(sources, steps) yield pipe pipe.close() @@ -115,7 +115,7 @@ def test_tensor_adapter_preserves_incoming_transformations_and_returns_snapshots def test_tensor_snapshot_accepts_passthrough_buffer_omitted_from_previous_call(): """Optional pass-through buffers can return after a call that did not supply them.""" original = ProxyArray(wp.zeros((2,), dtype=wp.float32, device="cpu")) - pipe = RenderBufferPipe({"value": original}, []) + pipe = ProcessingBufferPipe({"value": original}, []) try: pipe({}) current = ProxyArray(wp.full((2,), 3.0, dtype=wp.float32, device="cpu")) @@ -136,9 +136,9 @@ def prepare(inputs): destination = ProxyArray(wp.empty_like(inputs["value"].warp)) return PreparedBufferOperation({"value": destination}, lambda *_: calls.append(True)) - pipe = RenderBufferPipe( + pipe = ProcessingBufferPipe( {"source": source}, - [RenderBufferStepCfg(prepare=prepare, inputs={"value": "source"}, outputs={"value": "result"})], + [ProcessingBufferStepCfg(prepare=prepare, inputs={"value": "source"}, outputs={"value": "result"})], ) try: shape = (1, 1, 1, 1) if invalid == "shape" else source.shape @@ -163,10 +163,10 @@ def prepare(inputs, *, name): destination = destination.warp return PreparedBufferOperation({"value": destination}, lambda *_: None, close=lambda: closed.append(name)) - first = RenderBufferStepCfg( + first = ProcessingBufferStepCfg( prepare=prepare, inputs={"value": "source"}, outputs={"value": "middle"}, params={"name": "first"} ) - second = RenderBufferStepCfg( + second = ProcessingBufferStepCfg( prepare=prepare, inputs={"value": "middle"}, outputs={"value": "result"}, params={"name": "second"} ) if invalid == "missing_dependency": @@ -181,7 +181,7 @@ def prepare(inputs, *, name): else: message = "outputs must be ProxyArray" with pytest.raises(TypeError if invalid == "output_type" else ValueError, match=message): - RenderBufferPipe({"source": source}, [first, second]) + ProcessingBufferPipe({"source": source}, [first, second]) assert closed == (["second", "first"] if invalid in ("output_roles", "output_type") else ["first"]) @@ -209,10 +209,10 @@ def close(): return PreparedBufferOperation({"value": destination}, apply, reset=reset, close=close) - pipe = RenderBufferPipe( + pipe = ProcessingBufferPipe( {"value": source}, [ - RenderBufferStepCfg( + ProcessingBufferStepCfg( prepare=prepare, inputs={"value": "value"}, outputs={"value": "value"}, params={"name": name} ) for name in ("first", "second") diff --git a/source/isaaclab_ppisp/docs/README.md b/source/isaaclab_ppisp/docs/README.md index 8e9d0c269481..4d66cdfb2d58 100644 --- a/source/isaaclab_ppisp/docs/README.md +++ b/source/isaaclab_ppisp/docs/README.md @@ -1,7 +1,7 @@ # Isaac Lab PPISP This extension provides PPISP (Physically Plausible Image Signal Processing) as -an operation in the shared render-buffer pipeline used by cameras and observations. +an operation in the shared buffer pipeline used by cameras and observations. `PpispStepCfg` reads scene-linear `rgb_radiance` from a float32 `ProxyArray` with shape `(N, H, W, 3)` and writes uint8 `rgb`. For camera processing, configure @@ -17,8 +17,8 @@ ppisp_cfg = ppisp_cfg_from_usd_camera(source_camera_prim) step = PpispStepCfg(params={"cfg": ppisp_cfg}) ``` -The same step can run in `isaaclab.utils.images.RenderBufferPipe` outside a camera: -`RenderBufferPipe({"rgb_radiance": radiance_buffer}, [step])`, where `radiance_buffer` +The same step can run in `isaaclab.utils.processing.ProcessingBufferPipe` outside a camera: +`ProcessingBufferPipe({"rgb_radiance": radiance_buffer}, [step])`, where `radiance_buffer` is an allocated `ProxyArray`. Preparation validates its shape and dtype and allocates the persistent output once on the same device. Input/output bindings can be changed independently of PPISP math. For RGBA, diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/processor.py b/source/isaaclab_ppisp/isaaclab_ppisp/processor.py index db438e534f35..99dcf630cf77 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/processor.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/processor.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""PPISP operation for camera and observation render-buffer pipes.""" +"""PPISP operation for camera and observation buffer pipes.""" from __future__ import annotations @@ -14,7 +14,7 @@ import warp as wp from isaaclab.utils import configclass -from isaaclab.utils.images import PreparedBufferOperation, RenderBufferStepCfg +from isaaclab.utils.processing import PreparedBufferOperation, ProcessingBufferStepCfg from isaaclab.utils.warp import ProxyArray from .cfg import PpispCfg @@ -71,7 +71,7 @@ def apply(inputs: Mapping[str, ProxyArray], outputs: Mapping[str, ProxyArray]) - @configclass -class PpispStepCfg(RenderBufferStepCfg): +class PpispStepCfg(ProcessingBufferStepCfg): """Route scene-linear radiance through PPISP into a public RGB buffer. Supply concrete settings with ``params={"cfg": ppisp_cfg}``. To produce RGBA, set diff --git a/source/isaaclab_ppisp/test/test_ppisp_kernels.py b/source/isaaclab_ppisp/test/test_ppisp_kernels.py index 0324423a6e7a..f4c437ce4441 100644 --- a/source/isaaclab_ppisp/test/test_ppisp_kernels.py +++ b/source/isaaclab_ppisp/test/test_ppisp_kernels.py @@ -41,7 +41,7 @@ ) from isaaclab.test.utils import DeviceScope, test_devices -from isaaclab.utils.images import RenderBufferPipe +from isaaclab.utils.processing import ProcessingBufferPipe from isaaclab.utils.warp import ProxyArray wp.init() @@ -313,7 +313,7 @@ def test_ppisp_pipe_rgb_crf_extreme_centers_match_reference(): ) ) - pipe = RenderBufferPipe( + pipe = ProcessingBufferPipe( {"rgb_radiance": ProxyArray(hdr_color)}, [PpispStepCfg(params={"cfg": cfg})], ) @@ -348,7 +348,7 @@ def test_ppisp_controller_pipes_share_weights_for_rgb_and_rgba(device): cfg = PpispCfg() shared_weights = PpispControllerWeights(weights) pipes = [ - RenderBufferPipe( + ProcessingBufferPipe( {"rgb_radiance": ProxyArray(hdr)}, [PpispStepCfg(params={"cfg": cfg, "channels": channels, "controller_weights": shared_weights})], ) @@ -379,13 +379,13 @@ def test_ppisp_pipe_rejects_incompatible_input_storage(shape, dtype, message): """Reject invalid image channels and scalar types before preparing numerical execution.""" source = ProxyArray(wp.empty(shape, dtype=dtype, device="cpu")) with pytest.raises(ValueError, match=message): - RenderBufferPipe({"rgb_radiance": source}, [PpispStepCfg()]) + ProcessingBufferPipe({"rgb_radiance": source}, [PpispStepCfg()]) def test_ppisp_pipe_requires_resolved_configuration(): """An image operation must not discover its settings from a camera during preparation.""" source = ProxyArray(wp.empty((1, 2, 2, 3), dtype=wp.float32, device="cpu")) with pytest.raises(ValueError, match="resolve USD camera attributes"): - RenderBufferPipe( + ProcessingBufferPipe( {"rgb_radiance": source}, [PpispStepCfg(params={"cfg": PpispCfg(camera_prim_path="/World/Camera")})] ) From d00516dd56c56183f76e9f20ce0790fd18de41f1 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Fri, 16 Oct 2026 16:21:45 -0400 Subject: [PATCH 18/20] Rename tensor processors and configure camera exposure explicitly --- docs/source/api/lab/isaaclab.managers.rst | 4 +- .../api/lab/isaaclab.sensors.camera.rst | 4 + docs/source/api/lab/isaaclab.utils.rst | 14 ++-- docs/source/concepts/sensors/camera.rst | 73 +++++++++---------- examples/sensors/ppisp_camera.py | 2 + .../feat-render-buffer-pipe.changed.rst | 10 +-- .../isaaclab/envs/mdp/sensor_observations.py | 12 +-- .../isaaclab/managers/manager_base.py | 4 +- .../isaaclab/managers/manager_term_cfg.py | 14 ++-- .../isaaclab/managers/observation_manager.py | 48 ++++++------ .../isaaclab/renderers/output_contract.py | 2 - .../isaaclab/sensors/camera/__init__.pyi | 2 + .../isaaclab/sensors/camera/camera.py | 14 +++- .../isaaclab/sensors/camera/camera_cfg.py | 17 ++++- .../isaaclab/sensors/camera/camera_data.py | 11 --- .../camera/camera_exposure.py} | 37 +++++----- .../sensors/camera/camera_exposure_cfg.py | 34 +++++++++ source/isaaclab/isaaclab/utils/__init__.pyi | 10 +-- .../isaaclab/utils/modifiers/__init__.pyi | 23 ------ source/isaaclab/isaaclab/utils/processing.py | 8 +- .../__init__.py | 34 ++++----- .../utils/tensor_processors/__init__.pyi | 23 ++++++ .../dict_tensor_processor_adapter.py} | 26 +++---- .../processors.py} | 24 +++--- .../tensor_processor.py} | 38 +++++----- .../tensor_processor_cfg.py} | 44 +++++------ .../test/envs/test_buffer_observations.py | 30 ++++---- .../test/managers/test_manager_base.py | 4 +- .../test/managers/test_observation_manager.py | 66 +++++++++-------- ...a_overrides.py => test_camera_exposure.py} | 14 ++-- .../renderers/test_camera_output_contract.py | 42 +++++------ .../generate_synthetic_gaussian_asset.py | 4 +- .../test_camera_ppisp_gaussian_ovrtx.py | 4 +- .../isaaclab/test/utils/test_configclass.py | 4 +- source/isaaclab/test/utils/test_modifiers.py | 26 +++---- .../test/utils/test_processing_buffer_pipe.py | 26 +++---- .../dict-tensor-processors.changed.rst | 2 + .../changelog.d/dict-tensor-processors.major | 0 .../managers/observation_manager.py | 42 +++++------ .../isaaclab_experimental/utils/__init__.pyi | 4 +- .../__init__.py | 12 +-- .../processors.py} | 12 +-- .../tensor_processor.py} | 22 +++--- .../tensor_processor_cfg.py} | 14 ++-- .../feat-render-buffer-pipe.changed.rst | 6 +- .../renderers/newton_warp_renderer.py | 5 +- .../renderers/newton_warp_renderer_cfg.py | 1 - .../feat-render-buffer-pipe.changed.rst | 6 +- .../isaaclab_ov/renderers/ovrtx_renderer.py | 15 ---- .../renderers/ovrtx_renderer_cfg.py | 1 - .../isaaclab_ov/renderers/ovrtx_usd.py | 3 +- .../test/test_ovrtx_renderer_contract.py | 14 ++-- source/isaaclab_ov/test/test_ovrtx_usd.py | 6 +- .../feat-render-buffer-pipe.changed.rst | 6 +- .../renderers/isaac_rtx_renderer.py | 23 +----- .../renderers/isaac_rtx_renderer_cfg.py | 1 - .../test_isaac_rtx_renderer_contract.py | 21 +++--- .../feat-render-buffer-pipe.changed.rst | 7 +- source/isaaclab_ppisp/docs/README.md | 12 ++- .../isaaclab_ppisp/processor.py | 6 +- .../isaaclab_ppisp/test/test_ppisp_kernels.py | 41 +++++++---- 61 files changed, 530 insertions(+), 504 deletions(-) rename source/isaaclab/isaaclab/{renderers/rtx_camera_overrides.py => sensors/camera/camera_exposure.py} (57%) create mode 100644 source/isaaclab/isaaclab/sensors/camera/camera_exposure_cfg.py delete mode 100644 source/isaaclab/isaaclab/utils/modifiers/__init__.pyi rename source/isaaclab/isaaclab/utils/{modifiers => tensor_processors}/__init__.py (53%) create mode 100644 source/isaaclab/isaaclab/utils/tensor_processors/__init__.pyi rename source/isaaclab/isaaclab/utils/{modifiers/processing_buffer_pipe.py => tensor_processors/dict_tensor_processor_adapter.py} (70%) rename source/isaaclab/isaaclab/utils/{modifiers/modifier.py => tensor_processors/processors.py} (92%) rename source/isaaclab/isaaclab/utils/{modifiers/modifier_base.py => tensor_processors/tensor_processor.py} (56%) rename source/isaaclab/isaaclab/utils/{modifiers/modifier_cfg.py => tensor_processors/tensor_processor_cfg.py} (64%) rename source/isaaclab/test/renderers/{test_rtx_camera_overrides.py => test_camera_exposure.py} (79%) create mode 100644 source/isaaclab_experimental/changelog.d/dict-tensor-processors.changed.rst create mode 100644 source/isaaclab_experimental/changelog.d/dict-tensor-processors.major rename source/isaaclab_experimental/isaaclab_experimental/utils/{modifiers => tensor_processors}/__init__.py (50%) rename source/isaaclab_experimental/isaaclab_experimental/utils/{modifiers/modifier.py => tensor_processors/processors.py} (83%) rename source/isaaclab_experimental/isaaclab_experimental/utils/{modifiers/modifier_base.py => tensor_processors/tensor_processor.py} (67%) rename source/isaaclab_experimental/isaaclab_experimental/utils/{modifiers/modifier_cfg.py => tensor_processors/tensor_processor_cfg.py} (65%) diff --git a/docs/source/api/lab/isaaclab.managers.rst b/docs/source/api/lab/isaaclab.managers.rst index d378386b86de..5a5f94d5abb0 100644 --- a/docs/source/api/lab/isaaclab.managers.rst +++ b/docs/source/api/lab/isaaclab.managers.rst @@ -103,7 +103,7 @@ On each recorded sample, each environment retains its lag with probability 0.8 a one. Zero redraws on every sample. Holding the lag keeps the latency constant while frames continue to advance; it does not freeze the returned frame. Reset always redraws the selected environments' lags. -The processing order is observation function, modifiers, noise, clipping, scaling, delay, then history. +The processing order is observation function, tensor processors, noise, clipping, scaling, delay, then history. The ``delay_min_lag`` and ``delay_max_lag`` field names and delay placement follow `mjlab's observation configuration `_. ``delay_hold_prob`` follows mjlab's lag-retention semantics, with a default of 1.0 to preserve Isaac Lab's @@ -131,7 +131,7 @@ The manager checks the signature once during initialization. Only an explicitly ``out=None`` enables destination writes; accepting arbitrary ``**kwargs`` does not. The manager probes the ordinary call to infer shape, dtype and device, which must remain constant, then allocates a fresh contiguous destination for each computation. ``out`` is reserved and must not appear in term ``params``. -Custom modifiers, noise and history retain their usual snapshot protections. Built-in +Custom tensor processors, noise and history retain their usual snapshot protections. Built-in :class:`~isaaclab.envs.mdp.observations.image_rgb` terms use this interface automatically to avoid a second full-image copy for normalized uint8 images. Existing observation configurations require no changes. diff --git a/docs/source/api/lab/isaaclab.sensors.camera.rst b/docs/source/api/lab/isaaclab.sensors.camera.rst index e86b1ada9795..1c4eb5072bd0 100644 --- a/docs/source/api/lab/isaaclab.sensors.camera.rst +++ b/docs/source/api/lab/isaaclab.sensors.camera.rst @@ -17,3 +17,7 @@ The following classes are part of the public :mod:`isaaclab.sensors.camera` API. .. autoclass:: CameraISPMode :show-inheritance: + +.. autoclass:: CameraExposureCfg + :members: + :exclude-members: __init__ diff --git a/docs/source/api/lab/isaaclab.utils.rst b/docs/source/api/lab/isaaclab.utils.rst index f81f3ad459d1..00d65e2f8cec 100644 --- a/docs/source/api/lab/isaaclab.utils.rst +++ b/docs/source/api/lab/isaaclab.utils.rst @@ -17,7 +17,7 @@ logger math mesh - modifiers + tensor_processors noise processing seed @@ -125,10 +125,10 @@ Mesh operations :imported-members: :show-inheritance: -Modifier operations -~~~~~~~~~~~~~~~~~~~ +Tensor processors +~~~~~~~~~~~~~~~~~ -.. automodule:: isaaclab.utils.modifiers +.. automodule:: isaaclab.utils.tensor_processors :members: :imported-members: :special-members: __call__ @@ -151,17 +151,17 @@ Processing operations .. automodule:: isaaclab.utils.processing -``ProcessingBufferPipe`` processes named ``ProxyArray`` buffers, including camera outputs and nonvisual +``DictTensorProcessor`` processes named ``ProxyArray`` buffers, including camera outputs and nonvisual sensor data. Operations define their own input shape, dtype, and semantic requirements. .. autoclass:: isaaclab.utils.processing.PreparedBufferOperation :members: -.. autoclass:: isaaclab.utils.processing.ProcessingBufferStepCfg +.. autoclass:: isaaclab.utils.processing.DictTensorProcessorStepCfg :members: :exclude-members: __init__ -.. autoclass:: isaaclab.utils.processing.ProcessingBufferPipe +.. autoclass:: isaaclab.utils.processing.DictTensorProcessor :members: :special-members: __call__ diff --git a/docs/source/concepts/sensors/camera.rst b/docs/source/concepts/sensors/camera.rst index f9b2b9015cc4..274d9858c750 100644 --- a/docs/source/concepts/sensors/camera.rst +++ b/docs/source/concepts/sensors/camera.rst @@ -170,9 +170,6 @@ produce additional output names; its operation factories allocate those buffers. * - ``rgb_hdr`` - 3, ``float32`` - High-dynamic-range RGB using the active camera settings - * - ``rgb_radiance`` - - 3, ``float32`` - - Scene-linear RGB before exposure and response, in renderer-relative intensity units * - ``albedo`` - 4, ``uint8`` - Material base color @@ -202,15 +199,11 @@ produce additional output names; its operation factories allocate those buffers. non-colorized segmentation uses one ``int32`` ID channel. Label and prim-path mappings are stored in ``camera_data.info[output_name]``. -``rgb_hdr`` does not request exposure neutralization. ``rgb_radiance`` is the input for image -processing that applies its own exposure and camera response, such as PPISP. -A PPISP pipeline requests ``rgb_radiance`` as a private input automatically; add it to -``CameraCfg.data_types`` only when it should also be published in ``camera.data.output``. - -If both ``rgb_hdr`` and ``rgb_radiance`` are requested, they share the same scene-linear buffer. -On Isaac RTX and OVRTX, requesting radiance neutralizes exposure for the entire camera prim, so -``rgb_hdr`` also contains neutralized color. Use separate camera prims to obtain radiance and color -with the authored exposure; see the limitation in :ref:`camera-supported-annotators`. +``rgb_hdr`` uses the camera's active exposure settings. PPISP requires scene-linear HDR; +set ``CameraCfg.exposure=CameraExposureCfg()`` explicitly to disable RTX automatic exposure and +use neutral exposure. Leaving ``exposure=None`` preserves authored camera settings. Selecting an +output never changes exposure. PPISP requests ``rgb_hdr`` privately; include it in ``data_types`` +only to publish the raw input as well. .. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/camera-renderer-isaac-rtx.webp :align: center @@ -248,10 +241,6 @@ current support matrix is: - Yes - Yes - Yes - * - ``rgb_radiance`` - - Derived (see below) - - Derived (see below) - - Yes * - ``depth`` and both distance outputs - Yes - Yes @@ -283,13 +272,9 @@ current support matrix is: .. note:: - On Isaac RTX and OVRTX, ``rgb_radiance`` is not a native renderer output. These renderers derive - it from HDR color by authoring neutral exposure on the camera prim, so every output rendered from - that prim, including ``rgb``, ``rgba``, and ``rgb_hdr``, loses the authored exposure. This also - applies to other camera sensors that share the prim. A single render product cannot return both - authored-exposure color and ``rgb_radiance``; use a separate camera prim when both are required. - Newton Warp has no exposure model and is not affected. A native pre-exposure radiance output has - been requested from the RTX team (NVBug 6858736). + Explicit exposure settings affect every output from the camera prim, including other sensors + sharing it. Use separate camera prims when different exposures are required. Newton's HDR + output is already scene-linear and does not consume RTX exposure attributes. Querying an unsupported output fails during camera initialization. Renderer configuration controls semantic filters, segmentation colorization, and depth clipping where those options are @@ -326,6 +311,7 @@ background. Set a normalized RGB tuple to use a solid color for pixels that miss .. code-block:: python + from isaaclab.sensors.camera import CameraExposureCfg from isaaclab.utils import replace mask_camera = replace(front_camera, background_color=(0.0, 0.0, 0.0)) @@ -337,25 +323,27 @@ Post-render image processing ---------------------------- :attr:`~sensors.CameraCfg.pipeline` applies an ordered sequence of operations to named image buffers. -Each :class:`~isaaclab.utils.processing.ProcessingBufferStepCfg` maps operation input and output roles to buffer +Each :class:`~isaaclab.utils.processing.DictTensorProcessorStepCfg` maps operation input and output roles to buffer names. A step can read several channels, such as radiance and depth, and later steps read the preceding -step's results. The same :class:`~isaaclab.utils.processing.ProcessingBufferPipe` can run independently or serve +step's results. The same :class:`~isaaclab.utils.processing.DictTensorProcessor` can run independently or serve an observation consumer; operations receive explicit buffers and resources without discovering a camera, environment, or simulation singleton. The shared pipe accepts named ``ProxyArray`` buffers, including nonvisual sensor data; image layout and pixel meaning are requirements of individual operations. The camera resolves external inputs before renderer initialization, including inputs needed only by the pipeline. It publishes the names in ``data_types`` after processing. For example, this camera requests -private ``rgb_radiance`` from its renderer and publishes PPISP's processed ``rgb``: +private ``rgb_hdr`` from its renderer and publishes PPISP's processed ``rgb``: .. code-block:: python + from isaaclab.sensors.camera import CameraExposureCfg from isaaclab.utils import replace from isaaclab_ppisp import PpispCfg, PpispStepCfg processed_camera = replace( front_camera, data_types=["rgb"], + exposure=CameraExposureCfg(), pipeline=[PpispStepCfg(params={"cfg": PpispCfg(inputs={"exposureOffset": 1.5})})], ) @@ -408,6 +396,7 @@ through their ``request_render_inputs()`` and ``render_outputs`` interface: from isaaclab.managers import ObservationTermCfg, SceneEntityCfg from isaaclab_ppisp import PpispStepCfg + front_camera.exposure = CameraExposureCfg() policy_image = ObservationTermCfg( func=mdp.processed_sensor, params={ @@ -424,27 +413,27 @@ camera's published processed output, use an existing image term such as ``mdp.im The observation pipe executes on every term evaluation and returns a snapshot; its state belongs to that term rather than the camera's capture cadence. -For a single incoming tensor, :class:`~isaaclab.utils.modifiers.ProcessingBufferPipeModifierCfg` uses the +For a single incoming tensor, :class:`~isaaclab.utils.tensor_processors.DictTensorProcessorAdapterCfg` uses the same operations while preserving preceding observation transformations. This includes nonvisual observation tensors from sensors without the render-input interface. This form must preserve shape; use ``processed_sensor`` for multiple render inputs or changes to render-output shapes, or construct -a ``ProcessingBufferPipe`` directly for other named buffers. The first incoming tensor prepares the +a ``DictTensorProcessor`` directly for other named buffers. The first incoming tensor prepares the operations; later calls reuse their output storage. Its values must meet the operation's documented -input requirements, including any transformations from preceding modifiers: +input requirements, including any transformations from preceding tensor processors: .. code-block:: python - from isaaclab.utils import modifiers + from isaaclab.utils import tensor_processors # This camera publishes radiance for the ordinary image term to read. - raw_camera = replace(front_camera, data_types=["rgb_radiance"], pipeline=[]) + raw_camera = replace(front_camera, data_types=["rgb_hdr"], exposure=CameraExposureCfg(), pipeline=[]) policy_image = ObservationTermCfg( func=mdp.image_rgb, - params={"sensor_cfg": SceneEntityCfg("front_camera"), "data_type": "rgb_radiance", "normalize": False}, - modifiers=[ - modifiers.ProcessingBufferPipeModifierCfg( + params={"sensor_cfg": SceneEntityCfg("front_camera"), "data_type": "rgb_hdr", "normalize": False}, + tensor_processors=[ + tensor_processors.DictTensorProcessorAdapterCfg( pipeline=[PpispStepCfg()], - input_name="rgb_radiance", + input_name="rgb_hdr", output_name="rgb", ) ], @@ -466,6 +455,7 @@ operation: discovered_camera = replace( front_camera, data_types=["rgb"], + exposure=CameraExposureCfg(), pipeline=[PpispStepCfg(params={"cfg": ppisp_cfg})] if ppisp_cfg is not None else [], ) @@ -475,9 +465,10 @@ on the supplied stage. Alternatively, pass an already resolved camera prim to the operation. Static coefficients apply to the whole camera batch; controller weights may predict per-view exposure and color parameters. -Requesting ``rgb_radiance``, including a private pipeline request, still neutralizes camera exposure on -Isaac RTX and OVRTX as described in :ref:`camera-supported-annotators`. Processing configuration itself -does not change renderer settings. Newton Warp has no exposure model. +For either processing attachment, configure the source camera with +``exposure=CameraExposureCfg()`` before simulation initialization. PPISP's own exposure is set +through ``PpispCfg.inputs["exposureOffset"]`` (stops). Processing does not discover or change +renderer exposure settings. Run the ``ppisp-camera`` example for a complete PPISP workflow. It supports ``--renderer newton_renderer`` (Newton Warp, the default) and ``--renderer isaac_rtx`` (Isaac RTX). @@ -546,3 +537,9 @@ A runnable camera example is available as ``camera``: For saving output to disk, see :doc:`/source/how-to/save_camera_output`. For renderer selection and customization, see :doc:`/source/how-to/configure_rendering`. + +The observation processor API is ``isaaclab.utils.tensor_processors``. Custom stateful processors +inherit ``TensorProcessor`` and use ``TensorProcessorCfg``; attach configurations through +``ObservationTermCfg.tensor_processors``. These replace ``ModifierBase``, ``ModifierCfg``, and +``ObservationTermCfg.modifiers`` respectively. ``DictTensorProcessorAdapterCfg`` adapts a dictionary +processor pipeline to that single-tensor interface. diff --git a/examples/sensors/ppisp_camera.py b/examples/sensors/ppisp_camera.py index c2fcbf1b0c6d..653bcb405cc0 100644 --- a/examples/sensors/ppisp_camera.py +++ b/examples/sensors/ppisp_camera.py @@ -126,6 +126,7 @@ from isaaclab.assets import AssetBaseCfg, RigidObjectCfg from isaaclab.scene import InteractiveSceneCfg from isaaclab.sensors import CameraCfg +from isaaclab.sensors.camera import CameraExposureCfg from isaaclab.sim.spawners.materials import UsdPhysicsRigidBodyMaterialCfg from isaaclab.utils import configclass, instantiate @@ -374,6 +375,7 @@ def make_camera(camera_prim_path: str, *, ppisp_cfg: PpispCfg | None, width: int width=width, data_types=["rgb"], spawn=None, + exposure=CameraExposureCfg() if ppisp_cfg is not None else None, pipeline=[PpispStepCfg(params={"cfg": ppisp_cfg})] if ppisp_cfg is not None else [], renderer_cfg=make_renderer_cfg(), ) diff --git a/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst index e7515e13a8eb..1ae08535a7c5 100644 --- a/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst +++ b/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst @@ -1,5 +1,5 @@ -* **Breaking:** Replaced ``CameraCfg.isp_cfg`` with reusable buffer pipelines. Configure camera - processing with ``CameraCfg.pipeline=[PpispStepCfg(...)]``; use ``mdp.processed_sensor`` or - ``ProcessingBufferPipeModifierCfg`` for observation processing with the same operations. The shared - ``isaaclab.utils.processing.ProcessingBufferPipe`` accepts named ``ProxyArray`` buffers for both image and - nonvisual sensor data. +* **Breaking:** Added ``DictTensorProcessor`` for named buffers and renamed the modifiers API to + ``tensor_processors``, ``TensorProcessor``, and ``TensorProcessorCfg``. Replace observation + ``modifiers`` with ``tensor_processors``; use ``DictTensorProcessorAdapterCfg`` for dictionary operations. +* Replaced renderer-owned PPISP with camera and observation processors consuming ``rgb_hdr``. + Set ``CameraCfg.exposure=CameraExposureCfg()`` explicitly for neutral RTX HDR before PPISP. diff --git a/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py b/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py index af390bee049c..0c4e969953b4 100644 --- a/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py +++ b/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py @@ -14,7 +14,7 @@ import warp as wp from isaaclab.managers import ManagerTermBase, ObservationTermCfg, SceneEntityCfg -from isaaclab.utils.processing import ProcessingBufferPipe, ProcessingBufferStepCfg +from isaaclab.utils.processing import DictTensorProcessor, DictTensorProcessorStepCfg if TYPE_CHECKING: from isaaclab.envs import ManagerBasedEnv @@ -34,8 +34,8 @@ class processed_sensor(ManagerTermBase): returned tensors are snapshots, so later evaluations cannot overwrite retained history. Use this term for multiple render inputs or changed buffer dimensions. To process other - sensor data or the result of earlier observation modifiers, use - :class:`~isaaclab.utils.modifiers.ProcessingBufferPipeModifier` on an observation tensor. + sensor data or the result of earlier observation tensor_processors, use + :class:`~isaaclab.utils.tensor_processors.DictTensorProcessorAdapter` on an observation tensor. To read a camera's already processed public output, use the existing image observation terms. """ @@ -43,7 +43,7 @@ class processed_sensor(ManagerTermBase): def prepare_scene(cls, cfg: ObservationTermCfg, env: ManagerBasedEnv) -> None: sensor_cfg = cfg.params.get("sensor_cfg", SceneEntityCfg("camera")) output_name = cfg.params.get("output_name", "rgb") - requirements = ProcessingBufferPipe.required_source_names(cfg.params["pipeline"], [output_name]) + requirements = DictTensorProcessor.required_source_names(cfg.params["pipeline"], [output_name]) env.scene.sensors[sensor_cfg.name].request_render_inputs(requirements) def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedEnv): @@ -54,7 +54,7 @@ def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedEnv): device = wp.get_device(self.device) stream = wp.stream_from_torch(torch.cuda.current_stream(str(device))) if device.is_cuda else None with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): - self._pipe = ProcessingBufferPipe(buffers, cfg.params["pipeline"]) + self._pipe = DictTensorProcessor(buffers, cfg.params["pipeline"]) if self._output_name not in self._pipe.outputs: self._pipe.close() raise ValueError(f"Buffer processing does not provide output {self._output_name!r}.") @@ -62,7 +62,7 @@ def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedEnv): def __call__( self, env: ManagerBasedEnv, - pipeline: Sequence[ProcessingBufferStepCfg], + pipeline: Sequence[DictTensorProcessorStepCfg], sensor_cfg: SceneEntityCfg = SceneEntityCfg("camera"), output_name: str = "rgb", *, diff --git a/source/isaaclab/isaaclab/managers/manager_base.py b/source/isaaclab/isaaclab/managers/manager_base.py index 1411374f2d32..c010af425cc0 100644 --- a/source/isaaclab/isaaclab/managers/manager_base.py +++ b/source/isaaclab/isaaclab/managers/manager_base.py @@ -16,7 +16,7 @@ from ..physics import PhysicsEvent, PhysicsManager from ..utils import string as string_utils from ..utils import string_to_callable, to_dict -from ..utils.modifiers import ModifierCfg +from ..utils.tensor_processors import TensorProcessorCfg from .manager_term_cfg import ManagerTermBaseCfg from .scene_entity_cfg import SceneEntityCfg @@ -447,7 +447,7 @@ def _resolve_param_value( self._scene_entity_cfgs.append(value) elif isinstance(value, ManagerTermBaseCfg): self._process_term_cfg_at_play(f"{term_name}.{key}", value) - elif isinstance(value, ModifierCfg): + elif isinstance(value, TensorProcessorCfg): for field in fields(value): field_value = getattr(value, field.name) resolved_value = self._resolve_param_value( diff --git a/source/isaaclab/isaaclab/managers/manager_term_cfg.py b/source/isaaclab/isaaclab/managers/manager_term_cfg.py index 01df0af92391..985339b64e8a 100644 --- a/source/isaaclab/isaaclab/managers/manager_term_cfg.py +++ b/source/isaaclab/isaaclab/managers/manager_term_cfg.py @@ -14,8 +14,8 @@ import torch from ..utils import configclass -from ..utils.modifiers import ModifierCfg from ..utils.noise import NoiseCfg, NoiseModelCfg +from ..utils.tensor_processors import TensorProcessorCfg from .scene_entity_cfg import SceneEntityCfg if TYPE_CHECKING: @@ -167,15 +167,15 @@ class ObservationTermCfg(ManagerTermBaseCfg): shape (num_envs, obs_term_dim). """ - modifiers: list[ModifierCfg] | None = None - """The list of data modifiers to apply to the observation in order. Defaults to None, + tensor_processors: list[TensorProcessorCfg] | None = None + """The list of tensor processors to apply to the observation in order. Defaults to None, in which case no modifications will be applied. - Modifiers are applied in the order they are specified in the list. They can be stateless + Processors are applied in the order they are specified in the list. They can be stateless or stateful, and can be used to apply transformations to the observation data. For example, - a modifier can be used to normalize the observation data or to apply a rolling average. + a processor can be used to normalize the observation data or to apply a rolling average. - For more information on modifiers, see the :class:`~isaaclab.utils.modifiers.ModifierCfg` class. + For more information, see :class:`~isaaclab.utils.tensor_processors.TensorProcessorCfg`. """ noise: NoiseCfg | NoiseModelCfg | None = None @@ -206,7 +206,7 @@ class ObservationTermCfg(ManagerTermBaseCfg): delay_max_lag: int = 0 """Maximum observation delay, counted in recorded samples. Zero disables delay. - Set both lag bounds equal for a constant delay. Delay is applied after modifiers, noise, clipping, and + Set both lag bounds equal for a constant delay. Delay is applied after tensor processors, noise, clipping, and scaling, before observation history. Until enough samples have been recorded, the oldest available sample is returned. After reset, no data from the previous episode is returned. """ diff --git a/source/isaaclab/isaaclab/managers/observation_manager.py b/source/isaaclab/isaaclab/managers/observation_manager.py index 4f520715232e..867b9fafed56 100644 --- a/source/isaaclab/isaaclab/managers/observation_manager.py +++ b/source/isaaclab/isaaclab/managers/observation_manager.py @@ -18,7 +18,7 @@ from prettytable import PrettyTable from ..envs.utils.io_descriptors import _warn_io_descriptors_deprecated -from ..utils import instantiate, modifiers, noise, string_to_callable, to_dict +from ..utils import instantiate, noise, string_to_callable, tensor_processors, to_dict from ..utils.buffers import CircularBuffer, DelayBuffer from .manager_base import ManagerBase, ManagerTermBase from .manager_term_cfg import ManagerTermBaseCfg, ObservationGroupCfg, ObservationTermCfg @@ -156,7 +156,7 @@ def close(self) -> None: seen = set() for cfg in _iter_term_configs(self._group_obs_term_cfgs): resource = cfg.func - if isinstance(resource, (ManagerTermBase, modifiers.ModifierBase)) and id(resource) not in seen: + if isinstance(resource, (ManagerTermBase, tensor_processors.TensorProcessor)) and id(resource) not in seen: seen.add(id(resource)) resources.append(resource) errors = [] @@ -328,7 +328,7 @@ def _collect_io_descriptors(self, group_names_to_export: list[str] = ["policy"]) for k, v in term_cfg.__dict__.items(): # For now we do not add the noise modifier if k in [ - "modifiers", + "tensor_processors", "clip", "scale", "delay_min_lag", @@ -366,7 +366,7 @@ def _collect_io_descriptors(self, group_names_to_export: list[str] = ["policy"]) "flatten_history_dim", ]: formatted_item["overloads"][k] = v - elif k in ["modifiers", "description", "units"]: + elif k in ["tensor_processors", "description", "units"]: formatted_item["extras"][k] = v else: formatted_item[k] = v @@ -389,7 +389,7 @@ def reset(self, env_ids: Sequence[int] | None = None) -> dict[str, float]: self._group_obs_term_delay_buffer[group_name][term_name].reset(env_ids) if term_name in self._group_obs_term_history_buffer[group_name]: self._group_obs_term_history_buffer[group_name][term_name].reset(batch_ids=env_ids) - # call all modifiers that are classes + # call all tensor_processors that are classes for mod in self._group_obs_class_instances: mod.reset(env_ids=env_ids) @@ -433,7 +433,7 @@ def compute_group(self, group_name: str, update_history: bool = False) -> torch. The following steps are performed for each observation term: 1. Compute observation term by calling the function - 2. Apply custom modifiers in the order specified in :attr:`ObservationTermCfg.modifiers` + 2. Apply custom tensor_processors in the order specified in :attr:`ObservationTermCfg.tensor_processors` 3. Apply corruption/noise model based on :attr:`ObservationTermCfg.noise` 4. Apply clipping based on :attr:`ObservationTermCfg.clip` 5. Apply scaling based on :attr:`ObservationTermCfg.scale` @@ -470,7 +470,7 @@ def compute_group(self, group_name: str, update_history: bool = False) -> torch. term_cfgs = self._group_obs_term_cfgs[group_name] obs_terms = zip(group_term_names, term_cfgs) - # evaluate terms: compute, add noise, clip, scale, custom modifiers + # evaluate terms: compute, add noise, clip, scale, custom tensor_processors for term_name, term_cfg in obs_terms: output_spec = self._term_output_specs.get((group_name, term_name)) owned = output_spec is not None @@ -484,12 +484,12 @@ def compute_group(self, group_name: str, update_history: bool = False) -> torch. else: obs = term_cfg.func(self._env, **term_cfg.params) # apply post-processing - if term_cfg.modifiers is not None: - for modifier in term_cfg.modifiers: + if term_cfg.tensor_processors is not None: + for modifier in term_cfg.tensor_processors: owned = False # Custom callbacks may modify or retain their input. obs = obs.clone() - if isinstance(modifier.func, modifiers.ModifierBase): + if isinstance(modifier.func, tensor_processors.TensorProcessor): obs = modifier.func(obs) else: obs = modifier.func(obs, **modifier.params) @@ -615,9 +615,9 @@ def _prepare_terms(self): self._group_obs_term_delay_buffer: dict[str, dict[str, DelayBuffer]] = {} self._group_obs_term_history_buffer: dict[str, dict] = {} - # create a list to store classes instances, e.g., for modifiers and noise models + # create a list to store classes instances, e.g., for tensor_processors and noise models # we store it as a separate list to only call reset on them and prevent unnecessary calls - self._group_obs_class_instances: list[modifiers.ModifierBase | noise.NoiseModel] = [] + self._group_obs_class_instances: list[tensor_processors.TensorProcessor | noise.NoiseModel] = [] # make sure the simulation is playing since we compute obs dims which needs asset quantities if not self._env.sim.is_playing(): @@ -727,23 +727,23 @@ def _prepare_terms(self): # cast the scale into torch tensor term_cfg.scale = torch.tensor(term_cfg.scale, dtype=torch.float, device=self._env.device) - # prepare modifiers for each observation - if term_cfg.modifiers is not None: - # initialize list of modifiers for term - for mod_cfg in term_cfg.modifiers: - if not isinstance(mod_cfg, modifiers.ModifierCfg): + # prepare tensor_processors for each observation + if term_cfg.tensor_processors is not None: + # initialize list of tensor_processors for term + for mod_cfg in term_cfg.tensor_processors: + if not isinstance(mod_cfg, tensor_processors.TensorProcessorCfg): raise TypeError( f"Modifier configuration '{mod_cfg}' of observation term '{term_name}' is not of" - f" required type ModifierCfg, Received: '{type(mod_cfg)}'" + f" required type TensorProcessorCfg, Received: '{type(mod_cfg)}'" ) - # construct stateful modifiers with the observation size + # construct stateful tensor_processors with the observation size if inspect.isclass(mod_cfg.func): mod_cfg.func = mod_cfg.func(cfg=mod_cfg, data_dim=obs_dims, device=self._env.device) - if not isinstance(mod_cfg.func, modifiers.ModifierBase): + if not isinstance(mod_cfg.func, tensor_processors.TensorProcessor): raise TypeError( f"Modifier function '{mod_cfg.func}' for observation term '{term_name}'" - f" is not an instance of 'ModifierBase'. Received: '{type(mod_cfg.func)}'." + f" is not an instance of 'TensorProcessor'. Received: '{type(mod_cfg.func)}'." ) self._group_obs_class_instances.append(mod_cfg.func) @@ -823,10 +823,10 @@ def _prepare_terms(self): self._group_obs_term_history_buffer[group_name] = group_entry_history_buffer -def _iter_term_configs(value: object) -> Iterator[ManagerTermBaseCfg | modifiers.ModifierCfg]: +def _iter_term_configs(value: object) -> Iterator[ManagerTermBaseCfg | tensor_processors.TensorProcessorCfg]: """Visit manager-owned configurations without following term instances back into their configs.""" - if isinstance(value, (ManagerTermBaseCfg, modifiers.ModifierCfg, ObservationGroupCfg)): - if isinstance(value, (ManagerTermBaseCfg, modifiers.ModifierCfg)): + if isinstance(value, (ManagerTermBaseCfg, tensor_processors.TensorProcessorCfg, ObservationGroupCfg)): + if isinstance(value, (ManagerTermBaseCfg, tensor_processors.TensorProcessorCfg)): yield value for field in fields(value): if field.name != "func": diff --git a/source/isaaclab/isaaclab/renderers/output_contract.py b/source/isaaclab/isaaclab/renderers/output_contract.py index f62f4a5ef357..7ee8025b6693 100644 --- a/source/isaaclab/isaaclab/renderers/output_contract.py +++ b/source/isaaclab/isaaclab/renderers/output_contract.py @@ -27,8 +27,6 @@ class RenderBufferKind(StrEnum): RGBA = "rgba" RGB_HDR = "rgb_hdr" """HDR RGB using the camera's active rendering settings.""" - RGB_RADIANCE = "rgb_radiance" - """Scene-linear RGB before camera exposure and response, in renderer-relative intensity units.""" ALBEDO = "albedo" DEPTH = "depth" DISTANCE_TO_IMAGE_PLANE = "distance_to_image_plane" diff --git a/source/isaaclab/isaaclab/sensors/camera/__init__.pyi b/source/isaaclab/isaaclab/sensors/camera/__init__.pyi index afff9e01fe01..15454b8c29ef 100644 --- a/source/isaaclab/isaaclab/sensors/camera/__init__.pyi +++ b/source/isaaclab/isaaclab/sensors/camera/__init__.pyi @@ -6,6 +6,7 @@ __all__ = [ "Camera", "CameraCfg", + "CameraExposureCfg", "CameraData", "CameraISPMode", "RenderBufferKind", @@ -20,6 +21,7 @@ __all__ = [ from .camera import Camera from .camera_cfg import CameraCfg +from .camera_exposure_cfg import CameraExposureCfg from .camera_data import CameraData, RenderBufferKind, RenderBufferSpec from .camera_isp import CameraISPMode from .tiled_camera import TiledCamera diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index c5ee6dc3be57..188d421851e6 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -28,7 +28,7 @@ create_rotation_matrix_from_view, quat_from_matrix, ) -from ...utils.processing import ProcessingBufferPipe +from ...utils.processing import DictTensorProcessor from ...utils.warp import ProxyArray from ..sensor_base import SensorBase from .camera_data import CameraData, RenderBufferKind @@ -253,7 +253,7 @@ def __init__(self, cfg: CameraCfg): # Renderer requirements include private pipeline inputs and exclude produced public outputs. self._render_cfg = replace(self.cfg, data_types=self.cfg.render_data_types(), pipeline=[]) - self._buffer_pipe: ProcessingBufferPipe | None = None + self._buffer_pipe: DictTensorProcessor | None = None self._pipeline_capture: object | None = None self._render_camera_data: CameraData | None = None self._render_outputs: dict[str, ProxyArray] = {} @@ -703,6 +703,10 @@ def _initialize_impl(self): camera_prim_paths=cam_paths, view_count=self._num_envs, ) + if self.cfg.exposure is not None: + from .camera_exposure import apply_camera_exposure + + apply_camera_exposure(self.stage, list(cam_paths), self.cfg.exposure) self._renderer.prepare_cameras(self.stage, render_spec) # Stage preprocessing must happen before creating the view because the view keeps @@ -900,7 +904,7 @@ def _create_buffers(self): self._close_pipeline() self._pipeline_capture = None if self.cfg.pipeline: - self._buffer_pipe = ProcessingBufferPipe(self._render_outputs, self.cfg.pipeline) + self._buffer_pipe = DictTensorProcessor(self._render_outputs, self.cfg.pipeline) try: outputs = dict(self._buffer_pipe.outputs) alias_rgb = ( @@ -1195,6 +1199,10 @@ def _prepare_renderer(self): camera_prim_paths=cam_paths, view_count=num_views, ) + if self.cfg.exposure is not None: + from .camera_exposure import apply_camera_exposure + + apply_camera_exposure(self.stage, list(cam_paths), self.cfg.exposure) self._renderer.prepare_cameras(self.stage, render_spec) def _invalidate_initialize_callback(self, event): diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py index 507d32319a20..5a7ab6cc561c 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py @@ -13,9 +13,10 @@ from ...sim import FisheyeCameraCfg, PinholeCameraCfg from ...utils import configclass from ..sensor_base_cfg import SensorBaseCfg +from .camera_exposure_cfg import CameraExposureCfg if TYPE_CHECKING: - from ...utils.processing import ProcessingBufferStepCfg + from ...utils.processing import DictTensorProcessorStepCfg from .camera import Camera # Default values for the RTX-flavored fields kept on :class:`CameraCfg` for @@ -199,7 +200,15 @@ class OffsetCfg: renderer_cfg: RendererCfg = field(default_factory=RendererCfg) """Renderer configuration for camera sensor.""" - pipeline: list[ProcessingBufferStepCfg] = [] + exposure: CameraExposureCfg | None = None + """Explicit camera exposure, or None to preserve authored USD settings. + + Use ``CameraExposureCfg()`` for neutral HDR input to PPISP through either a camera + dictionary processor or an observation tensor processor. Applied before stage export; + changes affect every output from this camera prim. Newton ignores RTX exposure settings. + """ + + pipeline: list[DictTensorProcessorStepCfg] = [] """Ordered image operations applied after each newly published capture. Steps bind operation roles to named buffers and may consume several render outputs or preceding @@ -223,13 +232,13 @@ def render_data_types(self) -> list[str]: ) if not self.pipeline: return list(self.data_types) - from ...utils.processing import ProcessingBufferPipe + from ...utils.processing import DictTensorProcessor produced = {name for step in self.pipeline for name in step.outputs.values()} if "rgb" in produced and "rgba" not in produced and "rgba" in self.data_types: raise ValueError("A pipeline producing rgb must also produce rgba when rgba is requested.") published = [name for name in self.data_types if not (name == "rgb" and "rgba" in produced)] - return sorted(ProcessingBufferPipe.required_source_names(self.pipeline, published)) + return sorted(DictTensorProcessor.required_source_names(self.pipeline, published)) def __post_init__(self): """Forward deprecated RTX-flavored fields onto :attr:`renderer_cfg`. diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_data.py b/source/isaaclab/isaaclab/sensors/camera/camera_data.py index 7f99f0a7ad58..09b122afc94c 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_data.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_data.py @@ -151,9 +151,6 @@ def allocate( wrapped in a :class:`~isaaclab.utils.warp.ProxyArray`; call ``.torch`` on the result to obtain a zero-copy :class:`torch.Tensor` view. - When both are requested with matching layouts, ``rgb_hdr`` aliases ``rgb_radiance``: - radiance selects the camera's active HDR signal before exposure and response. - Args: data_types: Requested output names (typically :attr:`CameraCfg.data_types`). Every name must be a member of :class:`RenderBufferKind`. @@ -188,11 +185,6 @@ def allocate( allocated = requested.intersection(supported_specs) if rgb_alias: allocated.remove(RenderBufferKind.RGB) - hdr_alias = {RenderBufferKind.RGB_HDR, RenderBufferKind.RGB_RADIANCE} <= allocated and ( - supported_specs[RenderBufferKind.RGB_HDR] == supported_specs[RenderBufferKind.RGB_RADIANCE] - ) - if hdr_alias: - allocated.remove(RenderBufferKind.RGB_HDR) buffers: dict[str, ProxyArray] = {} for name, spec in supported_specs.items(): @@ -201,9 +193,6 @@ def allocate( shape = (num_views, height, width, spec.channels) buffers[str(name)] = ProxyArray(wp.zeros(shape, dtype=spec.dtype, device=device)) - if hdr_alias: - buffers[str(RenderBufferKind.RGB_HDR)] = buffers[str(RenderBufferKind.RGB_RADIANCE)] - if rgb_alias: # Zero-copy strided view into rgba: shape (N, H, W, 3), skipping the alpha channel. # Byte strides for a contiguous (N, H, W, 4) uint8 array are (H*W*4, W*4, 4, 1). diff --git a/source/isaaclab/isaaclab/renderers/rtx_camera_overrides.py b/source/isaaclab/isaaclab/sensors/camera/camera_exposure.py similarity index 57% rename from source/isaaclab/isaaclab/renderers/rtx_camera_overrides.py rename to source/isaaclab/isaaclab/sensors/camera/camera_exposure.py index 632f07adc8ee..f1d5ec0de55c 100644 --- a/source/isaaclab/isaaclab/renderers/rtx_camera_overrides.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_exposure.py @@ -3,32 +3,20 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Internal RTX camera exposure overrides for scene-linear sensor inputs. - -RTX has no native pre-exposure radiance output, so ``rgb_radiance`` is derived by authoring neutral -exposure on the camera prim, which affects every output rendered from it. This is a temporary -workaround shared by the Isaac RTX and OVRTX renderers, not public API. Remove it once RTX provides -a native radiance output (NVBug 6858736). -""" +"""Author explicitly configured exposure before renderer stage export.""" from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .camera_exposure_cfg import CameraExposureCfg -# Both RTX backends consume these schemas without requiring the schema plugin at authoring time. _CAMERA_EXPOSURE_SCHEMAS = ("OmniRtxCameraAutoExposureAPI_1", "OmniRtxCameraExposureAPI_1") -_NEUTRAL_CAMERA_EXPOSURE: tuple[tuple[str, str, Any], ...] = ( - ("exposure", "Float", 0.0), - ("exposure:fStop", "Float", 1.0), - ("exposure:iso", "Float", 0.0), - ("exposure:responsivity", "Float", 1.0), - ("exposure:time", "Float", 1.0), - ("omni:rtx:autoExposure:enabled", "Bool", False), -) -def _apply_rtx_exposure_overrides(stage: Any, prim_paths: list[str]) -> None: - """Disable RTX camera exposure so downstream processors receive scene-linear HDR. +def apply_camera_exposure(stage: Any, prim_paths: list[str], cfg: CameraExposureCfg) -> None: + """Apply camera exposure independently of requested render outputs. Existing API schemas are preserved. Schema metadata is authored directly so the RTX schema plugin does not need to be loaded yet. @@ -36,9 +24,18 @@ def _apply_rtx_exposure_overrides(stage: Any, prim_paths: list[str]) -> None: Args: stage: USD stage containing the camera prims. prim_paths: Resolved camera prim paths. + cfg: Explicit exposure settings shared by these camera prims. """ from pxr import Sdf + settings = ( + ("exposure", "Float", cfg.exposure), + ("exposure:fStop", "Float", cfg.f_stop), + ("exposure:iso", "Float", cfg.iso), + ("exposure:responsivity", "Float", cfg.responsivity), + ("exposure:time", "Float", cfg.time), + ("omni:rtx:autoExposure:enabled", "Bool", cfg.auto_exposure), + ) value_types = {"Bool": Sdf.ValueTypeNames.Bool, "Float": Sdf.ValueTypeNames.Float} for prim_path in prim_paths: prim = stage.GetPrimAtPath(prim_path) @@ -49,7 +46,7 @@ def _apply_rtx_exposure_overrides(stage: Any, prim_paths: list[str]) -> None: missing = [schema for schema in _CAMERA_EXPOSURE_SCHEMAS if schema not in existing] if missing: prim.SetMetadata("apiSchemas", Sdf.TokenListOp.Create(prependedItems=[*missing, *existing])) - for attr_name, sdf_type_name, value in _NEUTRAL_CAMERA_EXPOSURE: + for attr_name, sdf_type_name, value in settings: attr = prim.GetAttribute(attr_name) if not attr: attr = prim.CreateAttribute(attr_name, value_types[sdf_type_name], custom=False) diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_exposure_cfg.py b/source/isaaclab/isaaclab/sensors/camera/camera_exposure_cfg.py new file mode 100644 index 000000000000..d9e20aa9e44b --- /dev/null +++ b/source/isaaclab/isaaclab/sensors/camera/camera_exposure_cfg.py @@ -0,0 +1,34 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from isaaclab.utils import configclass + + +@configclass +class CameraExposureCfg: + """Explicit RTX camera exposure settings, applied before renderer stage export. + + Defaults disable automatic exposure and make HDR neutral for PPISP. Settings affect every + output from the camera prim. Newton does not consume RTX exposure attributes; its HDR output + is already scene-linear. Leave ``CameraCfg.exposure`` unset to preserve authored settings. + """ + + exposure: float = 0.0 + """Exposure compensation in stops.""" + + f_stop: float = 1.0 + """Exposure aperture ratio (separate from the lens depth-of-field aperture).""" + + iso: float = 0.0 + """RTX sensor ISO; zero selects neutral exposure.""" + + responsivity: float = 1.0 + """Sensor response multiplier.""" + + time: float = 1.0 + """Exposure duration in seconds.""" + + auto_exposure: bool = False + """Whether RTX automatic exposure is enabled.""" diff --git a/source/isaaclab/isaaclab/utils/__init__.pyi b/source/isaaclab/isaaclab/utils/__init__.pyi index 76dd49752ed0..09c6f837ddeb 100644 --- a/source/isaaclab/isaaclab/utils/__init__.pyi +++ b/source/isaaclab/isaaclab/utils/__init__.pyi @@ -31,8 +31,8 @@ __all__ = [ "create_trimesh_from_geom_shape", "convert_faces_to_triangles", "PRIMITIVE_MESH_TYPES", - "ModifierCfg", - "ModifierBase", + "TensorProcessorCfg", + "TensorProcessor", "DigitalFilter", "DigitalFilterCfg", "Integrator", @@ -87,9 +87,9 @@ from .mesh import ( convert_faces_to_triangles, PRIMITIVE_MESH_TYPES, ) -from .modifiers import ( - ModifierCfg, - ModifierBase, +from .tensor_processors import ( + TensorProcessorCfg, + TensorProcessor, DigitalFilter, DigitalFilterCfg, Integrator, diff --git a/source/isaaclab/isaaclab/utils/modifiers/__init__.pyi b/source/isaaclab/isaaclab/utils/modifiers/__init__.pyi deleted file mode 100644 index 9b0fa3c886da..000000000000 --- a/source/isaaclab/isaaclab/utils/modifiers/__init__.pyi +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -__all__ = [ - "ModifierCfg", - "ProcessingBufferPipeModifierCfg", - "ProcessingBufferPipeModifier", - "DigitalFilterCfg", - "IntegratorCfg", - "ModifierBase", - "DigitalFilter", - "Integrator", - "bias", - "clip", - "scale", -] - -from .modifier_cfg import ModifierCfg, DigitalFilterCfg, IntegratorCfg, ProcessingBufferPipeModifierCfg -from .processing_buffer_pipe import ProcessingBufferPipeModifier -from .modifier_base import ModifierBase -from .modifier import DigitalFilter, Integrator, bias, clip, scale diff --git a/source/isaaclab/isaaclab/utils/processing.py b/source/isaaclab/isaaclab/utils/processing.py index a6864042e27d..c313c4f956ec 100644 --- a/source/isaaclab/isaaclab/utils/processing.py +++ b/source/isaaclab/isaaclab/utils/processing.py @@ -39,7 +39,7 @@ class PreparedBufferOperation: @configclass -class ProcessingBufferStepCfg: +class DictTensorProcessorStepCfg: """Bind named buffers to the input/output roles of one processing operation. The factory ``prepare(inputs, **params)`` runs once during pipe construction and returns @@ -68,7 +68,7 @@ class ProcessingBufferStepCfg: """ -class ProcessingBufferPipe: +class DictTensorProcessor: """Apply an ordered sequence of operations to named :class:`ProxyArray` buffers. Buffers may hold images, LiDAR points, or other sensor and observation data. The pipe imposes @@ -88,7 +88,7 @@ class ProcessingBufferPipe: steps: Ordered operation configurations. """ - def __init__(self, source_buffers: Mapping[str, ProxyArray], steps: Sequence[ProcessingBufferStepCfg]): + def __init__(self, source_buffers: Mapping[str, ProxyArray], steps: Sequence[DictTensorProcessorStepCfg]): self._source_buffers = dict(source_buffers) self.required_inputs = self.required_source_names(steps) self._operations: list[ @@ -153,7 +153,7 @@ def _validate_buffer(name: str, value: ProxyArray, reference: ProxyArray) -> Non @staticmethod def required_source_names( - steps: Sequence[ProcessingBufferStepCfg], published_names: Sequence[str] = () + steps: Sequence[DictTensorProcessorStepCfg], published_names: Sequence[str] = () ) -> frozenset[str]: """Resolve external names without constructing operations or allocating buffer storage. diff --git a/source/isaaclab/isaaclab/utils/modifiers/__init__.py b/source/isaaclab/isaaclab/utils/tensor_processors/__init__.py similarity index 53% rename from source/isaaclab/isaaclab/utils/modifiers/__init__.py rename to source/isaaclab/isaaclab/utils/tensor_processors/__init__.py index d351b42e3875..d4230dd61886 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/__init__.py +++ b/source/isaaclab/isaaclab/utils/tensor_processors/__init__.py @@ -3,53 +3,53 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Sub-module containing different modifiers implementations. +"""Sub-module containing tensor processor implementations. -Modifiers are used to apply stateful or stateless modifications to tensor data. They take +Tensor processors are used to apply stateful or stateless modifications to tensor data. They take in a tensor and a configuration and return a tensor with the modification applied. This way users -can define custom operations to apply to a tensor. For instance, a modifier can be used to normalize +can define custom operations to apply to a tensor. For instance, a processor can be used to normalize the input data or to apply a rolling average. They are primarily used to apply custom operations in the :class:`~isaaclab.managers.ObservationManager` as an alternative to the built-in noise, clip and scale post-processing operations. For more details, see the :class:`~isaaclab.managers.ObservationTermCfg` class. -Usage with a function modifier: +Usage with a function processor: .. code-block:: python import torch - from isaaclab.utils import modifiers + from isaaclab.utils import tensor_processors # create a random tensor my_tensor = torch.rand(256, 128, device="cuda") - # create a modifier configuration - cfg = modifiers.ModifierCfg(func=modifiers.clip, params={"bounds": (0.0, torch.inf)}) + # create a processor configuration + cfg = tensor_processors.TensorProcessorCfg(func=tensor_processors.clip, params={"bounds": (0.0, torch.inf)}) - # apply the modifier - my_modified_tensor = cfg.func(my_tensor, cfg) + # apply the processor + my_modified_tensor = cfg.func(my_tensor, **cfg.params) -Usage with a class modifier: +Usage with a class processor: .. code-block:: python import torch - from isaaclab.utils import modifiers + from isaaclab.utils import tensor_processors # create a random tensor my_tensor = torch.rand(256, 128, device="cuda") - # create a modifier configuration + # create a processor configuration # a digital filter with a simple delay of 1 timestep - cfg = modifiers.DigitalFilterCfg(A=[0.0], B=[0.0, 1.0]) + cfg = tensor_processors.DigitalFilterCfg(A=[0.0], B=[0.0, 1.0]) - # create the modifier instance - my_modifier = modifiers.DigitalFilter(cfg, my_tensor.shape, "cuda") + # create the processor instance + my_processor = tensor_processors.DigitalFilter(cfg, my_tensor.shape, "cuda") - # apply the modifier as a callable object - my_modified_tensor = my_modifier(my_tensor) + # apply the processor as a callable object + my_modified_tensor = my_processor(my_tensor) """ diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/__init__.pyi b/source/isaaclab/isaaclab/utils/tensor_processors/__init__.pyi new file mode 100644 index 000000000000..1317c1d22978 --- /dev/null +++ b/source/isaaclab/isaaclab/utils/tensor_processors/__init__.pyi @@ -0,0 +1,23 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "TensorProcessorCfg", + "DictTensorProcessorAdapterCfg", + "DictTensorProcessorAdapter", + "DigitalFilterCfg", + "IntegratorCfg", + "TensorProcessor", + "DigitalFilter", + "Integrator", + "bias", + "clip", + "scale", +] + +from .tensor_processor_cfg import TensorProcessorCfg, DigitalFilterCfg, IntegratorCfg, DictTensorProcessorAdapterCfg +from .dict_tensor_processor_adapter import DictTensorProcessorAdapter +from .tensor_processor import TensorProcessor +from .processors import DigitalFilter, Integrator, bias, clip, scale diff --git a/source/isaaclab/isaaclab/utils/modifiers/processing_buffer_pipe.py b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_adapter.py similarity index 70% rename from source/isaaclab/isaaclab/utils/modifiers/processing_buffer_pipe.py rename to source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_adapter.py index 79edd26d74bd..7a3c9a1bd3cb 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/processing_buffer_pipe.py +++ b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_adapter.py @@ -13,46 +13,46 @@ import torch import warp as wp -from ..processing import ProcessingBufferPipe +from ..processing import DictTensorProcessor from ..warp import ProxyArray -from .modifier_base import ModifierBase +from .tensor_processor import TensorProcessor if TYPE_CHECKING: - from .modifier_cfg import ProcessingBufferPipeModifierCfg + from .tensor_processor_cfg import DictTensorProcessorAdapterCfg -class ProcessingBufferPipeModifier(ModifierBase): - """Apply a buffer pipe to the tensor received from the preceding modifier. +class DictTensorProcessorAdapter(TensorProcessor): + """Apply a buffer pipe to the tensor received from the preceding processor. - The modifier owns a pipe and returns a snapshot. Its output must preserve the input - shape, as required by :class:`ModifierBase`. The first incoming tensor prepares the pipe; + The processor owns a pipe and returns a snapshot. Its output must preserve the input + shape, as required by :class:`TensorProcessor`. The first incoming tensor prepares the pipe; subsequent calls reuse its output storage. Inputs may represent images or nonvisual sensor observations. For multiple render inputs or shape changes to render outputs, use :class:`~isaaclab.envs.mdp.processed_sensor` instead. """ - def __init__(self, cfg: ProcessingBufferPipeModifierCfg, data_dim: tuple[int, ...], device: str): + def __init__(self, cfg: DictTensorProcessorAdapterCfg, data_dim: tuple[int, ...], device: str): super().__init__(cfg, data_dim, device) - self._pipe: ProcessingBufferPipe | None = None + self._pipe: DictTensorProcessor | None = None self._closed = False def __call__(self, data: torch.Tensor) -> torch.Tensor: if self._closed: - raise RuntimeError("The buffer pipe modifier is closed.") + raise RuntimeError("The buffer pipe processor is closed.") if tuple(data.shape) != tuple(self._data_dim): raise ValueError( - f"Observation modifier input shape must be {tuple(self._data_dim)}, got {tuple(data.shape)}." + f"Observation processor input shape must be {tuple(self._data_dim)}, got {tuple(data.shape)}." ) if self._pipe is None: stream = wp.stream_from_torch(torch.cuda.current_stream(data.device)) if data.is_cuda else None with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): - pipe = ProcessingBufferPipe({self._cfg.input_name: ProxyArray(wp.from_torch(data))}, self._cfg.pipeline) + pipe = DictTensorProcessor({self._cfg.input_name: ProxyArray(wp.from_torch(data))}, self._cfg.pipeline) try: if self._cfg.output_name not in pipe.outputs: raise ValueError(f"Buffer processing does not provide output {self._cfg.output_name!r}.") if pipe.outputs[self._cfg.output_name].shape != tuple(self._data_dim): raise ValueError( - "Observation modifiers must preserve shape;" + "Observation tensor_processors must preserve shape;" " use a processing observation term for shape changes." ) except Exception: diff --git a/source/isaaclab/isaaclab/utils/modifiers/modifier.py b/source/isaaclab/isaaclab/utils/tensor_processors/processors.py similarity index 92% rename from source/isaaclab/isaaclab/utils/modifiers/modifier.py rename to source/isaaclab/isaaclab/utils/tensor_processors/processors.py index 2a6f456981b1..ff9a4a333f97 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/modifier.py +++ b/source/isaaclab/isaaclab/utils/tensor_processors/processors.py @@ -11,13 +11,13 @@ import torch from ..array import index_fill_ -from .modifier_base import ModifierBase +from .tensor_processor import TensorProcessor if TYPE_CHECKING: - from . import modifier_cfg + from . import tensor_processor_cfg ## -# Modifiers as functions +# Tensor processors as functions ## @@ -62,12 +62,12 @@ def bias(data: torch.Tensor, value: float) -> torch.Tensor: ## -# Sample of class based modifiers +# Sample of class based tensor_processors ## -class DigitalFilter(ModifierBase): - r"""Modifier used to apply digital filtering to the input data. +class DigitalFilter(TensorProcessor): + r"""Processor used to apply digital filtering to the input data. `Digital filters `_ are used to process discrete-time signals to extract useful parts of the signal, such as smoothing, noise reduction, or frequency separation. @@ -129,14 +129,14 @@ class DigitalFilter(ModifierBase): :math:`B = [1 - \alpha]`. """ - def __init__(self, cfg: modifier_cfg.DigitalFilterCfg, data_dim: tuple[int, ...], device: str): + def __init__(self, cfg: tensor_processor_cfg.DigitalFilterCfg, data_dim: tuple[int, ...], device: str): """Initializes digital filter. Args: cfg: Configuration parameters. data_dim: The dimensions of the data to be modified. First element is the batch size which usually corresponds to number of environments in the simulation. - device: The device to run the modifier on. + device: The device to run the processor on. Raises: ValueError: If filter coefficients are None. @@ -191,8 +191,8 @@ def __call__(self, data: torch.Tensor) -> torch.Tensor: return y_i -class Integrator(ModifierBase): - r"""Modifier that applies a numerical forward integration based on a middle Reimann sum. +class Integrator(TensorProcessor): + r"""Processor that applies a numerical forward integration based on a middle Reimann sum. An integrator is used to calculate the integral of a signal over time. The integral of a signal is the area under the curve of the signal. The integral can be approximated using numerical methods @@ -213,14 +213,14 @@ class Integrator(ModifierBase): :math:`\Delta t` is the time step between samples. """ - def __init__(self, cfg: modifier_cfg.IntegratorCfg, data_dim: tuple[int, ...], device: str): + def __init__(self, cfg: tensor_processor_cfg.IntegratorCfg, data_dim: tuple[int, ...], device: str): """Initializes the integrator configuration and state. Args: cfg: Integral parameters. data_dim: The dimensions of the data to be modified. First element is the batch size which usually corresponds to number of environments in the simulation. - device: The device to run the modifier on. + device: The device to run the processor on. """ # initialize parent class super().__init__(cfg, data_dim, device) diff --git a/source/isaaclab/isaaclab/utils/modifiers/modifier_base.py b/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor.py similarity index 56% rename from source/isaaclab/isaaclab/utils/modifiers/modifier_base.py rename to source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor.py index ea0594fe093f..fb0320994c6f 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/modifier_base.py +++ b/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor.py @@ -12,57 +12,59 @@ import torch if TYPE_CHECKING: - from .modifier_cfg import ModifierCfg + from .tensor_processor_cfg import TensorProcessorCfg -class ModifierBase(ABC): - """Base class for modifiers implemented as classes. +class TensorProcessor(ABC): + """Base class for tensor_processors implemented as classes. - Modifiers implementations can be functions or classes. If a modifier is a class, it should + Tensor processors implementations can be functions or classes. If a processor is a class, it should inherit from this class and implement the required methods. - A class implementation of a modifier can be used to store state information between calls. - This is useful for modifiers that require stateful operations, such as rolling averages + A class implementation of a processor can be used to store state information between calls. + This is useful for tensor_processors that require stateful operations, such as rolling averages or delays or decaying filters. Example pseudo-code to create and use the class: .. code-block:: python - from isaaclab.utils import modifiers + from isaaclab.utils import tensor_processors - # define custom keyword arguments to pass to ModifierCfg + # define custom keyword arguments to pass to TensorProcessorCfg kwarg_dict = {"arg_1": VAL_1, "arg_2": VAL_2} - # create modifier configuration object - # func is the class name of the modifier and params is the dictionary of arguments - modifier_config = modifiers.ModifierCfg(func=modifiers.ModifierBase, params=kwarg_dict) + # create processor configuration object + # func is the class name of the processor and params is the dictionary of arguments + processor_config = tensor_processors.TensorProcessorCfg( + func=tensor_processors.TensorProcessor, params=kwarg_dict + ) - # define modifier instance - my_modifier = modifiers.ModifierBase(cfg=modifier_config) + # define processor instance + my_processor = MyProcessor(processor_config, data_dim=(32, 4), device="cpu") """ - def __init__(self, cfg: ModifierCfg, data_dim: tuple[int, ...], device: str) -> None: - """Initializes the modifier class. + def __init__(self, cfg: TensorProcessorCfg, data_dim: tuple[int, ...], device: str) -> None: + """Initializes the processor class. Args: cfg: Configuration parameters. data_dim: The dimensions of the data to be modified. First element is the batch size which usually corresponds to number of environments in the simulation. - device: The device to run the modifier on. + device: The device to run the processor on. """ self._cfg = cfg self._data_dim = data_dim self._device = device def close(self) -> None: - """Release resources owned by this modifier.""" + """Release resources owned by this processor.""" pass @abstractmethod def reset(self, env_ids: Sequence[int] | None = None): - """Resets the Modifier. + """Resets the Processor. Args: env_ids: The environment ids. Defaults to None, in which case diff --git a/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py b/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor_cfg.py similarity index 64% rename from source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py rename to source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor_cfg.py index b2f234e8217c..12068a56a67f 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py +++ b/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor_cfg.py @@ -14,43 +14,43 @@ from .. import configclass if TYPE_CHECKING: - from ..processing import ProcessingBufferStepCfg - from .modifier import DigitalFilter, Integrator - from .modifier_base import ModifierBase - from .processing_buffer_pipe import ProcessingBufferPipeModifier + from ..processing import DictTensorProcessorStepCfg + from .dict_tensor_processor_adapter import DictTensorProcessorAdapter + from .processors import DigitalFilter, Integrator + from .tensor_processor import TensorProcessor @configclass -class ModifierCfg: - """Configuration parameters for function and class modifiers.""" +class TensorProcessorCfg: + """Configuration parameters for function and class tensor processors.""" - func: Callable[..., torch.Tensor] | type[ModifierBase] | str = MISSING - """Function or :class:`ModifierBase` class used by the modifier. + func: Callable[..., torch.Tensor] | type[TensorProcessor] | str = MISSING + """Function or :class:`TensorProcessor` class used by the processor. - Functions must take a tensor as their first argument. Classes must inherit from :class:`ModifierBase`; the + Functions must take a tensor as their first argument. Classes must inherit from :class:`TensorProcessor`; the observation manager constructs them with the configuration, observation dimensions, and device. """ params: dict[str, Any] = {} - """Parameters used by the modifier. Defaults to an empty dictionary. + """Parameters used by the processor. Defaults to an empty dictionary. - Function modifiers receive them as keyword arguments on each call. Class modifiers access them through this + Functions receive them as keyword arguments on each call. Classes access them through this configuration during construction; they are not forwarded to the constructed instance. """ @configclass -class ProcessingBufferPipeModifierCfg(ModifierCfg): +class DictTensorProcessorAdapterCfg(TensorProcessorCfg): """Bind an incoming observation tensor to a buffer pipe. - Operations are prepared from the first tensor received after preceding modifiers. + Operations are prepared from the first tensor received after preceding processors. Their documented input requirements define the expected array layout and value semantics. """ - func: type[ProcessingBufferPipeModifier] | str = "{DIR}.processing_buffer_pipe:ProcessingBufferPipeModifier" - """Modifier class consuming the incoming tensor.""" + func: type[DictTensorProcessorAdapter] | str = "{DIR}.dict_tensor_processor_adapter:DictTensorProcessorAdapter" + """Processor class consuming the incoming tensor.""" - pipeline: list[ProcessingBufferStepCfg] = MISSING + pipeline: list[DictTensorProcessorStepCfg] = MISSING """Ordered buffer processing operations.""" input_name: str = "rgb" @@ -61,13 +61,13 @@ class ProcessingBufferPipeModifierCfg(ModifierCfg): @configclass -class DigitalFilterCfg(ModifierCfg): - """Configuration parameters for a digital filter modifier. +class DigitalFilterCfg(TensorProcessorCfg): + """Configuration parameters for a digital filter processor. For more information, please check the :class:`DigitalFilter` class. """ - func: type[DigitalFilter] | str = "{DIR}.modifier:DigitalFilter" + func: type[DigitalFilter] | str = "{DIR}.processors:DigitalFilter" """The digital filter function to be called for applying the filter.""" A: list[float] = MISSING @@ -92,13 +92,13 @@ class DigitalFilterCfg(ModifierCfg): @configclass -class IntegratorCfg(ModifierCfg): - """Configuration parameters for an integrator modifier. +class IntegratorCfg(TensorProcessorCfg): + """Configuration parameters for an integrator processor. For more information, please check the :class:`Integrator` class. """ - func: type[Integrator] | str = "{DIR}.modifier:Integrator" + func: type[Integrator] | str = "{DIR}.processors:Integrator" """The integrator function to be called for applying the integrator.""" dt: float = MISSING diff --git a/source/isaaclab/test/envs/test_buffer_observations.py b/source/isaaclab/test/envs/test_buffer_observations.py index fb2d03909696..b9259e9d5ddf 100644 --- a/source/isaaclab/test/envs/test_buffer_observations.py +++ b/source/isaaclab/test/envs/test_buffer_observations.py @@ -20,8 +20,8 @@ ObservationManager, ObservationTermCfg, ) -from isaaclab.utils import configclass, modifiers -from isaaclab.utils.processing import PreparedBufferOperation, ProcessingBufferStepCfg +from isaaclab.utils import configclass, tensor_processors +from isaaclab.utils.processing import DictTensorProcessorStepCfg, PreparedBufferOperation from isaaclab.utils.warp import ProxyArray pytestmark = pytest.mark.unit @@ -107,7 +107,7 @@ def close(self): def test_pipe_modifier_uses_preceding_modifier_output(): - """A pipe consumes the current non-image observation after preceding modifiers.""" + """A pipe consumes the current non-image observation after preceding tensor_processors.""" ranges = torch.ones((2, 4)) preparations = [] @@ -125,11 +125,11 @@ class Policy(ObservationGroupCfg): concatenate_terms = False ranges = ObservationTermCfg( func=_read_ranges, - modifiers=[ - modifiers.ModifierCfg(func=modifiers.bias, params={"value": 3.0}), - modifiers.ProcessingBufferPipeModifierCfg( + tensor_processors=[ + tensor_processors.TensorProcessorCfg(func=tensor_processors.bias, params={"value": 3.0}), + tensor_processors.DictTensorProcessorAdapterCfg( pipeline=[ - ProcessingBufferStepCfg( + DictTensorProcessorStepCfg( prepare=prepare, inputs={"ranges": "ranges"}, outputs={"ranges": "ranges"}, @@ -172,10 +172,10 @@ def prepare(inputs): ) return PreparedBufferOperation({"color": output}, operation.apply, close=lambda: lifecycle.append("close")) - cfg = modifiers.ProcessingBufferPipeModifierCfg( - pipeline=[ProcessingBufferStepCfg(prepare=prepare, inputs={"color": "rgb"}, outputs={"color": "rgb"})] + cfg = tensor_processors.DictTensorProcessorAdapterCfg( + pipeline=[DictTensorProcessorStepCfg(prepare=prepare, inputs={"color": "rgb"}, outputs={"color": "rgb"})] ) - modifier = modifiers.ProcessingBufferPipeModifier(cfg, tuple(image.shape), "cpu") + modifier = tensor_processors.DictTensorProcessorAdapter(cfg, tuple(image.shape), "cpu") modifier.reset([1]) with pytest.raises(ValueError, match="input shape"): modifier(image[:, :, ::2]) @@ -191,7 +191,7 @@ def prepare(inputs): modifier(image) assert lifecycle == ["prepare", "close", "prepare", "close"] - unused_modifier = modifiers.ProcessingBufferPipeModifier(cfg, tuple(image.shape), "cpu") + unused_modifier = tensor_processors.DictTensorProcessorAdapter(cfg, tuple(image.shape), "cpu") unused_modifier.close() with pytest.raises(RuntimeError, match="closed"): unused_modifier(image) @@ -231,7 +231,7 @@ def close(): requests = [] camera = SimpleNamespace( request_render_inputs=lambda names: requests.append(set(names)), - render_outputs={"rgb_radiance": ProxyArray(wp.from_torch(radiance)), "depth": ProxyArray(wp.from_torch(depth))}, + render_outputs={"rgb_hdr": ProxyArray(wp.from_torch(radiance)), "depth": ProxyArray(wp.from_torch(depth))}, ) sensors = {"camera": camera} env = _env(scene=SimpleNamespace(sensors=sensors, keys=sensors.keys), lifecycle=lifecycle) @@ -248,9 +248,9 @@ class Policy(ObservationGroupCfg): func="isaaclab.envs.mdp:processed_sensor", params={ "pipeline": [ - ProcessingBufferStepCfg( + DictTensorProcessorStepCfg( prepare=prepare, - inputs={"color": "rgb_radiance", "depth": "depth"}, + inputs={"color": "rgb_hdr", "depth": "depth"}, outputs={"color": "rgb"}, ) ] @@ -267,7 +267,7 @@ class Observations: cfg = Observations() ObservationManager.prepare_scene(cfg, env) - assert requests == [{"rgb_radiance", "depth"}] + assert requests == [{"rgb_hdr", "depth"}] manager = ObservationManager(cfg, env) assert manager.group_obs_dim["policy"] == [(2, 2, 3)] result = manager.compute()["policy"]["image"] diff --git a/source/isaaclab/test/managers/test_manager_base.py b/source/isaaclab/test/managers/test_manager_base.py index bd11ccfcfc27..a123273d3791 100644 --- a/source/isaaclab/test/managers/test_manager_base.py +++ b/source/isaaclab/test/managers/test_manager_base.py @@ -24,7 +24,7 @@ from isaaclab.managers import ManagerTermBase, ManagerTermBaseCfg, SceneEntityCfg from isaaclab.managers.manager_base import ManagerBase from isaaclab.physics import PhysicsEvent -from isaaclab.utils import configclass, modifiers, to_dict, update_from_dict +from isaaclab.utils import configclass, tensor_processors, to_dict, update_from_dict pytestmark = pytest.mark.unit @@ -84,7 +84,7 @@ class NestedFieldTermCfg(ManagerTermBaseCfg): """Manager term with owned and opaque fields outside ``params``.""" nested_term: ManagerTermBaseCfg = ManagerTermBaseCfg(func=increment_dummy1_by_one) - modifier: modifiers.ModifierCfg = modifiers.ModifierCfg(func=increment_dummy1_by_one) + modifier: tensor_processors.TensorProcessorCfg = tensor_processors.TensorProcessorCfg(func=increment_dummy1_by_one) metadata: dict[str, str] = {"func": f"{__name__}:reset_dummy2_to_zero"} opaque: OpaqueCfg = OpaqueCfg() diff --git a/source/isaaclab/test/managers/test_observation_manager.py b/source/isaaclab/test/managers/test_observation_manager.py index fe00e34d64fd..000f56e3b55f 100644 --- a/source/isaaclab/test/managers/test_observation_manager.py +++ b/source/isaaclab/test/managers/test_observation_manager.py @@ -23,7 +23,7 @@ ObservationTermCfg, RewardTermCfg, ) -from isaaclab.utils import DelayBuffer, configclass, modifiers, noise, to_dict, update_from_dict, validate +from isaaclab.utils import DelayBuffer, configclass, noise, tensor_processors, to_dict, update_from_dict, validate pytestmark = pytest.mark.unit @@ -580,12 +580,12 @@ class PolicyCfg(ObservationGroupCfg): @pytest.mark.parametrize("position_term", [pos_w_data, destination_position]) def test_modifier_compute(setup_env, position_term): env = setup_env - """Test the observation computation with modifiers.""" + """Test the observation computation with tensor_processors.""" - modifier_1 = modifiers.ModifierCfg(func=modifiers.bias, params={"value": 1.0}) - modifier_2 = modifiers.ModifierCfg(func=modifiers.scale, params={"multiplier": 2.0}) - modifier_3 = modifiers.ModifierCfg(func=modifiers.clip, params={"bounds": (-0.5, 0.5)}) - modifier_4 = modifiers.IntegratorCfg(dt=env.dt) + modifier_1 = tensor_processors.TensorProcessorCfg(func=tensor_processors.bias, params={"value": 1.0}) + modifier_2 = tensor_processors.TensorProcessorCfg(func=tensor_processors.scale, params={"multiplier": 2.0}) + modifier_3 = tensor_processors.TensorProcessorCfg(func=tensor_processors.clip, params={"bounds": (-0.5, 0.5)}) + modifier_4 = tensor_processors.IntegratorCfg(dt=env.dt) @configclass class MyObservationManagerCfg: @@ -596,19 +596,19 @@ class PolicyCfg(ObservationGroupCfg): """Test config class for policy observation group.""" concatenate_terms = False - term_1 = ObservationTermCfg(func=position_term, modifiers=[]) - term_2 = ObservationTermCfg(func=position_term, modifiers=[modifier_1]) - term_3 = ObservationTermCfg(func=position_term, modifiers=[modifier_1, modifier_4]) + term_1 = ObservationTermCfg(func=position_term, tensor_processors=[]) + term_2 = ObservationTermCfg(func=position_term, tensor_processors=[modifier_1]) + term_3 = ObservationTermCfg(func=position_term, tensor_processors=[modifier_1, modifier_4]) @configclass class CriticCfg(ObservationGroupCfg): """Test config class for critic observation group""" concatenate_terms = False - term_1 = ObservationTermCfg(func=position_term, modifiers=[]) - term_2 = ObservationTermCfg(func=position_term, modifiers=[modifier_1]) - term_3 = ObservationTermCfg(func=position_term, modifiers=[modifier_1, modifier_2]) - term_4 = ObservationTermCfg(func=position_term, modifiers=[modifier_1, modifier_2, modifier_3]) + term_1 = ObservationTermCfg(func=position_term, tensor_processors=[]) + term_2 = ObservationTermCfg(func=position_term, tensor_processors=[modifier_1]) + term_3 = ObservationTermCfg(func=position_term, tensor_processors=[modifier_1, modifier_2]) + term_4 = ObservationTermCfg(func=position_term, tensor_processors=[modifier_1, modifier_2, modifier_3]) policy: ObservationGroupCfg = PolicyCfg() critic: ObservationGroupCfg = CriticCfg() @@ -695,7 +695,7 @@ def test_modifier_invalid_config(setup_env): env = setup_env """Test modifier initialization with invalid config.""" - modifier = modifiers.ModifierCfg(func=modifiers.clip, params={"min": -0.5, "max": 0.5}) + modifier = tensor_processors.TensorProcessorCfg(func=tensor_processors.clip, params={"min": -0.5, "max": 0.5}) @configclass class MyObservationManagerCfg: @@ -706,7 +706,7 @@ class PolicyCfg(ObservationGroupCfg): """Test config class for policy observation group.""" concatenate_terms = False - term_1 = ObservationTermCfg(func=pos_w_data, modifiers=[modifier]) + term_1 = ObservationTermCfg(func=pos_w_data, tensor_processors=[modifier]) policy: ObservationGroupCfg = PolicyCfg() @@ -800,10 +800,10 @@ def __init__(self, num_envs: int = 2) -> None: self.observation = torch.arange(num_envs, dtype=torch.float32).unsqueeze(-1) -class StatefulBiasModifier(modifiers.ModifierBase): +class StatefulBiasModifier(tensor_processors.TensorProcessor): """Stateful modifier used to verify lazy callable resolution.""" - def __init__(self, cfg: modifiers.ModifierCfg, data_dim: tuple[int, ...], device: str) -> None: + def __init__(self, cfg: tensor_processors.TensorProcessorCfg, data_dim: tuple[int, ...], device: str) -> None: super().__init__(cfg, data_dim, device) self.value = cfg.params["value"] self.reset_count = 0 @@ -842,33 +842,37 @@ def test_class_modifier_roundtrip_preserves_func_and_params(): """Reproduce #6067 with a class modifier and non-empty parameters.""" cfg = HistoryObservationsCfg() cfg.policy.history_length = None - cfg.policy.dummy.modifiers = [modifiers.ModifierCfg(func=StatefulBiasModifier, params={"value": 2.0})] + cfg.policy.dummy.tensor_processors = [ + tensor_processors.TensorProcessorCfg(func=StatefulBiasModifier, params={"value": 2.0}) + ] update_from_dict(cfg, to_dict(cfg)) term_cfg = cfg.policy.dummy - assert term_cfg.modifiers is not None - modifier_cfg = term_cfg.modifiers[0] - assert isinstance(modifier_cfg, modifiers.ModifierCfg) - assert isinstance(modifier_cfg.func, str) - assert modifier_cfg.params == {"value": 2.0} + assert term_cfg.tensor_processors is not None + tensor_processor_cfg = term_cfg.tensor_processors[0] + assert isinstance(tensor_processor_cfg, tensor_processors.TensorProcessorCfg) + assert isinstance(tensor_processor_cfg.func, str) + assert tensor_processor_cfg.params == {"value": 2.0} env = DummyEnv() manager = ObservationManager(cfg, cast("ManagerBasedEnv", env)) prepared_term_cfg = manager.cfg.policy.dummy - assert prepared_term_cfg.modifiers is not None - prepared_modifier_cfg = prepared_term_cfg.modifiers[0] - assert isinstance(prepared_modifier_cfg.func, StatefulBiasModifier) + assert prepared_term_cfg.tensor_processors is not None + prepared_tensor_processor_cfg = prepared_term_cfg.tensor_processors[0] + assert isinstance(prepared_tensor_processor_cfg.func, StatefulBiasModifier) observations = manager.compute()["policy"] torch.testing.assert_close(observations, env.observation + 2.0) manager.reset() - assert prepared_modifier_cfg.func.reset_count == 1 + assert prepared_tensor_processor_cfg.func.reset_count == 1 -def test_stateless_modifier_cfg_roundtrip_preserves_signature_validation(): +def test_stateless_tensor_processor_cfg_roundtrip_preserves_signature_validation(): """A stateless modifier remains callable and inspectable after a configuration round-trip.""" cfg = HistoryObservationsCfg() cfg.policy.history_length = None - cfg.policy.dummy.modifiers = [modifiers.ModifierCfg(func=modifiers.bias, params={"value": 2.0})] + cfg.policy.dummy.tensor_processors = [ + tensor_processors.TensorProcessorCfg(func=tensor_processors.bias, params={"value": 2.0}) + ] update_from_dict(cfg, to_dict(cfg)) env = DummyEnv() @@ -881,10 +885,10 @@ def test_class_modifier_validates_constructed_instance(): """Class modifier validation checks the constructed object.""" cfg = HistoryObservationsCfg() cfg.policy.history_length = None - cfg.policy.dummy.modifiers = [modifiers.ModifierCfg(func=InvalidModifier)] + cfg.policy.dummy.tensor_processors = [tensor_processors.TensorProcessorCfg(func=InvalidModifier)] update_from_dict(cfg, to_dict(cfg)) - with pytest.raises(TypeError, match="is not an instance of 'ModifierBase'"): + with pytest.raises(TypeError, match="is not an instance of 'TensorProcessor'"): ObservationManager(cfg, cast("ManagerBasedEnv", DummyEnv())) diff --git a/source/isaaclab/test/renderers/test_rtx_camera_overrides.py b/source/isaaclab/test/renderers/test_camera_exposure.py similarity index 79% rename from source/isaaclab/test/renderers/test_rtx_camera_overrides.py rename to source/isaaclab/test/renderers/test_camera_exposure.py index 73fab8a08d23..2167edf3edb7 100644 --- a/source/isaaclab/test/renderers/test_rtx_camera_overrides.py +++ b/source/isaaclab/test/renderers/test_camera_exposure.py @@ -5,12 +5,16 @@ """Tests for the RTX camera-exposure overrides used for scene-linear color outputs.""" +import pytest + from pxr import Sdf, Usd, UsdGeom -from isaaclab.renderers.rtx_camera_overrides import _apply_rtx_exposure_overrides +from isaaclab.sensors.camera import CameraExposureCfg +from isaaclab.sensors.camera.camera_exposure import apply_camera_exposure -def test_apply_rtx_exposure_overrides_resets_authored_exposure_values(): +@pytest.mark.parametrize("exposure, iso", [(0.0, 0.0), (2.0, 200.0)]) +def test_camera_exposure_applies_explicit_settings(exposure, iso): """Scene-linear outputs require neutral RTX-side exposure on the camera prim.""" stage = Usd.Stage.CreateInMemory() cam = UsdGeom.Camera.Define(stage, "/World/Camera") @@ -23,11 +27,11 @@ def test_apply_rtx_exposure_overrides_resets_authored_exposure_values(): prim.CreateAttribute("exposure:time", Sdf.ValueTypeNames.Float).Set(1.0 / 60.0) prim.CreateAttribute("omni:rtx:autoExposure:enabled", Sdf.ValueTypeNames.Bool).Set(True) - _apply_rtx_exposure_overrides(stage, ["/World/Camera"]) + apply_camera_exposure(stage, ["/World/Camera"], CameraExposureCfg(exposure=exposure, iso=iso)) - assert prim.GetAttribute("exposure").Get() == 0.0 + assert prim.GetAttribute("exposure").Get() == exposure assert prim.GetAttribute("exposure:fStop").Get() == 1.0 - assert prim.GetAttribute("exposure:iso").Get() == 0.0 + assert prim.GetAttribute("exposure:iso").Get() == iso assert prim.GetAttribute("exposure:responsivity").Get() == 1.0 assert prim.GetAttribute("exposure:time").Get() == 1.0 assert prim.GetAttribute("omni:rtx:autoExposure:enabled").Get() is False diff --git a/source/isaaclab/test/renderers/test_camera_output_contract.py b/source/isaaclab/test/renderers/test_camera_output_contract.py index 3a4129bb47aa..8a25c4d2e0de 100644 --- a/source/isaaclab/test/renderers/test_camera_output_contract.py +++ b/source/isaaclab/test/renderers/test_camera_output_contract.py @@ -25,7 +25,7 @@ from isaaclab.sim import PinholeCameraCfg from isaaclab.test.utils import DeviceScope, test_devices from isaaclab.utils import clone, validate -from isaaclab.utils.processing import PreparedBufferOperation, ProcessingBufferStepCfg +from isaaclab.utils.processing import DictTensorProcessorStepCfg, PreparedBufferOperation from isaaclab.utils.warp import ProxyArray pytestmark = [pytest.mark.integration, pytest.mark.rendering] @@ -192,7 +192,6 @@ def test_newton_warp_supported_output_types_key_set(): RenderBufferKind.RGB, RenderBufferKind.RGBA, RenderBufferKind.RGB_HDR, - RenderBufferKind.RGB_RADIANCE, RenderBufferKind.ALBEDO, RenderBufferKind.DEPTH, RenderBufferKind.DISTANCE_TO_CAMERA, @@ -202,7 +201,6 @@ def test_newton_warp_supported_output_types_key_set(): RenderBufferKind.INSTANCE_SEGMENTATION, } assert specs[RenderBufferKind.RGB_HDR] == RenderBufferSpec(3, wp.float32) - assert specs[RenderBufferKind.RGB_RADIANCE] == specs[RenderBufferKind.RGB_HDR] def test_camera_cfg_rejects_outputs_unsupported_by_renderer(): @@ -248,7 +246,7 @@ def test_newton_warp_segmentation_spec_follows_colorize_flags(colorize): assert specs[kind] == expected -@pytest.mark.parametrize("data_type", [RenderBufferKind.RGB_HDR, RenderBufferKind.RGB_RADIANCE]) +@pytest.mark.parametrize("data_type", [RenderBufferKind.RGB_HDR]) def test_newton_warp_wraps_requested_hdr_output(data_type): """Newton wires both linear color outputs to its HDR output slot.""" pytest.importorskip("isaaclab_newton") @@ -280,13 +278,12 @@ def _make_camera_cfg(data_types: list[str]) -> CameraCfg: def test_camera_data_allocates_supported_subset_and_aliases_color(): - """CameraData allocates requested + supported outputs, aliasing rgb into rgba and rgb_hdr into rgb_radiance.""" - cfg = _make_camera_cfg(["rgb", "rgba", "rgb_hdr", "rgb_radiance", "depth", "instance_segmentation"]) + """CameraData allocates requested + supported outputs, aliasing rgb into rgba.""" + cfg = _make_camera_cfg(["rgb", "rgba", "rgb_hdr", "depth", "instance_segmentation"]) specs = { RenderBufferKind.RGBA: RenderBufferSpec(4, wp.uint8), RenderBufferKind.RGB: RenderBufferSpec(3, wp.uint8), RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), - RenderBufferKind.RGB_RADIANCE: RenderBufferSpec(3, wp.float32), RenderBufferKind.DEPTH: RenderBufferSpec(1, wp.float32), RenderBufferKind.NORMALS: RenderBufferSpec(3, wp.float32), } @@ -294,15 +291,14 @@ def test_camera_data_allocates_supported_subset_and_aliases_color(): data_types=cfg.data_types, height=8, width=16, num_views=2, device="cpu", supported_specs=specs ) - assert set(data.output.keys()) == {"rgba", "rgb", "rgb_hdr", "rgb_radiance", "depth"} + assert set(data.output.keys()) == {"rgba", "rgb", "rgb_hdr", "depth"} assert data.output["rgba"].shape == (2, 8, 16, 4) assert data.output["rgba"].dtype == wp.uint8 assert data.output["depth"].shape == (2, 8, 16, 1) assert data.output["depth"].dtype == wp.float32 assert data.output["rgb"].warp.ptr == data.output["rgba"].warp.ptr - assert data.output["rgb_hdr"] is data.output["rgb_radiance"] - assert data.output["rgb_radiance"].shape == (2, 8, 16, 3) - assert data.output["rgb_radiance"].dtype == wp.float32 + assert data.output["rgb_hdr"].shape == (2, 8, 16, 3) + assert data.output["rgb_hdr"].dtype == wp.float32 assert data.image_shape == (8, 16) assert data.info == dict.fromkeys(data.output) @@ -335,15 +331,14 @@ def test_every_camera_prepares_before_the_shared_stage_export(monkeypatch): from isaaclab.physics import PhysicsEvent from isaaclab.renderers import RendererCfg - from isaaclab.renderers.rtx_camera_overrides import _apply_rtx_exposure_overrides - from isaaclab.sensors.camera import Camera + from isaaclab.sensors.camera import Camera, CameraExposureCfg from isaaclab.sensors.camera import camera as camera_module from isaaclab.sensors.sensor_base import SensorBase specs = { RenderBufferKind.RGB: RenderBufferSpec(3, wp.uint8), RenderBufferKind.RGBA: RenderBufferSpec(4, wp.uint8), - RenderBufferKind.RGB_RADIANCE: RenderBufferSpec(3, wp.float32), + RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), } exported_iso = [] prepare_error = None @@ -351,8 +346,6 @@ def test_every_camera_prepares_before_the_shared_stage_export(monkeypatch): def prepare_cameras(stage, spec): if prepare_error is not None: raise prepare_error - if "rgb_radiance" in spec.cfg.data_types: - _apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) def export_stage(stage, num_envs): exported_iso.append(prims[1].GetAttribute("exposure:iso").Get()) @@ -387,7 +380,7 @@ def prepare_identity(inputs): def make_camera(prim, data_types): camera = Camera.__new__(Camera) pipeline = ( - [ProcessingBufferStepCfg(prepare_identity, {"image": "rgb_radiance"}, {"image": "processed_radiance"})] + [DictTensorProcessorStepCfg(prepare_identity, {"image": "rgb_hdr"}, {"image": "processed_radiance"})] if data_types == ["processed_radiance"] else [] ) @@ -398,6 +391,7 @@ def make_camera(prim, data_types): width=3, spawn=None, pipeline=pipeline, + exposure=CameraExposureCfg() if pipeline else None, renderer_cfg=renderer_cfg, ) camera._render_cfg = replace(camera.cfg, data_types=camera.cfg.render_data_types(), pipeline=[]) @@ -486,18 +480,18 @@ def close(): width=1, data_types=["rgb", "depth"], pipeline=[ - ProcessingBufferStepCfg(prepare_color, {"radiance": "rgb_radiance", "range": "depth"}, {"color": "rgba"}) + DictTensorProcessorStepCfg(prepare_color, {"radiance": "rgb_hdr", "range": "depth"}, {"color": "rgba"}) ], ) specs = { - RenderBufferKind.RGB_RADIANCE: RenderBufferSpec(3, wp.float32), + RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), RenderBufferKind.DEPTH: RenderBufferSpec(1, wp.float32), } capture = {"frame": 1} radiance = 0.2 def read_output(render_data, camera_data): - camera_data.output["rgb_radiance"].torch.fill_(radiance) + camera_data.output["rgb_hdr"].torch.fill_(radiance) camera_data.output["depth"].torch.fill_(0.5) for name in camera_data.info: camera_data.info[name] = {"capture": capture} @@ -530,9 +524,9 @@ def read_output(render_data, camera_data): camera._update_buffers_impl(mask) assert set(camera.data.output) == {"rgb", "depth"} - assert set(camera.render_outputs) == {"rgb_radiance", "depth"} + assert set(camera.render_outputs) == {"rgb_hdr", "depth"} torch.testing.assert_close(camera.data.output["rgb"].torch, torch.full((2, 1, 1, 3), 70, dtype=torch.uint8)) - torch.testing.assert_close(camera.render_outputs["rgb_radiance"].torch, torch.full((2, 1, 1, 3), 0.2)) + torch.testing.assert_close(camera.render_outputs["rgb_hdr"].torch, torch.full((2, 1, 1, 3), 0.2)) assert camera.data.output["depth"] is camera.render_outputs["depth"] assert camera.data.info["rgb"]["capture"] is capture @@ -583,9 +577,9 @@ def test_camera_requests_private_observation_inputs_before_initialization(): camera._render_outputs = {} camera._is_initialized = False camera._clear_callbacks = lambda: None - camera.request_render_inputs(["depth", "rgb_radiance"]) + camera.request_render_inputs(["depth", "rgb_hdr"]) assert camera.cfg.data_types == ["rgb"] - assert set(camera._render_cfg.data_types) == {"rgb", "depth", "rgb_radiance"} + assert set(camera._render_cfg.data_types) == {"rgb", "depth", "rgb_hdr"} camera._is_initialized = True camera.request_render_inputs(["depth"]) with pytest.raises(RuntimeError, match="before initialization"): diff --git a/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py b/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py index 945492c850e0..a7ec17715a4b 100644 --- a/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py +++ b/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py @@ -31,7 +31,7 @@ import isaaclab.sim as sim_utils from isaaclab.assets import AssetBaseCfg, RigidObjectCfg from isaaclab.scene import InteractiveScene, InteractiveSceneCfg -from isaaclab.sensors.camera import Camera, CameraCfg +from isaaclab.sensors.camera import Camera, CameraCfg, CameraExposureCfg from isaaclab.terrains import TerrainImporterCfg from isaaclab.utils import configclass @@ -727,6 +727,7 @@ def render_synthetic_gaussian_scene( width=width, data_types=data_types, spawn=None, + exposure=CameraExposureCfg(), pipeline=[PpispStepCfg(params={"cfg": isp_cfg})], renderer_cfg=renderer_cfg, ) @@ -819,6 +820,7 @@ def _render_synthetic_gaussian_camera( width=width, data_types=data_types, spawn=None, + exposure=CameraExposureCfg(), pipeline=[PpispStepCfg(params={"cfg": ppisp_cfg})], renderer_cfg=renderer_cfg, ) diff --git a/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py b/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py index f58b8eb9e93c..c40a7e79cbbb 100644 --- a/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py +++ b/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py @@ -39,7 +39,7 @@ and the PhysX backend requires Kit (``carb``) to bootstrap. * Requests ``"rgb_hdr"`` in ``data_types`` because the test asserts the raw HDR source with :func:`assert_ppisp_lifts_exposure`. The PPISP camera pipeline - requests its private ``"rgb_radiance"`` input automatically, so + requests its private ``"rgb_hdr"`` input automatically, so ``"rgb_hdr"`` is not required just to enable PPISP. """ @@ -91,7 +91,7 @@ def _ovrtx_ppisp_cfg() -> PpispCfg: """Expose the unexposed scene-linear Gaussian signal into the CRF's useful range.""" cfg = make_aggressive_ppisp_cfg() - # rgb_radiance has a center near 0.12; +2 EV places it near 0.48 before the + # rgb_hdr has a center near 0.12; +2 EV places it near 0.48 before the # response curve. The shared fixture's -5 EV targets legacy exposed HDR. cfg.inputs["exposureOffset"] = 2.0 return cfg diff --git a/source/isaaclab/test/utils/test_configclass.py b/source/isaaclab/test/utils/test_configclass.py index 7d53871de683..a8fbfb09ad13 100644 --- a/source/isaaclab/test/utils/test_configclass.py +++ b/source/isaaclab/test/utils/test_configclass.py @@ -79,7 +79,7 @@ def double(x): @configclass -class ModifierCfg: +class TensorProcessorCfg: params: dict[str, Any] = {"A": 1, "B": 2} @@ -111,7 +111,7 @@ class BasicDemoCfg: device_id: int = 0 env: EnvCfg = EnvCfg() robot_default_state: RobotDefaultStateCfg = RobotDefaultStateCfg() - list_config = [ModifierCfg(), ModifierCfg(params={"A": 3, "B": 4})] + list_config = [TensorProcessorCfg(), TensorProcessorCfg(params={"A": 3, "B": 4})] @configclass diff --git a/source/isaaclab/test/utils/test_modifiers.py b/source/isaaclab/test/utils/test_modifiers.py index c131ce76addf..7ac0700b15a5 100644 --- a/source/isaaclab/test/utils/test_modifiers.py +++ b/source/isaaclab/test/utils/test_modifiers.py @@ -8,7 +8,7 @@ import pytest import torch -import isaaclab.utils.modifiers as modifiers +import isaaclab.utils.tensor_processors as tensor_processors from isaaclab.test.utils import test_devices from isaaclab.utils import configclass @@ -17,9 +17,9 @@ @configclass class ModifierTestCfg: - """Configuration for testing modifiers.""" + """Configuration for testing tensor_processors.""" - cfg: modifiers.ModifierCfg = MISSING + cfg: tensor_processors.TensorProcessorCfg = MISSING init_data: torch.Tensor = MISSING result: torch.Tensor = MISSING num_iter: int = 10 @@ -28,17 +28,17 @@ class ModifierTestCfg: @pytest.mark.parametrize( "func, params, result", [ - (modifiers.scale, {"multiplier": 2.0}, [2.0, 4.0, 6.0]), - (modifiers.bias, {"value": 1.0}, [2.0, 3.0, 4.0]), - (modifiers.clip, {"bounds": (1.5, 2.5)}, [1.5, 2.0, 2.5]), - (modifiers.clip, {"bounds": (1.5, None)}, [1.5, 2.0, 3.0]), - (modifiers.clip, {"bounds": (None, 2.5)}, [1.0, 2.0, 2.5]), + (tensor_processors.scale, {"multiplier": 2.0}, [2.0, 4.0, 6.0]), + (tensor_processors.bias, {"value": 1.0}, [2.0, 3.0, 4.0]), + (tensor_processors.clip, {"bounds": (1.5, 2.5)}, [1.5, 2.0, 2.5]), + (tensor_processors.clip, {"bounds": (1.5, None)}, [1.5, 2.0, 3.0]), + (tensor_processors.clip, {"bounds": (None, 2.5)}, [1.0, 2.0, 2.5]), ], ids=["scale", "bias", "clip", "clip_no_upper_bound", "clip_no_lower_bound"], ) -def test_stateless_modifiers(func, params, result): - """Test the stateless scale, bias, and clip modifiers.""" - cfg = modifiers.ModifierCfg(func=func, params=params) +def test_stateless_tensor_processors(func, params, result): + """Test the stateless scale, bias, and clip tensor_processors.""" + cfg = tensor_processors.TensorProcessorCfg(func=func, params=params) output = cfg.func(torch.tensor([1.0, 2.0, 3.0]), **cfg.params) assert torch.allclose(output, torch.tensor(result)) @@ -54,7 +54,7 @@ def test_digital_filter(device): # create test config test_cfg = ModifierTestCfg( - cfg=modifiers.DigitalFilterCfg(A=A, B=B), init_data=init_data, result=result, num_iter=16 + cfg=tensor_processors.DigitalFilterCfg(A=A, B=B), init_data=init_data, result=result, num_iter=16 ) # create a modifier instance @@ -89,7 +89,7 @@ def test_integral(device): # create test config test_cfg = ModifierTestCfg( - cfg=modifiers.IntegratorCfg(dt=dt), + cfg=tensor_processors.IntegratorCfg(dt=dt), init_data=init_data, result=result, num_iter=6, diff --git a/source/isaaclab/test/utils/test_processing_buffer_pipe.py b/source/isaaclab/test/utils/test_processing_buffer_pipe.py index 5278aec38c9d..a0200679795b 100644 --- a/source/isaaclab/test/utils/test_processing_buffer_pipe.py +++ b/source/isaaclab/test/utils/test_processing_buffer_pipe.py @@ -11,7 +11,7 @@ import torch import warp as wp -from isaaclab.utils.processing import PreparedBufferOperation, ProcessingBufferPipe, ProcessingBufferStepCfg +from isaaclab.utils.processing import DictTensorProcessor, DictTensorProcessorStepCfg, PreparedBufferOperation from isaaclab.utils.warp import ProxyArray pytestmark = pytest.mark.unit @@ -48,15 +48,15 @@ def composite_pipe(): "range": ProxyArray(wp.zeros((2, 2, 2, 1), dtype=wp.float32, device="cpu")), } steps = [ - ProcessingBufferStepCfg(prepare=_prepare_scale, inputs={"value": "raw_color"}, outputs={"value": "color"}), - ProcessingBufferStepCfg( + DictTensorProcessorStepCfg(prepare=_prepare_scale, inputs={"value": "raw_color"}, outputs={"value": "color"}), + DictTensorProcessorStepCfg( prepare=_prepare_composite, inputs={"color": "color", "depth": "range"}, outputs={"value": "color"}, ), ] - assert ProcessingBufferPipe.required_source_names(steps, ["color", "labels"]) == {"raw_color", "range", "labels"} - pipe = ProcessingBufferPipe(sources, steps) + assert DictTensorProcessor.required_source_names(steps, ["color", "labels"]) == {"raw_color", "range", "labels"} + pipe = DictTensorProcessor(sources, steps) yield pipe pipe.close() @@ -115,7 +115,7 @@ def test_tensor_adapter_preserves_incoming_transformations_and_returns_snapshots def test_tensor_snapshot_accepts_passthrough_buffer_omitted_from_previous_call(): """Optional pass-through buffers can return after a call that did not supply them.""" original = ProxyArray(wp.zeros((2,), dtype=wp.float32, device="cpu")) - pipe = ProcessingBufferPipe({"value": original}, []) + pipe = DictTensorProcessor({"value": original}, []) try: pipe({}) current = ProxyArray(wp.full((2,), 3.0, dtype=wp.float32, device="cpu")) @@ -136,9 +136,9 @@ def prepare(inputs): destination = ProxyArray(wp.empty_like(inputs["value"].warp)) return PreparedBufferOperation({"value": destination}, lambda *_: calls.append(True)) - pipe = ProcessingBufferPipe( + pipe = DictTensorProcessor( {"source": source}, - [ProcessingBufferStepCfg(prepare=prepare, inputs={"value": "source"}, outputs={"value": "result"})], + [DictTensorProcessorStepCfg(prepare=prepare, inputs={"value": "source"}, outputs={"value": "result"})], ) try: shape = (1, 1, 1, 1) if invalid == "shape" else source.shape @@ -163,10 +163,10 @@ def prepare(inputs, *, name): destination = destination.warp return PreparedBufferOperation({"value": destination}, lambda *_: None, close=lambda: closed.append(name)) - first = ProcessingBufferStepCfg( + first = DictTensorProcessorStepCfg( prepare=prepare, inputs={"value": "source"}, outputs={"value": "middle"}, params={"name": "first"} ) - second = ProcessingBufferStepCfg( + second = DictTensorProcessorStepCfg( prepare=prepare, inputs={"value": "middle"}, outputs={"value": "result"}, params={"name": "second"} ) if invalid == "missing_dependency": @@ -181,7 +181,7 @@ def prepare(inputs, *, name): else: message = "outputs must be ProxyArray" with pytest.raises(TypeError if invalid == "output_type" else ValueError, match=message): - ProcessingBufferPipe({"source": source}, [first, second]) + DictTensorProcessor({"source": source}, [first, second]) assert closed == (["second", "first"] if invalid in ("output_roles", "output_type") else ["first"]) @@ -209,10 +209,10 @@ def close(): return PreparedBufferOperation({"value": destination}, apply, reset=reset, close=close) - pipe = ProcessingBufferPipe( + pipe = DictTensorProcessor( {"value": source}, [ - ProcessingBufferStepCfg( + DictTensorProcessorStepCfg( prepare=prepare, inputs={"value": "value"}, outputs={"value": "value"}, params={"name": name} ) for name in ("first", "second") diff --git a/source/isaaclab_experimental/changelog.d/dict-tensor-processors.changed.rst b/source/isaaclab_experimental/changelog.d/dict-tensor-processors.changed.rst new file mode 100644 index 000000000000..be55076a3cac --- /dev/null +++ b/source/isaaclab_experimental/changelog.d/dict-tensor-processors.changed.rst @@ -0,0 +1,2 @@ +* **Breaking:** Renamed the modifiers API to ``tensor_processors``, ``TensorProcessor``, and + ``TensorProcessorCfg``. Use ``ObservationTermCfg.tensor_processors`` for observation processing. diff --git a/source/isaaclab_experimental/changelog.d/dict-tensor-processors.major b/source/isaaclab_experimental/changelog.d/dict-tensor-processors.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_experimental/isaaclab_experimental/managers/observation_manager.py b/source/isaaclab_experimental/isaaclab_experimental/managers/observation_manager.py index 051aaa8a6ff6..1477eca19793 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/managers/observation_manager.py +++ b/source/isaaclab_experimental/isaaclab_experimental/managers/observation_manager.py @@ -59,7 +59,7 @@ from isaaclab.managers.manager_term_cfg import ObservationGroupCfg, ObservationTermCfg from isaaclab.utils import instantiate, to_dict -from isaaclab_experimental.utils import modifiers, noise +from isaaclab_experimental.utils import noise, tensor_processors from isaaclab_experimental.utils.buffers import CircularBuffer from isaaclab_experimental.utils.torch_utils import clone_obs_buffer @@ -350,7 +350,7 @@ def _collect_io_descriptors(self, group_names_to_export: list[str] = ["policy"]) func(self._env, **term_cfg.params, inspect=True) desc = func._descriptor.__dict__.copy() overloads = {} - for k in ["modifiers", "clip", "scale", "history_length", "flatten_history_dim"]: + for k in ["tensor_processors", "clip", "scale", "history_length", "flatten_history_dim"]: if hasattr(term_cfg, k): overloads[k] = getattr(term_cfg, k) desc.update(overloads) @@ -374,7 +374,7 @@ def _collect_io_descriptors(self, group_names_to_export: list[str] = ["policy"]) v = v.detach().cpu().numpy().tolist() if k in ["scale", "clip", "history_length", "flatten_history_dim"]: formatted_item["overloads"][k] = v - elif k in ["modifiers", "description", "units"]: + elif k in ["tensor_processors", "description", "units"]: formatted_item["extras"][k] = v else: formatted_item[k] = v @@ -409,7 +409,7 @@ def reset( for term_name in self._group_obs_term_names[group_name]: if term_name in self._group_obs_term_history_buffer[group_name]: self._group_obs_term_history_buffer[group_name][term_name].reset(env_mask=env_mask) - # call all modifiers/noise models that are classes + # call all tensor_processors/noise models that are classes for mod in self._group_obs_class_instances: mod.reset(env_mask=env_mask) @@ -463,7 +463,7 @@ def compute_group(self, group_name: str, update_history: bool = False) -> torch. The following steps are performed for each observation term: 1. Compute observation term by calling the function - 2. Apply custom modifiers in the order specified in :attr:`ObservationTermCfg.modifiers` + 2. Apply custom tensor_processors in the order specified in :attr:`ObservationTermCfg.tensor_processors` 3. Apply corruption/noise model based on :attr:`ObservationTermCfg.noise` 4. Apply clipping based on :attr:`ObservationTermCfg.clip` 5. Apply scaling based on :attr:`ObservationTermCfg.scale` @@ -500,14 +500,14 @@ def compute_group(self, group_name: str, update_history: bool = False) -> torch. # Persistent per-term obs dict (pre-allocated in _prepare_terms). group_obs = self._group_obs_dict[group_name] - # evaluate terms: compute, add noise, clip, scale, custom modifiers + # evaluate terms: compute, add noise, clip, scale, custom tensor_processors for term_name, term_cfg in zip(group_term_names, self._group_obs_term_cfgs[group_name]): # compute term's value into pre-allocated Warp output term_cfg.func(self._env, term_cfg.out_wp, **term_cfg.params) - # apply custom modifiers (in-place on out_wp) - if term_cfg.modifiers is not None: - for modifier in term_cfg.modifiers: + # apply custom tensor_processors (in-place on out_wp) + if term_cfg.tensor_processors is not None: + for modifier in term_cfg.tensor_processors: modifier.func(term_cfg.out_wp, **modifier.params) # apply noise (Warp in-place on out_wp) @@ -605,9 +605,9 @@ def _prepare_terms(self): # noqa: C901 self._group_obs_concatenate_dim: dict[str, int] = dict() self._group_obs_term_history_buffer: dict[str, dict] = dict() - # create a list to store classes instances, e.g., for modifiers and noise models + # create a list to store classes instances, e.g., for tensor_processors and noise models # we store it as a separate list to only call reset on them and prevent unnecessary calls - self._group_obs_class_instances: list[modifiers.ModifierBase | noise.NoiseModel] = list() + self._group_obs_class_instances: list[tensor_processors.TensorProcessor | noise.NoiseModel] = list() # Persistent Warp output buffers for concatenated 2D groups (optional fast-path). # For other cases (non-concat groups, history outputs, non-2D concat dims), we allocate per-term outputs. @@ -735,27 +735,27 @@ def _prepare_terms(self): # noqa: C901 ) term_cfg.scale_wp = wp.array(scale_vals, dtype=wp.float32, device=self._env.device) - # prepare modifiers for each observation - if term_cfg.modifiers is not None: - # initialize list of modifiers for term - for mod_cfg in term_cfg.modifiers: + # prepare tensor_processors for each observation + if term_cfg.tensor_processors is not None: + # initialize list of tensor_processors for term + for mod_cfg in term_cfg.tensor_processors: # check if class modifier and initialize with observation size when adding - if isinstance(mod_cfg, modifiers.ModifierCfg): - # to list of modifiers + if isinstance(mod_cfg, tensor_processors.TensorProcessorCfg): + # to list of tensor_processors if inspect.isclass(mod_cfg.func): - if not issubclass(mod_cfg.func, modifiers.ModifierBase): + if not issubclass(mod_cfg.func, tensor_processors.TensorProcessor): raise TypeError( f"Modifier function '{mod_cfg.func}' for observation term '{term_name}'" - f" is not a subclass of 'ModifierBase'. Received: '{type(mod_cfg.func)}'." + f" is not a subclass of 'TensorProcessor'. Received: '{type(mod_cfg.func)}'." ) mod_cfg.func = mod_cfg.func(cfg=mod_cfg, data_dim=obs_dims, device=self._env.device) - # add to list of class modifiers + # add to list of class tensor_processors self._group_obs_class_instances.append(mod_cfg.func) else: raise TypeError( f"Modifier configuration '{mod_cfg}' of observation term '{term_name}' is not of" - f" required type ModifierCfg, Received: '{type(mod_cfg)}'" + f" required type TensorProcessorCfg, Received: '{type(mod_cfg)}'" ) # check if function is callable diff --git a/source/isaaclab_experimental/isaaclab_experimental/utils/__init__.pyi b/source/isaaclab_experimental/isaaclab_experimental/utils/__init__.pyi index de2860732514..f4aaa2e64681 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/utils/__init__.pyi +++ b/source/isaaclab_experimental/isaaclab_experimental/utils/__init__.pyi @@ -9,7 +9,7 @@ __all__ = [ "WarpGraphCache", "clone_obs_buffer", "buffers", - "modifiers", + "tensor_processors", "noise", "warp", ] @@ -17,4 +17,4 @@ __all__ = [ from .manager_call_switch import ManagerCallMode, ManagerCallSwitch from .torch_utils import clone_obs_buffer from .warp_graph_cache import WarpGraphCache -from . import buffers, modifiers, noise, warp +from . import buffers, tensor_processors, noise, warp diff --git a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/__init__.py b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/__init__.py similarity index 50% rename from source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/__init__.py rename to source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/__init__.py index 3563b166642a..cfe63ea07f87 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/__init__.py +++ b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/__init__.py @@ -3,19 +3,19 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Warp-native modifier implementations (experimental). +"""Warp-native processor implementations (experimental). Re-exports stable configs and base classes, then overrides the function-based -modifiers (``scale``, ``bias``, ``clip``) with Warp-native versions that +tensor processors (``scale``, ``bias``, ``clip``) with Warp-native versions that operate in-place on ``wp.array``. Calling convention (matches Warp MDP terms):: - modifier.func(data_wp, **params) -> None # in-place on wp.array + processor.func(data_wp, **params) -> None # in-place on wp.array """ -from .modifier import bias, clip, scale # noqa: F401 -from .modifier_base import ModifierBase # noqa: F401 +from .processors import bias, clip, scale # noqa: F401 +from .tensor_processor import TensorProcessor # noqa: F401 # Override with Warp-native implementations -from .modifier_cfg import ModifierCfg # noqa: F401 +from .tensor_processor_cfg import TensorProcessorCfg # noqa: F401 diff --git a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier.py b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/processors.py similarity index 83% rename from source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier.py rename to source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/processors.py index 887db5e4ee06..1980f53ca093 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier.py +++ b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/processors.py @@ -3,12 +3,12 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Warp-native function-based modifiers (experimental). +"""Warp-native function-based tensor_processors (experimental). -Each modifier takes a ``wp.array`` as its first argument and operates **in-place** +Each processor takes a ``wp.array`` as its first argument and operates **in-place** via ``wp.launch``. The calling convention mirrors Warp MDP terms:: - modifier.func(data_wp, **params) -> None + processor.func(data_wp, **params) -> None """ from __future__ import annotations @@ -27,7 +27,7 @@ def _scale_kernel(data: wp.array(dtype=wp.float32, ndim=2), multiplier: wp.float def scale(data: wp.array, multiplier: float) -> None: """Scale all elements of *data* by *multiplier* in-place. - Warp-native drop-in replacement for :func:`isaaclab.utils.modifiers.scale`. + Warp-native drop-in replacement for :func:`isaaclab.utils.tensor_processors.scale`. Args: data: The observation buffer to modify. Shape ``(num_envs, D)``. @@ -48,7 +48,7 @@ def _bias_kernel(data: wp.array(dtype=wp.float32, ndim=2), value: wp.float32): def bias(data: wp.array, value: float) -> None: """Add a uniform *value* to all elements of *data* in-place. - Warp-native drop-in replacement for :func:`isaaclab.utils.modifiers.bias`. + Warp-native drop-in replacement for :func:`isaaclab.utils.tensor_processors.bias`. Args: data: The observation buffer to modify. Shape ``(num_envs, D)``. @@ -69,7 +69,7 @@ def _clip_kernel(data: wp.array(dtype=wp.float32, ndim=2), lo: wp.float32, hi: w def clip(data: wp.array, bounds: tuple[float | None, float | None]) -> None: """Clamp all elements of *data* to [lo, hi] in-place. - Warp-native drop-in replacement for :func:`isaaclab.utils.modifiers.clip`. + Warp-native drop-in replacement for :func:`isaaclab.utils.tensor_processors.clip`. Args: data: The observation buffer to modify. Shape ``(num_envs, D)``. diff --git a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier_base.py b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor.py similarity index 67% rename from source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier_base.py rename to source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor.py index 083e1576e891..8b393460c456 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier_base.py +++ b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Warp-native modifier base class (experimental).""" +"""Warp-native processor base class (experimental).""" from __future__ import annotations @@ -13,29 +13,29 @@ import warp as wp if TYPE_CHECKING: - from .modifier_cfg import ModifierCfg + from .tensor_processor_cfg import TensorProcessorCfg -class ModifierBase(ABC): - """Base class for Warp-native class-based modifiers. +class TensorProcessor(ABC): + """Base class for Warp-native class-based tensor_processors. - Experimental fork of :class:`isaaclab.utils.modifiers.ModifierBase` adapted for the + Experimental fork of :class:`isaaclab.utils.tensor_processors.TensorProcessor` adapted for the Warp-first calling convention. Subclasses operate **in-place** on ``wp.array`` buffers and return ``None``. - A class implementation of a modifier can be used to store state information between - calls. This is useful for modifiers that require stateful operations, such as + A class implementation of a processor can be used to store state information between + calls. This is useful for tensor_processors that require stateful operations, such as rolling averages, delays, or decaying filters. """ - def __init__(self, cfg: ModifierCfg, data_dim: tuple[int, ...], device: str) -> None: - """Initializes the modifier class. + def __init__(self, cfg: TensorProcessorCfg, data_dim: tuple[int, ...], device: str) -> None: + """Initializes the processor class. Args: cfg: Configuration parameters. data_dim: The dimensions of the data to be modified. First element is the batch size (number of environments). - device: The device to run the modifier on. + device: The device to run the processor on. """ self._cfg = cfg self._data_dim = data_dim @@ -43,7 +43,7 @@ def __init__(self, cfg: ModifierCfg, data_dim: tuple[int, ...], device: str) -> @abstractmethod def reset(self, env_mask: wp.array | None = None): - """Resets the modifier. + """Resets the processor. Args: env_mask: Boolean env mask of shape ``(num_envs,)`` selecting environments diff --git a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier_cfg.py b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor_cfg.py similarity index 65% rename from source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier_cfg.py rename to source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor_cfg.py index 7cc22eaee6a9..26bb86e33e25 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier_cfg.py +++ b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor_cfg.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Warp-native modifier configuration (experimental).""" +"""Warp-native processor configuration (experimental).""" from collections.abc import Callable from dataclasses import MISSING @@ -13,23 +13,23 @@ @configclass -class ModifierCfg: - """Configuration parameters for Warp-native modifiers. +class TensorProcessorCfg: + """Configuration parameters for Warp-native tensor processors. - Experimental fork of :class:`isaaclab.utils.modifiers.ModifierCfg` adapted for the - Warp-first calling convention where modifier functions operate **in-place** on a + Experimental fork of :class:`isaaclab.utils.tensor_processors.TensorProcessorCfg` adapted for the + Warp-first calling convention where processor functions operate **in-place** on a ``wp.array`` buffer and return ``None``. """ func: Callable[..., None] = MISSING - """Function or callable class used by modifier. + """Function or callable class used by processor. The function must take a ``wp.array`` as the first argument and operate on it **in-place** (no return value). The remaining arguments are specified in the :attr:`params` attribute. It also supports callable classes that implement ``__call__()``. In this case the - class should inherit from :class:`ModifierBase` and implement the required methods. + class should inherit from :class:`TensorProcessor` and implement the required methods. """ params: dict[str, Any] = dict() diff --git a/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst index 6e9cc7742b12..d3f07a2f90bb 100644 --- a/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst +++ b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst @@ -1,3 +1,3 @@ -* **Breaking:** Moved PPISP processing from the renderer to ``CameraCfg.pipeline``. - Replace ``CameraCfg.isp_cfg`` with ``PpispStepCfg``; use ``rgb_radiance`` for its scene-linear - input, equivalent to ``rgb_hdr`` on Newton. +* **Breaking:** Replaced renderer-owned PPISP with ``DictTensorProcessor`` operations. + Replace ``CameraCfg.isp_cfg`` with ``PpispStepCfg`` and configure + ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py index 609beeb276f8..6e26ae05b3ce 100644 --- a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py +++ b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py @@ -83,9 +83,8 @@ class RenderData: # :class:`~isaaclab_newton.renderers.segmentation.NewtonSegmentationMapper`. _OUTPUT_MAP: dict[str, tuple[str, type]] = { str(RenderBufferKind.RGBA): ("color_image", wp.uint32), + # Newton applies no camera exposure, so its HDR color is already scene-linear. str(RenderBufferKind.RGB_HDR): ("hdr_color_image", wp.vec3f), - # Newton applies no camera exposure, so its HDR color is already pre-exposure radiance. - str(RenderBufferKind.RGB_RADIANCE): ("hdr_color_image", wp.vec3f), str(RenderBufferKind.ALBEDO): ("albedo_image", wp.uint32), str(RenderBufferKind.NORMALS): ("normals_image", wp.vec3f), } @@ -241,7 +240,7 @@ def get_output(self, output_name: str) -> wp.array: return self._seg_dests[output_name][0] elif output_name == RenderBufferKind.RGBA: return self.outputs.color_image - elif output_name in (RenderBufferKind.RGB_HDR, RenderBufferKind.RGB_RADIANCE): + elif output_name == RenderBufferKind.RGB_HDR: return self.outputs.hdr_color_image elif output_name == RenderBufferKind.ALBEDO: return self.outputs.albedo_image diff --git a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py index 6b1d93e4c1c4..cb828f3cee92 100644 --- a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py +++ b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py @@ -136,7 +136,6 @@ def segmentation_spec(colorize: bool) -> RenderBufferSpec: RenderBufferKind.RGBA: RenderBufferSpec(4, wp.uint8), RenderBufferKind.RGB: RenderBufferSpec(3, wp.uint8), RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), - RenderBufferKind.RGB_RADIANCE: RenderBufferSpec(3, wp.float32), RenderBufferKind.ALBEDO: RenderBufferSpec(4, wp.uint8), RenderBufferKind.DEPTH: RenderBufferSpec(1, wp.float32), RenderBufferKind.DISTANCE_TO_CAMERA: RenderBufferSpec(1, wp.float32), diff --git a/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst index 6cbb5cc3bc3c..d3f07a2f90bb 100644 --- a/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst +++ b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst @@ -1,3 +1,3 @@ -* **Breaking:** Moved PPISP processing from the renderer to ``CameraCfg.pipeline``. - Replace ``CameraCfg.isp_cfg`` with ``PpispStepCfg``; its ``rgb_radiance`` input neutralizes - exposure for all color outputs sharing the camera prim. +* **Breaking:** Replaced renderer-owned PPISP with ``DictTensorProcessor`` operations. + Replace ``CameraCfg.isp_cfg`` with ``PpispStepCfg`` and configure + ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index e1d2d896d27f..964865597dc8 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -74,7 +74,6 @@ from isaaclab.cloner import ClonePlan from isaaclab.cloner import path as cloner_path from isaaclab.renderers import BaseRenderer, RenderBufferKind, RenderBufferSpec -from isaaclab.renderers.rtx_camera_overrides import _apply_rtx_exposure_overrides from isaaclab.scene_data import SceneDataFormat from isaaclab.sim import SimulationContext from isaaclab.utils.warp import ProxyArray @@ -416,16 +415,6 @@ def visual_material_writer(self): """Return a writer factory once this renderer has a scene consumer.""" return self._create_visual_material_writer if self._initialized_scene else None - def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: - """Neutralize camera exposure when ``rgb_radiance`` is requested. - - The RTX core OVRTX embeds has no pre-exposure radiance output, so neutral ``exposure:*`` - values and ``OmniRtxCameraExposureAPI_1`` are authored on the camera prims. Otherwise, the - camera prim's authored exposure is left alone. - """ - if "rgb_radiance" in spec.cfg.data_types and spec.camera_prim_paths: - _apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) - def prepare_stage(self, stage: Any, num_envs: int) -> None: """Prepare the USD stage for OVRTX before :meth:`create_render_data`. @@ -863,10 +852,6 @@ def set_outputs(self, render_data: OVRTXCameraRenderData, output_data: dict[str, render_data.warp_buffers = { name: proxy.warp for name, proxy in output_data.items() if name != str(RenderBufferKind.RGB) } - # Radiance reads the same HdrColor var; exposure is neutralized in :meth:`prepare_cameras`. - radiance = render_data.warp_buffers.get(str(RenderBufferKind.RGB_RADIANCE)) - if radiance is not None: - render_data.warp_buffers.setdefault(str(RenderBufferKind.RGB_HDR), radiance) def _update_camera_legacy( self, diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py index 6b5958caf040..12eba4d96726 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py @@ -133,7 +133,6 @@ def segmentation_spec(colorize: bool) -> RenderBufferSpec: RenderBufferKind.RGBA: RenderBufferSpec(4, wp.uint8), RenderBufferKind.RGB: RenderBufferSpec(3, wp.uint8), RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), - RenderBufferKind.RGB_RADIANCE: RenderBufferSpec(3, wp.float32), RenderBufferKind.ALBEDO: RenderBufferSpec(4, wp.uint8), RenderBufferKind.SIMPLE_SHADING_CONSTANT_DIFFUSE: RenderBufferSpec(3, wp.uint8), RenderBufferKind.SIMPLE_SHADING_DIFFUSE_MDL: RenderBufferSpec(3, wp.uint8), diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py index f74190e45f0e..3883f1e9a91f 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py @@ -32,7 +32,6 @@ "simple_shading_diffuse_mdl": ("LdrColor", "LdrColor"), "simple_shading_full_mdl": ("LdrColor", "LdrColor"), "rgb_hdr": ("HdrColor", "HdrColor"), - "rgb_radiance": ("HdrColor", "HdrColor"), "albedo": ("albedo", "DiffuseAlbedoSD"), "depth": ("depth", "DistanceToImagePlaneSD"), "distance_to_image_plane": ("depth", "DistanceToImagePlaneSD"), @@ -198,7 +197,7 @@ def build_render_product_as_string( # Workaround: ovrtx 0.5 misses GPU-read transform changes, so eco mode freezes output after 500 frames. render_mode_lines.append("bool omni:rtx:rt:ecoMode:enabled = false") api_schemas = ["OmniRtxSettingsCommonAdvancedAPI_1"] - if not {"rgb_hdr", "rgb_radiance"}.isdisjoint(data_types): + if "rgb_hdr" in data_types: # OVRTX 0.5 reads this per product. Its default bypasses HdrColor for Gaussian pixels. api_schemas.append("OmniRtxSettingsParticleFieldAPI_1") render_mode_lines.append("bool omni:rtx:rtpt:gaussian:skipTonemapping:enabled = false") diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index 1f8ad68e193c..bb473c1815be 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -537,7 +537,7 @@ def capture(value, indices): assert renderer._camera_render_data == [] -@pytest.mark.parametrize("data_types", [["rgb_hdr"], ["rgb_radiance"], ["rgb_hdr", "rgb_radiance"]]) +@pytest.mark.parametrize("data_types", [["rgb_hdr"]]) def test_ovrtx_hdr_color_fills_requested_hdr_outputs(monkeypatch, data_types): """Radiance and HDR read one HdrColor var into the camera's own buffer.""" renderer = _make_ovrtx_renderer_without_backend() @@ -563,9 +563,9 @@ def test_ovrtx_hdr_color_fills_requested_hdr_outputs(monkeypatch, data_types): np.testing.assert_array_equal(data.output[name].warp.numpy(), 2.0) -@pytest.mark.parametrize("data_types", [["rgb_hdr"], ["rgb_radiance"]]) -def test_ovrtx_prepare_cameras_neutralizes_exposure_for_radiance(data_types): - """Radiance neutralizes camera exposure; standalone HDR keeps the authored camera settings.""" +@pytest.mark.parametrize("data_types", [["rgb"], ["rgb_hdr"]]) +def test_ovrtx_prepare_cameras_preserves_authored_exposure(data_types): + """Selecting render outputs preserves authored camera exposure.""" from pxr import Sdf, Usd, UsdGeom stage = Usd.Stage.CreateInMemory() @@ -574,7 +574,7 @@ def test_ovrtx_prepare_cameras_neutralizes_exposure_for_radiance(data_types): spec = types.SimpleNamespace(cfg=types.SimpleNamespace(data_types=data_types), camera_prim_paths=("/World/Camera",)) _make_ovrtx_renderer_without_backend().prepare_cameras(stage, spec) - assert camera.GetAttribute("exposure:iso").Get() == (0.0 if "rgb_radiance" in data_types else 100.0) + assert camera.GetAttribute("exposure:iso").Get() == 100.0 def test_ovrtx_set_outputs_wraps_caller_torch_zero_copy(): @@ -705,7 +705,7 @@ def test_ovrtx_radiance_is_extracted_on_the_output_device(monkeypatch): renderer._device = "cpu" render_data = _make_ovrtx_camera_render_data() data = CameraData.allocate( - data_types=["rgb_radiance"], + data_types=["rgb_hdr"], height=8, width=16, num_views=2, @@ -734,7 +734,7 @@ def extract_on_output_device(render_data, source_buffer, output_buffer): frame = types.SimpleNamespace(render_vars={render_data.render_var_keys["HdrColor"]: source}) renderer._process_render_frame(render_data, frame, render_data.warp_buffers) - np.testing.assert_array_equal(data.output["rgb_radiance"].warp.numpy(), 2.0) + np.testing.assert_array_equal(data.output["rgb_hdr"].warp.numpy(), 2.0) def test_launch_extract_all_tiles_rejects_wider_output_channels(): diff --git a/source/isaaclab_ov/test/test_ovrtx_usd.py b/source/isaaclab_ov/test/test_ovrtx_usd.py index cb5989c0b6b0..b4e57277ddd4 100644 --- a/source/isaaclab_ov/test/test_ovrtx_usd.py +++ b/source/isaaclab_ov/test/test_ovrtx_usd.py @@ -219,10 +219,10 @@ def test_ovrtx_authors_one_render_var_per_requested_data_type(): def test_ovrtx_data_types_sharing_a_source_author_one_render_var(): - """``rgb``/``rgba``, ``depth``/``distance_to_image_plane``, and ``rgb_hdr``/``rgb_radiance`` collapse onto one + """``rgb``/``rgba``, ``depth``/``distance_to_image_plane``, and ``rgb_hdr`` collapse onto one render var each.""" render_var_configs = get_render_var_configs( - ["rgb", "rgba", "depth", "distance_to_image_plane", "rgb_hdr", "rgb_radiance"], + ["rgb", "rgba", "depth", "distance_to_image_plane", "rgb_hdr"], render_scope_name="RenderCamera_0", ) @@ -312,7 +312,7 @@ def test_render_product_pins_device_ids_to_the_requested_cuda_device(camera_spec @pytest.mark.parametrize( ("data_types", "routes_hdr"), - [(["rgb"], False), (["rgb", "rgb_hdr"], True), (["rgb", "rgb_radiance"], True)], + [(["rgb"], False), (["rgb", "rgb_hdr"], True)], ) def test_render_product_routes_gaussian_hdr_only_for_hdr_outputs(camera_spec, render_data, data_types, routes_hdr): """Route Gaussian HDR when an HDR output is requested, without changing the camera outputs.""" diff --git a/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst index 6cbb5cc3bc3c..d3f07a2f90bb 100644 --- a/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst +++ b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst @@ -1,3 +1,3 @@ -* **Breaking:** Moved PPISP processing from the renderer to ``CameraCfg.pipeline``. - Replace ``CameraCfg.isp_cfg`` with ``PpispStepCfg``; its ``rgb_radiance`` input neutralizes - exposure for all color outputs sharing the camera prim. +* **Breaking:** Replaced renderer-owned PPISP with ``DictTensorProcessor`` operations. + Replace ``CameraCfg.isp_cfg`` with ``PpispStepCfg`` and configure + ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index 51899fe2341e..9a32e7589c2c 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -24,7 +24,6 @@ from isaaclab.app.settings_manager import get_settings_manager from isaaclab.renderers import BaseRenderer, RenderBufferKind, RenderBufferSpec from isaaclab.renderers.camera_render_spec import CameraRenderSpec -from isaaclab.renderers.rtx_camera_overrides import _apply_rtx_exposure_overrides from isaaclab.sim import SimulationContext from isaaclab.sim.utils import enable_extension from isaaclab.utils.version import get_isaac_sim_version @@ -50,8 +49,8 @@ from .isaac_rtx_renderer_cfg import SIMPLE_SHADING_MODES, IsaacRtxRendererCfg -# Outputs read from the HdrColor AOV. Radiance differs only by the neutral camera exposure. -_HDR_DATA_TYPES = frozenset({str(RenderBufferKind.RGB_HDR), str(RenderBufferKind.RGB_RADIANCE)}) +# Outputs read from the HdrColor AOV. +_HDR_DATA_TYPES = frozenset({str(RenderBufferKind.RGB_HDR)}) # RTX simple-shading constants. # @@ -194,20 +193,10 @@ def visual_material_writer(self): return FabricVisualMaterialWriter def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: - """Neutralize camera exposure when ``rgb_radiance`` is requested. - - RTX has no pre-exposure radiance output, so neutral ``exposure:*`` values and - ``OmniRtxCameraExposureAPI_1`` are authored on the camera prims. Otherwise, the - camera prim's authored exposure is left alone. - - :attr:`~isaaclab.sensors.camera.CameraCfg.background_color` is applied - per-render-product in :meth:`create_render_data` via USD attributes. - """ + """Route Gaussian HDR through the camera HDR output.""" if not _HDR_DATA_TYPES.isdisjoint(spec.cfg.data_types): # Gaussian splats otherwise skip tonemapping and never reach the HdrColor AOV. get_settings_manager().set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) - if "rgb_radiance" in spec.cfg.data_types and spec.camera_prim_paths: - _apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: """Publish the per-output Replicator layout this RTX backend writes. @@ -293,8 +282,7 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: if isaac_sim_version.major >= 6: simple_shading_mode = self._resolve_simple_shading_mode(spec) needs_color_render = any( - data_type in spec.cfg.data_types - for data_type in ("rgb", "rgba", str(RenderBufferKind.RGB_HDR), str(RenderBufferKind.RGB_RADIANCE)) + data_type in spec.cfg.data_types for data_type in ("rgb", "rgba", str(RenderBufferKind.RGB_HDR)) ) has_gui = settings.get("/isaaclab/has_gui") if simple_shading_mode is None and (not needs_color_render or has_gui): @@ -514,9 +502,6 @@ def _resolve_simple_shading_mode(self, spec: CameraRenderSpec) -> int | None: def set_outputs(self, render_data: IsaacRtxRenderData, output_data: dict[str, ProxyArray]): """Store reference to output buffers for writing during render. See :meth:`~isaaclab.renderers.base_renderer.BaseRenderer.set_outputs`.""" - radiance = output_data.get(str(RenderBufferKind.RGB_RADIANCE)) - if radiance is not None and str(RenderBufferKind.RGB_HDR) not in output_data: - output_data = {**output_data, str(RenderBufferKind.RGB_HDR): radiance} render_data.output_data = output_data def update_transforms(self) -> None: diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py index 644f677e79d3..b3efb5f8dff0 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py @@ -221,7 +221,6 @@ def segmentation_spec(colorize: bool) -> RenderBufferSpec: RenderBufferKind.RGBA: RenderBufferSpec(4, wp.uint8), RenderBufferKind.RGB: RenderBufferSpec(3, wp.uint8), RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), - RenderBufferKind.RGB_RADIANCE: RenderBufferSpec(3, wp.float32), RenderBufferKind.ALBEDO: RenderBufferSpec(4, wp.uint8), RenderBufferKind.DEPTH: RenderBufferSpec(1, wp.float32), RenderBufferKind.DISTANCE_TO_IMAGE_PLANE: RenderBufferSpec(1, wp.float32), diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py index d58f7f2e64a1..95c37c9bae6b 100644 --- a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py @@ -64,7 +64,6 @@ def test_isaac_rtx_supported_output_types_include_rgb_hdr(monkeypatch, isaac_sim specs = renderer.supported_output_types() assert specs[RenderBufferKind.RGB_HDR] == RenderBufferSpec(3, wp.float32) - assert specs[RenderBufferKind.RGB_RADIANCE] == specs[RenderBufferKind.RGB_HDR] requires_6_0 = [ RenderBufferKind.ALBEDO, RenderBufferKind.SIMPLE_SHADING_CONSTANT_DIFFUSE, @@ -107,9 +106,7 @@ def test_renderer_marks_rtx_sensor_rendering_when_created(monkeypatch): settings.set.assert_any_call("/physics/fabricUpdateTransformations", True) -@pytest.mark.parametrize( - ("data_types", "routes_hdr"), [(["rgb"], False), (["rgb_hdr"], True), (["rgb_radiance"], True)] -) +@pytest.mark.parametrize(("data_types", "routes_hdr"), [(["rgb"], False), (["rgb_hdr"], True)]) def test_prepare_cameras_routes_gaussian_hdr_for_hdr_cameras(monkeypatch, data_types, routes_hdr): """Gaussian HDR routing is enabled only for cameras that read HDR color.""" _install_omni_stubs(monkeypatch) @@ -124,9 +121,9 @@ def test_prepare_cameras_routes_gaussian_hdr_for_hdr_cameras(monkeypatch, data_t assert (hdr_call in settings.set.call_args_list) is routes_hdr -@pytest.mark.parametrize("data_types", [["rgb_hdr"], ["rgb_radiance"]]) -def test_prepare_cameras_neutralizes_exposure_for_radiance(monkeypatch, data_types): - """Radiance neutralizes camera exposure; standalone HDR keeps the authored camera settings.""" +@pytest.mark.parametrize("data_types", [["rgb"], ["rgb_hdr"]]) +def test_prepare_cameras_preserves_authored_exposure(monkeypatch, data_types): + """Selecting render outputs preserves authored camera exposure.""" _install_omni_stubs(monkeypatch) from isaaclab_physx.renderers.isaac_rtx_renderer import IsaacRtxRenderer @@ -138,12 +135,12 @@ def test_prepare_cameras_neutralizes_exposure_for_radiance(monkeypatch, data_typ spec = SimpleNamespace(cfg=SimpleNamespace(data_types=data_types), camera_prim_paths=("/World/Camera",)) IsaacRtxRenderer.__new__(IsaacRtxRenderer).prepare_cameras(stage, spec) - assert camera.GetAttribute("exposure:iso").Get() == (0.0 if "rgb_radiance" in data_types else 100.0) + assert camera.GetAttribute("exposure:iso").Get() == 100.0 -@pytest.mark.parametrize("data_types", [["rgb_radiance"], ["rgb_radiance", "rgb_hdr"]]) -def test_radiance_reads_the_hdr_annotator_into_its_output(monkeypatch, data_types): - """Radiance and HDR attach one HdrColor annotator and extract it into the camera's own buffers.""" +@pytest.mark.parametrize("data_types", [["rgb_hdr"]]) +def test_hdr_reads_the_hdr_annotator_into_its_output(monkeypatch, data_types): + """HDR attaches one HdrColor annotator and extract it into the camera's own buffers.""" replicator, syntheticdata = _install_omni_stubs(monkeypatch) monkeypatch.setattr(syntheticdata, "SyntheticData", MagicMock(), raising=False) @@ -290,7 +287,7 @@ def test_create_render_data_uses_unique_sdf_safe_render_product_name(monkeypatch pytest.param(["rgb", "simple_shading_full_mdl"], 3, False, id="rgb_keeps_path_tracing"), pytest.param(["rgba", "simple_shading_full_mdl"], 3, False, id="rgba_keeps_path_tracing"), pytest.param(["rgb_hdr", "simple_shading_full_mdl"], 3, False, id="rgb_hdr_keeps_path_tracing"), - pytest.param(["rgb_radiance", "simple_shading_full_mdl"], 3, False, id="radiance_keeps_path_tracing"), + pytest.param(["rgb_hdr", "simple_shading_full_mdl"], 3, False, id="radiance_keeps_path_tracing"), ], ) def test_simple_shading_configures_its_render_product( diff --git a/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst index 5edaa8601455..d3f07a2f90bb 100644 --- a/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst +++ b/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst @@ -1,4 +1,3 @@ -* **Breaking:** Moved PPISP into camera and observation pipelines through ``PpispStepCfg``. - Resolve USD settings explicitly with ``ppisp_cfg_from_usd_camera`` or ``resolve_and_normalize`` - before configuring the step. Replaced ``apply_rtx_exposure_overrides`` with renderer-managed - exposure when requesting ``rgb_radiance``. +* **Breaking:** Replaced renderer-owned PPISP with ``DictTensorProcessor`` operations. + Replace ``CameraCfg.isp_cfg`` with ``PpispStepCfg`` and configure + ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_ppisp/docs/README.md b/source/isaaclab_ppisp/docs/README.md index 4d66cdfb2d58..277b033b4110 100644 --- a/source/isaaclab_ppisp/docs/README.md +++ b/source/isaaclab_ppisp/docs/README.md @@ -3,10 +3,14 @@ This extension provides PPISP (Physically Plausible Image Signal Processing) as an operation in the shared buffer pipeline used by cameras and observations. -`PpispStepCfg` reads scene-linear `rgb_radiance` from a float32 `ProxyArray` with +`PpispStepCfg` reads scene-linear `rgb_hdr` from a float32 `ProxyArray` with shape `(N, H, W, 3)` and writes uint8 `rgb`. For camera processing, configure `CameraCfg.pipeline=[PpispStepCfg(params={"cfg": ppisp_cfg})]`. The camera requests -required radiance internally; only the requested public outputs are published. +required HDR internally; only the requested public outputs are published. Set +`CameraCfg.exposure=CameraExposureCfg()` (from `isaaclab.sensors.camera`) before initialization +for neutral RTX HDR input. Output selection does not alter exposure. The same setting is required +when PPISP runs only as an observation tensor processor through `DictTensorProcessorAdapterCfg`. +Set PPISP exposure in stops through `PpispCfg.inputs["exposureOffset"]`. Resolve authored settings explicitly before constructing the pipeline: @@ -17,8 +21,8 @@ ppisp_cfg = ppisp_cfg_from_usd_camera(source_camera_prim) step = PpispStepCfg(params={"cfg": ppisp_cfg}) ``` -The same step can run in `isaaclab.utils.processing.ProcessingBufferPipe` outside a camera: -`ProcessingBufferPipe({"rgb_radiance": radiance_buffer}, [step])`, where `radiance_buffer` +The same step can run in `isaaclab.utils.processing.DictTensorProcessor` outside a camera: +`DictTensorProcessor({"rgb_hdr": radiance_buffer}, [step])`, where `radiance_buffer` is an allocated `ProxyArray`. Preparation validates its shape and dtype and allocates the persistent output once on the same device. Input/output bindings can be changed independently of PPISP math. For RGBA, diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/processor.py b/source/isaaclab_ppisp/isaaclab_ppisp/processor.py index 99dcf630cf77..bfc3d3b95e92 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/processor.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/processor.py @@ -14,7 +14,7 @@ import warp as wp from isaaclab.utils import configclass -from isaaclab.utils.processing import PreparedBufferOperation, ProcessingBufferStepCfg +from isaaclab.utils.processing import DictTensorProcessorStepCfg, PreparedBufferOperation from isaaclab.utils.warp import ProxyArray from .cfg import PpispCfg @@ -71,7 +71,7 @@ def apply(inputs: Mapping[str, ProxyArray], outputs: Mapping[str, ProxyArray]) - @configclass -class PpispStepCfg(ProcessingBufferStepCfg): +class PpispStepCfg(DictTensorProcessorStepCfg): """Route scene-linear radiance through PPISP into a public RGB buffer. Supply concrete settings with ``params={"cfg": ppisp_cfg}``. To produce RGBA, set @@ -80,6 +80,6 @@ class PpispStepCfg(ProcessingBufferStepCfg): """ prepare = "isaaclab_ppisp:prepare_ppisp" - inputs: dict[str, str] = {"radiance": "rgb_radiance"} + inputs: dict[str, str] = {"radiance": "rgb_hdr"} outputs: dict[str, str] = {"color": "rgb"} params: dict[str, Any] = {"cfg": PpispCfg()} diff --git a/source/isaaclab_ppisp/test/test_ppisp_kernels.py b/source/isaaclab_ppisp/test/test_ppisp_kernels.py index f4c437ce4441..b46e58475883 100644 --- a/source/isaaclab_ppisp/test/test_ppisp_kernels.py +++ b/source/isaaclab_ppisp/test/test_ppisp_kernels.py @@ -41,7 +41,7 @@ ) from isaaclab.test.utils import DeviceScope, test_devices -from isaaclab.utils.processing import ProcessingBufferPipe +from isaaclab.utils.processing import DictTensorProcessor from isaaclab.utils.warp import ProxyArray wp.init() @@ -289,7 +289,8 @@ def bounded_softplus(raw: float, min_value: float) -> float: return np.maximum(y, 0.0) ** gamma -def test_ppisp_pipe_rgb_crf_extreme_centers_match_reference(): +@pytest.mark.parametrize("attachment", ["dictionary", "tensor"]) +def test_ppisp_pipe_rgb_crf_extreme_centers_match_reference(attachment): """A CPU RGB pipe preserves the reference curve, including saturated CRF centers.""" import numpy as np @@ -313,12 +314,24 @@ def test_ppisp_pipe_rgb_crf_extreme_centers_match_reference(): ) ) - pipe = ProcessingBufferPipe( - {"rgb_radiance": ProxyArray(hdr_color)}, + pipe = DictTensorProcessor( + {"rgb_hdr": ProxyArray(hdr_color)}, [PpispStepCfg(params={"cfg": cfg})], ) - rgb = pipe({"rgb_radiance": ProxyArray(hdr_color)})["rgb"] - rgb_np = rgb.warp.numpy()[0, 0] + if attachment == "dictionary": + rgb_np = pipe({"rgb_hdr": ProxyArray(hdr_color)})["rgb"].warp.numpy()[0, 0] + else: + from isaaclab.utils.tensor_processors import DictTensorProcessorAdapter, DictTensorProcessorAdapterCfg + + processor = DictTensorProcessorAdapter( + DictTensorProcessorAdapterCfg( + pipeline=[PpispStepCfg(params={"cfg": cfg})], input_name="rgb_hdr", output_name="rgb" + ), + hdr_data.shape, + "cpu", + ) + rgb_np = processor(wp.to_torch(hdr_color)).numpy()[0, 0] + processor.close() for channel, suffix in enumerate("RGB"): expected = _crf_reference( ramp, @@ -348,8 +361,8 @@ def test_ppisp_controller_pipes_share_weights_for_rgb_and_rgba(device): cfg = PpispCfg() shared_weights = PpispControllerWeights(weights) pipes = [ - ProcessingBufferPipe( - {"rgb_radiance": ProxyArray(hdr)}, + DictTensorProcessor( + {"rgb_hdr": ProxyArray(hdr)}, [PpispStepCfg(params={"cfg": cfg, "channels": channels, "controller_weights": shared_weights})], ) for channels in (3, 4) @@ -357,16 +370,16 @@ def test_ppisp_controller_pipes_share_weights_for_rgb_and_rgba(device): # Use different pixels from preparation and honor prior work on a non-default Torch stream. with torch.cuda.stream(torch.cuda.Stream(device=device)): incoming = torch.from_numpy(hdr_data * 0.5).to(device) * 3.0 - observation = pipes[0].apply_to_tensor(incoming, input_name="rgb_radiance", output_name="rgb") + observation = pipes[0].apply_to_tensor(incoming, input_name="rgb_hdr", output_name="rgb") results = [observation.cpu().numpy()] - results.append(pipes[1]({"rgb_radiance": ProxyArray(hdr)})["rgb"].warp.numpy()) + results.append(pipes[1]({"rgb_hdr": ProxyArray(hdr)})["rgb"].warp.numpy()) for result, input_scale in zip(results, (1.5, 1.0), strict=True): expected = np.floor(_crf_reference(ramp * input_scale * 2.0, 0.013659, 0.013659, 0.378165, 0.0) * 255.0) np.testing.assert_allclose(result[0, 0, :, :3].astype(float), np.repeat(expected[:, None], 3, axis=1), atol=1.0) assert (results[1][..., 3] == 255).all() pipes[0].close() hdr.zero_() - np.testing.assert_array_equal(pipes[1]({"rgb_radiance": ProxyArray(hdr)})["rgb"].warp.numpy()[..., :3], 0) + np.testing.assert_array_equal(pipes[1]({"rgb_hdr": ProxyArray(hdr)})["rgb"].warp.numpy()[..., :3], 0) np.testing.assert_array_equal(weights.numpy(), weights_np) pipes[1].close() @@ -379,13 +392,13 @@ def test_ppisp_pipe_rejects_incompatible_input_storage(shape, dtype, message): """Reject invalid image channels and scalar types before preparing numerical execution.""" source = ProxyArray(wp.empty(shape, dtype=dtype, device="cpu")) with pytest.raises(ValueError, match=message): - ProcessingBufferPipe({"rgb_radiance": source}, [PpispStepCfg()]) + DictTensorProcessor({"rgb_hdr": source}, [PpispStepCfg()]) def test_ppisp_pipe_requires_resolved_configuration(): """An image operation must not discover its settings from a camera during preparation.""" source = ProxyArray(wp.empty((1, 2, 2, 3), dtype=wp.float32, device="cpu")) with pytest.raises(ValueError, match="resolve USD camera attributes"): - ProcessingBufferPipe( - {"rgb_radiance": source}, [PpispStepCfg(params={"cfg": PpispCfg(camera_prim_path="/World/Camera")})] + DictTensorProcessor( + {"rgb_hdr": source}, [PpispStepCfg(params={"cfg": PpispCfg(camera_prim_path="/World/Camera")})] ) From d10d8bf9fbb861f5c8f1f20426c6b490ca765d32 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Fri, 16 Oct 2026 16:55:10 -0400 Subject: [PATCH 19/20] Simplify dictionary processor interfaces and colocate tensor processors --- docs/source/api/lab/isaaclab.utils.rst | 28 +- docs/source/concepts/sensors/camera.rst | 67 +++-- examples/sensors/ppisp_camera.py | 4 +- .../feat-render-buffer-pipe.changed.rst | 2 + .../isaaclab/envs/mdp/sensor_observations.py | 9 +- .../isaaclab/sensors/camera/camera.py | 6 +- .../isaaclab/sensors/camera/camera_cfg.py | 15 +- source/isaaclab/isaaclab/utils/__init__.pyi | 4 + source/isaaclab/isaaclab/utils/processing.py | 251 ------------------ .../utils/tensor_processors/__init__.pyi | 5 + .../utils/tensor_processors/_pipeline.py | 170 ++++++++++++ .../dict_tensor_processor.py | 47 ++++ .../dict_tensor_processor_adapter.py | 8 +- .../dict_tensor_processor_cfg.py | 40 +++ .../tensor_processors/tensor_processor.py | 31 +-- .../tensor_processors/tensor_processor_cfg.py | 4 +- .../test/envs/test_buffer_observations.py | 101 +++---- .../renderers/test_camera_output_contract.py | 52 ++-- .../generate_synthetic_gaussian_asset.py | 6 +- ..._pipe.py => test_dict_tensor_processor.py} | 173 +++++++----- .../feat-render-buffer-pipe.changed.rst | 4 +- .../feat-render-buffer-pipe.changed.rst | 4 +- .../feat-render-buffer-pipe.changed.rst | 4 +- .../feat-render-buffer-pipe.changed.rst | 4 +- source/isaaclab_ppisp/docs/README.md | 27 +- .../isaaclab_ppisp/isaaclab_ppisp/__init__.py | 2 +- .../isaaclab_ppisp/__init__.pyi | 7 +- .../isaaclab_ppisp/isaaclab_ppisp/pipeline.py | 2 +- .../isaaclab_ppisp/processor.py | 119 ++++----- .../isaaclab_ppisp/processor_cfg.py | 26 ++ .../isaaclab_ppisp/test/test_ppisp_kernels.py | 27 +- 31 files changed, 650 insertions(+), 599 deletions(-) delete mode 100644 source/isaaclab/isaaclab/utils/processing.py create mode 100644 source/isaaclab/isaaclab/utils/tensor_processors/_pipeline.py create mode 100644 source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor.py create mode 100644 source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_cfg.py rename source/isaaclab/test/utils/{test_processing_buffer_pipe.py => test_dict_tensor_processor.py} (59%) create mode 100644 source/isaaclab_ppisp/isaaclab_ppisp/processor_cfg.py diff --git a/docs/source/api/lab/isaaclab.utils.rst b/docs/source/api/lab/isaaclab.utils.rst index 00d65e2f8cec..4e8029d2ba7b 100644 --- a/docs/source/api/lab/isaaclab.utils.rst +++ b/docs/source/api/lab/isaaclab.utils.rst @@ -128,6 +128,15 @@ Mesh operations Tensor processors ~~~~~~~~~~~~~~~~~ +The processor interfaces live together in :mod:`isaaclab.utils.tensor_processors`: + +* ``TensorProcessor``: tensor to tensor. +* ``DictTensorProcessor``: dictionary of named tensors to dictionary of named tensors. + +Dictionary tensors use ``ProxyArray`` for Warp and Torch access without copying. Processors own +any persistent outputs and state. ``DictTensorProcessorCfg`` declares input/output keys for camera +setup. Camera and observation consumers compose processors and preserve untouched keys. + .. automodule:: isaaclab.utils.tensor_processors :members: :imported-members: @@ -146,25 +155,6 @@ Noise operations :show-inheritance: :exclude-members: __init__, func -Processing operations -~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: isaaclab.utils.processing - -``DictTensorProcessor`` processes named ``ProxyArray`` buffers, including camera outputs and nonvisual -sensor data. Operations define their own input shape, dtype, and semantic requirements. - -.. autoclass:: isaaclab.utils.processing.PreparedBufferOperation - :members: - -.. autoclass:: isaaclab.utils.processing.DictTensorProcessorStepCfg - :members: - :exclude-members: __init__ - -.. autoclass:: isaaclab.utils.processing.DictTensorProcessor - :members: - :special-members: __call__ - Seed operations ~~~~~~~~~~~~~~~ diff --git a/docs/source/concepts/sensors/camera.rst b/docs/source/concepts/sensors/camera.rst index 274d9858c750..4440b5722b29 100644 --- a/docs/source/concepts/sensors/camera.rst +++ b/docs/source/concepts/sensors/camera.rst @@ -155,7 +155,7 @@ Output types For raw outputs, the active renderer validates the requested names and allocates the channel count and data type declared by its :class:`~isaaclab.renderers.RenderBufferSpec`. A camera pipeline may -produce additional output names; its operation factories allocate those buffers. +produce additional output names; its processors allocate those buffers. .. list-table:: Common output contracts :header-rows: 1 @@ -322,13 +322,14 @@ scene. Post-render image processing ---------------------------- -:attr:`~sensors.CameraCfg.pipeline` applies an ordered sequence of operations to named image buffers. -Each :class:`~isaaclab.utils.processing.DictTensorProcessorStepCfg` maps operation input and output roles to buffer -names. A step can read several channels, such as radiance and depth, and later steps read the preceding -step's results. The same :class:`~isaaclab.utils.processing.DictTensorProcessor` can run independently or serve -an observation consumer; operations receive explicit buffers and resources without discovering a camera, -environment, or simulation singleton. The shared pipe accepts named ``ProxyArray`` buffers, including -nonvisual sensor data; image layout and pixel meaning are requirements of individual operations. +:attr:`~sensors.CameraCfg.pipeline` composes dictionary processors after rendering. +:class:`~isaaclab.utils.tensor_processors.DictTensorProcessor` takes a dictionary of named tensors +and returns a dictionary of named tensors. It lives beside +:class:`~isaaclab.utils.tensor_processors.TensorProcessor`, whose call takes and returns one tensor. +:class:`~isaaclab.utils.tensor_processors.DictTensorProcessorCfg` declares the actual input and output +keys needed for renderer setup. There are no separate operation roles or prepared-operation objects. +The shared dictionary interface also accepts nonvisual sensor data; each processor defines its +layout and value semantics. The camera resolves external inputs before renderer initialization, including inputs needed only by the pipeline. It publishes the names in ``data_types`` after processing. For example, this camera requests @@ -338,30 +339,42 @@ private ``rgb_hdr`` from its renderer and publishes PPISP's processed ``rgb``: from isaaclab.sensors.camera import CameraExposureCfg from isaaclab.utils import replace - from isaaclab_ppisp import PpispCfg, PpispStepCfg + from isaaclab_ppisp import PpispCfg, PpispProcessorCfg processed_camera = replace( front_camera, data_types=["rgb"], exposure=CameraExposureCfg(), - pipeline=[PpispStepCfg(params={"cfg": PpispCfg(inputs={"exposureOffset": 1.5})})], + pipeline=[PpispProcessorCfg(params={"cfg": PpispCfg(inputs={"exposureOffset": 1.5})})], ) PPISP (Physically Plausible Image Signal Processing) applies responsivity, exposure, vignetting, color correction, and a camera response function. It consumes float32 scene-linear radiance and produces -uint8 color. Set ``outputs={"color": "rgba"}`` and ``params={"cfg": ppisp_cfg, "channels": 4}`` on the +uint8 color. Set ``outputs=["rgba"]`` and ``params={"cfg": ppisp_cfg, "channels": 4}`` on the step for RGBA output. A pipeline-produced RGBA buffer also supplies a zero-copy RGB view when ``rgb`` is requested. A pipeline producing only RGB cannot satisfy an explicit RGBA request. -For a custom operation, provide a factory ``prepare(inputs, **params)`` returning -:class:`~isaaclab.utils.processing.PreparedBufferOperation`. The factory receives actual -:class:`~isaaclab.utils.warp.ProxyArray` inputs, inspects their shape, dtype, and device, and allocates -its output buffers and workspace once. Return the output arrays in the prepared operation, then write -each supplied destination in ``apply(inputs, outputs)``. Both mappings use operation role names; the -step configuration maps those roles to camera-buffer names. Preparation must not read input pixels or -retain their addresses for execution: each ``apply`` call receives the current inputs. -Operations document their expected image layout and pixel meaning. PPISP requires NHWC float32 -scene-linear radiance and produces NHWC uint8 color; a float32 array alone does not establish radiance. +For custom dictionary processing, subclass ``DictTensorProcessor``. Its constructor receives +``(cfg, data)`` with actual ``ProxyArray`` tensors. Allocate persistent output tensors in +``self.outputs`` without reading input values or advancing state. Implement ``__call__(data)`` +to fill and return those named outputs, and override ``reset`` or ``close`` only when needed. +The pipeline passes the current dictionary to each processor and merges its returned outputs; +untouched keys pass through. Shape, dtype and device come from the arrays directly. + +PPISP is a regular ``PpispProcessor`` subclass. It can also be constructed and called directly: + +.. code-block:: python + + from isaaclab.utils import instantiate + + cfg = PpispProcessorCfg() + processor = instantiate(cfg, {"rgb_hdr": hdr_tensor}) + color = processor({"rgb_hdr": hdr_tensor})["rgb"] + processor.close() + +Here ``hdr_tensor`` is a scene-linear float32 ``ProxyArray`` with shape ``(N, H, W, 3)``. +The returned color is uint8; copy it before another call if its values must be retained. + Published camera outputs must keep the camera's batch size, height, width, and NHWC layout so pose and calibration metadata remain valid. Intermediate buffers may change shape. Use ``processed_sensor`` below when the final observation should have a different image shape or layout. @@ -373,8 +386,8 @@ buffers and raw buffers retain their storage until camera reinitialization, but their contents; clone values that must survive another capture. Each camera owns its pipe and operation workspace. Operations may share explicitly supplied immutable model resources. Configuration copying must preserve the identity of borrowed resources: PPISP provides -``PpispControllerWeights`` for device weights. Custom factories can capture a resource in a closure -or use a wrapper with identity-preserving ``__deepcopy__``; raw Warp allocations are not configuration values. +``PpispControllerWeights`` for device weights. Custom processors can use a resource wrapper with +identity-preserving ``__deepcopy__``; raw Warp allocations are not configuration values. Pipelines run after scalar and batched captures. When an asynchronous renderer republishes a capture, the camera retains its processed result and matching capture metadata. Camera reset forwards only the @@ -394,14 +407,14 @@ through their ``request_render_inputs()`` and ``render_outputs`` interface: from isaaclab.envs import mdp from isaaclab.managers import ObservationTermCfg, SceneEntityCfg - from isaaclab_ppisp import PpispStepCfg + from isaaclab_ppisp import PpispProcessorCfg front_camera.exposure = CameraExposureCfg() policy_image = ObservationTermCfg( func=mdp.processed_sensor, params={ "sensor_cfg": SceneEntityCfg("front_camera"), - "pipeline": [PpispStepCfg()], + "pipeline": [PpispProcessorCfg()], "output_name": "rgb", }, ) @@ -432,7 +445,7 @@ input requirements, including any transformations from preceding tensor processo params={"sensor_cfg": SceneEntityCfg("front_camera"), "data_type": "rgb_hdr", "normalize": False}, tensor_processors=[ tensor_processors.DictTensorProcessorAdapterCfg( - pipeline=[PpispStepCfg()], + pipeline=[PpispProcessorCfg()], input_name="rgb_hdr", output_name="rgb", ) @@ -449,14 +462,14 @@ operation: .. code-block:: python - from isaaclab_ppisp import PpispDiscoveryMode, PpispStepCfg, resolve_and_normalize + from isaaclab_ppisp import PpispDiscoveryMode, PpispProcessorCfg, resolve_and_normalize ppisp_cfg = resolve_and_normalize(PpispDiscoveryMode.AUTO_CAMERA, stage, "/World/Camera") discovered_camera = replace( front_camera, data_types=["rgb"], exposure=CameraExposureCfg(), - pipeline=[PpispStepCfg(params={"cfg": ppisp_cfg})] if ppisp_cfg is not None else [], + pipeline=[PpispProcessorCfg(params={"cfg": ppisp_cfg})] if ppisp_cfg is not None else [], ) ``AUTO_CAMERA`` checks the supplied camera prim; ``AUTO_ANY`` can fall back to the first PPISP camera diff --git a/examples/sensors/ppisp_camera.py b/examples/sensors/ppisp_camera.py index 653bcb405cc0..ad3a8da4a8db 100644 --- a/examples/sensors/ppisp_camera.py +++ b/examples/sensors/ppisp_camera.py @@ -120,7 +120,7 @@ import matplotlib.pyplot as plt import numpy as np import torch -from isaaclab_ppisp import PpispCfg, PpispStepCfg, ppisp_cfg_from_usd_camera +from isaaclab_ppisp import PpispCfg, PpispProcessorCfg, ppisp_cfg_from_usd_camera import isaaclab.sim as sim_utils from isaaclab.assets import AssetBaseCfg, RigidObjectCfg @@ -376,7 +376,7 @@ def make_camera(camera_prim_path: str, *, ppisp_cfg: PpispCfg | None, width: int data_types=["rgb"], spawn=None, exposure=CameraExposureCfg() if ppisp_cfg is not None else None, - pipeline=[PpispStepCfg(params={"cfg": ppisp_cfg})] if ppisp_cfg is not None else [], + pipeline=[PpispProcessorCfg(params={"cfg": ppisp_cfg})] if ppisp_cfg is not None else [], renderer_cfg=make_renderer_cfg(), ) return instantiate(camera_cfg) diff --git a/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst index 1ae08535a7c5..393b39b971a2 100644 --- a/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst +++ b/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst @@ -3,3 +3,5 @@ ``modifiers`` with ``tensor_processors``; use ``DictTensorProcessorAdapterCfg`` for dictionary operations. * Replaced renderer-owned PPISP with camera and observation processors consuming ``rgb_hdr``. Set ``CameraCfg.exposure=CameraExposureCfg()`` explicitly for neutral RTX HDR before PPISP. +* Colocated dictionary and single-tensor processors and removed preparation wrappers and role mappings. + Dictionary processors now own their output tensors and expose a direct dictionary-to-dictionary call. diff --git a/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py b/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py index 0c4e969953b4..e20e37369c09 100644 --- a/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py +++ b/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py @@ -14,7 +14,8 @@ import warp as wp from isaaclab.managers import ManagerTermBase, ObservationTermCfg, SceneEntityCfg -from isaaclab.utils.processing import DictTensorProcessor, DictTensorProcessorStepCfg +from isaaclab.utils.tensor_processors import DictTensorProcessorCfg +from isaaclab.utils.tensor_processors._pipeline import _DictTensorProcessorPipeline if TYPE_CHECKING: from isaaclab.envs import ManagerBasedEnv @@ -43,7 +44,7 @@ class processed_sensor(ManagerTermBase): def prepare_scene(cls, cfg: ObservationTermCfg, env: ManagerBasedEnv) -> None: sensor_cfg = cfg.params.get("sensor_cfg", SceneEntityCfg("camera")) output_name = cfg.params.get("output_name", "rgb") - requirements = DictTensorProcessor.required_source_names(cfg.params["pipeline"], [output_name]) + requirements = _DictTensorProcessorPipeline.required_source_names(cfg.params["pipeline"], [output_name]) env.scene.sensors[sensor_cfg.name].request_render_inputs(requirements) def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedEnv): @@ -54,7 +55,7 @@ def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedEnv): device = wp.get_device(self.device) stream = wp.stream_from_torch(torch.cuda.current_stream(str(device))) if device.is_cuda else None with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): - self._pipe = DictTensorProcessor(buffers, cfg.params["pipeline"]) + self._pipe = _DictTensorProcessorPipeline(buffers, cfg.params["pipeline"]) if self._output_name not in self._pipe.outputs: self._pipe.close() raise ValueError(f"Buffer processing does not provide output {self._output_name!r}.") @@ -62,7 +63,7 @@ def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedEnv): def __call__( self, env: ManagerBasedEnv, - pipeline: Sequence[DictTensorProcessorStepCfg], + pipeline: Sequence[DictTensorProcessorCfg], sensor_cfg: SceneEntityCfg = SceneEntityCfg("camera"), output_name: str = "rgb", *, diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 188d421851e6..4bc05a216f3c 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -28,7 +28,7 @@ create_rotation_matrix_from_view, quat_from_matrix, ) -from ...utils.processing import DictTensorProcessor +from ...utils.tensor_processors._pipeline import _DictTensorProcessorPipeline from ...utils.warp import ProxyArray from ..sensor_base import SensorBase from .camera_data import CameraData, RenderBufferKind @@ -253,7 +253,7 @@ def __init__(self, cfg: CameraCfg): # Renderer requirements include private pipeline inputs and exclude produced public outputs. self._render_cfg = replace(self.cfg, data_types=self.cfg.render_data_types(), pipeline=[]) - self._buffer_pipe: DictTensorProcessor | None = None + self._buffer_pipe: _DictTensorProcessorPipeline | None = None self._pipeline_capture: object | None = None self._render_camera_data: CameraData | None = None self._render_outputs: dict[str, ProxyArray] = {} @@ -904,7 +904,7 @@ def _create_buffers(self): self._close_pipeline() self._pipeline_capture = None if self.cfg.pipeline: - self._buffer_pipe = DictTensorProcessor(self._render_outputs, self.cfg.pipeline) + self._buffer_pipe = _DictTensorProcessorPipeline(self._render_outputs, self.cfg.pipeline) try: outputs = dict(self._buffer_pipe.outputs) alias_rgb = ( diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py index 5a7ab6cc561c..20f0310d8cab 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py @@ -16,7 +16,7 @@ from .camera_exposure_cfg import CameraExposureCfg if TYPE_CHECKING: - from ...utils.processing import DictTensorProcessorStepCfg + from ...utils.tensor_processors import DictTensorProcessorCfg from .camera import Camera # Default values for the RTX-flavored fields kept on :class:`CameraCfg` for @@ -208,11 +208,12 @@ class OffsetCfg: changes affect every output from this camera prim. Newton ignores RTX exposure settings. """ - pipeline: list[DictTensorProcessorStepCfg] = [] + pipeline: list[DictTensorProcessorCfg] = [] """Ordered image operations applied after each newly published capture. - Steps bind operation roles to named buffers and may consume several render outputs or preceding - results. The renderer is configured with the pipeline's external dependencies before initialization. + Processors consume and return dictionaries of named tensors. They may combine several render + outputs or preceding results. The renderer receives the pipeline's external dependencies + before initialization. Only :attr:`data_types` are published in :attr:`CameraData.output`; unprocessed inputs remain available through :attr:`Camera.render_outputs`. Operations borrow their inputs read-only and own persistent result buffers. Camera reset and teardown reset and close the operations. @@ -232,13 +233,13 @@ def render_data_types(self) -> list[str]: ) if not self.pipeline: return list(self.data_types) - from ...utils.processing import DictTensorProcessor + from ...utils.tensor_processors._pipeline import _DictTensorProcessorPipeline - produced = {name for step in self.pipeline for name in step.outputs.values()} + produced = {name for step in self.pipeline for name in step.outputs} if "rgb" in produced and "rgba" not in produced and "rgba" in self.data_types: raise ValueError("A pipeline producing rgb must also produce rgba when rgba is requested.") published = [name for name in self.data_types if not (name == "rgb" and "rgba" in produced)] - return sorted(DictTensorProcessor.required_source_names(self.pipeline, published)) + return sorted(_DictTensorProcessorPipeline.required_source_names(self.pipeline, published)) def __post_init__(self): """Forward deprecated RTX-flavored fields onto :attr:`renderer_cfg`. diff --git a/source/isaaclab/isaaclab/utils/__init__.pyi b/source/isaaclab/isaaclab/utils/__init__.pyi index 09c6f837ddeb..99f7ef906c6b 100644 --- a/source/isaaclab/isaaclab/utils/__init__.pyi +++ b/source/isaaclab/isaaclab/utils/__init__.pyi @@ -33,6 +33,8 @@ __all__ = [ "PRIMITIVE_MESH_TYPES", "TensorProcessorCfg", "TensorProcessor", + "DictTensorProcessor", + "DictTensorProcessorCfg", "DigitalFilter", "DigitalFilterCfg", "Integrator", @@ -88,6 +90,8 @@ from .mesh import ( PRIMITIVE_MESH_TYPES, ) from .tensor_processors import ( + DictTensorProcessor, + DictTensorProcessorCfg, TensorProcessorCfg, TensorProcessor, DigitalFilter, diff --git a/source/isaaclab/isaaclab/utils/processing.py b/source/isaaclab/isaaclab/utils/processing.py deleted file mode 100644 index c313c4f956ec..000000000000 --- a/source/isaaclab/isaaclab/utils/processing.py +++ /dev/null @@ -1,251 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Named buffer processing shared by sensors and observations.""" - -from __future__ import annotations - -from builtins import ExceptionGroup -from collections.abc import Callable, Mapping, Sequence -from dataclasses import MISSING, dataclass -from types import MappingProxyType -from typing import Any - -import torch -import warp as wp - -from .configclass import configclass -from .string import string_to_callable -from .warp import ProxyArray - - -@dataclass(frozen=True) -class PreparedBufferOperation: - """One operation with persistent output buffers and resolved resources. - - ``apply`` receives input and destination dictionaries keyed by operation roles, not sensor - output names. Preparation allocates the outputs once. Inputs are borrowed read-only by - convention; outputs must use independent storage. The operation must fill each supplied - destination and enqueue GPU work on the current Warp stream. The pipe retains the outputs - and controls execution order; private workspace belongs to the operation. - """ - - outputs: Mapping[str, ProxyArray] - apply: Callable[[Mapping[str, ProxyArray], Mapping[str, ProxyArray]], None] - reset: Callable[[Sequence[int] | None], None] | None = None - close: Callable[[], None] | None = None - - -@configclass -class DictTensorProcessorStepCfg: - """Bind named buffers to the input/output roles of one processing operation. - - The factory ``prepare(inputs, **params)`` runs once during pipe construction and returns - a :class:`PreparedBufferOperation` with allocated output buffers. ``inputs`` contains actual - :class:`ProxyArray` objects keyed by operation roles. Factories inspect array shape, dtype - and device directly; each operation documents its expected layout and value semantics. - Factories should not read input values during preparation or retain input pointers for execution: - ``apply`` receives the current inputs on every call. Scene resources must be supplied explicitly. - """ - - prepare: Callable[..., PreparedBufferOperation] | str = MISSING - """Operation factory or an import string in ``module:attribute`` form.""" - - inputs: dict[str, str] = MISSING - """Map operation input roles to buffer names read from the preceding stage.""" - - outputs: dict[str, str] = MISSING - """Map operation output roles to names created or replaced in the bundle.""" - - params: dict[str, Any] = {} - """Resolved operation parameters passed to the factory. - - Configurations are deep-copied by ``configclass`` and managers. Borrowed device allocations - or models need a resource wrapper that preserves identity through ``__deepcopy__``, or a - factory closure retaining the resource. Do not put a raw Warp allocation in this dictionary. - """ - - -class DictTensorProcessor: - """Apply an ordered sequence of operations to named :class:`ProxyArray` buffers. - - Buffers may hold images, LiDAR points, or other sensor and observation data. The pipe imposes - no image layout; each operation defines its input requirements and allocates its outputs. - - A pipe retains persistent output storage allocated by its prepared operations. Create one instance per - consumer; instances are not reentrant. Borrowed inputs are never modified by the pipe, and - untouched channels pass through by reference. Returned buffers remain valid until the next - call, reset or close; consumers retaining an observation must take a snapshot. - - Buffer dependencies are resolved in order, so an operation reads preceding replacements. - GPU execution uses the caller's current Warp stream without a device-wide synchronization. - The caller owns producer readiness and keeps borrowed storage alive until consumption ends. - - Args: - source_buffers: Available external buffers, used to prepare operations and validate later inputs. - steps: Ordered operation configurations. - """ - - def __init__(self, source_buffers: Mapping[str, ProxyArray], steps: Sequence[DictTensorProcessorStepCfg]): - self._source_buffers = dict(source_buffers) - self.required_inputs = self.required_source_names(steps) - self._operations: list[ - tuple[PreparedBufferOperation, dict[str, str], dict[str, str], Mapping[str, ProxyArray]] - ] = [] - self._closed = False - self.outputs: Mapping[str, ProxyArray] = MappingProxyType({}) - available = dict(source_buffers) - produced: set[str] = set() - try: - for step in steps: - inputs, outputs = dict(step.inputs), dict(step.outputs) - if not outputs or len(set(outputs.values())) != len(outputs): - raise ValueError("A processing step must bind distinct output names to its output roles.") - missing = set(inputs.values()) - available.keys() - if missing: - raise ValueError(f"Missing input buffers for preparation: {sorted(missing)}.") - prepare = string_to_callable(step.prepare) if isinstance(step.prepare, str) else step.prepare - input_buffers = {role: available[name] for role, name in inputs.items()} - operation = prepare(MappingProxyType(input_buffers), **step.params) - if not isinstance(operation, PreparedBufferOperation): - raise TypeError("A buffer operation factory must return PreparedBufferOperation.") - # Own the operation before checking its outputs so failed validation releases its resources. - output_buffers = {} - destinations = MappingProxyType(output_buffers) - self._operations.append((operation, inputs, outputs, destinations)) - output_buffers.update(operation.outputs) - if destinations.keys() != outputs.keys(): - raise ValueError("Operation output roles must match the step's output bindings.") - if any(not isinstance(buffer, ProxyArray) for buffer in destinations.values()): - raise TypeError("Prepared operation outputs must be ProxyArray objects.") - devices = {str(buffer.device) for buffer in (*input_buffers.values(), *destinations.values())} - if len(devices) > 1: - raise ValueError("A buffer operation must use one device for its inputs and outputs.") - for role, name in outputs.items(): - available[name] = destinations[role] - produced.update(outputs.values()) - except Exception as error: - try: - self.close() - except Exception as cleanup_error: - raise error from cleanup_error - raise - self.outputs = MappingProxyType(available) - self.produced_outputs = frozenset(produced) - - @property - def has_reset_callbacks(self) -> bool: - """Whether any operation needs selected batch entries reset.""" - return any(operation.reset is not None for operation, *_ in self._operations) - - @staticmethod - def _validate_buffer(name: str, value: ProxyArray, reference: ProxyArray) -> None: - """Check replacement storage against the actual buffer used to prepare the pipe.""" - if not isinstance(value, ProxyArray): - raise TypeError(f"Buffer {name!r} must be a ProxyArray.") - if value.shape != reference.shape or value.dtype != reference.dtype or value.device != reference.device: - raise ValueError( - f"Buffer {name!r} must match shape, dtype and device" - f" {reference.shape}, {reference.dtype}, {reference.device}." - ) - - @staticmethod - def required_source_names( - steps: Sequence[DictTensorProcessorStepCfg], published_names: Sequence[str] = () - ) -> frozenset[str]: - """Resolve external names without constructing operations or allocating buffer storage. - - Call this before producer initialization to request private processing inputs. Public - pass-through outputs also require a source, even when no operation reads them. - """ - required: set[str] = set() - produced: set[str] = set() - for step in steps: - required.update(set(step.inputs.values()) - produced) - produced.update(step.outputs.values()) - required.update(set(published_names) - produced) - return frozenset(required) - - def __call__(self, buffers: Mapping[str, ProxyArray]) -> dict[str, ProxyArray]: - """Execute the pipe on ready source buffers and return the resulting bundle.""" - if self._closed: - raise RuntimeError("Cannot execute a closed buffer pipe.") - missing = self.required_inputs - buffers.keys() - if missing: - raise ValueError(f"Missing required buffers: {sorted(missing)}.") - for name in self._source_buffers.keys() & buffers.keys(): - self._validate_buffer(name, buffers[name], self._source_buffers[name]) - current = dict(buffers) - for operation, inputs, outputs, destinations in self._operations: - operation.apply({role: current[name] for role, name in inputs.items()}, destinations) - current.update({name: destinations[role] for role, name in outputs.items()}) - # Release replaced source references, such as the preceding observation's tensor. - self._source_buffers.update({name: buffers[name] for name in self._source_buffers.keys() & buffers.keys()}) - self.outputs = MappingProxyType(dict(current)) - return current - - def apply_to_tensor( - self, - data: torch.Tensor, - *, - input_name: str, - output_name: str, - auxiliary_buffers: Mapping[str, ProxyArray] | None = None, - ) -> torch.Tensor: - """Process an incoming observation tensor and return an independently owned snapshot. - - The supplied tensor is authoritative: earlier observation transformations are preserved. - Auxiliary inputs must already be ready on the current Warp/Torch stream. CUDA work is - ordered with the caller's Torch stream; no host/device transfer is performed. - """ - if auxiliary_buffers is not None and input_name in auxiliary_buffers: - raise ValueError("Auxiliary buffers must not replace the incoming observation tensor.") - buffers = dict(auxiliary_buffers or {}) - buffers[input_name] = ProxyArray(wp.from_torch(data)) - return self.as_tensor(buffers, output_name=output_name) - - def as_tensor( - self, buffers: Mapping[str, ProxyArray], *, output_name: str, out: torch.Tensor | None = None - ) -> torch.Tensor: - """Execute on the current Torch stream and snapshot one output for an observation. - - Source buffers must be ready on the current Warp or Torch stream. Stream ordering - is established without a device-wide synchronization; no host/device copy is made. - A supplied ``out`` must match the prepared output's shape, dtype and device and is overwritten in place. - """ - if self._closed: - raise RuntimeError("Cannot execute a closed buffer pipe.") - reference = self.outputs[output_name] if output_name in self.produced_outputs else buffers[output_name] - if out is not None: - self._validate_buffer("out", ProxyArray(wp.from_torch(out)), reference) - device = wp.get_device(reference.device) - stream = wp.stream_from_torch(torch.cuda.current_stream(str(device))) if device.is_cuda else None - with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): - result = self(buffers)[output_name].torch - return result.clone() if out is None else out.copy_(result) - - def reset(self, env_ids: Sequence[int] | None = None) -> None: - """Reset operation-owned state for selected batch entries.""" - for operation, *_ in self._operations: - if operation.reset is not None: - operation.reset(env_ids) - - def close(self) -> None: - """Release operation resources; repeated calls have no effect.""" - if self._closed: - return - self._closed = True - operations, self._operations = self._operations, [] - self._source_buffers.clear() - self.outputs = MappingProxyType({}) - errors = [] - for operation, *_ in reversed(operations): - if operation.close is not None: - try: - operation.close() - except Exception as error: - errors.append(error) - if errors: - raise ExceptionGroup("Failed to close buffer operations.", errors) diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/__init__.pyi b/source/isaaclab/isaaclab/utils/tensor_processors/__init__.pyi index 1317c1d22978..50ca97dd9829 100644 --- a/source/isaaclab/isaaclab/utils/tensor_processors/__init__.pyi +++ b/source/isaaclab/isaaclab/utils/tensor_processors/__init__.pyi @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD-3-Clause __all__ = [ + "DictTensorProcessor", + "DictTensorProcessorCfg", "TensorProcessorCfg", "DictTensorProcessorAdapterCfg", "DictTensorProcessorAdapter", @@ -21,3 +23,6 @@ from .tensor_processor_cfg import TensorProcessorCfg, DigitalFilterCfg, Integrat from .dict_tensor_processor_adapter import DictTensorProcessorAdapter from .tensor_processor import TensorProcessor from .processors import DigitalFilter, Integrator, bias, clip, scale + +from .dict_tensor_processor import DictTensorProcessor +from .dict_tensor_processor_cfg import DictTensorProcessorCfg diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/_pipeline.py b/source/isaaclab/isaaclab/utils/tensor_processors/_pipeline.py new file mode 100644 index 000000000000..b5cd8c927f62 --- /dev/null +++ b/source/isaaclab/isaaclab/utils/tensor_processors/_pipeline.py @@ -0,0 +1,170 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from builtins import ExceptionGroup +from collections.abc import Mapping, Sequence + +import torch +import warp as wp + +from ..configclass import instantiate +from ..warp import ProxyArray +from .dict_tensor_processor import DictTensorProcessor +from .dict_tensor_processor_cfg import DictTensorProcessorCfg + + +class _DictTensorProcessorPipeline: + """Compose dictionary processors in order, preserving untouched input keys. + + Each processor receives current named tensors and returns its named outputs. Preparation + constructs processors once; calls reuse their output storage. Each consumer owns its own + pipeline. Input buffers are borrowed, and returned mappings are independent snapshots of + references, not tensor copies. Use ``as_tensor`` when an observation must retain values. + """ + + def __init__(self, source_buffers: Mapping[str, ProxyArray], processors: Sequence[DictTensorProcessorCfg]): + self._source_buffers = dict(source_buffers) + self.required_inputs = self.required_source_names(processors) + self._processors: list[tuple[DictTensorProcessor, dict[str, ProxyArray]]] = [] + self._closed = False + self.outputs = dict(source_buffers) + self.produced_outputs = frozenset(name for cfg in processors for name in cfg.outputs) + try: + for cfg in processors: + if not cfg.outputs or len(set(cfg.outputs)) != len(cfg.outputs): + raise ValueError("A processor must declare distinct output names.") + missing = set(cfg.inputs) - self.outputs.keys() + if missing: + raise ValueError(f"Missing input buffers for preparation: {sorted(missing)}.") + processor = instantiate(cfg, dict(self.outputs)) + if not isinstance(processor, DictTensorProcessor): + raise TypeError("Dictionary processors must inherit DictTensorProcessor.") + self._processors.append((processor, dict(processor.outputs))) + if set(processor.outputs) != set(cfg.outputs): + raise ValueError("Processor outputs must match its declared output names.") + if any(not isinstance(buffer, ProxyArray) for buffer in processor.outputs.values()): + raise TypeError("Processor outputs must be ProxyArray objects.") + devices = {str(self.outputs[name].device) for name in cfg.inputs} + devices.update(str(buffer.device) for buffer in processor.outputs.values()) + if len(devices) > 1: + raise ValueError("A processor must use one device for its inputs and outputs.") + self.outputs.update(processor.outputs) + except Exception as error: + try: + self.close() + except Exception as cleanup_error: + raise error from cleanup_error + raise + + @property + def has_reset_callbacks(self) -> bool: + """Whether any processor owns resettable state.""" + return any(type(processor).reset is not DictTensorProcessor.reset for processor, _ in self._processors) + + @staticmethod + def _validate_buffer(name: str, value: ProxyArray, reference: ProxyArray) -> None: + if not isinstance(value, ProxyArray): + raise TypeError(f"Buffer {name!r} must be a ProxyArray.") + if value.shape != reference.shape or value.dtype != reference.dtype or value.device != reference.device: + raise ValueError( + f"Buffer {name!r} must match shape, dtype and device" + f" {reference.shape}, {reference.dtype}, {reference.device}." + ) + + @staticmethod + def required_source_names( + processors: Sequence[DictTensorProcessorCfg], published_names: Sequence[str] = () + ) -> frozenset[str]: + """Resolve external keys before allocating buffers or initializing a renderer.""" + required: set[str] = set() + produced: set[str] = set() + for cfg in processors: + required.update(set(cfg.inputs) - produced) + produced.update(cfg.outputs) + required.update(set(published_names) - produced) + return frozenset(required) + + def __call__(self, data: dict[str, ProxyArray]) -> dict[str, ProxyArray]: + """Apply processors and return their results with untouched inputs passed through.""" + if self._closed: + raise RuntimeError("Cannot execute a closed processor pipeline.") + missing = self.required_inputs - data.keys() + if missing: + raise ValueError(f"Missing required buffers: {sorted(missing)}.") + for name in self._source_buffers.keys() & data.keys(): + self._validate_buffer(name, data[name], self._source_buffers[name]) + current = dict(data) + for processor, outputs in self._processors: + result = processor(dict(current)) + if result.keys() != outputs.keys() or any(result[name] is not output for name, output in outputs.items()): + raise ValueError("Processors must return their persistent named output buffers.") + current.update(result) + self._source_buffers.update({name: data[name] for name in self._source_buffers.keys() & data.keys()}) + self.outputs = dict(current) + return current + + def apply_to_tensor( + self, + data: torch.Tensor, + *, + input_name: str, + output_name: str, + auxiliary_buffers: Mapping[str, ProxyArray] | None = None, + ) -> torch.Tensor: + """Process an incoming observation tensor and return an independently owned snapshot. + + The supplied tensor is authoritative: earlier observation transformations are preserved. + Auxiliary inputs must already be ready on the current Warp/Torch stream. CUDA work is + ordered with the caller's Torch stream; no host/device transfer is performed. + """ + if auxiliary_buffers is not None and input_name in auxiliary_buffers: + raise ValueError("Auxiliary buffers must not replace the incoming observation tensor.") + buffers = dict(auxiliary_buffers or {}) + buffers[input_name] = ProxyArray(wp.from_torch(data)) + return self.as_tensor(buffers, output_name=output_name) + + def as_tensor( + self, buffers: Mapping[str, ProxyArray], *, output_name: str, out: torch.Tensor | None = None + ) -> torch.Tensor: + """Execute on the current Torch stream and snapshot one output for an observation. + + Source buffers must be ready on the current Warp or Torch stream. Stream ordering + is established without a device-wide synchronization; no host/device copy is made. + A supplied ``out`` must match the prepared output's shape, dtype and device and is overwritten in place. + """ + if self._closed: + raise RuntimeError("Cannot execute a closed buffer pipe.") + reference = self.outputs[output_name] if output_name in self.produced_outputs else buffers[output_name] + if out is not None: + self._validate_buffer("out", ProxyArray(wp.from_torch(out)), reference) + device = wp.get_device(reference.device) + stream = wp.stream_from_torch(torch.cuda.current_stream(str(device))) if device.is_cuda else None + with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): + result = self(buffers)[output_name].torch + return result.clone() if out is None else out.copy_(result) + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + """Reset processor-owned state for selected batch entries.""" + for processor, _ in self._processors: + processor.reset(env_ids) + + def close(self) -> None: + """Close processors in reverse order, including after failed construction.""" + if self._closed: + return + self._closed = True + processors, self._processors = self._processors, [] + self._source_buffers.clear() + self.outputs = {} + errors = [] + for processor, _ in reversed(processors): + try: + processor.close() + except Exception as error: + errors.append(error) + if errors: + raise ExceptionGroup("Failed to close dictionary processors.", errors) diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor.py b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor.py new file mode 100644 index 000000000000..1ee875cdabe0 --- /dev/null +++ b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor.py @@ -0,0 +1,47 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Sequence +from typing import TYPE_CHECKING + +from ..warp import ProxyArray + +if TYPE_CHECKING: + from .dict_tensor_processor_cfg import DictTensorProcessorCfg + + +class DictTensorProcessor(ABC): + """Transform a dictionary of named tensors into a dictionary of named tensors. + + This is the dictionary counterpart of :class:`TensorProcessor`. Tensor values use + :class:`ProxyArray` so processors can use Warp or Torch without copying. Inputs are + borrowed read-only. Constructors inspect their shape, dtype and device and allocate + persistent outputs without reading input values or advancing processor state. + + Subclasses populate ``outputs`` with the buffers they produce, implement ``__call__``, + and override ``reset`` or ``close`` when they own state or external resources. Calls + return those same output buffers; values remain valid until the next call. Consumers + that retain results must copy them. GPU work uses the caller's current Warp stream. + """ + + def __init__(self, cfg: DictTensorProcessorCfg, data: dict[str, ProxyArray]) -> None: + self.cfg = cfg + self.outputs: dict[str, ProxyArray] = {} + + @abstractmethod + def __call__(self, data: dict[str, ProxyArray]) -> dict[str, ProxyArray]: + """Process current inputs and return the named outputs without mutating inputs.""" + raise NotImplementedError + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + """Reset state for selected batch entries, or all entries when None.""" + pass + + def close(self) -> None: + """Release owned resources; repeated calls must have no effect.""" + pass diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_adapter.py b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_adapter.py index 7a3c9a1bd3cb..7b079b815f2f 100644 --- a/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_adapter.py +++ b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_adapter.py @@ -13,8 +13,8 @@ import torch import warp as wp -from ..processing import DictTensorProcessor from ..warp import ProxyArray +from ._pipeline import _DictTensorProcessorPipeline from .tensor_processor import TensorProcessor if TYPE_CHECKING: @@ -33,7 +33,7 @@ class DictTensorProcessorAdapter(TensorProcessor): def __init__(self, cfg: DictTensorProcessorAdapterCfg, data_dim: tuple[int, ...], device: str): super().__init__(cfg, data_dim, device) - self._pipe: DictTensorProcessor | None = None + self._pipe: _DictTensorProcessorPipeline | None = None self._closed = False def __call__(self, data: torch.Tensor) -> torch.Tensor: @@ -46,7 +46,9 @@ def __call__(self, data: torch.Tensor) -> torch.Tensor: if self._pipe is None: stream = wp.stream_from_torch(torch.cuda.current_stream(data.device)) if data.is_cuda else None with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): - pipe = DictTensorProcessor({self._cfg.input_name: ProxyArray(wp.from_torch(data))}, self._cfg.pipeline) + pipe = _DictTensorProcessorPipeline( + {self._cfg.input_name: ProxyArray(wp.from_torch(data))}, self._cfg.pipeline + ) try: if self._cfg.output_name not in pipe.outputs: raise ValueError(f"Buffer processing does not provide output {self._cfg.output_name!r}.") diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_cfg.py b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_cfg.py new file mode 100644 index 000000000000..119afafac7d5 --- /dev/null +++ b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_cfg.py @@ -0,0 +1,40 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from dataclasses import MISSING +from typing import TYPE_CHECKING, Any + +from ..configclass import configclass + +if TYPE_CHECKING: + from .dict_tensor_processor import DictTensorProcessor + + +@configclass +class DictTensorProcessorCfg: + """Configure one dictionary processor and declare its producer dependencies. + + Names are actual dictionary keys, with no separate operation roles. Cameras use these + declarations before renderer initialization. The class receives ``(cfg, data)`` and + owns its output storage, state, and lifecycle, just like a single-tensor processor. + """ + + class_type: type[DictTensorProcessor] | str = MISSING + """Processor class, or a resolvable import string.""" + + inputs: list[str] = MISSING + """Dictionary keys required from the producer or preceding processors.""" + + outputs: list[str] = MISSING + """Dictionary keys produced or replaced by this processor.""" + + params: dict[str, Any] = {} + """Processor-specific settings, accessible through ``cfg.params``. + + Configurations are copied. Explicitly shared device resources must preserve identity + through ``__deepcopy__``; do not place raw Warp allocations in this dictionary. + """ diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor.py b/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor.py index fb0320994c6f..cdf0d681c0ab 100644 --- a/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor.py +++ b/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor.py @@ -16,33 +16,12 @@ class TensorProcessor(ABC): - """Base class for tensor_processors implemented as classes. - - Tensor processors implementations can be functions or classes. If a processor is a class, it should - inherit from this class and implement the required methods. - - A class implementation of a processor can be used to store state information between calls. - This is useful for tensor_processors that require stateful operations, such as rolling averages - or delays or decaying filters. - - Example pseudo-code to create and use the class: - - .. code-block:: python - - from isaaclab.utils import tensor_processors - - # define custom keyword arguments to pass to TensorProcessorCfg - kwarg_dict = {"arg_1": VAL_1, "arg_2": VAL_2} - - # create processor configuration object - # func is the class name of the processor and params is the dictionary of arguments - processor_config = tensor_processors.TensorProcessorCfg( - func=tensor_processors.TensorProcessor, params=kwarg_dict - ) - - # define processor instance - my_processor = MyProcessor(processor_config, data_dim=(32, 4), device="cpu") + """Transform one tensor into one tensor, optionally retaining state between calls. + This is the single-tensor counterpart of :class:`DictTensorProcessor`. Subclasses + implement ``__call__`` and ``reset``; override ``close`` when external resources need + cleanup. Observation processors preserve the input shape. Configuration and dimensions + are supplied by the observation manager once during construction. """ def __init__(self, cfg: TensorProcessorCfg, data_dim: tuple[int, ...], device: str) -> None: diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor_cfg.py b/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor_cfg.py index 12068a56a67f..2561e8ad7aff 100644 --- a/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor_cfg.py +++ b/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor_cfg.py @@ -14,8 +14,8 @@ from .. import configclass if TYPE_CHECKING: - from ..processing import DictTensorProcessorStepCfg from .dict_tensor_processor_adapter import DictTensorProcessorAdapter + from .dict_tensor_processor_cfg import DictTensorProcessorCfg from .processors import DigitalFilter, Integrator from .tensor_processor import TensorProcessor @@ -50,7 +50,7 @@ class DictTensorProcessorAdapterCfg(TensorProcessorCfg): func: type[DictTensorProcessorAdapter] | str = "{DIR}.dict_tensor_processor_adapter:DictTensorProcessorAdapter" """Processor class consuming the incoming tensor.""" - pipeline: list[DictTensorProcessorStepCfg] = MISSING + pipeline: list[DictTensorProcessorCfg] = MISSING """Ordered buffer processing operations.""" input_name: str = "rgb" diff --git a/source/isaaclab/test/envs/test_buffer_observations.py b/source/isaaclab/test/envs/test_buffer_observations.py index b9259e9d5ddf..f74856539b3b 100644 --- a/source/isaaclab/test/envs/test_buffer_observations.py +++ b/source/isaaclab/test/envs/test_buffer_observations.py @@ -21,7 +21,7 @@ ObservationTermCfg, ) from isaaclab.utils import configclass, tensor_processors -from isaaclab.utils.processing import DictTensorProcessorStepCfg, PreparedBufferOperation +from isaaclab.utils.tensor_processors import DictTensorProcessor, DictTensorProcessorCfg from isaaclab.utils.warp import ProxyArray pytestmark = pytest.mark.unit @@ -69,18 +69,20 @@ def _depth_weighted_half( result[n, y, x, c] = color[n, y, 2 * x, c] * depth[n, y, 2 * x, 0] -def _prepare_gain(inputs, gain): - result = ProxyArray(wp.empty_like(inputs["color"].warp)) +class _Gain(DictTensorProcessor): + def __init__(self, cfg, data): + super().__init__(cfg, data) + self.outputs[cfg.outputs[0]] = ProxyArray(wp.empty_like(data[cfg.inputs[0]].warp)) - def apply(inputs, outputs): + def __call__(self, data): + output = self.outputs[self.cfg.outputs[0]] wp.launch( _gain, - dim=outputs["color"].shape, - inputs=[inputs["color"].warp, gain, outputs["color"].warp], - device=result.device, + dim=output.shape, + inputs=[data[self.cfg.inputs[0]].warp, self.cfg.params.get("gain", 2.0), output.warp], + device=output.device, ) - - return PreparedBufferOperation({"color": result}, apply) + return dict(self.outputs) def _read_ranges(env): @@ -111,14 +113,15 @@ def test_pipe_modifier_uses_preceding_modifier_output(): ranges = torch.ones((2, 4)) preparations = [] - def prepare(inputs, gain): - preparations.append(inputs["ranges"].shape) - result = ProxyArray(wp.empty_like(inputs["ranges"].warp)) - - def apply(inputs, outputs): - torch.mul(inputs["ranges"].torch, gain, out=outputs["ranges"].torch) + class RangeGain(DictTensorProcessor): + def __init__(self, cfg, data): + super().__init__(cfg, data) + preparations.append(data["ranges"].shape) + self.outputs["ranges"] = ProxyArray(wp.empty_like(data["ranges"].warp)) - return PreparedBufferOperation({"ranges": result}, apply) + def __call__(self, data): + torch.mul(data["ranges"].torch, self.cfg.params["gain"], out=self.outputs["ranges"].torch) + return dict(self.outputs) @configclass class Policy(ObservationGroupCfg): @@ -129,10 +132,10 @@ class Policy(ObservationGroupCfg): tensor_processors.TensorProcessorCfg(func=tensor_processors.bias, params={"value": 3.0}), tensor_processors.DictTensorProcessorAdapterCfg( pipeline=[ - DictTensorProcessorStepCfg( - prepare=prepare, - inputs={"ranges": "ranges"}, - outputs={"ranges": "ranges"}, + DictTensorProcessorCfg( + class_type=RangeGain, + inputs=["ranges"], + outputs=["ranges"], params={"gain": 2.0}, ) ], @@ -162,18 +165,18 @@ def test_pipe_modifier_retries_failed_preparation_and_stays_closed(): lifecycle = [] valid_shape = False - def prepare(inputs): - lifecycle.append("prepare") - operation = _prepare_gain(inputs, 2.0) - output = ( - operation.outputs["color"] - if valid_shape - else ProxyArray(wp.empty((2, 2, 2, 3), dtype=wp.float32, device="cpu")) - ) - return PreparedBufferOperation({"color": output}, operation.apply, close=lambda: lifecycle.append("close")) + class TrackingGain(_Gain): + def __init__(self, cfg, data): + super().__init__(cfg, data) + lifecycle.append("prepare") + if not valid_shape: + self.outputs["rgb"] = ProxyArray(wp.empty((2, 2, 2, 3), dtype=wp.float32, device="cpu")) + + def close(self): + lifecycle.append("close") cfg = tensor_processors.DictTensorProcessorAdapterCfg( - pipeline=[DictTensorProcessorStepCfg(prepare=prepare, inputs={"color": "rgb"}, outputs={"color": "rgb"})] + pipeline=[DictTensorProcessorCfg(class_type=TrackingGain, inputs=["rgb"], outputs=["rgb"])] ) modifier = tensor_processors.DictTensorProcessorAdapter(cfg, tuple(image.shape), "cpu") modifier.reset([1]) @@ -202,30 +205,30 @@ def test_processed_sensor_declares_private_inputs_and_reports_transformed_shape( """Preparation resolves private AOVs before binding; manager dimensions describe the processed output.""" lifecycle = [] - def prepare(inputs): - color = inputs["color"] - n, h, w, c = color.shape - output = ProxyArray(wp.empty((n, h, w // 2, c), dtype=wp.float32, device=color.device)) + class DepthWeighted(DictTensorProcessor): + def __init__(self, cfg, data): + super().__init__(cfg, data) + color = data["rgb_hdr"] + n, h, w, c = color.shape + self.outputs["rgb"] = ProxyArray(wp.empty((n, h, w // 2, c), dtype=wp.float32, device=color.device)) - def apply(inputs, outputs): + def __call__(self, data): + output = self.outputs["rgb"] wp.launch( _depth_weighted_half, dim=output.shape, - inputs=[inputs["color"].warp, inputs["depth"].warp, outputs["color"].warp], + inputs=[data["rgb_hdr"].warp, data["depth"].warp, output.warp], device=output.device, ) + return dict(self.outputs) - def close(): + def reset(self, env_ids=None): + lifecycle.append(("reset", env_ids)) + + def close(self): lifecycle.append(("close",)) raise ValueError("operation close failed") - return PreparedBufferOperation( - {"color": output}, - apply, - reset=lambda env_ids: lifecycle.append(("reset", env_ids)), - close=close, - ) - radiance = torch.full((2, 2, 4, 3), 2.0) depth = torch.tensor([1.0, 9.0, 3.0, 9.0]).reshape(1, 1, 4, 1).expand(2, 2, 4, 1).contiguous() requests = [] @@ -248,10 +251,10 @@ class Policy(ObservationGroupCfg): func="isaaclab.envs.mdp:processed_sensor", params={ "pipeline": [ - DictTensorProcessorStepCfg( - prepare=prepare, - inputs={"color": "rgb_hdr", "depth": "depth"}, - outputs={"color": "rgb"}, + DictTensorProcessorCfg( + class_type=DepthWeighted, + inputs=["rgb_hdr", "depth"], + outputs=["rgb"], ) ] }, @@ -277,7 +280,7 @@ class Observations: torch.testing.assert_close(manager.compute()["policy"]["image"], expected * 2) torch.testing.assert_close(result, expected) manager.reset([1]) - with pytest.raises(ExceptionGroup, match="Failed to close buffer operations") as error: + with pytest.raises(ExceptionGroup, match="Failed to close dictionary processors") as error: manager.close() assert str(error.value.exceptions[0]) == "operation close failed" manager.close() diff --git a/source/isaaclab/test/renderers/test_camera_output_contract.py b/source/isaaclab/test/renderers/test_camera_output_contract.py index 8a25c4d2e0de..34f133010b75 100644 --- a/source/isaaclab/test/renderers/test_camera_output_contract.py +++ b/source/isaaclab/test/renderers/test_camera_output_contract.py @@ -25,7 +25,7 @@ from isaaclab.sim import PinholeCameraCfg from isaaclab.test.utils import DeviceScope, test_devices from isaaclab.utils import clone, validate -from isaaclab.utils.processing import DictTensorProcessorStepCfg, PreparedBufferOperation +from isaaclab.utils.tensor_processors import DictTensorProcessor, DictTensorProcessorCfg from isaaclab.utils.warp import ProxyArray pytestmark = [pytest.mark.integration, pytest.mark.rendering] @@ -370,17 +370,19 @@ def export_stage(stage, num_envs): lambda path, **kwargs: SimpleNamespace(count=2, prims=[stage.GetPrimAtPath(path)] * 2, close=lambda: None), ) - def prepare_identity(inputs): - output = ProxyArray(wp.empty_like(inputs["image"].warp)) - return PreparedBufferOperation( - {"image": output}, - lambda inputs, outputs: wp.copy(outputs["image"].warp, inputs["image"].warp), - ) + class Identity(DictTensorProcessor): + def __init__(self, cfg, data): + super().__init__(cfg, data) + self.outputs["processed_radiance"] = ProxyArray(wp.empty_like(data["rgb_hdr"].warp)) + + def __call__(self, data): + wp.copy(self.outputs["processed_radiance"].warp, data["rgb_hdr"].warp) + return dict(self.outputs) def make_camera(prim, data_types): camera = Camera.__new__(Camera) pipeline = ( - [DictTensorProcessorStepCfg(prepare_identity, {"image": "rgb_hdr"}, {"image": "processed_radiance"})] + [DictTensorProcessorCfg(Identity, ["rgb_hdr"], ["processed_radiance"])] if data_types == ["processed_radiance"] else [] ) @@ -450,38 +452,34 @@ def test_camera_pipeline_keeps_raw_inputs_and_processes_only_new_captures(monkey events = [] output_shape = (2, 1, 1, 4) close_fails = False - reset_enabled = True - def prepare_color(inputs): - output = ProxyArray(wp.empty(output_shape, dtype=wp.uint8, device=inputs["radiance"].device)) + class Color(DictTensorProcessor): + def __init__(self, cfg, data): + super().__init__(cfg, data) + self.outputs["rgba"] = ProxyArray(wp.empty(output_shape, dtype=wp.uint8, device=data["rgb_hdr"].device)) - def apply(inputs, outputs): + def __call__(self, data): events.append("apply") - output = outputs["color"].torch - output[..., :3].copy_(100 * (inputs["radiance"].torch + inputs["range"].torch)) + output = self.outputs["rgba"].torch + output[..., :3].copy_(100 * (data["rgb_hdr"].torch + data["depth"].torch)) output[..., 3] = 255 + return dict(self.outputs) + + def reset(self, env_ids=None): + events.append(tuple(env_ids)) - def close(): + def close(self): events.append("close") if close_fails: raise RuntimeError("processor close failed") - return PreparedBufferOperation( - {"color": output}, - apply, - reset=(lambda ids: events.append(tuple(ids))) if reset_enabled else None, - close=close, - ) - cfg = CameraCfg( prim_path="/World/Camera", spawn=None, height=1, width=1, data_types=["rgb", "depth"], - pipeline=[ - DictTensorProcessorStepCfg(prepare_color, {"radiance": "rgb_hdr", "range": "depth"}, {"color": "rgba"}) - ], + pipeline=[DictTensorProcessorCfg(Color, ["rgb_hdr", "depth"], ["rgba"])], ) specs = { RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), @@ -547,7 +545,7 @@ def read_output(render_data, camera_data): assert camera._buffer_pipe is None assert events[-2:] == ["close", "close"] output_shape = (2, 1, 1, 4) - reset_enabled = False + monkeypatch.setattr(Color, "reset", DictTensorProcessor.reset) camera._create_buffers() # Stateless operations must not materialize indices: CUDA nonzero would synchronize the host. with monkeypatch.context() as patch: @@ -559,7 +557,7 @@ def read_output(render_data, camera_data): camera._renderer.cleanup = lambda data: events.append("renderer-close") camera._view.close = lambda: events.append("view-close") close_fails = True - with pytest.raises(ExceptionGroup, match="close buffer operations"): + with pytest.raises(ExceptionGroup, match="close dictionary processors"): camera._invalidate_initialize_callback(None) assert events[-3:] == ["close", "renderer-close", "view-close"] assert not camera.is_initialized diff --git a/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py b/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py index a7ec17715a4b..16aa58f8754e 100644 --- a/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py +++ b/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py @@ -24,7 +24,7 @@ from typing import TYPE_CHECKING import torch -from isaaclab_ppisp import PpispCfg, PpispStepCfg, normalize_ppisp_cfg, ppisp_cfg_from_usd_camera +from isaaclab_ppisp import PpispCfg, PpispProcessorCfg, normalize_ppisp_cfg, ppisp_cfg_from_usd_camera from pxr import Gf, Sdf, Usd, UsdGeom, Vt @@ -728,7 +728,7 @@ def render_synthetic_gaussian_scene( data_types=data_types, spawn=None, exposure=CameraExposureCfg(), - pipeline=[PpispStepCfg(params={"cfg": isp_cfg})], + pipeline=[PpispProcessorCfg(params={"cfg": isp_cfg})], renderer_cfg=renderer_cfg, ) camera = Camera(cfg) @@ -821,7 +821,7 @@ def _render_synthetic_gaussian_camera( data_types=data_types, spawn=None, exposure=CameraExposureCfg(), - pipeline=[PpispStepCfg(params={"cfg": ppisp_cfg})], + pipeline=[PpispProcessorCfg(params={"cfg": ppisp_cfg})], renderer_cfg=renderer_cfg, ) camera = Camera(cfg) diff --git a/source/isaaclab/test/utils/test_processing_buffer_pipe.py b/source/isaaclab/test/utils/test_dict_tensor_processor.py similarity index 59% rename from source/isaaclab/test/utils/test_processing_buffer_pipe.py rename to source/isaaclab/test/utils/test_dict_tensor_processor.py index a0200679795b..afcdf3145240 100644 --- a/source/isaaclab/test/utils/test_processing_buffer_pipe.py +++ b/source/isaaclab/test/utils/test_dict_tensor_processor.py @@ -11,34 +11,36 @@ import torch import warp as wp -from isaaclab.utils.processing import DictTensorProcessor, DictTensorProcessorStepCfg, PreparedBufferOperation +from isaaclab.utils.tensor_processors import DictTensorProcessor, DictTensorProcessorCfg +from isaaclab.utils.tensor_processors._pipeline import _DictTensorProcessorPipeline from isaaclab.utils.warp import ProxyArray pytestmark = pytest.mark.unit -def _prepare_scale(inputs, *, factor=2.0): - destination = ProxyArray(wp.empty_like(inputs["value"].warp)) +class _Scale(DictTensorProcessor): + def __init__(self, cfg, data): + super().__init__(cfg, data) + self.outputs[cfg.outputs[0]] = ProxyArray(wp.empty_like(data[cfg.inputs[0]].warp)) - def apply(inputs, outputs): - torch.mul(inputs["value"].torch, factor, out=outputs["value"].torch) - - return PreparedBufferOperation({"value": destination}, apply) + def __call__(self, data): + torch.mul( + data[self.cfg.inputs[0]].torch, + self.cfg.params.get("factor", 2.0), + out=self.outputs[self.cfg.outputs[0]].torch, + ) + return dict(self.outputs) -def _prepare_composite(inputs): - """Combine color and depth, retaining only one spatial sample per batch entry.""" - color = inputs["color"] - destination = ProxyArray(wp.empty((color.shape[0], 1, 1, 3), dtype=color.dtype, device=color.device)) +class _Composite(DictTensorProcessor): + def __init__(self, cfg, data): + super().__init__(cfg, data) + color = data["color"] + self.outputs["color"] = ProxyArray(wp.empty((color.shape[0], 1, 1, 3), dtype=color.dtype, device=color.device)) - def apply(inputs, outputs): - torch.sub( - inputs["color"].torch[:, :1, :1], - inputs["depth"].torch[:, :1, :1], - out=outputs["value"].torch, - ) - - return PreparedBufferOperation({"value": destination}, apply) + def __call__(self, data): + torch.sub(data["color"].torch[:, :1, :1], data["range"].torch[:, :1, :1], out=self.outputs["color"].torch) + return dict(self.outputs) @pytest.fixture @@ -48,15 +50,19 @@ def composite_pipe(): "range": ProxyArray(wp.zeros((2, 2, 2, 1), dtype=wp.float32, device="cpu")), } steps = [ - DictTensorProcessorStepCfg(prepare=_prepare_scale, inputs={"value": "raw_color"}, outputs={"value": "color"}), - DictTensorProcessorStepCfg( - prepare=_prepare_composite, - inputs={"color": "color", "depth": "range"}, - outputs={"value": "color"}, + DictTensorProcessorCfg(class_type=_Scale, inputs=["raw_color"], outputs=["color"]), + DictTensorProcessorCfg( + class_type=_Composite, + inputs=["color", "range"], + outputs=["color"], ), ] - assert DictTensorProcessor.required_source_names(steps, ["color", "labels"]) == {"raw_color", "range", "labels"} - pipe = DictTensorProcessor(sources, steps) + assert _DictTensorProcessorPipeline.required_source_names(steps, ["color", "labels"]) == { + "raw_color", + "range", + "labels", + } + pipe = _DictTensorProcessorPipeline(sources, steps) yield pipe pipe.close() @@ -115,7 +121,7 @@ def test_tensor_adapter_preserves_incoming_transformations_and_returns_snapshots def test_tensor_snapshot_accepts_passthrough_buffer_omitted_from_previous_call(): """Optional pass-through buffers can return after a call that did not supply them.""" original = ProxyArray(wp.zeros((2,), dtype=wp.float32, device="cpu")) - pipe = DictTensorProcessor({"value": original}, []) + pipe = _DictTensorProcessorPipeline({"value": original}, []) try: pipe({}) current = ProxyArray(wp.full((2,), 3.0, dtype=wp.float32, device="cpu")) @@ -132,13 +138,14 @@ def test_invalid_source_buffers_are_rejected_before_processing(invalid): calls = [] source = ProxyArray(wp.zeros((2, 1, 1, 1), dtype=wp.float32, device="cpu")) - def prepare(inputs): - destination = ProxyArray(wp.empty_like(inputs["value"].warp)) - return PreparedBufferOperation({"value": destination}, lambda *_: calls.append(True)) + class CountCalls(_Scale): + def __call__(self, data): + calls.append(True) + return super().__call__(data) - pipe = DictTensorProcessor( + pipe = _DictTensorProcessorPipeline( {"source": source}, - [DictTensorProcessorStepCfg(prepare=prepare, inputs={"value": "source"}, outputs={"value": "result"})], + [DictTensorProcessorCfg(class_type=CountCalls, inputs=["source"], outputs=["result"])], ) try: shape = (1, 1, 1, 1) if invalid == "shape" else source.shape @@ -151,72 +158,69 @@ def prepare(inputs): pipe.close() -@pytest.mark.parametrize("invalid", ["missing_dependency", "output_roles", "duplicate_outputs", "output_type"]) -def test_invalid_bindings_release_prepared_operations(invalid): - """Construction failure closes owned operations, including one whose outputs could not be bound.""" +@pytest.mark.parametrize("invalid", ["missing_dependency", "output_names", "duplicate_outputs", "output_type"]) +def test_invalid_declarations_release_processors(invalid): + """Construction failure closes each successfully constructed processor in reverse order.""" closed = [] source = ProxyArray(wp.zeros((2, 1, 1, 1), dtype=wp.float32, device="cpu")) - def prepare(inputs, *, name): - destination = ProxyArray(wp.empty_like(inputs["value"].warp)) - if invalid == "output_type" and name == "second": - destination = destination.warp - return PreparedBufferOperation({"value": destination}, lambda *_: None, close=lambda: closed.append(name)) + class Tracked(_Scale): + def __init__(self, cfg, data): + super().__init__(cfg, data) + if cfg.params["name"] == "second": + if invalid == "output_names": + self.outputs = {"wrong": next(iter(self.outputs.values()))} + elif invalid == "output_type": + self.outputs[cfg.outputs[0]] = self.outputs[cfg.outputs[0]].warp - first = DictTensorProcessorStepCfg( - prepare=prepare, inputs={"value": "source"}, outputs={"value": "middle"}, params={"name": "first"} - ) - second = DictTensorProcessorStepCfg( - prepare=prepare, inputs={"value": "middle"}, outputs={"value": "result"}, params={"name": "second"} - ) + def close(self): + closed.append(self.cfg.params["name"]) + + first = DictTensorProcessorCfg(Tracked, ["source"], ["middle"], {"name": "first"}) + second = DictTensorProcessorCfg(Tracked, ["middle"], ["result"], {"name": "second"}) if invalid == "missing_dependency": - second.inputs = {"value": "absent"} + second.inputs = ["absent"] message = "Missing input buffers for preparation" - elif invalid == "output_roles": - second.outputs = {"wrong": "result"} - message = "output roles" + elif invalid == "output_names": + message = "declared output names" elif invalid == "duplicate_outputs": - second.outputs = {"value": "result", "other": "result"} + second.outputs = ["result", "result"] message = "distinct output names" else: message = "outputs must be ProxyArray" with pytest.raises(TypeError if invalid == "output_type" else ValueError, match=message): - DictTensorProcessor({"source": source}, [first, second]) - assert closed == (["second", "first"] if invalid in ("output_roles", "output_type") else ["first"]) + _DictTensorProcessorPipeline({"source": source}, [first, second]) + assert closed == (["second", "first"] if invalid in ("output_names", "output_type") else ["first"]) @pytest.mark.parametrize("close_fails", [False, True], ids=["normal_close", "failing_close"]) def test_reset_selects_batch_entries_and_close_releases_operations_in_reverse_order(close_fails): - """State belongs to each operation; selected resets and idempotent teardown preserve its lifecycle.""" + """State belongs to each processor; selected resets and idempotent teardown preserve its lifecycle.""" closed = [] source = ProxyArray(wp.zeros((2, 1, 1, 1), dtype=wp.float32, device="cpu")) - def prepare(inputs, *, name): - destination = ProxyArray(wp.empty_like(inputs["value"].warp)) - history = torch.zeros(inputs["value"].shape) + class History(_Scale): + def __init__(self, cfg, data): + super().__init__(cfg, data) + self.history = torch.zeros(data["value"].shape) - def apply(inputs, outputs): - history.add_(inputs["value"].torch) - outputs["value"].torch.copy_(history) + def __call__(self, data): + self.history.add_(data["value"].torch) + self.outputs["value"].torch.copy_(self.history) + return dict(self.outputs) - def reset(env_ids): - history[slice(None) if env_ids is None else env_ids] = 0 + def reset(self, env_ids=None): + self.history[slice(None) if env_ids is None else env_ids] = 0 - def close(): + def close(self): + name = self.cfg.params["name"] closed.append(name) if close_fails and name == "second": - raise RuntimeError("Operation cleanup failed") - - return PreparedBufferOperation({"value": destination}, apply, reset=reset, close=close) + raise RuntimeError("Processor cleanup failed") - pipe = DictTensorProcessor( + pipe = _DictTensorProcessorPipeline( {"value": source}, - [ - DictTensorProcessorStepCfg( - prepare=prepare, inputs={"value": "value"}, outputs={"value": "value"}, params={"name": name} - ) - for name in ("first", "second") - ], + [DictTensorProcessorCfg(History, ["value"], ["value"], {"name": name}) for name in ("first", "second")], ) source = {"value": ProxyArray(wp.ones(source.shape, dtype=wp.float32, device="cpu"))} try: @@ -238,3 +242,26 @@ def close(): pipe(source) with pytest.raises(RuntimeError, match="closed"): pipe.as_tensor(source, output_name="value") + + +@pytest.mark.parametrize("replacement", ["missing", "new_storage"]) +def test_processor_cannot_replace_published_output_storage(replacement): + """Camera consumers retain prepared tensors, so missing or replaced outputs must fail.""" + + class ReplacesOutput(_Scale): + def __call__(self, data): + super().__call__(data) + if replacement == "missing": + return {} + self.outputs["result"] = ProxyArray(wp.empty_like(data["source"].warp)) + return dict(self.outputs) + + source = ProxyArray(wp.ones((2,), dtype=wp.float32, device="cpu")) + pipe = _DictTensorProcessorPipeline( + {"source": source}, [DictTensorProcessorCfg(ReplacesOutput, ["source"], ["result"])] + ) + try: + with pytest.raises(ValueError, match="persistent named output buffers"): + pipe({"source": source}) + finally: + pipe.close() diff --git a/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst index d3f07a2f90bb..5832bdc80d5d 100644 --- a/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst +++ b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst @@ -1,3 +1,3 @@ -* **Breaking:** Replaced renderer-owned PPISP with ``DictTensorProcessor`` operations. - Replace ``CameraCfg.isp_cfg`` with ``PpispStepCfg`` and configure +* **Breaking:** Replaced renderer-owned PPISP with dictionary processors. + Replace ``CameraCfg.isp_cfg`` with ``PpispProcessorCfg`` and configure ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst index d3f07a2f90bb..5832bdc80d5d 100644 --- a/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst +++ b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst @@ -1,3 +1,3 @@ -* **Breaking:** Replaced renderer-owned PPISP with ``DictTensorProcessor`` operations. - Replace ``CameraCfg.isp_cfg`` with ``PpispStepCfg`` and configure +* **Breaking:** Replaced renderer-owned PPISP with dictionary processors. + Replace ``CameraCfg.isp_cfg`` with ``PpispProcessorCfg`` and configure ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst index d3f07a2f90bb..5832bdc80d5d 100644 --- a/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst +++ b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst @@ -1,3 +1,3 @@ -* **Breaking:** Replaced renderer-owned PPISP with ``DictTensorProcessor`` operations. - Replace ``CameraCfg.isp_cfg`` with ``PpispStepCfg`` and configure +* **Breaking:** Replaced renderer-owned PPISP with dictionary processors. + Replace ``CameraCfg.isp_cfg`` with ``PpispProcessorCfg`` and configure ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst index d3f07a2f90bb..5832bdc80d5d 100644 --- a/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst +++ b/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst @@ -1,3 +1,3 @@ -* **Breaking:** Replaced renderer-owned PPISP with ``DictTensorProcessor`` operations. - Replace ``CameraCfg.isp_cfg`` with ``PpispStepCfg`` and configure +* **Breaking:** Replaced renderer-owned PPISP with dictionary processors. + Replace ``CameraCfg.isp_cfg`` with ``PpispProcessorCfg`` and configure ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_ppisp/docs/README.md b/source/isaaclab_ppisp/docs/README.md index 277b033b4110..044c281c9fba 100644 --- a/source/isaaclab_ppisp/docs/README.md +++ b/source/isaaclab_ppisp/docs/README.md @@ -1,11 +1,13 @@ # Isaac Lab PPISP This extension provides PPISP (Physically Plausible Image Signal Processing) as -an operation in the shared buffer pipeline used by cameras and observations. +a `DictTensorProcessor` used by cameras and observations. `DictTensorProcessor` and +`TensorProcessor` live together in `isaaclab.utils.tensor_processors`; their call signatures are +dictionary-to-dictionary and tensor-to-tensor respectively. -`PpispStepCfg` reads scene-linear `rgb_hdr` from a float32 `ProxyArray` with +`PpispProcessorCfg` reads scene-linear `rgb_hdr` from a float32 `ProxyArray` with shape `(N, H, W, 3)` and writes uint8 `rgb`. For camera processing, configure -`CameraCfg.pipeline=[PpispStepCfg(params={"cfg": ppisp_cfg})]`. The camera requests +`CameraCfg.pipeline=[PpispProcessorCfg(params={"cfg": ppisp_cfg})]`. The camera requests required HDR internally; only the requested public outputs are published. Set `CameraCfg.exposure=CameraExposureCfg()` (from `isaaclab.sensors.camera`) before initialization for neutral RTX HDR input. Output selection does not alter exposure. The same setting is required @@ -15,18 +17,16 @@ Set PPISP exposure in stops through `PpispCfg.inputs["exposureOffset"]`. Resolve authored settings explicitly before constructing the pipeline: ```python -from isaaclab_ppisp import PpispStepCfg, ppisp_cfg_from_usd_camera +from isaaclab_ppisp import PpispProcessorCfg, ppisp_cfg_from_usd_camera ppisp_cfg = ppisp_cfg_from_usd_camera(source_camera_prim) -step = PpispStepCfg(params={"cfg": ppisp_cfg}) +step = PpispProcessorCfg(params={"cfg": ppisp_cfg}) ``` -The same step can run in `isaaclab.utils.processing.DictTensorProcessor` outside a camera: -`DictTensorProcessor({"rgb_hdr": radiance_buffer}, [step])`, where `radiance_buffer` -is an allocated `ProxyArray`. Preparation validates its shape and dtype and allocates -the persistent output once on the same device. Input/output bindings can be changed -independently of PPISP math. For RGBA, -use `outputs={"color": "rgba"}` and `params={"cfg": ppisp_cfg, "channels": 4}`. +`PpispProcessor` can also run outside a camera. Construct it with its configuration and an +actual dictionary of `ProxyArray` tensors; construction validates shape and dtype and allocates +persistent output once on the same device. Input/output names can be changed independently of PPISP math. For RGBA, +use `outputs=["rgba"]` and `params={"cfg": ppisp_cfg, "channels": 4}`. PPISP consumes the preceding operation's radiance, so an earlier exposure or scale operation changes its result. The producer must supply scene-linear color before camera exposure and response. Array storage cannot establish that semantic contract; @@ -38,3 +38,8 @@ shared, read-only device weights; configuration copies preserve this resource's caller owns upload synchronization, and mutable controller workspace stays private. The controller requires CUDA; static PPISP also runs on CPU. `PpispPipeline` remains available for direct numerical use with explicitly supplied Warp buffers. + +For direct use, construct `PpispProcessor(cfg, {"rgb_hdr": hdr_tensor})`, then call +`processor({"rgb_hdr": hdr_tensor})` to obtain `{"rgb": color_tensor}`. It owns its persistent +output and numerical workspace; call `processor.close()` when finished. No preparation factory +or prepared-operation wrapper is needed. diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py index fb6d434387d7..8afc7ae3e769 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py @@ -6,7 +6,7 @@ """Post-render PPISP (Physically Plausible Image Signal Processing) for IsaacLab. Provides a shared render-buffer operation that converts scene-linear radiance to RGB or RGBA. -Camera and observation pipelines use :class:`PpispStepCfg` with concrete settings. USD discovery +Camera and observation pipelines use :class:`PpispProcessorCfg` with concrete settings. USD discovery helpers remain explicit, and renderer backends do not own PPISP execution or controller workspace. """ diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi index 7b7b417455eb..646c40c9387e 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi +++ b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi @@ -10,7 +10,7 @@ __all__ = [ "PpispControllerWeights", "PpispDiscoveryMode", "PpispPipeline", - "PpispStepCfg", + "PpispProcessorCfg", "apply_ppisp_to_rgba", "apply_ppisp_to_rgba_with_controller_params", "auto_any_ppisp_cfg", @@ -21,7 +21,7 @@ __all__ = [ "normalize_ppisp_cfg", "ppisp_cfg_from_usd_camera", "ppisp_cfg_from_usd_stage", - "prepare_ppisp", + "PpispProcessor", "resolve_and_normalize", ] @@ -41,4 +41,5 @@ from .cfg import ( ) from .kernels import apply_ppisp_to_rgba, apply_ppisp_to_rgba_with_controller_params, compute_ppisp_controller_params from .pipeline import PpispControllerWeights, PpispPipeline -from .processor import PpispStepCfg, prepare_ppisp +from .processor import PpispProcessor +from .processor_cfg import PpispProcessorCfg diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py b/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py index 3f6d51d8cd96..66d52c4a9151 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py @@ -46,7 +46,7 @@ class PpispPipeline: This numerical object performs no camera, scene, or renderer discovery. Resolve USD settings with :func:`~isaaclab_ppisp.ppisp_cfg_from_usd_camera` before construction. Each instance owns its controller workspace and is not reentrant; coefficients apply to its whole image batch. - :func:`~isaaclab_ppisp.prepare_ppisp` creates an independent instance for each buffer-pipe stage. + :class:`~isaaclab_ppisp.PpispProcessor` owns an independent instance for each consumer. Args: cfg: Concrete PPISP settings, normalized in place for direct numerical use. diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/processor.py b/source/isaaclab_ppisp/isaaclab_ppisp/processor.py index bfc3d3b95e92..34080bf4f0f3 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/processor.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/processor.py @@ -3,83 +3,72 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""PPISP operation for camera and observation buffer pipes.""" +"""PPISP dictionary processor shared by cameras and observations.""" from __future__ import annotations -from collections.abc import Mapping from copy import deepcopy -from typing import Any import warp as wp -from isaaclab.utils import configclass -from isaaclab.utils.processing import DictTensorProcessorStepCfg, PreparedBufferOperation +from isaaclab.utils.tensor_processors import DictTensorProcessor from isaaclab.utils.warp import ProxyArray from .cfg import PpispCfg -from .pipeline import PpispControllerWeights, PpispPipeline +from .pipeline import PpispPipeline +from .processor_cfg import PpispProcessorCfg -def prepare_ppisp( - inputs: Mapping[str, ProxyArray], - *, - cfg: PpispCfg, - channels: int = 3, - controller_weights: PpispControllerWeights | None = None, -) -> PreparedBufferOperation: - """Prepare PPISP and allocate its persistent output from an actual radiance buffer. +class PpispProcessor(DictTensorProcessor): + """Transform scene-linear HDR into uint8 color using persistent PPISP storage. - The producer must supply scene-linear color before camera exposure and response. This semantic - requirement cannot be inferred from the array's dtype, shape, or pixel values. - - Args: - inputs: One ``radiance`` input: float32 with shape ``(N, H, W, 3)``. - cfg: Concrete configuration. Resolve USD camera attributes before preparing the pipe. - channels: Number of output channels, three for RGB or four for opaque RGBA. - controller_weights: Optional read-only device weights shared across operations. Each operation - retains its own mutable controller workspace; weights must be ready on the calling stream. - - Returns: - A prepared operation owning a uint8 ``color`` output with three or four channels. - """ - if set(inputs) != {"radiance"}: - raise ValueError("PPISP requires exactly one input role named 'radiance'.") - source = inputs["radiance"] - if not isinstance(source, ProxyArray): - raise TypeError("PPISP radiance must be a ProxyArray.") - if len(source.shape) != 4 or source.shape[-1] != 3 or any(size <= 0 for size in source.shape): - raise ValueError("PPISP radiance must have shape (N, H, W, 3) with positive dimensions.") - if source.dtype is not wp.float32: - raise ValueError("PPISP radiance must have dtype wp.float32.") - if channels not in (3, 4): - raise ValueError("PPISP color output requires three RGB or four RGBA channels.") - if not isinstance(cfg, PpispCfg) or cfg.camera_prim_path is not None: - raise ValueError("PPISP stages require a concrete PpispCfg; resolve USD camera attributes before preparation.") - pipeline = PpispPipeline(deepcopy(cfg), controller_weights=controller_weights) - try: - pipeline.initialize(source.shape, source.device) - color = ProxyArray(wp.empty((*source.shape[:3], channels), dtype=wp.uint8, device=source.device)) - - def apply(inputs: Mapping[str, ProxyArray], outputs: Mapping[str, ProxyArray]) -> None: - pipeline.apply(inputs["radiance"].warp, outputs["color"].warp) - - return PreparedBufferOperation(outputs={"color": color}, apply=apply, close=pipeline.close) - except Exception: - pipeline.close() - raise - - -@configclass -class PpispStepCfg(DictTensorProcessorStepCfg): - """Route scene-linear radiance through PPISP into a public RGB buffer. - - Supply concrete settings with ``params={"cfg": ppisp_cfg}``. To produce RGBA, set - ``outputs={"color": "rgba"}`` and ``params={"cfg": ppisp_cfg, "channels": 4}``. - Both camera and observation pipes use the same operation implementation. + Input is float32 NHWC with three channels. It must represent color before exposure and + camera response; array storage cannot establish that semantic contract. Output has three + RGB or four RGBA channels. Controllers may share immutable weights via configuration, + while each processor owns its controller workspace and output buffer. """ - prepare = "isaaclab_ppisp:prepare_ppisp" - inputs: dict[str, str] = {"radiance": "rgb_hdr"} - outputs: dict[str, str] = {"color": "rgb"} - params: dict[str, Any] = {"cfg": PpispCfg()} + def __init__(self, cfg: PpispProcessorCfg, data: dict[str, ProxyArray]) -> None: + super().__init__(cfg, data) + if len(cfg.inputs) != 1 or len(cfg.outputs) != 1: + raise ValueError("PPISP requires exactly one input and one output name.") + self._input_name, self._output_name = cfg.inputs[0], cfg.outputs[0] + source = data[self._input_name] + if not isinstance(source, ProxyArray): + raise TypeError("PPISP input must be a ProxyArray.") + if len(source.shape) != 4 or source.shape[-1] != 3 or any(size <= 0 for size in source.shape): + raise ValueError("PPISP input must have shape (N, H, W, 3) with positive dimensions.") + if source.dtype is not wp.float32: + raise ValueError("PPISP input must have dtype wp.float32.") + channels = cfg.params.get("channels", 3) + if channels not in (3, 4): + raise ValueError("PPISP color output requires three RGB or four RGBA channels.") + ppisp_cfg = cfg.params["cfg"] + if not isinstance(ppisp_cfg, PpispCfg) or ppisp_cfg.camera_prim_path is not None: + raise ValueError("PPISP requires a concrete PpispCfg; resolve USD camera attributes before construction.") + self._shape, self._device = source.shape, source.device + self._closed = False + self._pipeline = PpispPipeline(deepcopy(ppisp_cfg), controller_weights=cfg.params.get("controller_weights")) + try: + self._pipeline.initialize(source.shape, source.device) + self.outputs[self._output_name] = ProxyArray( + wp.empty((*source.shape[:3], channels), dtype=wp.uint8, device=source.device) + ) + except Exception: + self.close() + raise + + def __call__(self, data: dict[str, ProxyArray]) -> dict[str, ProxyArray]: + """Read the current input and return the persistent named color tensor.""" + if self._closed: + raise RuntimeError("Cannot execute a closed PPISP processor.") + source = data[self._input_name] + if source.shape != self._shape or source.dtype is not wp.float32 or source.device != self._device: + raise ValueError("PPISP input must match the shape, dtype and device used during construction.") + self._pipeline.apply(source.warp, self.outputs[self._output_name].warp) + return dict(self.outputs) + + def close(self) -> None: + if not self._closed: + self._closed = True + self._pipeline.close() diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/processor_cfg.py b/source/isaaclab_ppisp/isaaclab_ppisp/processor_cfg.py new file mode 100644 index 000000000000..876ebd243a98 --- /dev/null +++ b/source/isaaclab_ppisp/isaaclab_ppisp/processor_cfg.py @@ -0,0 +1,26 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from typing import Any + +from isaaclab.utils import configclass +from isaaclab.utils.tensor_processors import DictTensorProcessorCfg + +from .cfg import PpispCfg + + +@configclass +class PpispProcessorCfg(DictTensorProcessorCfg): + """Process one scene-linear HDR tensor into RGB or RGBA. + + ``inputs`` and ``outputs`` each contain one actual dictionary key. Supply concrete PPISP + settings in ``params={"cfg": ppisp_cfg}``. Add ``"channels": 4`` for RGBA. The source camera + needs explicit neutral exposure; this processor never changes renderer settings. + """ + + class_type = "{DIR}.processor:PpispProcessor" + inputs: list[str] = ["rgb_hdr"] + outputs: list[str] = ["rgb"] + params: dict[str, Any] = {"cfg": PpispCfg()} diff --git a/source/isaaclab_ppisp/test/test_ppisp_kernels.py b/source/isaaclab_ppisp/test/test_ppisp_kernels.py index b46e58475883..352ea6335c83 100644 --- a/source/isaaclab_ppisp/test/test_ppisp_kernels.py +++ b/source/isaaclab_ppisp/test/test_ppisp_kernels.py @@ -9,7 +9,7 @@ from isaaclab_ppisp import ( PpispCfg, PpispControllerWeights, - PpispStepCfg, + PpispProcessorCfg, apply_ppisp_to_rgba, apply_ppisp_to_rgba_with_controller_params, compute_ppisp_controller_params, @@ -41,7 +41,7 @@ ) from isaaclab.test.utils import DeviceScope, test_devices -from isaaclab.utils.processing import DictTensorProcessor +from isaaclab.utils.tensor_processors._pipeline import _DictTensorProcessorPipeline from isaaclab.utils.warp import ProxyArray wp.init() @@ -314,18 +314,18 @@ def test_ppisp_pipe_rgb_crf_extreme_centers_match_reference(attachment): ) ) - pipe = DictTensorProcessor( - {"rgb_hdr": ProxyArray(hdr_color)}, - [PpispStepCfg(params={"cfg": cfg})], - ) if attachment == "dictionary": - rgb_np = pipe({"rgb_hdr": ProxyArray(hdr_color)})["rgb"].warp.numpy()[0, 0] + from isaaclab_ppisp import PpispProcessor + + processor = PpispProcessor(PpispProcessorCfg(params={"cfg": cfg}), {"rgb_hdr": ProxyArray(hdr_color)}) + rgb_np = processor({"rgb_hdr": ProxyArray(hdr_color)})["rgb"].warp.numpy()[0, 0] + processor.close() else: from isaaclab.utils.tensor_processors import DictTensorProcessorAdapter, DictTensorProcessorAdapterCfg processor = DictTensorProcessorAdapter( DictTensorProcessorAdapterCfg( - pipeline=[PpispStepCfg(params={"cfg": cfg})], input_name="rgb_hdr", output_name="rgb" + pipeline=[PpispProcessorCfg(params={"cfg": cfg})], input_name="rgb_hdr", output_name="rgb" ), hdr_data.shape, "cpu", @@ -344,7 +344,6 @@ def test_ppisp_pipe_rgb_crf_extreme_centers_match_reference(attachment): # float32 kernel math may land one byte away from the float64 reference at a floor boundary np.testing.assert_allclose(rgb_np[:, channel].astype(float), expected_bytes, atol=1.0, err_msg=suffix) np.testing.assert_array_equal(hdr_color.numpy(), hdr_data) - pipe.close() @pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @@ -361,9 +360,9 @@ def test_ppisp_controller_pipes_share_weights_for_rgb_and_rgba(device): cfg = PpispCfg() shared_weights = PpispControllerWeights(weights) pipes = [ - DictTensorProcessor( + _DictTensorProcessorPipeline( {"rgb_hdr": ProxyArray(hdr)}, - [PpispStepCfg(params={"cfg": cfg, "channels": channels, "controller_weights": shared_weights})], + [PpispProcessorCfg(params={"cfg": cfg, "channels": channels, "controller_weights": shared_weights})], ) for channels in (3, 4) ] @@ -392,13 +391,13 @@ def test_ppisp_pipe_rejects_incompatible_input_storage(shape, dtype, message): """Reject invalid image channels and scalar types before preparing numerical execution.""" source = ProxyArray(wp.empty(shape, dtype=dtype, device="cpu")) with pytest.raises(ValueError, match=message): - DictTensorProcessor({"rgb_hdr": source}, [PpispStepCfg()]) + _DictTensorProcessorPipeline({"rgb_hdr": source}, [PpispProcessorCfg()]) def test_ppisp_pipe_requires_resolved_configuration(): """An image operation must not discover its settings from a camera during preparation.""" source = ProxyArray(wp.empty((1, 2, 2, 3), dtype=wp.float32, device="cpu")) with pytest.raises(ValueError, match="resolve USD camera attributes"): - DictTensorProcessor( - {"rgb_hdr": source}, [PpispStepCfg(params={"cfg": PpispCfg(camera_prim_path="/World/Camera")})] + _DictTensorProcessorPipeline( + {"rgb_hdr": source}, [PpispProcessorCfg(params={"cfg": PpispCfg(camera_prim_path="/World/Camera")})] ) From 0f058c9552c099c6f927b72c5f46e0123bd5fcd3 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Fri, 16 Oct 2026 18:37:27 -0400 Subject: [PATCH 20/20] Clean up stale processor documentation --- docs/source/api/lab/isaaclab.managers.rst | 2 +- .../api/lab/isaaclab.sensors.camera.rst | 1 + docs/source/api/lab/isaaclab.utils.rst | 1 - docs/source/api/lab_ppisp/isaaclab_ppisp.rst | 23 ++++++++++ docs/source/concepts/sensors/camera.rst | 42 +++++++++---------- .../isaaclab/envs/mdp/sensor_observations.py | 10 ++--- .../isaaclab/managers/observation_manager.py | 8 ++-- .../isaaclab/sensors/camera/camera.py | 2 +- .../isaaclab/sensors/camera/camera_cfg.py | 6 +-- .../utils/tensor_processors/__init__.py | 17 ++++---- .../dict_tensor_processor_adapter.py | 8 ++-- .../tensor_processors/tensor_processor_cfg.py | 10 ++--- .../managers/observation_manager.py | 40 +++++++++--------- .../utils/tensor_processors/__init__.py | 7 +--- .../utils/tensor_processors/processors.py | 11 +++-- .../tensor_processors/tensor_processor.py | 4 +- .../renderers/isaac_rtx_renderer.py | 3 +- source/isaaclab_ppisp/docs/README.md | 16 +++---- .../isaaclab_ppisp/isaaclab_ppisp/__init__.py | 2 +- source/isaaclab_ppisp/isaaclab_ppisp/cfg.py | 2 +- .../isaaclab_ppisp/isaaclab_ppisp/kernels.py | 2 +- .../isaaclab_ppisp/isaaclab_ppisp/pipeline.py | 2 +- source/isaaclab_ppisp/pyproject.toml | 2 +- 23 files changed, 121 insertions(+), 100 deletions(-) diff --git a/docs/source/api/lab/isaaclab.managers.rst b/docs/source/api/lab/isaaclab.managers.rst index 5a5f94d5abb0..b4cd0fc7dfcc 100644 --- a/docs/source/api/lab/isaaclab.managers.rst +++ b/docs/source/api/lab/isaaclab.managers.rst @@ -115,7 +115,7 @@ For a delay buffer with no recorded sample after initialization or reset, the cu without recording it. Once recording starts, delays exceeding the available history return the oldest sample. Partial resets invalidate only the selected environments' histories. -Observation outputs are independent snapshots: later simulation steps, modifier updates, and resets +Observation outputs are independent snapshots: later simulation steps, processor updates, and resets do not overwrite previously returned tensors. The manager automatically copies borrowed storage before mutation or retention. Term and custom callback outputs are treated conservatively, without decorators or task-level copy settings. Clipping and scaling create independent storage when needed, which later diff --git a/docs/source/api/lab/isaaclab.sensors.camera.rst b/docs/source/api/lab/isaaclab.sensors.camera.rst index 1c4eb5072bd0..89ece542a6c0 100644 --- a/docs/source/api/lab/isaaclab.sensors.camera.rst +++ b/docs/source/api/lab/isaaclab.sensors.camera.rst @@ -13,6 +13,7 @@ The following classes are part of the public :mod:`isaaclab.sensors.camera` API. .. autosummary:: :nosignatures: + CameraExposureCfg CameraISPMode .. autoclass:: CameraISPMode diff --git a/docs/source/api/lab/isaaclab.utils.rst b/docs/source/api/lab/isaaclab.utils.rst index 4e8029d2ba7b..d6c9d85756ff 100644 --- a/docs/source/api/lab/isaaclab.utils.rst +++ b/docs/source/api/lab/isaaclab.utils.rst @@ -19,7 +19,6 @@ mesh tensor_processors noise - processing seed sensors string diff --git a/docs/source/api/lab_ppisp/isaaclab_ppisp.rst b/docs/source/api/lab_ppisp/isaaclab_ppisp.rst index 79b9565d1d77..ed2117d16a9c 100644 --- a/docs/source/api/lab_ppisp/isaaclab_ppisp.rst +++ b/docs/source/api/lab_ppisp/isaaclab_ppisp.rst @@ -14,10 +14,33 @@ The following classes are part of the public :mod:`isaaclab_ppisp` API. :nosignatures: PpispCfg + PpispProcessorCfg + PpispProcessor + PpispControllerWeights + PpispDiscoveryMode PpispPipeline .. autoclass:: PpispCfg :show-inheritance: +.. autoclass:: PpispProcessorCfg + :members: + :inherited-members: + :exclude-members: __init__ + :show-inheritance: + +.. autoclass:: PpispProcessor + :members: + :inherited-members: + :special-members: __call__ + :show-inheritance: + +.. autoclass:: PpispControllerWeights + :members: + +.. autoclass:: PpispDiscoveryMode + :members: + :show-inheritance: + .. autoclass:: PpispPipeline :show-inheritance: diff --git a/docs/source/concepts/sensors/camera.rst b/docs/source/concepts/sensors/camera.rst index 4440b5722b29..6a35e36cfc3a 100644 --- a/docs/source/concepts/sensors/camera.rst +++ b/docs/source/concepts/sensors/camera.rst @@ -311,7 +311,6 @@ background. Set a normalized RGB tuple to use a solid color for pixels that miss .. code-block:: python - from isaaclab.sensors.camera import CameraExposureCfg from isaaclab.utils import replace mask_camera = replace(front_camera, background_color=(0.0, 0.0, 0.0)) @@ -327,9 +326,8 @@ Post-render image processing and returns a dictionary of named tensors. It lives beside :class:`~isaaclab.utils.tensor_processors.TensorProcessor`, whose call takes and returns one tensor. :class:`~isaaclab.utils.tensor_processors.DictTensorProcessorCfg` declares the actual input and output -keys needed for renderer setup. There are no separate operation roles or prepared-operation objects. -The shared dictionary interface also accepts nonvisual sensor data; each processor defines its -layout and value semantics. +keys needed for renderer setup. The shared dictionary interface also accepts nonvisual sensor data; +each processor defines its layout and value semantics. The camera resolves external inputs before renderer initialization, including inputs needed only by the pipeline. It publishes the names in ``data_types`` after processing. For example, this camera requests @@ -351,8 +349,8 @@ private ``rgb_hdr`` from its renderer and publishes PPISP's processed ``rgb``: PPISP (Physically Plausible Image Signal Processing) applies responsivity, exposure, vignetting, color correction, and a camera response function. It consumes float32 scene-linear radiance and produces uint8 color. Set ``outputs=["rgba"]`` and ``params={"cfg": ppisp_cfg, "channels": 4}`` on the -step for RGBA output. A pipeline-produced RGBA buffer also supplies a zero-copy RGB view when ``rgb`` -is requested. A pipeline producing only RGB cannot satisfy an explicit RGBA request. +processor configuration for RGBA output. A pipeline-produced RGBA buffer also supplies a zero-copy +RGB view when ``rgb`` is requested. A pipeline producing only RGB cannot satisfy an explicit RGBA request. For custom dictionary processing, subclass ``DictTensorProcessor``. Its constructor receives ``(cfg, data)`` with actual ``ProxyArray`` tensors. Allocate persistent output tensors in @@ -361,7 +359,7 @@ to fill and return those named outputs, and override ``reset`` or ``close`` only The pipeline passes the current dictionary to each processor and merges its returned outputs; untouched keys pass through. Shape, dtype and device come from the arrays directly. -PPISP is a regular ``PpispProcessor`` subclass. It can also be constructed and called directly: +``PpispProcessor`` subclasses ``DictTensorProcessor`` and can also be constructed and called directly: .. code-block:: python @@ -381,25 +379,25 @@ below when the final observation should have a different image shape or layout. ``camera.render_outputs`` exposes borrowed raw renderer buffers, including private inputs. Their ``ProxyArray`` properties provide shape, dtype, and device. Reading raw outputs updates the camera -when due. Neither an operation nor a caller may mutate borrowed inputs. Public processed +when due. Neither a processor nor a caller may mutate borrowed inputs. Public processed buffers and raw buffers retain their storage until camera reinitialization, but later captures replace -their contents; clone values that must survive another capture. Each camera owns its pipe and operation -workspace. Operations may share explicitly supplied immutable model resources. +their contents; clone values that must survive another capture. Each camera owns its pipeline and processor +workspace. Processors may share explicitly supplied immutable model resources. Configuration copying must preserve the identity of borrowed resources: PPISP provides ``PpispControllerWeights`` for device weights. Custom processors can use a resource wrapper with identity-preserving ``__deepcopy__``; raw Warp allocations are not configuration values. Pipelines run after scalar and batched captures. When an asynchronous renderer republishes a capture, the camera retains its processed result and matching capture metadata. Camera reset forwards only the -selected environment indices to operation reset callbacks; teardown closes the operations. For an +selected environment indices to processor reset callbacks; teardown closes the processors. For an observation-only pipeline, declare raw dependencies with ``camera.request_render_inputs(names)`` before -simulation initialization. Additional requests after initialization fail because renderer setup is -already complete. Observation consumers use their own pipe and snapshot results needed across captures. +simulation initialization. Requests for new inputs after initialization fail because renderer setup is +already complete. Observation consumers use their own pipeline and snapshot results needed across captures. Processing sensor observations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Use :class:`~isaaclab.envs.mdp.processed_sensor` for observation-only processing, including operations +Use :class:`~isaaclab.envs.mdp.processed_sensor` for observation-only processing, including processors that combine several camera outputs or change image dimensions. The term currently binds sensors through their ``request_render_inputs()`` and ``render_outputs`` interface: @@ -423,23 +421,23 @@ Manager-based environments collect the term's private renderer inputs before sim standalone observation manager must call ``ObservationManager.prepare_scene(cfg, env)`` before reset. The term processes raw renderer outputs even when the camera has its own pipeline. To read the camera's published processed output, use an existing image term such as ``mdp.image_rgb`` instead. -The observation pipe executes on every term evaluation and returns a snapshot; its state belongs to +The observation pipeline executes on every term evaluation and returns a snapshot; its state belongs to that term rather than the camera's capture cadence. For a single incoming tensor, :class:`~isaaclab.utils.tensor_processors.DictTensorProcessorAdapterCfg` uses the -same operations while preserving preceding observation transformations. This includes nonvisual +same dictionary processors while preserving preceding observation transformations. This includes nonvisual observation tensors from sensors without the render-input interface. This form must preserve shape; use ``processed_sensor`` for multiple render inputs or changes to render-output shapes, or construct -a ``DictTensorProcessor`` directly for other named buffers. The first incoming tensor prepares the -operations; later calls reuse their output storage. Its values must meet the operation's documented -input requirements, including any transformations from preceding tensor processors: +a ``DictTensorProcessor`` directly for other named buffers. The first incoming tensor constructs the +dictionary processors; later calls reuse their output storage. Its values must meet each processor's +documented input requirements, including any transformations from preceding tensor processors: .. code-block:: python from isaaclab.utils import tensor_processors # This camera publishes radiance for the ordinary image term to read. - raw_camera = replace(front_camera, data_types=["rgb_hdr"], exposure=CameraExposureCfg(), pipeline=[]) + front_camera = replace(front_camera, data_types=["rgb_hdr"], exposure=CameraExposureCfg(), pipeline=[]) policy_image = ObservationTermCfg( func=mdp.image_rgb, params={"sensor_cfg": SceneEntityCfg("front_camera"), "data_type": "rgb_hdr", "normalize": False}, @@ -458,7 +456,7 @@ Migrating renderer-owned ISP configuration ``CameraCfg.isp_cfg`` no longer executes image processing. A non-None value raises a migration error; replace it with ``CameraCfg.pipeline`` as above. ``CameraISPMode`` remains importable for legacy configuration values, but new code should resolve USD attributes explicitly before constructing the -operation: +processor: .. code-block:: python @@ -475,7 +473,7 @@ operation: ``AUTO_CAMERA`` checks the supplied camera prim; ``AUTO_ANY`` can fall back to the first PPISP camera on the supplied stage. Alternatively, pass an already resolved camera prim to ``ppisp_cfg_from_usd_camera(camera_prim)``. Discovery has no implicit simulation or stage lookup inside -the operation. Static coefficients apply to the whole camera batch; controller weights may predict +the processor. Static coefficients apply to the whole camera batch; controller weights may predict per-view exposure and color parameters. For either processing attachment, configure the source camera with diff --git a/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py b/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py index e20e37369c09..8e7b1e478f6f 100644 --- a/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py +++ b/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py @@ -22,20 +22,20 @@ class processed_sensor(ManagerTermBase): - """Process named sensor render outputs with the shared buffer pipe. + """Apply dictionary processors to named sensor render outputs. The sensor must expose ``request_render_inputs`` and ``render_outputs``, as - :class:`~isaaclab.sensors.Camera` does. Operations define the supported buffer layouts and + :class:`~isaaclab.sensors.Camera` does. Processors define the supported tensor layouts and value semantics; outputs need not be visual images. Dependencies are requested before simulation initialization through :meth:`~isaaclab.managers.ObservationManager.prepare_scene`. They remain private to the renderer buffer bundle and need not appear in ``CameraCfg.data_types``. Each term owns - its own operation state and output storage. The pipe runs on every observation evaluation; + its own processor state and output storage. The pipeline runs on every observation evaluation; returned tensors are snapshots, so later evaluations cannot overwrite retained history. Use this term for multiple render inputs or changed buffer dimensions. To process other - sensor data or the result of earlier observation tensor_processors, use + sensor data or the result of earlier observation tensor processors, use :class:`~isaaclab.utils.tensor_processors.DictTensorProcessorAdapter` on an observation tensor. To read a camera's already processed public output, use the existing image observation terms. """ @@ -69,7 +69,7 @@ def __call__( *, out: torch.Tensor | None = None, ) -> torch.Tensor: - """Return a snapshot of the selected operation output. + """Return a snapshot of the selected processor output. The observation manager supplies ``out`` to receive the snapshot without another clone. Direct callers may omit it to allocate a new snapshot. diff --git a/source/isaaclab/isaaclab/managers/observation_manager.py b/source/isaaclab/isaaclab/managers/observation_manager.py index 867b9fafed56..51094dafeea6 100644 --- a/source/isaaclab/isaaclab/managers/observation_manager.py +++ b/source/isaaclab/isaaclab/managers/observation_manager.py @@ -60,8 +60,8 @@ class ObservationManager(ManagerBase): called in the order of the terms in the group. The functions are expected to return a tensor with shape (num_envs, ...). - If a noise model or custom modifier is registered for a term, the function is called to corrupt - the observation. The corruption function is expected to return a tensor with the same shape as the observation. + If a noise model or custom tensor processor is registered for a term, it transforms + the observation and returns a tensor with the same shape as the observation. The observations are clipped and scaled as per the configuration settings. Returned observations are independent snapshots, including dictionary entries and history. @@ -145,7 +145,7 @@ def prepare_scene(cls, cfg: object, env: ManagerBasedEnv) -> None: func.prepare_scene(term, env) def close(self) -> None: - """Close every owned term and modifier once, including nested configurations. + """Close every owned term and tensor processor once, including nested configurations. All resources are closed before any callback errors are raised. Repeated calls are safe. """ @@ -433,7 +433,7 @@ def compute_group(self, group_name: str, update_history: bool = False) -> torch. The following steps are performed for each observation term: 1. Compute observation term by calling the function - 2. Apply custom tensor_processors in the order specified in :attr:`ObservationTermCfg.tensor_processors` + 2. Apply custom tensor processors in the order specified in :attr:`ObservationTermCfg.tensor_processors` 3. Apply corruption/noise model based on :attr:`ObservationTermCfg.noise` 4. Apply clipping based on :attr:`ObservationTermCfg.clip` 5. Apply scaling based on :attr:`ObservationTermCfg.scale` diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 4bc05a216f3c..f81ffe898512 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -919,7 +919,7 @@ def _create_buffers(self): f" {expected_shape}; received {buffer.shape}. Use an observation pipe" " for shape-changing results, because camera calibration describes the rendered image." ) - # Preserve RGB's view semantics when an operation produces RGBA, without publishing stale raw RGB. + # Preserve RGB's view semantics when a processor produces RGBA, without publishing stale raw RGB. if alias_rgb: rgba = outputs["rgba"].warp if len(rgba.shape) != 4 or rgba.shape[-1] != 4: diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py index 20f0310d8cab..e540e8915702 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py @@ -209,14 +209,14 @@ class OffsetCfg: """ pipeline: list[DictTensorProcessorCfg] = [] - """Ordered image operations applied after each newly published capture. + """Ordered dictionary processors applied after each newly published capture. Processors consume and return dictionaries of named tensors. They may combine several render outputs or preceding results. The renderer receives the pipeline's external dependencies before initialization. Only :attr:`data_types` are published in :attr:`CameraData.output`; unprocessed inputs remain - available through :attr:`Camera.render_outputs`. Operations borrow their inputs read-only and - own persistent result buffers. Camera reset and teardown reset and close the operations. + available through :attr:`Camera.render_outputs`. Processors borrow their inputs read-only and + own persistent result buffers. Camera reset and teardown reset and close the processors. """ isp_cfg: Any | None = None diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/__init__.py b/source/isaaclab/isaaclab/utils/tensor_processors/__init__.py index d4230dd61886..f2f85320e7cf 100644 --- a/source/isaaclab/isaaclab/utils/tensor_processors/__init__.py +++ b/source/isaaclab/isaaclab/utils/tensor_processors/__init__.py @@ -3,16 +3,17 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Sub-module containing tensor processor implementations. +"""Stateful and stateless processing for tensors and dictionaries of named tensors. -Tensor processors are used to apply stateful or stateless modifications to tensor data. They take -in a tensor and a configuration and return a tensor with the modification applied. This way users -can define custom operations to apply to a tensor. For instance, a processor can be used to normalize -the input data or to apply a rolling average. +Single-tensor functions and :class:`TensorProcessor` subclasses take and return one tensor. +They apply custom operations in the :class:`~isaaclab.managers.ObservationManager`, such as +normalization or rolling averages. Configure them through +:attr:`~isaaclab.managers.ObservationTermCfg.tensor_processors`. -They are primarily used to apply custom operations in the :class:`~isaaclab.managers.ObservationManager` -as an alternative to the built-in noise, clip and scale post-processing operations. For more details, see -the :class:`~isaaclab.managers.ObservationTermCfg` class. +:class:`DictTensorProcessor` subclasses take and return dictionaries of named +:class:`~isaaclab.utils.warp.ProxyArray` tensors. They own persistent outputs and state for camera +pipelines and processed sensor observation terms. :class:`DictTensorProcessorAdapter` applies +these dictionary processors to an incoming observation tensor. Usage with a function processor: diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_adapter.py b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_adapter.py index 7b079b815f2f..4987652e9a79 100644 --- a/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_adapter.py +++ b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_adapter.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Buffer processing for an incoming observation tensor.""" +"""Dictionary processing for an incoming observation tensor.""" from __future__ import annotations @@ -22,10 +22,10 @@ class DictTensorProcessorAdapter(TensorProcessor): - """Apply a buffer pipe to the tensor received from the preceding processor. + """Apply dictionary processors to the tensor received from the preceding processor. - The processor owns a pipe and returns a snapshot. Its output must preserve the input - shape, as required by :class:`TensorProcessor`. The first incoming tensor prepares the pipe; + The adapter owns a processor pipeline and returns a snapshot. Its output must preserve the input + shape, as required by :class:`TensorProcessor`. The first incoming tensor initializes the pipeline; subsequent calls reuse its output storage. Inputs may represent images or nonvisual sensor observations. For multiple render inputs or shape changes to render outputs, use :class:`~isaaclab.envs.mdp.processed_sensor` instead. diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor_cfg.py b/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor_cfg.py index 2561e8ad7aff..5b48601a0b07 100644 --- a/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor_cfg.py +++ b/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor_cfg.py @@ -41,9 +41,9 @@ class TensorProcessorCfg: @configclass class DictTensorProcessorAdapterCfg(TensorProcessorCfg): - """Bind an incoming observation tensor to a buffer pipe. + """Bind an incoming observation tensor to a dictionary processor pipeline. - Operations are prepared from the first tensor received after preceding processors. + Dictionary processors are constructed from the first tensor received after preceding processors. Their documented input requirements define the expected array layout and value semantics. """ @@ -51,13 +51,13 @@ class DictTensorProcessorAdapterCfg(TensorProcessorCfg): """Processor class consuming the incoming tensor.""" pipeline: list[DictTensorProcessorCfg] = MISSING - """Ordered buffer processing operations.""" + """Ordered dictionary processor configurations.""" input_name: str = "rgb" - """Name assigned to the incoming tensor in the buffer bundle.""" + """Dictionary key assigned to the incoming tensor.""" output_name: str = "rgb" - """Name selected from the processed bundle.""" + """Dictionary key selected from the processed outputs.""" @configclass diff --git a/source/isaaclab_experimental/isaaclab_experimental/managers/observation_manager.py b/source/isaaclab_experimental/isaaclab_experimental/managers/observation_manager.py index 1477eca19793..5fed28b70c63 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/managers/observation_manager.py +++ b/source/isaaclab_experimental/isaaclab_experimental/managers/observation_manager.py @@ -32,11 +32,11 @@ The observation manager can be used to compute observations for all the groups or for a specific group. The observations are computed by calling the registered functions for each term in the group. The functions are -called in the order of the terms in the group. The functions are expected to return a tensor with shape -(num_envs, ...). +called in the order of the terms in the group. The functions write into pre-allocated Warp buffers +with shape (num_envs, ...) and return None. -If a noise model or custom modifier is registered for a term, the function is called to corrupt -the observation. The corruption function is expected to return a tensor with the same shape as the observation. +If a tensor processor or noise model is registered for a term, it updates the observation buffer +in place, preserving its shape. Tensor processors run before noise models. The observations are clipped and scaled as per the configuration settings. Experimental (Warp-first) note: @@ -133,11 +133,11 @@ class ObservationManager(ManagerBase): The observation manager can be used to compute observations for all the groups or for a specific group. The observations are computed by calling the registered functions for each term in the group. The functions are - called in the order of the terms in the group. The functions are expected to return a tensor with shape - (num_envs, ...). + called in the order of the terms in the group. The functions write into pre-allocated Warp buffers + with shape (num_envs, ...) and return None. - If a noise model or custom modifier is registered for a term, the function is called to corrupt - the observation. The corruption function is expected to return a tensor with the same shape as the observation. + If a tensor processor or noise model is registered for a term, it updates the observation buffer + in place, preserving its shape. Tensor processors run before noise models. The observations are clipped and scaled as per the configuration settings. Experimental (Warp-first) note: @@ -409,7 +409,7 @@ def reset( for term_name in self._group_obs_term_names[group_name]: if term_name in self._group_obs_term_history_buffer[group_name]: self._group_obs_term_history_buffer[group_name][term_name].reset(env_mask=env_mask) - # call all tensor_processors/noise models that are classes + # call all tensor processors/noise models that are classes for mod in self._group_obs_class_instances: mod.reset(env_mask=env_mask) @@ -457,13 +457,13 @@ def compute_group(self, group_name: str, update_history: bool = False) -> torch. """Computes the observations for a given group. The observations for a given group are computed by calling the registered functions for each - term in the group. The functions are called in the order of the terms in the group. The functions - are expected to return a tensor with shape (num_envs, ...). + term in the group. The functions are called in the order of the terms in the group and write into + pre-allocated Warp buffers with shape (num_envs, ...). The following steps are performed for each observation term: 1. Compute observation term by calling the function - 2. Apply custom tensor_processors in the order specified in :attr:`ObservationTermCfg.tensor_processors` + 2. Apply custom tensor processors in the order specified in :attr:`ObservationTermCfg.tensor_processors` 3. Apply corruption/noise model based on :attr:`ObservationTermCfg.noise` 4. Apply clipping based on :attr:`ObservationTermCfg.clip` 5. Apply scaling based on :attr:`ObservationTermCfg.scale` @@ -500,12 +500,12 @@ def compute_group(self, group_name: str, update_history: bool = False) -> torch. # Persistent per-term obs dict (pre-allocated in _prepare_terms). group_obs = self._group_obs_dict[group_name] - # evaluate terms: compute, add noise, clip, scale, custom tensor_processors + # evaluate terms: compute, apply tensor processors, add noise, clip, scale for term_name, term_cfg in zip(group_term_names, self._group_obs_term_cfgs[group_name]): # compute term's value into pre-allocated Warp output term_cfg.func(self._env, term_cfg.out_wp, **term_cfg.params) - # apply custom tensor_processors (in-place on out_wp) + # apply custom tensor processors (in-place on out_wp) if term_cfg.tensor_processors is not None: for modifier in term_cfg.tensor_processors: modifier.func(term_cfg.out_wp, **modifier.params) @@ -605,7 +605,7 @@ def _prepare_terms(self): # noqa: C901 self._group_obs_concatenate_dim: dict[str, int] = dict() self._group_obs_term_history_buffer: dict[str, dict] = dict() - # create a list to store classes instances, e.g., for tensor_processors and noise models + # create a list to store class instances, e.g., for tensor processors and noise models # we store it as a separate list to only call reset on them and prevent unnecessary calls self._group_obs_class_instances: list[tensor_processors.TensorProcessor | noise.NoiseModel] = list() @@ -735,13 +735,13 @@ def _prepare_terms(self): # noqa: C901 ) term_cfg.scale_wp = wp.array(scale_vals, dtype=wp.float32, device=self._env.device) - # prepare tensor_processors for each observation + # prepare tensor processors for each observation if term_cfg.tensor_processors is not None: - # initialize list of tensor_processors for term + # initialize list of tensor processors for term for mod_cfg in term_cfg.tensor_processors: - # check if class modifier and initialize with observation size when adding + # check if class processor and initialize with observation size when adding if isinstance(mod_cfg, tensor_processors.TensorProcessorCfg): - # to list of tensor_processors + # to list of tensor processors if inspect.isclass(mod_cfg.func): if not issubclass(mod_cfg.func, tensor_processors.TensorProcessor): raise TypeError( @@ -750,7 +750,7 @@ def _prepare_terms(self): # noqa: C901 ) mod_cfg.func = mod_cfg.func(cfg=mod_cfg, data_dim=obs_dims, device=self._env.device) - # add to list of class tensor_processors + # add to list of class tensor processors self._group_obs_class_instances.append(mod_cfg.func) else: raise TypeError( diff --git a/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/__init__.py b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/__init__.py index cfe63ea07f87..fa6f870c2a67 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/__init__.py +++ b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/__init__.py @@ -5,9 +5,8 @@ """Warp-native processor implementations (experimental). -Re-exports stable configs and base classes, then overrides the function-based -tensor processors (``scale``, ``bias``, ``clip``) with Warp-native versions that -operate in-place on ``wp.array``. +Provides Warp-native configuration and base classes, plus function-based tensor +processors (``scale``, ``bias``, ``clip``) that operate in-place on ``wp.array``. Calling convention (matches Warp MDP terms):: @@ -16,6 +15,4 @@ from .processors import bias, clip, scale # noqa: F401 from .tensor_processor import TensorProcessor # noqa: F401 - -# Override with Warp-native implementations from .tensor_processor_cfg import TensorProcessorCfg # noqa: F401 diff --git a/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/processors.py b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/processors.py index 1980f53ca093..6406befb1dc4 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/processors.py +++ b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/processors.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Warp-native function-based tensor_processors (experimental). +"""Warp-native function-based tensor processors (experimental). Each processor takes a ``wp.array`` as its first argument and operates **in-place** via ``wp.launch``. The calling convention mirrors Warp MDP terms:: @@ -27,7 +27,8 @@ def _scale_kernel(data: wp.array(dtype=wp.float32, ndim=2), multiplier: wp.float def scale(data: wp.array, multiplier: float) -> None: """Scale all elements of *data* by *multiplier* in-place. - Warp-native drop-in replacement for :func:`isaaclab.utils.tensor_processors.scale`. + Warp-native counterpart of :func:`isaaclab.utils.tensor_processors.scale`, updating the supplied + array in place and returning None. Args: data: The observation buffer to modify. Shape ``(num_envs, D)``. @@ -48,7 +49,8 @@ def _bias_kernel(data: wp.array(dtype=wp.float32, ndim=2), value: wp.float32): def bias(data: wp.array, value: float) -> None: """Add a uniform *value* to all elements of *data* in-place. - Warp-native drop-in replacement for :func:`isaaclab.utils.tensor_processors.bias`. + Warp-native counterpart of :func:`isaaclab.utils.tensor_processors.bias`, updating the supplied + array in place and returning None. Args: data: The observation buffer to modify. Shape ``(num_envs, D)``. @@ -69,7 +71,8 @@ def _clip_kernel(data: wp.array(dtype=wp.float32, ndim=2), lo: wp.float32, hi: w def clip(data: wp.array, bounds: tuple[float | None, float | None]) -> None: """Clamp all elements of *data* to [lo, hi] in-place. - Warp-native drop-in replacement for :func:`isaaclab.utils.tensor_processors.clip`. + Warp-native counterpart of :func:`isaaclab.utils.tensor_processors.clip`, updating the supplied + array in place and returning None. Args: data: The observation buffer to modify. Shape ``(num_envs, D)``. diff --git a/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor.py b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor.py index 8b393460c456..ccfb87ab6fc9 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor.py +++ b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor.py @@ -17,14 +17,14 @@ class TensorProcessor(ABC): - """Base class for Warp-native class-based tensor_processors. + """Base class for Warp-native class-based tensor processors. Experimental fork of :class:`isaaclab.utils.tensor_processors.TensorProcessor` adapted for the Warp-first calling convention. Subclasses operate **in-place** on ``wp.array`` buffers and return ``None``. A class implementation of a processor can be used to store state information between - calls. This is useful for tensor_processors that require stateful operations, such as + calls. This is useful for tensor processors that require stateful operations, such as rolling averages, delays, or decaying filters. """ diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index 9a32e7589c2c..8d7c222818af 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -374,8 +374,7 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: annotator = rep.AnnotatorRegistry.get_annotator("rgb", device=spec.device, do_array_copy=False) annotators["rgba"] = annotator elif annotator_type in _HDR_DATA_TYPES: - # Radiance reads the same HdrColor AOV through the ``rgb_hdr`` alias set in - # :meth:`set_outputs`; exposure is neutralized in :meth:`prepare_cameras`. + # HdrColor reflects the exposure configured on the camera prim. if str(RenderBufferKind.RGB_HDR) not in annotators: annotator = rep.AnnotatorRegistry.get_annotator("HdrColor", device=spec.device, do_array_copy=False) annotators[str(RenderBufferKind.RGB_HDR)] = annotator diff --git a/source/isaaclab_ppisp/docs/README.md b/source/isaaclab_ppisp/docs/README.md index 044c281c9fba..37ee029329db 100644 --- a/source/isaaclab_ppisp/docs/README.md +++ b/source/isaaclab_ppisp/docs/README.md @@ -5,7 +5,7 @@ a `DictTensorProcessor` used by cameras and observations. `DictTensorProcessor` `TensorProcessor` live together in `isaaclab.utils.tensor_processors`; their call signatures are dictionary-to-dictionary and tensor-to-tensor respectively. -`PpispProcessorCfg` reads scene-linear `rgb_hdr` from a float32 `ProxyArray` with +`PpispProcessor` reads scene-linear `rgb_hdr` from a float32 `ProxyArray` with shape `(N, H, W, 3)` and writes uint8 `rgb`. For camera processing, configure `CameraCfg.pipeline=[PpispProcessorCfg(params={"cfg": ppisp_cfg})]`. The camera requests required HDR internally; only the requested public outputs are published. Set @@ -20,26 +20,26 @@ Resolve authored settings explicitly before constructing the pipeline: from isaaclab_ppisp import PpispProcessorCfg, ppisp_cfg_from_usd_camera ppisp_cfg = ppisp_cfg_from_usd_camera(source_camera_prim) -step = PpispProcessorCfg(params={"cfg": ppisp_cfg}) +processor_cfg = PpispProcessorCfg(params={"cfg": ppisp_cfg}) ``` `PpispProcessor` can also run outside a camera. Construct it with its configuration and an actual dictionary of `ProxyArray` tensors; construction validates shape and dtype and allocates persistent output once on the same device. Input/output names can be changed independently of PPISP math. For RGBA, use `outputs=["rgba"]` and `params={"cfg": ppisp_cfg, "channels": 4}`. -PPISP consumes the preceding operation's radiance, so an earlier exposure or scale -operation changes its result. The producer must supply scene-linear color before +PPISP consumes the preceding processor's radiance, so an earlier exposure or scale +processor changes its result. The producer must supply scene-linear color before camera exposure and response. Array storage cannot establish that semantic contract; float32 display-encoded RGB must not be supplied as radiance. -Each operation owns its controller scratch and output buffers. Optional +Each processor owns its controller scratch and output buffers. Optional `params["controller_weights"]=PpispControllerWeights(device_weights)` can supply shared, read-only device weights; configuration copies preserve this resource's identity. The caller owns upload synchronization, and mutable controller workspace stays private. The controller requires CUDA; static PPISP also runs on CPU. `PpispPipeline` remains available for direct numerical use with explicitly supplied Warp buffers. -For direct use, construct `PpispProcessor(cfg, {"rgb_hdr": hdr_tensor})`, then call +For direct use, construct `PpispProcessor(processor_cfg, {"rgb_hdr": hdr_tensor})`, then call `processor({"rgb_hdr": hdr_tensor})` to obtain `{"rgb": color_tensor}`. It owns its persistent -output and numerical workspace; call `processor.close()` when finished. No preparation factory -or prepared-operation wrapper is needed. +output and numerical workspace; copy values that must survive another call, and call +`processor.close()` when finished. diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py index 8afc7ae3e769..244159b63550 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py @@ -5,7 +5,7 @@ """Post-render PPISP (Physically Plausible Image Signal Processing) for IsaacLab. -Provides a shared render-buffer operation that converts scene-linear radiance to RGB or RGBA. +Provides a dictionary processor that converts scene-linear radiance to RGB or RGBA. Camera and observation pipelines use :class:`PpispProcessorCfg` with concrete settings. USD discovery helpers remain explicit, and renderer backends do not own PPISP execution or controller workspace. """ diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py b/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py index d5c9079b6b45..d9416602171b 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py @@ -337,7 +337,7 @@ def resolve_and_normalize( ) -> PpispCfg | None: """Resolve explicit settings or a discovery mode using the caller's USD stage. - Call this before constructing a PPISP stage. The processing operation receives only concrete + Call this before constructing a PPISP processor. The processor receives only concrete settings and never looks up a camera or stage. Legacy ``CameraISPMode`` string-enum values remain accepted for migration without importing the camera package. diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py b/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py index 631a3e5f6e87..010761850428 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py @@ -550,7 +550,7 @@ def _apply_ppisp_kernel( For each pixel, the model applies HDR responsivity scaling, exposure scaling, per-channel vignetting, color homography correction, CRF tone - mapping, and uint8 RGBA packing. The first tensor dimension is the + mapping, and uint8 RGB or RGBA packing. The first tensor dimension is the camera/environment index, so image-space effects use local ``height_id`` and ``width_id`` coordinates per camera. """ diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py b/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py index 66d52c4a9151..04bdea699a9f 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py @@ -36,7 +36,7 @@ def __post_init__(self) -> None: raise ValueError("PPISP controller weights require the expected one-dimensional float32 buffer.") def __deepcopy__(self, memo: dict) -> PpispControllerWeights: - # Config cloning must share model resources, while each operation creates its own workspace. + # Config cloning must share model resources, while each processor creates its own workspace. return self diff --git a/source/isaaclab_ppisp/pyproject.toml b/source/isaaclab_ppisp/pyproject.toml index ade02233204e..52b8d53c60b9 100644 --- a/source/isaaclab_ppisp/pyproject.toml +++ b/source/isaaclab_ppisp/pyproject.toml @@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta" [project] name = "isaaclab_ppisp" version = "0.3.1" -description = "Extension providing a renderer-backend-agnostic PPISP pipeline that converts HDR scene-linear color to LDR RGBA at the end of a render tick." +description = "PPISP processors that convert scene-linear HDR to LDR RGB or RGBA for cameras and observations." license = {text = "BSD-3-Clause"} authors = [{name = "Isaac Lab Project Developers"}] maintainers = [{name = "Isaac Lab Project Developers"}]