From 5c0b18d92b31be460f52f3e22fb117bea8f42eb6 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Fri, 16 Oct 2026 19:18:35 -0400 Subject: [PATCH] Move PPISP processing from renderers into camera pipelines --- .../api/lab/isaaclab.sensors.camera.rst | 5 + docs/source/api/lab_ppisp/isaaclab_ppisp.rst | 23 ++ docs/source/concepts/sensors/camera.rst | 107 ++++--- examples/sensors/ppisp_camera.py | 6 +- .../feat-render-buffer-pipe.changed.rst | 3 + .../feat-render-buffer-pipe.fixed.rst | 2 + .../changelog.d/feat-render-buffer-pipe.major | 0 .../isaaclab/renderers/output_contract.py | 1 + .../isaaclab/renderers/render_context.py | 23 +- .../isaaclab/sensors/camera/__init__.pyi | 2 + .../isaaclab/sensors/camera/camera.py | 248 ++++++++++------ .../isaaclab/sensors/camera/camera_cfg.py | 59 ++-- .../isaaclab/sensors/camera/camera_data.py | 5 + .../sensors/camera/camera_exposure.py | 53 ++++ .../sensors/camera/camera_exposure_cfg.py | 34 +++ .../isaaclab/sensors/camera/camera_isp.py | 17 +- .../isaaclab/isaaclab/sensors/sensor_base.py | 2 +- .../isaaclab/sim/simulation_context.py | 9 + .../test/renderers/test_camera_exposure.py} | 20 +- .../renderers/test_camera_output_contract.py | 264 +++++++++++++++++- .../test_simulation_render_context.py | 2 + .../generate_synthetic_gaussian_asset.py | 29 +- .../test_camera_ppisp_gaussian_ovrtx.py | 45 ++- .../test/sensors/test_opencv_distortion.py | 1 + .../test/sensors/test_sensor_destructors.py | 1 + .../feat-render-buffer-pipe.changed.rst | 3 + .../changelog.d/feat-render-buffer-pipe.major | 0 .../renderers/newton_warp_renderer.py | 98 +------ .../test_newton_manager_abstraction.py | 2 +- .../feat-render-buffer-pipe.changed.rst | 3 + .../changelog.d/feat-render-buffer-pipe.major | 0 .../isaaclab_ov/renderers/ovrtx_renderer.py | 101 +------ .../isaaclab_ov/renderers/ovrtx_usd.py | 13 +- .../test/test_ovrtx_renderer_contract.py | 131 ++++----- source/isaaclab_ov/test/test_ovrtx_usd.py | 29 +- .../test/test_ovrtx_visual_material_sync.py | 7 +- .../feat-render-buffer-pipe.changed.rst | 3 + .../changelog.d/feat-render-buffer-pipe.major | 0 .../renderers/isaac_rtx_renderer.py | 124 ++------ .../test_isaac_rtx_renderer_contract.py | 101 ++++++- .../feat-render-buffer-pipe.changed.rst | 3 + .../changelog.d/feat-render-buffer-pipe.major | 0 source/isaaclab_ppisp/docs/README.md | 38 ++- .../isaaclab_ppisp/isaaclab_ppisp/__init__.py | 10 +- .../isaaclab_ppisp/__init__.pyi | 11 +- source/isaaclab_ppisp/isaaclab_ppisp/cfg.py | 47 ++-- .../isaaclab_ppisp/isaaclab_ppisp/kernels.py | 48 ++-- .../isaaclab_ppisp/isaaclab_ppisp/pipeline.py | 147 +++++----- .../isaaclab_ppisp/processor.py | 74 +++++ .../isaaclab_ppisp/processor_cfg.py | 26 ++ .../isaaclab_ppisp/rtx_camera_overrides.py | 96 ------- source/isaaclab_ppisp/pyproject.toml | 2 +- source/isaaclab_ppisp/test/test_ppisp.py | 5 +- .../isaaclab_ppisp/test/test_ppisp_kernels.py | 82 +++++- 54 files changed, 1335 insertions(+), 830 deletions(-) create mode 100644 source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst create mode 100644 source/isaaclab/changelog.d/feat-render-buffer-pipe.fixed.rst create mode 100644 source/isaaclab/changelog.d/feat-render-buffer-pipe.major create mode 100644 source/isaaclab/isaaclab/sensors/camera/camera_exposure.py create mode 100644 source/isaaclab/isaaclab/sensors/camera/camera_exposure_cfg.py rename source/{isaaclab_ppisp/test/test_rtx_camera_overrides.py => isaaclab/test/renderers/test_camera_exposure.py} (68%) create mode 100644 source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst create mode 100644 source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.major create mode 100644 source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst create mode 100644 source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.major create mode 100644 source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst create mode 100644 source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.major create mode 100644 source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst create mode 100644 source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.major create mode 100644 source/isaaclab_ppisp/isaaclab_ppisp/processor.py create mode 100644 source/isaaclab_ppisp/isaaclab_ppisp/processor_cfg.py delete mode 100644 source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py diff --git a/docs/source/api/lab/isaaclab.sensors.camera.rst b/docs/source/api/lab/isaaclab.sensors.camera.rst index e86b1ada9795..89ece542a6c0 100644 --- a/docs/source/api/lab/isaaclab.sensors.camera.rst +++ b/docs/source/api/lab/isaaclab.sensors.camera.rst @@ -13,7 +13,12 @@ The following classes are part of the public :mod:`isaaclab.sensors.camera` API. .. autosummary:: :nosignatures: + CameraExposureCfg CameraISPMode .. autoclass:: CameraISPMode :show-inheritance: + +.. autoclass:: CameraExposureCfg + :members: + :exclude-members: __init__ diff --git a/docs/source/api/lab_ppisp/isaaclab_ppisp.rst b/docs/source/api/lab_ppisp/isaaclab_ppisp.rst index 79b9565d1d77..ed2117d16a9c 100644 --- a/docs/source/api/lab_ppisp/isaaclab_ppisp.rst +++ b/docs/source/api/lab_ppisp/isaaclab_ppisp.rst @@ -14,10 +14,33 @@ The following classes are part of the public :mod:`isaaclab_ppisp` API. :nosignatures: PpispCfg + PpispProcessorCfg + PpispProcessor + PpispControllerWeights + PpispDiscoveryMode PpispPipeline .. autoclass:: PpispCfg :show-inheritance: +.. autoclass:: PpispProcessorCfg + :members: + :inherited-members: + :exclude-members: __init__ + :show-inheritance: + +.. autoclass:: PpispProcessor + :members: + :inherited-members: + :special-members: __call__ + :show-inheritance: + +.. autoclass:: PpispControllerWeights + :members: + +.. autoclass:: PpispDiscoveryMode + :members: + :show-inheritance: + .. autoclass:: PpispPipeline :show-inheritance: diff --git a/docs/source/concepts/sensors/camera.rst b/docs/source/concepts/sensors/camera.rst index 7d71e7bc11ab..a8b470e216a1 100644 --- a/docs/source/concepts/sensors/camera.rst +++ b/docs/source/concepts/sensors/camera.rst @@ -153,8 +153,9 @@ updating it adds frame-query overhead. Output types ------------ -The active renderer validates ``data_types`` and allocates the channel count and data type declared -by its :class:`~isaaclab.renderers.RenderBufferSpec`. +For raw outputs, the active renderer validates the requested names and allocates the channel count +and data type declared by its :class:`~isaaclab.renderers.RenderBufferSpec`. A camera pipeline may +produce additional output names; its processors allocate those buffers. .. list-table:: Common output contracts :header-rows: 1 @@ -168,7 +169,7 @@ by its :class:`~isaaclab.renderers.RenderBufferSpec`. - Low-dynamic-range color * - ``rgb_hdr`` - 3, ``float32`` - - Scene-linear high-dynamic-range color + - High-dynamic-range RGB using the active camera settings * - ``albedo`` - 4, ``uint8`` - Material base color @@ -198,6 +199,12 @@ by its :class:`~isaaclab.renderers.RenderBufferSpec`. non-colorized segmentation uses one ``int32`` ID channel. Label and prim-path mappings are stored in ``camera_data.info[output_name]``. +``rgb_hdr`` uses the camera's active exposure settings. PPISP requires scene-linear HDR; +set ``CameraCfg.exposure=CameraExposureCfg()`` explicitly to disable RTX automatic exposure and +use neutral exposure. Leaving ``exposure=None`` preserves authored camera settings. Selecting an +output never changes exposure. PPISP requests ``rgb_hdr`` privately; include it in ``data_types`` +only to publish the raw input as well. + .. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/camera-renderer-isaac-rtx.webp :align: center :figwidth: 100% @@ -263,6 +270,12 @@ current support matrix is: - Yes - No +.. note:: + + Explicit exposure settings affect every output from the camera prim, including other sensors + sharing it. Use separate camera prims when different exposures are required. Newton's HDR + output is already scene-linear and does not consume RTX exposure attributes. + Querying an unsupported output fails during camera initialization. Renderer configuration controls semantic filters, segmentation colorization, and depth clipping where those options are backend-specific. @@ -305,47 +318,72 @@ background. Set a normalized RGB tuple to use a solid color for pixels that miss The setting is per camera. Cameras with renderer-default and solid backgrounds can coexist in one scene. -Post-render image signal processing ------------------------------------ +Post-render image processing +---------------------------- -:attr:`~sensors.CameraCfg.isp_cfg` optionally applies an image signal processing (ISP) pass to the -renderer's scene-linear HDR output. The shipped implementation is PPISP (Physically Plausible Image -Signal Processing), which applies responsivity, exposure, vignetting, color correction, and a camera -response function before writing ``rgb`` or ``rgba``. +:attr:`~sensors.CameraCfg.pipeline` applies +:class:`~isaaclab.utils.tensor_processors.DictTensorProcessor` instances after rendering. +Each processor consumes named ``ProxyArray`` tensors and returns persistent named outputs; +its :class:`~isaaclab.utils.tensor_processors.DictTensorProcessorCfg` declares those keys. +The camera requests required renderer inputs privately and publishes the names in ``data_types``. -The field accepts: - -* ``None`` to disable post-render ISP. -* :class:`~isaaclab_ppisp.PpispCfg` for explicit coefficients or coefficients imported from a USD - camera. -* :class:`~sensors.CameraISPMode` to discover ``ppisp:*`` attributes on a camera prim. +For example, PPISP (Physically Plausible Image Signal Processing) converts scene-linear HDR +to uint8 color through exposure, vignetting, color correction, and a camera response function: .. code-block:: python - from isaaclab.sensors.camera import CameraCfg, CameraISPMode + from isaaclab.sensors.camera import CameraExposureCfg from isaaclab.utils import replace - from isaaclab_ppisp import PpispCfg + from isaaclab_ppisp import PpispCfg, PpispProcessorCfg + + processed_camera = replace( + front_camera, + data_types=["rgb"], + exposure=CameraExposureCfg(), + pipeline=[PpispProcessorCfg(params={"cfg": PpispCfg(inputs={"exposureOffset": 1.5})})], + ) - explicit_isp = replace(front_camera, data_types=["rgb"], isp_cfg=PpispCfg(inputs={"exposureOffset": 1.5})) +PPISP requires float32 scene-linear input with shape ``(N, H, W, 3)``. Configure neutral camera +exposure before simulation initialization, as above; PPISP's ``exposureOffset`` is measured in stops. +Include ``rgb_hdr`` in ``data_types`` to publish its raw input alongside the processed output. +For RGBA, use ``outputs=["rgba"]`` and ``params={"cfg": ppisp_cfg, "channels": 4}``. +A processed RGBA buffer can also provide a zero-copy RGB view; RGB alone cannot satisfy an RGBA request. - discovered_isp = replace(front_camera, data_types=["rgb"], isp_cfg=CameraISPMode.AUTO_CAMERA) +Published outputs must retain the camera's batch size, height, width, and NHWC layout so calibration +remains valid. Intermediate buffers may change shape. Processors borrow their inputs read-only, +and each camera owns its pipeline workspace. Outputs reuse storage across captures; copy values +that must survive another update. Explicitly shared immutable controller weights are supported +through :class:`~isaaclab_ppisp.PpispControllerWeights`. -``AUTO_CAMERA`` checks the first matched camera prim. ``AUTO_ANY`` falls back to the first PPISP -camera anywhere on the stage. Discovery happens once during camera construction. +The pipeline runs once per new capture, including batched captures. If an asynchronous renderer +republishes an old capture, its processed result and capture metadata are retained. Camera reset +forwards selected environment indices to processor reset callbacks; teardown closes processors. +Existing image observation terms, such as ``mdp.image_rgb``, read these processed camera outputs. -PPISP is composed by Isaac RTX, OVRTX, and Newton Warp. It requires ``rgb`` or ``rgba`` output. A -static configuration is shared by all cloned views in one camera batch; controller weights may -predict per-view exposure and color parameters, while the remaining coefficients stay shared. ISP -configuration and discovered USD attributes are fixed for the camera lifetime. +Migrating renderer-owned ISP configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. important:: +``CameraCfg.isp_cfg`` now raises a migration error for non-None values. Replace it with +``CameraCfg.pipeline`` and explicit neutral exposure as above. ``CameraISPMode`` remains +importable for legacy configuration values; new code resolves USD attributes before constructing +the processor: - With Isaac RTX and OVRTX, enabling ``isp_cfg`` makes PPISP the ISP authority. The renderer - disables RTX auto-exposure, authors neutral ``exposure:*`` values, and applies the - ``OmniRtxCameraAutoExposureAPI_1`` and ``OmniRtxCameraExposureAPI_1`` schemas on every matched - camera prim so RTX does not process the image a second time. Do not combine ``isp_cfg`` with - separately authored RTX exposure or tonemapping settings. When ``isp_cfg`` is ``None``, the - renderer leaves authored camera exposure unchanged. +.. code-block:: python + + from isaaclab_ppisp import PpispDiscoveryMode, resolve_and_normalize + + ppisp_cfg = resolve_and_normalize(PpispDiscoveryMode.AUTO_CAMERA, stage, "/World/Camera") + discovered_camera = replace( + front_camera, + data_types=["rgb"], + exposure=CameraExposureCfg(), + pipeline=[PpispProcessorCfg(params={"cfg": ppisp_cfg})] if ppisp_cfg is not None else [], + ) + +``AUTO_CAMERA`` checks the supplied camera; ``AUTO_ANY`` can fall back to another PPISP camera +on the stage. Alternatively, use ``ppisp_cfg_from_usd_camera(camera_prim)``. +The processor performs no camera or stage discovery. Static coefficients apply to the whole batch; +controller weights may predict per-view exposure and color parameters. Run the ``ppisp-camera`` example for a complete PPISP workflow. It supports ``--renderer newton_renderer`` (Newton Warp, the default) and ``--renderer isaac_rtx`` (Isaac RTX). @@ -416,3 +454,8 @@ A runnable camera example is available as ``camera``: For saving output to disk, see :doc:`/source/how-to/save_camera_output`. For renderer selection and customization, see :doc:`/source/how-to/configure_rendering`. + +The observation processor API is ``isaaclab.utils.tensor_processors``. Custom stateful processors +inherit ``TensorProcessor`` and use ``TensorProcessorCfg``; attach configurations through +``ObservationTermCfg.tensor_processors``. These replace ``ModifierBase``, ``ModifierCfg``, and +``ObservationTermCfg.modifiers`` respectively. diff --git a/examples/sensors/ppisp_camera.py b/examples/sensors/ppisp_camera.py index 79e95688eb37..ad3a8da4a8db 100644 --- a/examples/sensors/ppisp_camera.py +++ b/examples/sensors/ppisp_camera.py @@ -120,12 +120,13 @@ import matplotlib.pyplot as plt import numpy as np import torch -from isaaclab_ppisp.cfg import PpispCfg, ppisp_cfg_from_usd_camera +from isaaclab_ppisp import PpispCfg, PpispProcessorCfg, ppisp_cfg_from_usd_camera import isaaclab.sim as sim_utils from isaaclab.assets import AssetBaseCfg, RigidObjectCfg from isaaclab.scene import InteractiveSceneCfg from isaaclab.sensors import CameraCfg +from isaaclab.sensors.camera import CameraExposureCfg from isaaclab.sim.spawners.materials import UsdPhysicsRigidBodyMaterialCfg from isaaclab.utils import configclass, instantiate @@ -374,7 +375,8 @@ def make_camera(camera_prim_path: str, *, ppisp_cfg: PpispCfg | None, width: int width=width, data_types=["rgb"], spawn=None, - isp_cfg=ppisp_cfg, + exposure=CameraExposureCfg() if ppisp_cfg is not None else None, + pipeline=[PpispProcessorCfg(params={"cfg": ppisp_cfg})] if ppisp_cfg is not None else [], renderer_cfg=make_renderer_cfg(), ) return instantiate(camera_cfg) diff --git a/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..b39df51f5fb8 --- /dev/null +++ b/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst @@ -0,0 +1,3 @@ +* **Breaking:** Replaced renderer-owned PPISP with camera processors consuming ``rgb_hdr``. + Replace ``CameraCfg.isp_cfg`` with ``PpispProcessorCfg`` in ``CameraCfg.pipeline``. + Set ``CameraCfg.exposure=CameraExposureCfg()`` explicitly for neutral RTX HDR before PPISP. diff --git a/source/isaaclab/changelog.d/feat-render-buffer-pipe.fixed.rst b/source/isaaclab/changelog.d/feat-render-buffer-pipe.fixed.rst new file mode 100644 index 000000000000..db061e515934 --- /dev/null +++ b/source/isaaclab/changelog.d/feat-render-buffer-pipe.fixed.rst @@ -0,0 +1,2 @@ +* Avoided unnecessary CUDA synchronization during masked camera resets when image pipeline + operations had no reset callbacks. diff --git a/source/isaaclab/changelog.d/feat-render-buffer-pipe.major b/source/isaaclab/changelog.d/feat-render-buffer-pipe.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab/isaaclab/renderers/output_contract.py b/source/isaaclab/isaaclab/renderers/output_contract.py index fdd7d9e9f248..7ee8025b6693 100644 --- a/source/isaaclab/isaaclab/renderers/output_contract.py +++ b/source/isaaclab/isaaclab/renderers/output_contract.py @@ -26,6 +26,7 @@ class RenderBufferKind(StrEnum): RGB = "rgb" RGBA = "rgba" RGB_HDR = "rgb_hdr" + """HDR RGB using the camera's active rendering settings.""" ALBEDO = "albedo" DEPTH = "depth" DISTANCE_TO_IMAGE_PLANE = "distance_to_image_plane" diff --git a/source/isaaclab/isaaclab/renderers/render_context.py b/source/isaaclab/isaaclab/renderers/render_context.py index 9db971853300..c67807441ea4 100644 --- a/source/isaaclab/isaaclab/renderers/render_context.py +++ b/source/isaaclab/isaaclab/renderers/render_context.py @@ -9,8 +9,9 @@ import logging import warnings +import weakref from collections.abc import Sequence -from typing import Any +from typing import TYPE_CHECKING, Any import torch import warp as wp @@ -19,6 +20,9 @@ from .base_renderer import BaseRenderer, VisualMaterialBatch from .renderer_cfg import RendererCfg +if TYPE_CHECKING: + from ..sensors.camera.camera import Camera + logger = logging.getLogger(__name__) @@ -65,6 +69,7 @@ class RenderContext: __slots__ = ( "clone_contexts", "_backend_registry", + "_cameras", "_physics_initialized", "_prepared_renderer_ids", "_prepared_num_envs", @@ -82,6 +87,7 @@ def __init__(self, backend_registry: list[tuple[Any, Any]]) -> None: self.clone_contexts: set[type | str] = set() """Scene representations declared by camera renderers and visualizers before cloning.""" self._backend_registry = backend_registry + self._cameras: weakref.WeakValueDictionary[int, Camera] = weakref.WeakValueDictionary() self._physics_initialized: bool = False # Set to True after the first PHYSICS_READY callback fires. self._prepared_renderer_ids: set[int] = set() self._prepared_num_envs: int | None = None @@ -133,6 +139,19 @@ def ensure_initialize(self) -> None: for _cfg, renderer in self._renderer_entries: renderer.initialize() + def register_camera(self, camera: Camera) -> None: + """Include a camera in stage preparation without extending its lifetime.""" + self._cameras[id(camera)] = camera + + def unregister_camera(self, camera: Camera) -> None: + """Stop preparing a camera whose sensor callbacks have been cleared.""" + self._cameras.pop(id(camera), None) + + def prepare_cameras(self) -> None: + """Prepare every live camera before any sensor can export the shared renderer stage.""" + for camera in tuple(self._cameras.values()): + camera._prepare_renderer() + def register_visual_material(self, material: Any) -> None: """Register one initialized material asset for flat channel composition.""" if any(registered is material for registered in self._visual_materials): @@ -372,6 +391,8 @@ def close(self) -> None: Stage preparation remains valid for the renderer's lifetime across physics resets. + Live camera registrations survive a stop so their stage preparation runs again on reset. + Raises: RuntimeError: If a material writer failed to close. """ diff --git a/source/isaaclab/isaaclab/sensors/camera/__init__.pyi b/source/isaaclab/isaaclab/sensors/camera/__init__.pyi index afff9e01fe01..15454b8c29ef 100644 --- a/source/isaaclab/isaaclab/sensors/camera/__init__.pyi +++ b/source/isaaclab/isaaclab/sensors/camera/__init__.pyi @@ -6,6 +6,7 @@ __all__ = [ "Camera", "CameraCfg", + "CameraExposureCfg", "CameraData", "CameraISPMode", "RenderBufferKind", @@ -20,6 +21,7 @@ __all__ = [ from .camera import Camera from .camera_cfg import CameraCfg +from .camera_exposure_cfg import CameraExposureCfg from .camera_data import CameraData, RenderBufferKind, RenderBufferSpec from .camera_isp import CameraISPMode from .tiled_camera import TiledCamera diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 9ecf4fee19d2..6167f1126013 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -8,6 +8,8 @@ import logging import sys from collections.abc import Sequence +from copy import copy +from dataclasses import replace from typing import TYPE_CHECKING, Any, Literal, cast import numpy as np @@ -18,6 +20,7 @@ from ... import sim as sim_utils from ...app.logging_utils import force_log_level +from ...cloner import path as cloner_path from ...renderers import BaseRenderer, CameraRenderSpec from ...sim.views import FrameView from ...utils.math import ( @@ -25,11 +28,13 @@ create_rotation_matrix_from_view, quat_from_matrix, ) +from ...utils.tensor_processors._pipeline import _DictTensorProcessorPipeline from ...utils.warp import ProxyArray from ..sensor_base import SensorBase from .camera_data import CameraData, RenderBufferKind if TYPE_CHECKING: + from ...renderers.render_context import RenderContext from .camera_cfg import CameraCfg logger = logging.getLogger(__name__) @@ -224,6 +229,7 @@ def __init__(self, cfg: CameraCfg): RuntimeError: If no camera prim is found at the given path. ValueError: If the provided data types are not supported by the camera or active renderer. """ + self._render_context: RenderContext | None = None # perform check on supported data types self._check_supported_data_types(cfg) # initialize base class @@ -263,41 +269,17 @@ def __init__(self, cfg: CameraCfg): if sim_ctx is not None: sim_ctx.require_visual_shapes() - # An ISP (any ``isp_cfg`` other than ``None``) requires the HDR AOV; - # an explicit ``"rgb_hdr"`` in ``data_types`` also requires the - # HDR-routing flag flipped on the RTX-bearing backends. - require_hdr_output = "rgb_hdr" in self.cfg.data_types or self.cfg.isp_cfg is not None - - # TODO(follow-up PR): move this flag flip out of Camera. The cleanest path is - # an apply_pre_reset_settings() hook on RendererCfg (default no-op) that - # IsaacRtxRendererCfg overrides to flip /isaaclab/render/rtx_sensors. The - # flag must be set pre-sim.reset() because SimulationContext.is_rendering - # and several env classes read it before the renderer's __init__ runs. - renderer_type = getattr(self.cfg.renderer_cfg, "renderer_type", None) - if renderer_type == "isaac_rtx": - from ...app.settings_manager import get_settings_manager - - settings = get_settings_manager() - settings.set("/isaaclab/render/rtx_sensors", True) - settings.set("/physics/fabricUpdateTransformations", True) - if require_hdr_output: - settings.set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) - elif renderer_type == "ovrtx" and require_hdr_output: - from ...app.settings_manager import get_settings_manager - - get_settings_manager().set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) - # FIXME: settings.set is a no-op for ovrtx - # warning only since it affects only ParticleField3DGaussianSplat scene - logger.warning( - "OVRTX backend with PPISP/HDR requires /rtx/rtpt/gaussian/skipTonemapping/enabled to be false." - ) - + # Renderer requirements include private pipeline inputs and exclude produced public outputs. + self._render_cfg = replace(self.cfg, data_types=self.cfg.render_data_types(), pipeline=[]) + self._buffer_pipe: _DictTensorProcessorPipeline | None = None + self._pipeline_capture: object | None = None + self._render_camera_data: CameraData | None = None # UsdGeom Camera prim for the sensor self._sensor_prims: list[UsdGeom.Camera] = [] # Allocated in :meth:`_create_buffers` once the renderer's output contract is known. self._data: CameraData | None = None # The backend's ``__init__`` is its pre-physics phase, so it has to exist before - # ``sim.reset()``; sensor initialization only runs on ``PhysicsEvent.PHYSICS_READY``, which is + # ``sim.reset()``; sensor initialization only runs once physics is ready, which is # too late. Backends are shared per renderer config, so this stays cheap for many cameras. self._renderer: BaseRenderer | None = None if sim_ctx is not None: @@ -317,16 +299,19 @@ def __del__(self, _sys=sys): """ if _sys.is_finalizing() or _sys.meta_path is None: return - # unsubscribe callbacks - super().__del__() - - # release the frame view's backend state, getattr covers partial initialization - if getattr(self, "_view", None) is not None: - self._view.close() - self._view = None - # cleanup render resources (renderer may be None if never initialized) - if getattr(self, "_renderer", None) is not None: - self._renderer.cleanup(getattr(self, "_render_data", None)) + try: + # unsubscribe callbacks + super().__del__() + + # release the frame view's backend state, getattr covers partial initialization + if getattr(self, "_view", None) is not None: + self._view.close() + self._view = None + # cleanup render resources (renderer may be None if never initialized) + if getattr(self, "_renderer", None) is not None: + self._renderer.cleanup(getattr(self, "_render_data", None)) + finally: + self._close_pipeline() def __str__(self) -> str: """Returns: A string containing information about the instance.""" @@ -660,6 +645,9 @@ def reset(self, env_ids: Sequence[int] | slice | None = None, env_mask: wp.array self._renderer.reset(self._render_data, env_ids) # reset the timestamps super().reset(env_ids, env_mask) + if self._buffer_pipe is not None and self._buffer_pipe.has_reset_callbacks: + pipeline_env_ids = wp.to_torch(env_mask).nonzero().squeeze(-1) if env_mask is not None else env_ids + self._buffer_pipe.reset(pipeline_env_ids) # reset the data # note: this recomputation is useful if one performs events such as randomizations on the camera poses. if env_mask is not None: @@ -692,29 +680,10 @@ def _initialize_impl(self): sim_ctx = sim_utils.SimulationContext.instance() if sim_ctx is None: raise RuntimeError("SimulationContext is not initialized.") - # Normally created in ``__init__``; only missing when the camera was built without a simulation. - if self._renderer is None: - self._renderer = sim_ctx.get_or_create_backend(self.cfg.renderer_cfg) - sim_ctx.get_scene_data_provider().cameras[self.cfg.prim_path] = self - - # Build the render spec early — both the wrapper ISP (which delegates - # any renderer-side per-camera setup) and ``create_render_data`` consume - # it, and the prims are already authored at this point. - cam_paths = tuple(str(p.GetPath()) for p in sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) - render_spec = CameraRenderSpec( - cfg=self.cfg, - device=str(self._device), - num_instances=self._num_envs, - camera_prim_paths=cam_paths, - view_count=self._num_envs, - ) - - # Delegate per-camera USD setup to the renderer — must run **before** - # ``ensure_prepare_stage`` so renderers that snapshot the stage - # (ovrtx's ``stage.Export``) capture the resulting overrides in their - # exported USD. - self._renderer.prepare_cameras(self.stage, render_spec) + # Repeat the pre-export setup for cameras initialized outside the shared preparation pass. + render_spec = self._prepare_renderer(self._num_envs) + cam_paths = render_spec.camera_prim_paths # Stage preprocessing must happen before creating the view because the view keeps # references to prims located in the stage. @@ -758,7 +727,7 @@ def _prepare_camera(self, env_mask: wp.array) -> None: self._update_poses(env_mask=env_mask, frame_op=1) else: self._update_camera_state(env_mask=env_mask, frame_op=1) - self._renderer.prepare_capture(self._render_data, self._data, self._frame) + self._renderer.prepare_capture(self._render_data, self._render_camera_data, self._frame) def _update_buffers_impl(self, env_mask: wp.array): if not self._env_mask_has_any(env_mask): @@ -772,12 +741,13 @@ def _update_buffers_impl(self, env_mask: wp.array): sim_ctx.render_context.render_into_camera( renderer, self._render_data, - self._data, + self._render_camera_data, sim_ctx.get_physics_step_count(), ) else: renderer.render(self._render_data) - renderer.read_output(self._render_data, self._data) + renderer.read_output(self._render_data, self._render_camera_data) + self._apply_pipeline() @staticmethod def _update_buffers_batch_impl(sensors: Sequence[SensorBase]) -> None: @@ -799,9 +769,38 @@ def _update_buffers_batch_impl(sensors: Sequence[SensorBase]) -> None: return sim_ctx.render_context.render_into_cameras( - [(camera._renderer, camera._render_data, camera._data) for camera in ready], + [(camera._renderer, camera._render_data, camera._render_camera_data) for camera in ready], sim_ctx.get_physics_step_count(), ) + for camera in ready: + camera._apply_pipeline() + + def _close_pipeline(self) -> None: + """Release pipeline resources, including partially initialized cameras.""" + pipe = getattr(self, "_buffer_pipe", None) + self._buffer_pipe = None + if pipe is not None: + pipe.close() + + def _apply_pipeline(self) -> None: + """Process a new capture once and publish metadata matching its processed pixels.""" + if self._buffer_pipe is None: + return + info = self._render_camera_data.info + capture = next( + (value["capture"] for value in info.values() if isinstance(value, dict) and "capture" in value), None + ) + if capture is not None and capture is self._pipeline_capture: + return + self._buffer_pipe(self._render_camera_data.output) + self._pipeline_capture = capture + for name in self._data.output: + if name in self._buffer_pipe.produced_outputs or ( + name == "rgb" and "rgba" in self._buffer_pipe.produced_outputs + ): + self._data.info[name] = {"capture": capture} if capture is not None else None + else: + self._data.info[name] = info.get(name) """ Private Helpers @@ -817,7 +816,7 @@ def _check_supported_data_types(self, cfg: CameraCfg): ) # check if there is any intersection in unsupported types # reason: these use np structured data types which are not compatible with the camera buffer contract - common_elements = set(cfg.data_types) & Camera.UNSUPPORTED_TYPES + common_elements = set(cfg.render_data_types()) & Camera.UNSUPPORTED_TYPES if common_elements: # provide alternative fast counterparts fast_common_elements = [] @@ -840,7 +839,7 @@ def _create_buffers(self): known: list[str] = [] unknown: list[str] = [] unsupported: list[str] = [] - for name in self.cfg.data_types: + for name in self._render_cfg.data_types: try: if RenderBufferKind(name) in specs: known.append(name) @@ -872,7 +871,49 @@ def _create_buffers(self): self._data.create_buffers(self._view.count, str(self._device)) self._initialize_intrinsics() self._update_poses() - self._renderer.set_outputs(self._render_data, self._data.output) + self._render_camera_data = self._data + self._renderer.set_outputs(self._render_data, self._render_camera_data.output) + self._close_pipeline() + self._pipeline_capture = None + if self.cfg.pipeline: + self._buffer_pipe = _DictTensorProcessorPipeline(self._render_camera_data.output, self.cfg.pipeline) + try: + outputs = dict(self._buffer_pipe.outputs) + alias_rgb = ( + "rgba" in self._buffer_pipe.produced_outputs and "rgb" not in self._buffer_pipe.produced_outputs + ) + expected_shape = (self._view.count, self.cfg.height, self.cfg.width) + for name in self.cfg.data_types: + buffer = outputs["rgba" if alias_rgb and name == "rgb" else name] + if len(buffer.shape) != 4 or buffer.shape[:3] != expected_shape: + raise ValueError( + f"Camera pipeline output {name!r} must use NHWC with batch and spatial shape" + f" {expected_shape}; received {buffer.shape}." + " Camera calibration describes the rendered image dimensions." + ) + # Preserve RGB's view semantics when a processor produces RGBA, without publishing stale raw RGB. + if alias_rgb: + rgba = outputs["rgba"].warp + if len(rgba.shape) != 4 or rgba.shape[-1] != 4: + raise ValueError("A pipeline output named rgba must have shape (N, H, W, 4).") + rgb = wp.array( + ptr=rgba.ptr, + shape=(*rgba.shape[:-1], 3), + strides=rgba.strides, + dtype=rgba.dtype, + device=rgba.device, + copy=False, + ) + outputs["rgb"] = ProxyArray(rgb) + self._data = copy(self._render_camera_data) + self._data._output = {name: outputs[name] for name in self.cfg.data_types} + self._data.info = dict.fromkeys(self._data.output) + except Exception as error: + try: + self._close_pipeline() + except Exception as cleanup_error: + error.add_note(f"Failed to close camera pipeline after initialization failed: {cleanup_error}") + raise def _read_authored_opencv_intrinsics( self, prim: Usd.Prim, width: int, height: int, env_id: int @@ -1084,15 +1125,64 @@ def _env_mask_has_any(self, env_mask: wp.array) -> bool: Internal simulation callbacks. """ + def _register_callbacks(self): + """Register the sensor lifecycle and shared renderer stage preparation.""" + super()._register_callbacks() + self._render_context = sim_utils.SimulationContext.instance().render_context + self._render_context.register_camera(self) + + def _clear_callbacks(self) -> None: + """Release stage preparation along with the ordinary sensor subscriptions.""" + if self._render_context is not None: + self._render_context.unregister_camera(self) + self._render_context = None + super()._clear_callbacks() + + def _prepare_renderer(self, num_views: int | None = None) -> CameraRenderSpec | None: + """Apply this camera's renderer USD overrides before any camera initializes. + + A renderer that exports the stage when the first camera initializes, such as OVRTX, then sees + the overrides of every camera. + """ + if self._is_initialized: + return None + sim_ctx = sim_utils.SimulationContext.instance() + if self._renderer is None: + self._renderer = sim_ctx.get_or_create_backend(self.cfg.renderer_cfg) + cam_paths = tuple(str(p.GetPath()) for p in sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) + if num_views is None: + # Match the environment count that sensor initialization derives next. + clone_plan = sim_ctx.get_clone_plan() + if clone_plan is not None and cloner_path.match(self.cfg.prim_path, clone_plan.env_template) is not None: + num_views = len(clone_plan.topology.world_prototype_layout) + else: + num_views = len(cam_paths) + render_spec = CameraRenderSpec( + cfg=self._render_cfg, + device=str(sim_ctx.device), + num_instances=num_views, + camera_prim_paths=cam_paths, + view_count=num_views, + ) + if self.cfg.exposure is not None: + from .camera_exposure import apply_camera_exposure + + apply_camera_exposure(self.stage, list(cam_paths), self.cfg.exposure) + self._renderer.prepare_cameras(self.stage, render_spec) + return render_spec + def _invalidate_initialize_callback(self, event): """Invalidates the scene elements.""" - if self._renderer is not None and self._render_data is not None: - self._renderer.cleanup(self._render_data) - self._render_data = None - self._renderer = None - # call parent - super()._invalidate_initialize_callback(event) - # release backend state deterministically, then invalidate the view - if self._view is not None: - self._view.close() - self._view = None + try: + self._close_pipeline() + finally: + if self._renderer is not None and self._render_data is not None: + self._renderer.cleanup(self._render_data) + self._render_data = None + self._renderer = None + # call parent + super()._invalidate_initialize_callback(event) + # release backend state deterministically, then invalidate the view + if self._view is not None: + self._view.close() + self._view = None diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py index 57827d458b2e..0670ddef5510 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py @@ -13,9 +13,10 @@ from ...sim import FisheyeCameraCfg, PinholeCameraCfg from ...utils import configclass from ..sensor_base_cfg import SensorBaseCfg -from .camera_isp import CameraISPMode +from .camera_exposure_cfg import CameraExposureCfg if TYPE_CHECKING: + from ...utils.tensor_processors import DictTensorProcessorCfg from .camera import Camera # Default values for the RTX-flavored fields kept on :class:`CameraCfg` for @@ -206,29 +207,46 @@ class OffsetCfg: renderer_cfg: RendererCfg = field(default_factory=RendererCfg) """Renderer configuration for camera sensor.""" - isp_cfg: Any | CameraISPMode | None = None - """Post-render ISP cfg applied by the renderer backend after it produces HDR output. + exposure: CameraExposureCfg | None = None + """Explicit camera exposure, or None to preserve authored USD settings. - Defaults to ``None`` (ISP disabled). Auto-discovery is opt-in via a - :class:`CameraISPMode` sentinel — see below. + Use ``CameraExposureCfg()`` for neutral HDR input to a PPISP camera processor. Applied before stage export; + changes affect every output from this camera prim. Newton ignores RTX exposure settings. + """ - Accepted values: + pipeline: list[DictTensorProcessorCfg] = [] + """Ordered dictionary processors applied after each newly published capture. - * ``None`` — ISP disabled. No HDR AOV is requested and no RTX-side - tonemapping flags are flipped. - * A :class:`CameraISPMode` sentinel — the renderer backend walks the USD stage to - discover an ISP shader (e.g. via the :mod:`isaaclab_ppisp` package). - * A concrete ISP cfg dataclass (e.g. :class:`isaaclab_ppisp.PpispCfg`) — used directly. + Processors consume and return dictionaries of named tensors. They may combine several render + outputs or preceding results. The renderer receives the pipeline's external dependencies + before initialization. + Only :attr:`data_types` are published in :attr:`CameraData.output`; include ``rgb_hdr`` to expose + the unprocessed input. Processors borrow their inputs read-only and + own persistent result buffers. Camera reset and teardown reset and close the processors. + """ - The cfg applies once per Camera sensor batch. The PPISP Warp kernel takes - scalar coefficients, so every cloned view in a tiled batch shares the same - ISP configuration — there is no per-view ISP today. + isp_cfg: Any | None = None + """Retired renderer-owned ISP setting, retained to report a migration error. - :mod:`isaaclab.sensors.camera` does not depend on any ISP implementation; the - annotation is intentionally loose (``Any``) so the sensor layer can carry the - cfg through to a renderer that knows what to do with it. + Non-None values raise :class:`ValueError`. Configure :attr:`pipeline` with a PPISP step instead. """ + def render_data_types(self) -> list[str]: + """Resolve raw renderer inputs, including requested outputs not produced by the pipeline.""" + if self.isp_cfg is not None: + raise ValueError( + "CameraCfg.isp_cfg has been retired. Configure CameraCfg.pipeline with a PPISP step instead." + ) + if not self.pipeline: + return list(self.data_types) + from ...utils.tensor_processors._pipeline import _DictTensorProcessorPipeline + + produced = {name for step in self.pipeline for name in step.outputs} + if "rgb" in produced and "rgba" not in produced and "rgba" in self.data_types: + raise ValueError("A pipeline producing rgb must also produce rgba when rgba is requested.") + published = [name for name in self.data_types if not (name == "rgb" and "rgba" in produced)] + return sorted(_DictTensorProcessorPipeline.required_source_names(self.pipeline, published)) + def __post_init__(self): """Forward deprecated RTX-flavored fields onto :attr:`renderer_cfg`. @@ -236,6 +254,10 @@ def __post_init__(self): :class:`DeprecationWarning` and is copied onto ``self.renderer_cfg`` when that cfg defines the same-named field. """ + if self.isp_cfg is not None: + raise ValueError( + "CameraCfg.isp_cfg has been retired. Configure CameraCfg.pipeline with a PPISP step instead." + ) renderer_type = getattr(self.renderer_cfg, "renderer_type", None) if renderer_type == "default": from ...utils.backend_utils import get_default_renderer_cfg @@ -260,13 +282,14 @@ def __post_init__(self): def validate_config(self) -> None: """Validate the requested data types against the selected renderer contract.""" + raw_names = self.render_data_types() if self.renderer_cfg is None: return supported_specs = self.renderer_cfg.supported_output_types() if supported_specs is None: return supported = {str(kind) for kind in supported_specs} - unsupported = sorted(set(self.data_types) - supported) + unsupported = sorted(set(raw_names) - supported) if unsupported: raise ValueError( f"Renderer {type(self.renderer_cfg).__name__} only supports data types {sorted(supported)}, " diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_data.py b/source/isaaclab/isaaclab/sensors/camera/camera_data.py index 78933793a57e..45eeeb6837f2 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_data.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_data.py @@ -104,6 +104,11 @@ def output(self) -> dict[str, ProxyArray] | None: and C is the number of channels. Use ``.torch`` for a ``torch.Tensor`` view or ``.warp`` for the underlying ``wp.array``. + When :attr:`CameraCfg.pipeline` is configured, values are the pipeline results selected by + :attr:`CameraCfg.data_types`. Results preserve NHWC layout, batch size and image dimensions + so camera calibration remains applicable; channel count and dtype may change. Buffers are reused + at the next capture. Include ``rgb_hdr`` in :attr:`CameraCfg.data_types` to publish the raw input. + The format of the data is available in the `Replicator Documentation`_. For semantic-based data, this corresponds to the ``"data"`` key in the output of the sensor. diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_exposure.py b/source/isaaclab/isaaclab/sensors/camera/camera_exposure.py new file mode 100644 index 000000000000..f1d5ec0de55c --- /dev/null +++ b/source/isaaclab/isaaclab/sensors/camera/camera_exposure.py @@ -0,0 +1,53 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Author explicitly configured exposure before renderer stage export.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .camera_exposure_cfg import CameraExposureCfg + +_CAMERA_EXPOSURE_SCHEMAS = ("OmniRtxCameraAutoExposureAPI_1", "OmniRtxCameraExposureAPI_1") + + +def apply_camera_exposure(stage: Any, prim_paths: list[str], cfg: CameraExposureCfg) -> None: + """Apply camera exposure independently of requested render outputs. + + Existing API schemas are preserved. Schema metadata is authored directly so the RTX schema + plugin does not need to be loaded yet. + + Args: + stage: USD stage containing the camera prims. + prim_paths: Resolved camera prim paths. + cfg: Explicit exposure settings shared by these camera prims. + """ + from pxr import Sdf + + settings = ( + ("exposure", "Float", cfg.exposure), + ("exposure:fStop", "Float", cfg.f_stop), + ("exposure:iso", "Float", cfg.iso), + ("exposure:responsivity", "Float", cfg.responsivity), + ("exposure:time", "Float", cfg.time), + ("omni:rtx:autoExposure:enabled", "Bool", cfg.auto_exposure), + ) + value_types = {"Bool": Sdf.ValueTypeNames.Bool, "Float": Sdf.ValueTypeNames.Float} + for prim_path in prim_paths: + prim = stage.GetPrimAtPath(prim_path) + if not prim or not prim.IsValid(): + continue + current = prim.GetMetadata("apiSchemas") or Sdf.TokenListOp() + existing = list(current.prependedItems) + list(current.explicitItems) + list(current.appendedItems) + missing = [schema for schema in _CAMERA_EXPOSURE_SCHEMAS if schema not in existing] + if missing: + prim.SetMetadata("apiSchemas", Sdf.TokenListOp.Create(prependedItems=[*missing, *existing])) + for attr_name, sdf_type_name, value in settings: + attr = prim.GetAttribute(attr_name) + if not attr: + attr = prim.CreateAttribute(attr_name, value_types[sdf_type_name], custom=False) + attr.Set(value) diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_exposure_cfg.py b/source/isaaclab/isaaclab/sensors/camera/camera_exposure_cfg.py new file mode 100644 index 000000000000..d9e20aa9e44b --- /dev/null +++ b/source/isaaclab/isaaclab/sensors/camera/camera_exposure_cfg.py @@ -0,0 +1,34 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from isaaclab.utils import configclass + + +@configclass +class CameraExposureCfg: + """Explicit RTX camera exposure settings, applied before renderer stage export. + + Defaults disable automatic exposure and make HDR neutral for PPISP. Settings affect every + output from the camera prim. Newton does not consume RTX exposure attributes; its HDR output + is already scene-linear. Leave ``CameraCfg.exposure`` unset to preserve authored settings. + """ + + exposure: float = 0.0 + """Exposure compensation in stops.""" + + f_stop: float = 1.0 + """Exposure aperture ratio (separate from the lens depth-of-field aperture).""" + + iso: float = 0.0 + """RTX sensor ISO; zero selects neutral exposure.""" + + responsivity: float = 1.0 + """Sensor response multiplier.""" + + time: float = 1.0 + """Exposure duration in seconds.""" + + auto_exposure: bool = False + """Whether RTX automatic exposure is enabled.""" diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_isp.py b/source/isaaclab/isaaclab/sensors/camera/camera_isp.py index 8934198bbcf8..17a2256f6ffe 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_isp.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_isp.py @@ -3,12 +3,10 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Sentinel enum for :attr:`~isaaclab.sensors.camera.CameraCfg.isp_cfg`. +"""Legacy ISP discovery names retained for migration of saved configurations. -The renderer backend (which depends on the appropriate ISP implementation -package) is responsible for resolving these sentinels to a concrete cfg and -applying the ISP pipeline. :mod:`isaaclab.sensors.camera` carries the sentinel -through to the renderer without knowing about any specific ISP implementation. +``CameraCfg.isp_cfg`` no longer executes processing. Use ``CameraCfg.pipeline`` and +the discovery configuration provided by the image-processing package instead. """ from __future__ import annotations @@ -17,14 +15,7 @@ class CameraISPMode(StrEnum): - """Sentinel modes for :attr:`~isaaclab.sensors.camera.CameraCfg.isp_cfg`. - - Selects how the renderer backend discovers the ISP configuration from the - USD stage at camera initialization time. Discovery resolves one cfg for - the whole Camera sensor batch, using the first matched camera prim as the - camera-bound lookup target. ``isp_cfg=None`` disables ISP entirely; this - enum carries only the *active* discovery strategies. - """ + """Retired camera ISP discovery modes; use the processing package's discovery configuration.""" AUTO_CAMERA = "auto_camera" """Discover an ISP cfg bound to the batch's first matched camera prim. diff --git a/source/isaaclab/isaaclab/sensors/sensor_base.py b/source/isaaclab/isaaclab/sensors/sensor_base.py index 175515362097..3dbbcf91d69e 100644 --- a/source/isaaclab/isaaclab/sensors/sensor_base.py +++ b/source/isaaclab/isaaclab/sensors/sensor_base.py @@ -347,7 +347,7 @@ def _register_callbacks(self): def _invoke(callback_name, event): getattr(obj_ref, callback_name)(event) - # Backend-agnostic: PHYSICS_READY (init) and STOP (invalidate) + # Backend-agnostic: PHYSICS_READY initializes sensors; STOP invalidates them. self._initialize_handle = physics_mgr_cls.register_callback( lambda payload: PhysicsManager.safe_callback_invoke( _invoke, "_initialize_callback", payload, physics_manager=physics_mgr_cls diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index 34fb29889d99..a71bfccab9ad 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -262,6 +262,15 @@ def __init__(self, cfg: SimulationCfg | None = None): PhysicsEvent.PHYSICS_READY, order=5, ) + # Prepare every camera before SensorBase initializes any sensor at order 10. Renderers + # that export the stage during the first camera's initialization then see every override. + self.physics_manager.register_callback( + lambda _payload: PhysicsManager.safe_callback_invoke( + self._render_context.prepare_cameras, physics_manager=self.physics_manager + ), + PhysicsEvent.PHYSICS_READY, + order=9, + ) self.physics_manager.register_callback( lambda _payload: self._render_context.close(), PhysicsEvent.STOP, order=100 ) diff --git a/source/isaaclab_ppisp/test/test_rtx_camera_overrides.py b/source/isaaclab/test/renderers/test_camera_exposure.py similarity index 68% rename from source/isaaclab_ppisp/test/test_rtx_camera_overrides.py rename to source/isaaclab/test/renderers/test_camera_exposure.py index 77aafda82ef7..2167edf3edb7 100644 --- a/source/isaaclab_ppisp/test/test_rtx_camera_overrides.py +++ b/source/isaaclab/test/renderers/test_camera_exposure.py @@ -3,15 +3,19 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Tests for the RTX-side camera-exposure overrides applied when PPISP is active.""" +"""Tests for the RTX camera-exposure overrides used for scene-linear color outputs.""" -from isaaclab_ppisp import apply_rtx_exposure_overrides +import pytest from pxr import Sdf, Usd, UsdGeom +from isaaclab.sensors.camera import CameraExposureCfg +from isaaclab.sensors.camera.camera_exposure import apply_camera_exposure -def test_apply_rtx_exposure_overrides_resets_authored_exposure_values(): - """When PPISP is the ISP authority, the camera prim's RTX-side exposure must be neutral.""" + +@pytest.mark.parametrize("exposure, iso", [(0.0, 0.0), (2.0, 200.0)]) +def test_camera_exposure_applies_explicit_settings(exposure, iso): + """Scene-linear outputs require neutral RTX-side exposure on the camera prim.""" stage = Usd.Stage.CreateInMemory() cam = UsdGeom.Camera.Define(stage, "/World/Camera") prim = cam.GetPrim() @@ -23,15 +27,15 @@ def test_apply_rtx_exposure_overrides_resets_authored_exposure_values(): prim.CreateAttribute("exposure:time", Sdf.ValueTypeNames.Float).Set(1.0 / 60.0) prim.CreateAttribute("omni:rtx:autoExposure:enabled", Sdf.ValueTypeNames.Bool).Set(True) - apply_rtx_exposure_overrides(stage, ["/World/Camera"]) + apply_camera_exposure(stage, ["/World/Camera"], CameraExposureCfg(exposure=exposure, iso=iso)) - assert prim.GetAttribute("exposure").Get() == 0.0 + assert prim.GetAttribute("exposure").Get() == exposure assert prim.GetAttribute("exposure:fStop").Get() == 1.0 - assert prim.GetAttribute("exposure:iso").Get() == 0.0 + assert prim.GetAttribute("exposure:iso").Get() == iso assert prim.GetAttribute("exposure:responsivity").Get() == 1.0 assert prim.GetAttribute("exposure:time").Get() == 1.0 assert prim.GetAttribute("omni:rtx:autoExposure:enabled").Get() is False - # The PPISP API schemas must also be applied — that is what makes Kit's + # The RTX exposure API schemas must also be applied — that is what makes Kit's # USD watcher route the camera prim's ``exposure:*`` and # ``omni:rtx:autoExposure:*`` attributes into the per-camera carb # settings RTX consumes. diff --git a/source/isaaclab/test/renderers/test_camera_output_contract.py b/source/isaaclab/test/renderers/test_camera_output_contract.py index c13430483768..775800a3e478 100644 --- a/source/isaaclab/test/renderers/test_camera_output_contract.py +++ b/source/isaaclab/test/renderers/test_camera_output_contract.py @@ -6,10 +6,13 @@ """Tests for the renderer→camera output contract.""" import warnings +from builtins import ExceptionGroup +from dataclasses import replace from types import SimpleNamespace import numpy as np import pytest +import torch import warp as wp pytest.importorskip("isaaclab_physx") @@ -22,6 +25,8 @@ from isaaclab.sim import PinholeCameraCfg from isaaclab.test.utils import DeviceScope, test_devices from isaaclab.utils import clone, validate +from isaaclab.utils.tensor_processors import DictTensorProcessor, DictTensorProcessorCfg +from isaaclab.utils.warp import ProxyArray pytestmark = [pytest.mark.integration, pytest.mark.rendering] @@ -241,8 +246,9 @@ def test_newton_warp_segmentation_spec_follows_colorize_flags(colorize): assert specs[kind] == expected -def test_newton_warp_wraps_requested_rgb_hdr_output(): - """NewtonWarpRenderer wires requested RGB_HDR proxies to the Newton HDR output slot.""" +@pytest.mark.parametrize("data_type", [RenderBufferKind.RGB_HDR]) +def test_newton_warp_wraps_requested_hdr_output(data_type): + """Newton wires both linear color outputs to its HDR output slot.""" pytest.importorskip("isaaclab_newton") pytest.importorskip("newton") wp.init() @@ -255,10 +261,10 @@ def test_newton_warp_wraps_requested_rgb_hdr_output(): render_data = RenderData(fake_sensor, SimpleNamespace(cfg=camera_cfg)) hdr_proxy = ProxyArray(wp.zeros((2, 3, 4, 3), dtype=wp.float32, device="cpu")) - render_data.set_outputs({str(RenderBufferKind.RGB_HDR): hdr_proxy}) + render_data.set_outputs({str(data_type): hdr_proxy}) assert render_data.outputs.hdr_color_image is not None - assert render_data.get_output(RenderBufferKind.RGB_HDR) is render_data.outputs.hdr_color_image + assert render_data.get_output(data_type) is render_data.outputs.hdr_color_image def _make_camera_cfg(data_types: list[str]) -> CameraCfg: @@ -271,12 +277,13 @@ def _make_camera_cfg(data_types: list[str]) -> CameraCfg: ) -def test_camera_data_allocates_supported_subset_and_aliases_rgb(): - """CameraData allocates the intersection of requested + supported and aliases rgb into rgba.""" - cfg = _make_camera_cfg(["rgb", "rgba", "depth", "instance_segmentation"]) +def test_camera_data_allocates_supported_subset_and_aliases_color(): + """CameraData allocates requested + supported outputs, aliasing rgb into rgba.""" + cfg = _make_camera_cfg(["rgb", "rgba", "rgb_hdr", "depth", "instance_segmentation"]) specs = { RenderBufferKind.RGBA: RenderBufferSpec(4, wp.uint8), RenderBufferKind.RGB: RenderBufferSpec(3, wp.uint8), + RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), RenderBufferKind.DEPTH: RenderBufferSpec(1, wp.float32), RenderBufferKind.NORMALS: RenderBufferSpec(3, wp.float32), } @@ -284,14 +291,16 @@ def test_camera_data_allocates_supported_subset_and_aliases_rgb(): data_types=cfg.data_types, height=8, width=16, num_views=2, device="cpu", supported_specs=specs ) - assert set(data.output.keys()) == {"rgba", "rgb", "depth"} + assert set(data.output.keys()) == {"rgba", "rgb", "rgb_hdr", "depth"} assert data.output["rgba"].shape == (2, 8, 16, 4) assert data.output["rgba"].dtype == wp.uint8 assert data.output["depth"].shape == (2, 8, 16, 1) assert data.output["depth"].dtype == wp.float32 assert data.output["rgb"].warp.ptr == data.output["rgba"].warp.ptr + assert data.output["rgb_hdr"].shape == (2, 8, 16, 3) + assert data.output["rgb_hdr"].dtype == wp.float32 assert data.image_shape == (8, 16) - assert data.info == {"rgba": None, "rgb": None, "depth": None} + assert data.info == dict.fromkeys(data.output) def test_camera_data_allocate_raises_on_unknown_name(): @@ -308,3 +317,240 @@ def test_camera_data_allocate_raises_on_unknown_name(): ) assert "not_a_real_type" in str(exc_info.value) assert "RenderBufferKind" in str(exc_info.value) + + +def test_every_camera_prepares_before_the_shared_stage_export(monkeypatch): + """Every live camera prepares before stage export, including after stop and with callback error forwarding.""" + import weakref + + pytest.importorskip("isaaclab_newton") + pytest.importorskip("newton") + from isaaclab_newton.physics import NewtonCfg, NewtonManager + + from pxr import Sdf, UsdGeom + + from isaaclab.physics import PhysicsEvent + from isaaclab.renderers import RendererCfg + from isaaclab.sensors.camera import Camera, CameraExposureCfg + from isaaclab.sensors.camera import camera as camera_module + from isaaclab.sensors.sensor_base import SensorBase + + specs = { + RenderBufferKind.RGB: RenderBufferSpec(3, wp.uint8), + RenderBufferKind.RGBA: RenderBufferSpec(4, wp.uint8), + RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), + } + exported_iso = [] + prepare_error = None + + def prepare_cameras(stage, spec): + if prepare_error is not None: + raise prepare_error + + def export_stage(stage, num_envs): + exported_iso.append(prims[1].GetAttribute("exposure:iso").Get()) + + renderer = SimpleNamespace( + initialize=lambda: None, + supported_output_types=lambda: specs, + prepare_cameras=prepare_cameras, + prepare_stage=export_stage, + create_render_data=lambda spec: SimpleNamespace(spec=spec), + set_outputs=lambda data, outputs: None, + cleanup=lambda data: None, + close=lambda: None, + ) + renderer_cfg = RendererCfg(class_type=lambda cfg, sim: renderer, renderer_type="contract") + monkeypatch.setattr(SensorBase, "_initialize_impl", lambda self: None) + monkeypatch.setattr(Camera, "_initialize_intrinsics", lambda self: None) + monkeypatch.setattr(Camera, "_update_poses", lambda self: None) + monkeypatch.setattr( + camera_module, + "FrameView", + lambda path, **kwargs: SimpleNamespace(count=1, prims=[stage.GetPrimAtPath(path)], close=lambda: None), + ) + + def make_camera(prim, exposure=None): + camera = Camera.__new__(Camera) + camera.cfg = CameraCfg( + prim_path=str(prim.GetPath()), + data_types=["rgb_hdr"], + height=2, + width=3, + spawn=None, + exposure=exposure, + renderer_cfg=renderer_cfg, + ) + camera._render_cfg = replace(camera.cfg, data_types=camera.cfg.render_data_types(), pipeline=[]) + camera.stage = stage + camera._device = "cpu" + camera._num_envs = 1 + camera._is_global = True + camera._is_initialized = False + camera._sensor_prims = [] + camera._renderer = renderer + camera._render_data = None + camera._view = None + camera._register_callbacks() + return camera + + sim_cfg = sim_utils.SimulationCfg(device="cpu", physics=NewtonCfg()) + with sim_utils.build_simulation_context(sim_cfg=sim_cfg) as sim: + stage = sim.stage + prims = [UsdGeom.Camera.Define(stage, path).GetPrim() for path in ("/World/First", "/World/Second")] + iso = prims[1].CreateAttribute("exposure:iso", Sdf.ValueTypeNames.Float) + sim.get_or_create_backend(renderer_cfg) + cameras = [make_camera(prims[0]), make_camera(prims[1], CameraExposureCfg())] + discarded = make_camera(prims[0]) + discarded_ref = weakref.ref(discarded) + del discarded + assert discarded_ref() is None + + try: + for _ in range(2): + iso.Set(100.0) + sim.physics_manager.dispatch_event(PhysicsEvent.PHYSICS_READY) + assert all(camera.is_initialized for camera in cameras) + assert iso.Get() == 0.0 + sim.physics_manager.dispatch_event(PhysicsEvent.STOP) + assert exported_iso == [0.0] + + # External event buses store callback failures instead of propagating them directly. + errors = [] + monkeypatch.setattr(NewtonManager, "store_callback_exception", errors.append, raising=False) + prepare_error = ValueError("camera preparation failed") + sim.physics_manager.dispatch_event(PhysicsEvent.PHYSICS_READY) + assert errors and errors[0] is prepare_error + assert exported_iso == [0.0] + finally: + for camera in cameras: + camera.__del__() + + +def test_camera_pipeline_keeps_raw_inputs_and_processes_only_new_captures(monkeypatch): + """Camera publication preserves raw pixels, capture metadata and calibration-compatible geometry.""" + from isaaclab.sensors.camera import Camera + from isaaclab.sensors.sensor_base import SensorBase + from isaaclab.sim import SimulationContext + + events = [] + output_shape = (2, 1, 1, 4) + close_fails = False + + class Color(DictTensorProcessor): + def __init__(self, cfg, data): + super().__init__(cfg, data) + self.outputs["rgba"] = ProxyArray(wp.empty(output_shape, dtype=wp.uint8, device=data["rgb_hdr"].device)) + + def __call__(self, data): + events.append("apply") + output = self.outputs["rgba"].torch + output[..., :3].copy_(100 * (data["rgb_hdr"].torch + data["depth"].torch)) + output[..., 3] = 255 + return dict(self.outputs) + + def reset(self, env_ids=None): + events.append(tuple(env_ids)) + + def close(self): + events.append("close") + if close_fails: + raise RuntimeError("processor close failed") + + cfg = CameraCfg( + prim_path="/World/Camera", + spawn=None, + height=1, + width=1, + data_types=["rgb", "depth"], + pipeline=[DictTensorProcessorCfg(Color, ["rgb_hdr", "depth"], ["rgba"])], + ) + specs = { + RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), + RenderBufferKind.DEPTH: RenderBufferSpec(1, wp.float32), + } + capture = {"frame": 1} + radiance = 0.2 + + def read_output(render_data, camera_data): + camera_data.output["rgb_hdr"].torch.fill_(radiance) + camera_data.output["depth"].torch.fill_(0.5) + for name in camera_data.info: + camera_data.info[name] = {"capture": capture} + + camera = Camera.__new__(Camera) + camera.cfg = cfg + camera._render_cfg = replace(cfg, data_types=cfg.render_data_types(), pipeline=[]) + camera._view = SimpleNamespace(count=2, close=lambda: None) + camera._device = "cpu" + camera._num_envs = 2 + camera._is_global = False + camera._is_initialized = True + camera._render_data = None + camera._clear_callbacks = lambda: None + camera._renderer = SimpleNamespace( + supported_output_types=lambda: specs, + set_outputs=lambda data, outputs: None, + render=lambda data: None, + read_output=read_output, + reset=lambda data, ids: None, + cleanup=lambda data: None, + ) + monkeypatch.setattr(SimulationContext, "instance", staticmethod(lambda: None)) + monkeypatch.setattr(SensorBase, "reset", lambda self, ids, mask: None) + monkeypatch.setattr(camera, "_initialize_intrinsics", lambda: None) + monkeypatch.setattr(camera, "_update_poses", lambda **kwargs: None) + monkeypatch.setattr(camera, "_prepare_camera", lambda mask: None) + monkeypatch.setattr(camera, "_update_outdated_buffers", lambda: None) + camera._create_buffers() + mask = wp.ones(2, dtype=wp.bool, device="cpu") + camera._update_buffers_impl(mask) + + assert set(camera.data.output) == {"rgb", "depth"} + assert set(camera._render_camera_data.output) == {"rgb_hdr", "depth"} + torch.testing.assert_close(camera.data.output["rgb"].torch, torch.full((2, 1, 1, 3), 70, dtype=torch.uint8)) + torch.testing.assert_close(camera._render_camera_data.output["rgb_hdr"].torch, torch.full((2, 1, 1, 3), 0.2)) + assert camera.data.output["depth"] is camera._render_camera_data.output["depth"] + assert camera.data.info["rgb"]["capture"] is capture + + camera._update_buffers_impl(mask) + assert events == ["apply"] + capture = {"frame": 2} + radiance = 0.4 + camera._update_buffers_impl(mask) + torch.testing.assert_close(camera.data.output["rgb"].torch, torch.full((2, 1, 1, 3), 90, dtype=torch.uint8)) + assert events == ["apply", "apply"] + reset_mask = wp.array([False, True], dtype=wp.bool, device="cpu") + camera.reset(env_mask=reset_mask) + assert events[-1] == (1,) + # A rebuilt pipe cannot publish resized pixels against unchanged camera calibration. + output_shape = (2, 2, 1, 4) + with pytest.raises(ValueError, match="calibration"): + camera._create_buffers() + assert camera._buffer_pipe is None + assert events[-2:] == ["close", "close"] + output_shape = (2, 1, 1, 4) + monkeypatch.setattr(Color, "reset", DictTensorProcessor.reset) + camera._create_buffers() + # Stateless operations must not materialize indices: CUDA nonzero would synchronize the host. + with monkeypatch.context() as patch: + patch.setattr( + torch.Tensor, "nonzero", lambda self: pytest.fail("Stateless camera reset materialized mask indices") + ) + camera.reset(env_mask=reset_mask) + camera._render_data = object() + camera._renderer.cleanup = lambda data: events.append("renderer-close") + camera._view.close = lambda: events.append("view-close") + close_fails = True + with pytest.raises(ExceptionGroup, match="close dictionary processors"): + camera._invalidate_initialize_callback(None) + assert events[-3:] == ["close", "renderer-close", "view-close"] + assert not camera.is_initialized + assert camera._renderer is camera._render_data is camera._view is camera._buffer_pipe is None + camera.__del__() + + +def test_camera_retired_isp_setting_reports_pipeline_migration(): + """Legacy camera ISP configuration fails before it can silently skip image processing.""" + with pytest.raises(ValueError, match="CameraCfg.pipeline"): + CameraCfg(isp_cfg="auto_any") diff --git a/source/isaaclab/test/renderers/test_simulation_render_context.py b/source/isaaclab/test/renderers/test_simulation_render_context.py index 854df5465dde..5bc6f50e8ac6 100644 --- a/source/isaaclab/test/renderers/test_simulation_render_context.py +++ b/source/isaaclab/test/renderers/test_simulation_render_context.py @@ -296,6 +296,8 @@ def __init__(self, renderer, name, update_period=0.0): self._data = CameraData() self._data.create_buffers(2, "cpu") self._data.info = {} + self._render_camera_data = self._data + self._buffer_pipe = None self._frame = ProxyArray(wp.zeros(2, dtype=wp.int64, device="cpu")) self._ALL_INDICES = wp.array([0, 1], dtype=wp.int32, device="cpu") self._ALL_ENV_MASK = wp.ones(2, dtype=wp.bool, device="cpu") diff --git a/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py b/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py index 0756f2b72d26..16aa58f8754e 100644 --- a/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py +++ b/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py @@ -24,15 +24,14 @@ from typing import TYPE_CHECKING import torch -from isaaclab_ppisp import PpispCfg, normalize_ppisp_cfg +from isaaclab_ppisp import PpispCfg, PpispProcessorCfg, normalize_ppisp_cfg, ppisp_cfg_from_usd_camera from pxr import Gf, Sdf, Usd, UsdGeom, Vt import isaaclab.sim as sim_utils from isaaclab.assets import AssetBaseCfg, RigidObjectCfg from isaaclab.scene import InteractiveScene, InteractiveSceneCfg -from isaaclab.sensors.camera import Camera, CameraCfg -from isaaclab.sensors.camera.camera_isp import CameraISPMode +from isaaclab.sensors.camera import Camera, CameraCfg, CameraExposureCfg from isaaclab.terrains import TerrainImporterCfg from isaaclab.utils import configclass @@ -728,7 +727,8 @@ def render_synthetic_gaussian_scene( width=width, data_types=data_types, spawn=None, - isp_cfg=isp_cfg, + exposure=CameraExposureCfg(), + pipeline=[PpispProcessorCfg(params={"cfg": isp_cfg})], renderer_cfg=renderer_cfg, ) camera = Camera(cfg) @@ -736,7 +736,9 @@ def render_synthetic_gaussian_scene( for _ in range(stabilisation_steps): sim.step() camera.update(sim_dt) - outputs = {name: tensor.clone().detach().cpu().to(torch.float32) for name, tensor in camera.data.output.items()} + outputs = { + name: tensor.torch.clone().detach().cpu().to(torch.float32) for name, tensor in camera.data.output.items() + } del camera return outputs @@ -756,9 +758,7 @@ def render_synthetic_gaussian_scene_with_static_ppisp_attrs( ) -> dict[str, torch.Tensor]: """Render the synthesised gaussian asset through authored static PPISP camera attributes. - The camera uses :class:`CameraISPMode.AUTO_CAMERA`; renderer backends must - discover the camera-authored PPISP attributes and route them through their - PPISP workflow. + Resolve the camera-authored attributes before constructing the shared PPISP image operation. """ with fresh_synthetic_gaussian_interactive_scene(usd_path, sim_cfg, num_envs=num_envs) as sim: author_static_ppisp_camera_attrs(sim.stage, ppisp_cfg=ppisp_cfg) @@ -769,7 +769,7 @@ def render_synthetic_gaussian_scene_with_static_ppisp_attrs( width=width, sim_dt=sim_dt, stabilisation_steps=stabilisation_steps, - isp_cfg=CameraISPMode.AUTO_CAMERA, + ppisp_cfg=ppisp_cfg_from_usd_camera(sim.stage.GetPrimAtPath(_camera_path_for_env(0))), sim=sim, ) @@ -797,7 +797,7 @@ def render_synthetic_gaussian_scene_with_controller_ppisp_attrs( width=width, sim_dt=sim_dt, stabilisation_steps=stabilisation_steps, - isp_cfg=CameraISPMode.AUTO_CAMERA, + ppisp_cfg=ppisp_cfg_from_usd_camera(sim.stage.GetPrimAtPath(_camera_path_for_env(0))), sim=sim, ) @@ -810,7 +810,7 @@ def _render_synthetic_gaussian_camera( width: int, sim_dt: float, stabilisation_steps: int, - isp_cfg: PpispCfg | CameraISPMode | None, + ppisp_cfg: PpispCfg, sim: SimulationContext, ) -> dict[str, torch.Tensor]: cfg = CameraCfg( @@ -820,7 +820,8 @@ def _render_synthetic_gaussian_camera( width=width, data_types=data_types, spawn=None, - isp_cfg=isp_cfg, + exposure=CameraExposureCfg(), + pipeline=[PpispProcessorCfg(params={"cfg": ppisp_cfg})], renderer_cfg=renderer_cfg, ) camera = Camera(cfg) @@ -828,6 +829,8 @@ def _render_synthetic_gaussian_camera( for _ in range(stabilisation_steps): sim.step() camera.update(sim_dt) - outputs = {name: tensor.clone().detach().cpu().to(torch.float32) for name, tensor in camera.data.output.items()} + outputs = { + name: tensor.torch.clone().detach().cpu().to(torch.float32) for name, tensor in camera.data.output.items() + } del camera return outputs diff --git a/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py b/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py index 1371bebe5b0e..c40a7e79cbbb 100644 --- a/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py +++ b/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py @@ -3,12 +3,12 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Validate the camera PPISP wrapper applied to a 3D Gaussian (NuRec / +"""Validate the camera PPISP pipeline applied to a 3D Gaussian (NuRec / ParticleField) scene through the ``ovrtx`` renderer. The asset is synthesised at test time by :mod:`generate_synthetic_gaussian_asset` -and rendered via :func:`generate_synthetic_gaussian_asset.render_synthetic_gaussian_scene`. -The aggressive wrapper PPISP cfg +and rendered through the camera's shared render-buffer pipeline. +The aggressive PPISP cfg (:func:`generate_synthetic_gaussian_asset.make_aggressive_ppisp_cfg`) intentionally engages every feature past its subtle-correction defaults so each can be asserted independently: @@ -38,8 +38,8 @@ * Uses Newton physics because ``ovrtx`` is incompatible with Kit/Isaac Sim and the PhysX backend requires Kit (``carb``) to bootstrap. * Requests ``"rgb_hdr"`` in ``data_types`` because the test asserts the raw - HDR source with :func:`assert_ppisp_lifts_exposure`. The PPISP render path - itself also allocates an internal HDR buffer when ``isp_cfg`` is set, so + HDR source with :func:`assert_ppisp_lifts_exposure`. The PPISP camera pipeline + requests its private ``"rgb_hdr"`` input automatically, so ``"rgb_hdr"`` is not required just to enable PPISP. """ @@ -57,10 +57,10 @@ make_aggressive_ppisp_cfg, make_neutral_ppisp_cfg, make_synthetic_gaussian_usd, - render_synthetic_gaussian_scene, render_synthetic_gaussian_scene_with_controller_ppisp_attrs, render_synthetic_gaussian_scene_with_static_ppisp_attrs, ) +from isaaclab_ppisp import PpispCfg from isaaclab.sim import SimulationCfg @@ -87,17 +87,14 @@ SIM_DT = 1.0 / 60.0 MULTI_TILE_COUNT = 4 -# Mark the gaussian-on-OVRTX tests xfail by default: the wrapper-side -# ``carb.settings.set_bool`` call that disables RTX-side tonemapping is a -# no-op for the kit-less ovrtx backend, and the equivalent must be applied -# externally before launching pytest. -_XFAIL_OVRTX_GAUSSIAN_PPISP = pytest.mark.xfail( - reason=( - "kit-less ovrtx cannot toggle RTX-side tonemapping at runtime; the equivalent must be " - "applied externally before launching pytest" - ), - strict=False, -) + +def _ovrtx_ppisp_cfg() -> PpispCfg: + """Expose the unexposed scene-linear Gaussian signal into the CRF's useful range.""" + cfg = make_aggressive_ppisp_cfg() + # rgb_hdr has a center near 0.12; +2 EV places it near 0.48 before the + # response curve. The shared fixture's -5 EV targets legacy exposed HDR. + cfg.inputs["exposureOffset"] = 2.0 + return cfg def _ovrtx_sim_cfg(device: str) -> SimulationCfg: @@ -110,12 +107,11 @@ def _ovrtx_sim_cfg(device: str) -> SimulationCfg: @pytest.mark.parametrize("device", ["cuda:0"]) @_SKIP_MISSING_OVRTX -@_XFAIL_OVRTX_GAUSSIAN_PPISP def test_camera_ppisp_authored_static_attrs_are_applied_on_synthetic_gaussians_ovrtx(device): """OVRTX must apply camera-authored static PPISP attributes.""" with tempfile.TemporaryDirectory(prefix="isaaclab-synth-gauss-") as tmpdir: asset_path = make_synthetic_gaussian_usd(f"{tmpdir}/synthetic_gaussians.usda") - aggressive_cfg = make_aggressive_ppisp_cfg() + aggressive_cfg = _ovrtx_ppisp_cfg() neutral = render_synthetic_gaussian_scene_with_static_ppisp_attrs( asset_path, @@ -141,12 +137,11 @@ def test_camera_ppisp_authored_static_attrs_are_applied_on_synthetic_gaussians_o @pytest.mark.parametrize("device", ["cuda:0"]) @_SKIP_MISSING_OVRTX -@_XFAIL_OVRTX_GAUSSIAN_PPISP def test_camera_ppisp_controller_matches_static_attrs_on_synthetic_gaussians_ovrtx(device): """OVRTX controller output must match the equivalent static PPISP cfg.""" with tempfile.TemporaryDirectory(prefix="isaaclab-synth-gauss-") as tmpdir: asset_path = make_synthetic_gaussian_usd(f"{tmpdir}/synthetic_gaussians.usda") - ppisp_cfg = make_aggressive_ppisp_cfg() + ppisp_cfg = _ovrtx_ppisp_cfg() static = render_synthetic_gaussian_scene_with_static_ppisp_attrs( asset_path, @@ -171,7 +166,10 @@ def test_camera_ppisp_controller_matches_static_attrs_on_synthetic_gaussians_ovr @pytest.mark.parametrize("device", ["cuda:0"]) @_SKIP_MISSING_OVRTX -@_XFAIL_OVRTX_GAUSSIAN_PPISP +@pytest.mark.xfail( + reason="OVRTX renders cloned Gaussian tiles black in native RGB and radiance, including without a camera pipeline", + strict=False, +) def test_camera_ppisp_wrapper_signatures_on_synthetic_gaussians_ovrtx_multitile(device): """Multi-tile wrapper PPISP via ``ovrtx`` must hold the same invariants independently for every tile. @@ -184,10 +182,11 @@ def test_camera_ppisp_wrapper_signatures_on_synthetic_gaussians_ovrtx_multitile( """ with tempfile.TemporaryDirectory(prefix="isaaclab-synth-gauss-") as tmpdir: asset_path = make_synthetic_gaussian_usd(f"{tmpdir}/synthetic_gaussians.usda") - output = render_synthetic_gaussian_scene( + output = render_synthetic_gaussian_scene_with_static_ppisp_attrs( asset_path, sim_cfg=_ovrtx_sim_cfg(device), renderer_cfg=OVRTXRendererCfg(), + ppisp_cfg=_ovrtx_ppisp_cfg(), data_types=["rgb", "rgb_hdr"], num_envs=MULTI_TILE_COUNT, sim_dt=SIM_DT, diff --git a/source/isaaclab/test/sensors/test_opencv_distortion.py b/source/isaaclab/test/sensors/test_opencv_distortion.py index 5419d47b337b..adbeabc18b09 100644 --- a/source/isaaclab/test/sensors/test_opencv_distortion.py +++ b/source/isaaclab/test/sensors/test_opencv_distortion.py @@ -193,6 +193,7 @@ def _camera_for_prims(prims, width=640, height=480, device="cpu"): fake._ALL_ENV_MASK = wp.ones(len(prims), dtype=wp.bool, device=device) fake._is_outdated = wp.zeros(len(prims), dtype=wp.bool, device=device) # attributes touched by ``__del__``/``_clear_callbacks`` when the fake object is garbage collected + fake._render_context = None fake._initialize_handle = None fake._invalidate_initialize_handle = None fake._prim_deletion_handle = None diff --git a/source/isaaclab/test/sensors/test_sensor_destructors.py b/source/isaaclab/test/sensors/test_sensor_destructors.py index a5d057d9b8f3..b4bbb7245d64 100644 --- a/source/isaaclab/test/sensors/test_sensor_destructors.py +++ b/source/isaaclab/test/sensors/test_sensor_destructors.py @@ -77,6 +77,7 @@ def cleanup(self, _render_data): view = _View() renderer = _Renderer() camera = object.__new__(Camera) + camera._render_context = None camera._view = view camera._renderer = renderer camera._render_data = None diff --git a/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..5832bdc80d5d --- /dev/null +++ b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst @@ -0,0 +1,3 @@ +* **Breaking:** Replaced renderer-owned PPISP with dictionary processors. + Replace ``CameraCfg.isp_cfg`` with ``PpispProcessorCfg`` and configure + ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.major b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py index 9fb230320d80..77b075973766 100644 --- a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py +++ b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py @@ -11,7 +11,7 @@ import warnings from dataclasses import dataclass from functools import partial -from typing import TYPE_CHECKING, Any, NoReturn +from typing import TYPE_CHECKING, Any import newton import warp as wp @@ -27,8 +27,6 @@ from .segmentation import NewtonSegmentationMapper, NewtonSegmentationMapping if TYPE_CHECKING: - from isaaclab_ppisp import PpispPipeline - from isaaclab.sensors.camera.camera_data import CameraData from isaaclab.utils.warp import ProxyArray @@ -68,21 +66,6 @@ def _update_camera_rays(intrinsics: wp.array(dtype=wp.mat33f), rays: wp.array4d( rays[0, y, x, 1] = wp.normalize(direction) -_PPISP_IMPORT_ERROR_MESSAGE = ( - "isaaclab_ppisp is required when CameraCfg.isp_cfg is set. " - "Run `uv sync` from the Isaac Lab source checkout, or install the Isaac Lab wheel " - "with `uv pip install isaaclab`." -) - - -def _raise_missing_ppisp_error(exc: ModuleNotFoundError) -> NoReturn: - # Only translate missing isaaclab_ppisp imports into the optional-dependency hint; - # unrelated missing modules should surface unchanged for easier debugging. - if exc.name != "isaaclab_ppisp" and not (exc.name and exc.name.startswith("isaaclab_ppisp.")): - raise exc - raise ModuleNotFoundError(_PPISP_IMPORT_ERROR_MESSAGE, name="isaaclab_ppisp") from exc - - class RenderData: # Back-compat alias for callers of ``RenderData.OutputNames``. OutputNames = RenderBufferKind @@ -100,6 +83,7 @@ class RenderData: # :class:`~isaaclab_newton.renderers.segmentation.NewtonSegmentationMapper`. _OUTPUT_MAP: dict[str, tuple[str, type]] = { str(RenderBufferKind.RGBA): ("color_image", wp.uint32), + # Newton applies no camera exposure, so its HDR color is already scene-linear. str(RenderBufferKind.RGB_HDR): ("hdr_color_image", wp.vec3f), str(RenderBufferKind.ALBEDO): ("albedo_image", wp.uint32), str(RenderBufferKind.NORMALS): ("normals_image", wp.vec3f), @@ -183,27 +167,6 @@ def __init__( # OpenCV lens-distortion model (``spawn.distortion``), consumed by :meth:`_build_distortion_rays` # to trace distorted per-pixel rays instead of the centered, square-pixel pinhole field. self._distortion = spawn.distortion if spawn is not None else None - # Post-render PPISP pipeline composed when ``spec.cfg.isp_cfg`` is set. - # ``isp_cfg`` is already fully normalized by ``prepare_cameras`` by the time it reaches here. - self.ppisp_pipeline: PpispPipeline | None = None - if spec.cfg.isp_cfg is not None: - try: - from isaaclab_ppisp import PpispPipeline - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - self.ppisp_pipeline = PpispPipeline(spec.cfg.isp_cfg) - self._hdr_scratch_wp: wp.array | None = None - """Internal HDR scratch buffer allocated when PPISP is composed but the - user did not request ``"rgb_hdr"`` in ``data_types``. Also exposed to - the Newton sensor through :attr:`CameraOutputs.hdr_color_image` as a - vec3f reinterpretation of this same backing storage.""" - self._ppisp_hdr_source: wp.array | None = None - """PPISP HDR source bound once in :meth:`set_outputs` from the caller's - ``rgb_hdr`` output or :attr:`_hdr_scratch_wp`.""" - self._ppisp_rgba_dest: wp.array | None = None - """PPISP LDR destination bound once in :meth:`set_outputs` from the - caller's ``rgba`` output.""" def _view(self, proxy: ProxyArray, dtype: type, shape: tuple[int, ...]) -> wp.array: """Alias the caller's output buffer as a ``(world_count, 1, H, W)`` warp array of ``dtype``. @@ -262,24 +225,6 @@ def set_outputs(self, output_data: dict[str, ProxyArray]): # requested segmentation outputs are remapped from this single buffer in :meth:`_convert_segmentation`. if self._seg_dests: self.outputs.shape_index_image = wp.zeros(shape, dtype=wp.uint32, device=self.newton_sensor.model.device) - # When PPISP is composed but the user did not request the raw HDR AOV, - # allocate an internal HDR scratch buffer and route a vec3f-shaped view - # of it as the Newton sensor's ``hdr_color_image`` so the renderer - # fills it directly. - if self.ppisp_pipeline is not None and self.outputs.hdr_color_image is None: - ref_proxy = next(iter(output_data.values())) - self._hdr_scratch_wp = wp.zeros( - (self.newton_sensor.model.world_count, self.height, self.width, 3), - dtype=wp.float32, - device=ref_proxy.device, - ) - self.outputs.hdr_color_image = wp.array( - ptr=self._hdr_scratch_wp.ptr, - dtype=wp.vec3f, - shape=shape, - device=self._hdr_scratch_wp.device, - copy=False, - ) # Newton clears HDR to black regardless of clear_color. A hit mask preserves dark geometry. if ( self.outputs.hdr_color_image is not None @@ -287,17 +232,6 @@ def set_outputs(self, output_data: dict[str, ProxyArray]): and self.outputs.shape_index_image is None ): self.outputs.shape_index_image = wp.zeros(shape, dtype=wp.uint32, device=self.newton_sensor.model.device) - # Bind the two warp arrays the per-frame PPISP dispatch needs. - if self.ppisp_pipeline is not None: - if str(RenderBufferKind.RGBA) not in output_data: - raise ValueError( - "Newton renderer ISP requires 'rgba' (or 'rgb', which aliases into rgba) as the" - " LDR output destination, but neither was provided. Add 'rgb' or 'rgba' to" - " Camera.cfg.data_types when isp_cfg is set." - ) - hdr_proxy = output_data.get(str(RenderBufferKind.RGB_HDR)) - self._ppisp_hdr_source = hdr_proxy.warp if hdr_proxy is not None else self._hdr_scratch_wp - self._ppisp_rgba_dest = output_data[str(RenderBufferKind.RGBA)].warp def get_output(self, output_name: str) -> wp.array: if output_name in self._depth_dests: @@ -510,15 +444,10 @@ def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: return self.cfg.supported_output_types() def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: - """Resolve the camera's PPISP cfg before rendering. - - :mod:`isaaclab.sensors.camera` does not depend on PPISP; the renderer - owns the sentinel-resolution + cfg-normalization step. Newton has no - USD-side overrides to author beyond this. - - Also captures the USD ``stage`` so the segmentation mapper can read the scene's - :class:`UsdSemantics.LabelsAPI` labels when a segmentation output is requested. + """Capture the USD stage for the segmentation mapper. + The mapper reads the scene's :class:`UsdSemantics.LabelsAPI` labels when a segmentation + output is requested. Newton authors no camera overrides. """ self._stage = stage if spec.cfg.spawn is None or spec.cfg.spawn.distortion is None: @@ -537,15 +466,6 @@ def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: stacklevel=2, ) break - if spec.cfg.isp_cfg is None: - return - try: - from isaaclab_ppisp import resolve_and_normalize - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - camera_prim_path = spec.camera_prim_paths[0] if spec.camera_prim_paths else None - spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) def prepare_stage(self, stage: Any, num_envs: int) -> None: """No-op for Newton Warp - uses Newton scene directly without stage export. @@ -640,14 +560,6 @@ def render(self, render_data: RenderData): use_cuda_graph=self.cfg.use_cuda_graph and backend.model.tri_count == 0, ) - # Post-render PPISP: HDR scene-linear → LDR RGBA. Source/destination - # tensors were bound once in ``set_outputs``. - if render_data.ppisp_pipeline is not None: - render_data.ppisp_pipeline.apply( - render_data._ppisp_hdr_source, - render_data._ppisp_rgba_dest, - ) - def _launch_render(self, render_data: RenderData) -> None: """Launch the tiled-camera render kernels for sensor graph capture.""" # default_render_config is shared state across all Newton sensors, so set max_distance diff --git a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py index c5f219d72a1f..b52726ae3998 100644 --- a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py +++ b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py @@ -312,7 +312,7 @@ def read(_data): monkeypatch.setattr(renderer, "_launch_render", read) for _ in range(2): - render_data = SimpleNamespace(graph=None, ppisp_pipeline=None) + render_data = SimpleNamespace(graph=None) renderer.render(render_data) assert render_data.graph is None assert len(refits) == 1 diff --git a/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..5832bdc80d5d --- /dev/null +++ b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst @@ -0,0 +1,3 @@ +* **Breaking:** Replaced renderer-owned PPISP with dictionary processors. + Replace ``CameraCfg.isp_cfg`` with ``PpispProcessorCfg`` and configure + ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.major b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index 3f6a9db1367e..bb32a67a3d05 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -33,7 +33,7 @@ from collections import deque from collections.abc import Callable, Iterable, Iterator, Sequence from pathlib import Path -from typing import TYPE_CHECKING, Any, NoReturn, cast +from typing import TYPE_CHECKING, Any, cast logger = logging.getLogger(__name__) @@ -122,7 +122,6 @@ ) if TYPE_CHECKING: - from isaaclab_ppisp import PpispPipeline from ovrtx import AttributeBinding, Operation, PendingFetch, RenderProductSetOutputs from isaaclab.renderers.base_renderer import VisualMaterialBatch @@ -159,11 +158,6 @@ } -_PPISP_IMPORT_ERROR_MESSAGE = ( - "isaaclab_ppisp is required when CameraCfg.isp_cfg is set. " - "Run `uv sync` from the Isaac Lab source checkout, or install the Isaac Lab wheel " - "with `uv pip install isaaclab`." -) _READ_GPU_TRANSFORMS_ENV = "ISAAC_LAB_OVRTX_READ_GPU_TRANSFORMS" @@ -191,14 +185,6 @@ def ovrtx_use_ovstage_enabled() -> bool: return value == "1" -def _raise_missing_ppisp_error(exc: ModuleNotFoundError) -> NoReturn: - # Only translate missing isaaclab_ppisp imports into the optional-dependency hint; - # unrelated missing modules should surface unchanged for easier debugging. - if exc.name != "isaaclab_ppisp" and not (exc.name and exc.name.startswith("isaaclab_ppisp.")): - raise exc - raise ModuleNotFoundError(_PPISP_IMPORT_ERROR_MESSAGE, name="isaaclab_ppisp") from exc - - def _read_gpu_transforms_enabled() -> bool: """Return whether OVRTX should read GPU transforms from its internal transform cache.""" value = os.environ.get(_READ_GPU_TRANSFORMS_ENV, "1").strip() @@ -344,16 +330,6 @@ def __init__(self, spec: CameraRenderSpec, render_scope_name: str): # Populated for "semantic_segmentation" (with an "idToLabels" mapping) and # "instance_segmentation" (with "idToLabels" and "idToSemantics" mappings). self.renderer_info: dict[str, Any] = {} - # Post-render PPISP pipeline composed when ``spec.cfg.isp_cfg`` is set. - # ``isp_cfg`` is already fully normalized by ``prepare_cameras`` by the time it reaches here. - self.ppisp_pipeline: PpispPipeline | None = None - if spec.cfg.isp_cfg is not None: - try: - from isaaclab_ppisp import PpispPipeline - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - self.ppisp_pipeline = PpispPipeline(spec.cfg.isp_cfg) def cleanup(self) -> None: """Release this camera's native resources and buffers. Safe to call repeatedly. @@ -372,7 +348,6 @@ def cleanup(self) -> None: self.intrinsic_bindings.clear() self.warp_buffers.clear() self.renderer_info.clear() - self.ppisp_pipeline = None class OVRTXRenderer(BaseRenderer): @@ -451,29 +426,6 @@ def visual_material_writer(self): """Return a writer factory once this renderer has a scene consumer.""" return self._create_visual_material_writer if self._initialized_scene else None - def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: - """Resolve the camera's PPISP cfg and apply OVRTX-specific USD overrides. - - When ``spec.cfg.isp_cfg`` is set, resolves it (sentinel discovery + - normalization) via :func:`isaaclab_ppisp.resolve_and_normalize` so - :mod:`isaaclab` does not need to know about PPISP. Then pins - ``exposure:*`` to neutral and applies ``OmniRtxCameraExposureAPI_1`` so - the RTX exposure model OVRTX embeds does not compound on top of the - ISP. Without an ISP, the camera prim's authored exposure is left alone. - """ - if spec.cfg.isp_cfg is None: - return - try: - from isaaclab_ppisp import apply_rtx_exposure_overrides, resolve_and_normalize - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - camera_prim_path = spec.camera_prim_paths[0] - spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) - if spec.cfg.isp_cfg is None: - return - apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) - def prepare_stage(self, stage: Any, num_envs: int) -> None: """Load, clone, and bind the shared scene before any camera products are registered.""" self._clone_plan = self._sim.get_clone_plan() @@ -781,23 +733,6 @@ def set_outputs(self, render_data: OVRTXCameraRenderData, output_data: dict[str, render_data.warp_buffers = { name: proxy.warp for name, proxy in output_data.items() if name != str(RenderBufferKind.RGB) } - # When PPISP is composed but the user did not request the raw HDR AOV, - # allocate an internal HDR scratch buffer under "rgb_hdr" so both the - # HdrColor extractor and PPISP dispatch can use the same buffer map. - if render_data.ppisp_pipeline is not None and str(RenderBufferKind.RGB_HDR) not in render_data.warp_buffers: - ref_proxy = next(iter(output_data.values())) - render_data.warp_buffers[str(RenderBufferKind.RGB_HDR)] = wp.zeros( - (render_data.num_envs, render_data.height, render_data.width, 3), - dtype=wp.float32, - device=ref_proxy.device, - ) - if render_data.ppisp_pipeline is not None: - if str(RenderBufferKind.RGBA) not in render_data.warp_buffers: - raise ValueError( - "OVRTX renderer ISP requires 'rgba' (or 'rgb', which aliases into rgba) as the" - " LDR output destination, but neither was provided. Add 'rgb' or 'rgba' to" - " Camera.cfg.data_types when isp_cfg is set." - ) def prepare_capture(self, render_data: OVRTXCameraRenderData, camera_data: CameraData, frame: ProxyArray) -> None: """Snapshot delayed-image metadata without changing live camera state.""" @@ -1100,24 +1035,12 @@ def _extract_hdr_color_tiles( return if tiled_data.dtype not in (wp.float16, wp.float32): raise TypeError(f"Unsupported OVRTX HdrColor dtype: {tiled_data.dtype}.") - self._launch_extract_all_tiles(render_data, tiled_data, output_buffers["rgb_hdr"]) - - def _prepare_ppisp_hdr_source( - self, render_data: OVRTXCameraRenderData, tiled_data: wp.array, output_buffers: dict - ) -> wp.array: - """Return the PPISP HdrColor source on the output buffer device.""" - if render_data.ppisp_pipeline is None: - return tiled_data - - output_device = str(output_buffers[str(RenderBufferKind.RGB_HDR)].device) - if str(tiled_data.device) == output_device: - return tiled_data - - # The render product pins ``deviceIds`` to this renderer's CUDA device, so the mapping - # normally lands on the output device already. This stays as a fallback for the case OVRTX - # reports as "deviceIds ... not in the active device set" and falls back to automatic - # assignment. - return wp.clone(tiled_data, device=output_device) + output = output_buffers["rgb_hdr"] + if tiled_data.device != output.device: + # OVRTX may fall back to automatic device assignment when the requested deviceIds + # is outside its active device set. Extract on the destination's device in that case. + tiled_data = wp.clone(tiled_data, device=output.device) + self._launch_extract_all_tiles(render_data, tiled_data, output) def _process_render_frame(self, render_data: OVRTXCameraRenderData, frame, output_buffers: dict) -> None: """Extract RGB, depth, albedo, and semantic from a single render frame into output_buffers.""" @@ -1130,7 +1053,7 @@ def _process_render_frame(self, render_data: OVRTXCameraRenderData, frame, outpu if ldr_color is not None: buffer_key = None - if render_data.ppisp_pipeline is None and "rgba" in output_buffers: + if "rgba" in output_buffers: buffer_key = "rgba" else: # The output buffers must contain only one simple shading data type at most after resolution of the data @@ -1165,7 +1088,6 @@ def _process_render_frame(self, render_data: OVRTXCameraRenderData, frame, outpu hdr_color = frame.render_vars.get(render_data.render_var_keys[_HDR_COLOR_VAR]) if hdr_color is not None and "rgb_hdr" in output_buffers: with self._map_render_var_to_dlpack(hdr_color) as tiled_hdr_data: - tiled_hdr_data = self._prepare_ppisp_hdr_source(render_data, tiled_hdr_data, output_buffers) self._extract_hdr_color_tiles(render_data, tiled_hdr_data, output_buffers) self._process_id_segmentation_render_var( @@ -1281,13 +1203,6 @@ def _process_render_products( data.warp_buffers, ) - # Post-render PPISP uses each camera's own HDR source and RGBA destination. - if data.ppisp_pipeline is not None: - data.ppisp_pipeline.apply( - data.warp_buffers[str(RenderBufferKind.RGB_HDR)], - data.warp_buffers[str(RenderBufferKind.RGBA)], - ) - def _close_legacy(self) -> None: """Release the renderer's tensor bindings. See :meth:`close`.""" diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py index ef8d77bb2fd5..0dc3ec97367c 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py @@ -155,8 +155,7 @@ def build_render_product_as_string( resolved camera path after runtime cloning. Args: - spec: Camera configuration, environment count, and camera paths. ISP configurations - automatically request the HDR render variable in addition to the configured outputs. + spec: Camera configuration, environment count, and camera paths. render_data: Camera's render scope and product identity. device_id: CUDA device index the render product is pinned to, so its render var buffers are allocated on the same device as the Warp kernels that read them. @@ -167,8 +166,6 @@ def build_render_product_as_string( Render product USD layer, including the USDA header and default prim metadata. """ data_types = list(spec.cfg.data_types) - if spec.cfg.isp_cfg is not None and "rgb_hdr" not in data_types: - data_types.append("rgb_hdr") tiled_width, tiled_height = render_data.num_cols * render_data.width, render_data.num_rows * render_data.height render_var_configs = get_render_var_configs(data_types, render_data.render_scope_name) minimal_mode = next( @@ -199,6 +196,12 @@ def build_render_product_as_string( # Workaround: ovrtx 0.5 misses GPU-read transform changes, so eco mode freezes output after 500 frames. render_mode_lines.append("bool omni:rtx:rt:ecoMode:enabled = false") + api_schemas = ["OmniRtxSettingsCommonAdvancedAPI_1"] + if "rgb_hdr" in data_types: + # OVRTX 0.5 reads this per product. Its default bypasses HdrColor for Gaussian pixels. + api_schemas.append("OmniRtxSettingsParticleFieldAPI_1") + render_mode_lines.append("bool omni:rtx:rtpt:gaussian:skipTonemapping:enabled = false") + api_schemas_block = ", ".join(f'"{schema}"' for schema in api_schemas) render_mode_block = "\n ".join(render_mode_lines) ordered_vars = ", ".join(f"<{path}>" for path, _, _ in render_var_configs) @@ -216,7 +219,7 @@ def build_render_product_as_string( def Scope "{render_data.render_scope_name}" {{ def RenderProduct "RenderProduct" ( - prepend apiSchemas = ["OmniRtxSettingsCommonAdvancedAPI_1"] + prepend apiSchemas = [{api_schemas_block}] ) {{ rel camera = [<{render_data.camera_paths[0]}>] uint[] deviceIds = [{device_id}] diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index 8fb07184d6c9..ab0062f85c32 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -203,13 +203,11 @@ def test_ovrtx_render_submits_every_product_and_routes_requested_outputs( cameras = [_make_ovrtx_camera_render_data() for _ in range(2 if batch else 1)] products = {} processed = [] - postprocessed = [] submissions = [] published_ordinals = [] for index, camera in enumerate(cameras): camera.render_product_path = f"/Render/Camera{index}" camera.warp_buffers = {str(RenderBufferKind.RGB_HDR): object(), str(RenderBufferKind.RGBA): object()} - camera.ppisp_pipeline = types.SimpleNamespace(apply=lambda *buffers: postprocessed.append(buffers)) products[camera.render_product_path] = types.SimpleNamespace(frames=[object()]) renderer._render_product_paths = [*products, "/Render/UnrequestedCamera"] if missing_output == "product": @@ -236,15 +234,10 @@ def advance_write_floor(*, ordinal): assert processed == [ (camera, products[camera.render_product_path].frames[0], camera.warp_buffers) for camera in cameras ] - assert postprocessed == [ - (camera.warp_buffers[str(RenderBufferKind.RGB_HDR)], camera.warp_buffers[str(RenderBufferKind.RGBA)]) - for camera in cameras - ] else: with pytest.raises(RuntimeError, match=cameras[-1].render_product_path): render(request) assert not processed - assert not postprocessed assert len(submissions) == 1 # Unrequested products are submitted too, and only the requested cameras are read back. @@ -558,6 +551,60 @@ def capture(value, indices): assert renderer._camera_render_data == [] +@pytest.mark.parametrize("cross_device", [False, True]) +def test_ovrtx_hdr_color_is_extracted_on_the_output_device(monkeypatch, cross_device): + """HDR fills the camera's own buffer, copying a mapped source from another device before extraction.""" + renderer = _make_ovrtx_renderer_without_backend() + renderer._device = "cpu" + data = CameraData.allocate( + data_types=["rgb_hdr"], + height=8, + width=16, + num_views=2, + device="cpu", + supported_specs=renderer.supported_output_types(), + ) + render_data = _make_ovrtx_camera_render_data() + renderer.set_outputs(render_data, data.output) + + pixels = wp.full((8, 32, 4), 2.0, dtype=wp.float32, device="cpu") + source = types.SimpleNamespace(device="cuda:1", dtype=wp.float32) if cross_device else pixels + clone_calls = [] + + def fake_clone(src, *, device): + clone_calls.append((src, device)) + return pixels + + extract = renderer._launch_extract_all_tiles + + def extract_on_output_device(render_data, source_buffer, output_buffer): + expected = [(source, output_buffer.device)] if cross_device else [] + assert clone_calls == expected, "Copy mapped HDR buffers from another device before extracting them." + extract(render_data, source_buffer, output_buffer) + + monkeypatch.setattr(wp, "clone", fake_clone) + monkeypatch.setattr(renderer, "_launch_extract_all_tiles", extract_on_output_device) + monkeypatch.setattr(renderer, "_map_render_var_to_dlpack", lambda source: contextlib.nullcontext(source)) + frame = types.SimpleNamespace(render_vars={render_data.render_var_keys["HdrColor"]: source}) + renderer._process_render_frame(render_data, frame, render_data.warp_buffers) + + np.testing.assert_array_equal(data.output["rgb_hdr"].warp.numpy(), 2.0) + + +@pytest.mark.parametrize("data_types", [["rgb"], ["rgb_hdr"]]) +def test_ovrtx_prepare_cameras_preserves_authored_exposure(data_types): + """Selecting render outputs preserves authored camera exposure.""" + from pxr import Sdf, Usd, UsdGeom + + stage = Usd.Stage.CreateInMemory() + camera = UsdGeom.Camera.Define(stage, "/World/Camera").GetPrim() + camera.CreateAttribute("exposure:iso", Sdf.ValueTypeNames.Float).Set(100.0) + spec = types.SimpleNamespace(cfg=types.SimpleNamespace(data_types=data_types), camera_prim_paths=("/World/Camera",)) + _make_ovrtx_renderer_without_backend().prepare_cameras(stage, spec) + + assert camera.GetAttribute("exposure:iso").Get() == 100.0 + + def test_ovrtx_set_outputs_wraps_caller_torch_zero_copy(): """OVRTXRenderer.set_outputs publishes warp views over the caller's warp storage.""" renderer = _make_ovrtx_renderer_without_backend() @@ -591,47 +638,6 @@ def test_ovrtx_set_outputs_wraps_caller_torch_zero_copy(): assert "rgb" not in render_data.warp_buffers -def test_ovrtx_set_outputs_routes_ppisp_buffers_through_warp_buffers(): - """OVRTXRenderer.set_outputs stores PPISP source/destination in warp_buffers.""" - renderer = _make_ovrtx_renderer_without_backend() - - cfg = _make_camera_cfg(["rgb"]) - data = CameraData.allocate( - data_types=cfg.data_types, - height=8, - width=16, - num_views=2, - device="cpu", - supported_specs=renderer.supported_output_types(), - ) - render_data = _make_ovrtx_camera_render_data() - render_data.ppisp_pipeline = object() - renderer.set_outputs(render_data, data.output) - - assert render_data.warp_buffers["rgba"].ptr == data.output["rgba"].warp.ptr - assert "rgb_hdr" in render_data.warp_buffers - assert render_data.warp_buffers["rgb_hdr"].shape == (2, 8, 16, 3) - assert render_data.warp_buffers["rgb_hdr"].dtype is wp.float32 - - -def test_ovrtx_process_frame_skips_ldr_rgba_when_ppisp_is_active(): - """PPISP owns RGBA output, so OVRTX LdrColor should not pre-fill it.""" - - class FailingRenderVar: - def map(self, *args, **kwargs): - raise AssertionError("PPISP RGBA output must not read OVRTX LdrColor") - - renderer = _make_ovrtx_renderer_without_backend() - render_data = _make_ovrtx_camera_render_data() - render_data.ppisp_pipeline = object() - key = "LdrColor" - if ovrtx_renderer_module.uses_prim_path_render_vars(ovrtx_renderer_module.OVRTX_VERSION): - key = f"/{render_data.render_scope_name}/Vars/{key}" - frame = types.SimpleNamespace(render_vars={key: FailingRenderVar()}) - - renderer._process_render_frame(render_data, frame, {"rgba": object()}) - - # Render-var keys depend only on the OVRTX version; ``use_ovstage`` only selects the registration path. @pytest.mark.parametrize(("use_ovstage", "version"), [(False, "0.4"), (True, "0.5")]) def test_ovrtx_process_frame_reads_authored_camera_render_vars(monkeypatch, use_ovstage, version): @@ -712,33 +718,6 @@ def fake_map(self, render_var): render_data.cleanup() -def test_ovrtx_ppisp_hdr_source_is_cloned_to_output_device(monkeypatch): - """PPISP HdrColor source is moved to the HDR output buffer device.""" - - class FakeArray: - device = "cuda:1" - - class OutputArray: - device = "cuda:0" - - cloned = object() - clone_calls = [] - - def fake_clone(src, *, device): - clone_calls.append((src, device)) - return cloned - - monkeypatch.setattr(wp, "clone", fake_clone) - - renderer = _make_ovrtx_renderer_without_backend() - render_data = _make_ovrtx_camera_render_data() - render_data.ppisp_pipeline = object() - source = FakeArray() - - assert renderer._prepare_ppisp_hdr_source(render_data, source, {"rgb_hdr": OutputArray()}) is cloned - assert clone_calls == [(source, "cuda:0")] - - def test_launch_extract_all_tiles_rejects_wider_output_channels(): """An output wider than the tiled input would read out of bounds, so it must raise before launching.""" renderer = _make_ovrtx_renderer_without_backend() @@ -881,7 +860,6 @@ def test_ovrtx_cleanup_releases_only_the_given_render_data(monkeypatch, cleanup_ renderer._camera_render_data.append(render_data) render_data.warp_buffers = {"rgba": wp.zeros((8, 16, 4), dtype=wp.uint8, device="cpu")} render_data.renderer_info = {"semantic_segmentation": {"idToLabels": {}}} - render_data.ppisp_pipeline = object() if use_ovstage: render_data.camera_xform_query = "to_remove" render_data.resources.callback(renderer.backend.paths.destroy_path_list, "to_remove") @@ -928,7 +906,6 @@ def test_ovrtx_cleanup_releases_only_the_given_render_data(monkeypatch, cleanup_ assert render_data.intrinsic_bindings == [] assert render_data.warp_buffers == {} assert render_data.renderer_info == {} - assert render_data.ppisp_pipeline is None assert render_data.pending is render_data.ready is None assert renderer._render_product_paths == ["/RenderCamera_0/RenderProduct_camera"] assert renderer._initialized_scene is True diff --git a/source/isaaclab_ov/test/test_ovrtx_usd.py b/source/isaaclab_ov/test/test_ovrtx_usd.py index ded8dc4feffb..b4e57277ddd4 100644 --- a/source/isaaclab_ov/test/test_ovrtx_usd.py +++ b/source/isaaclab_ov/test/test_ovrtx_usd.py @@ -219,14 +219,17 @@ def test_ovrtx_authors_one_render_var_per_requested_data_type(): def test_ovrtx_data_types_sharing_a_source_author_one_render_var(): - """``rgb``/``rgba`` and ``depth``/``distance_to_image_plane`` collapse onto one render var each.""" + """``rgb``/``rgba``, ``depth``/``distance_to_image_plane``, and ``rgb_hdr`` collapse onto one + render var each.""" render_var_configs = get_render_var_configs( - ["rgb", "rgba", "depth", "distance_to_image_plane"], render_scope_name="RenderCamera_0" + ["rgb", "rgba", "depth", "distance_to_image_plane", "rgb_hdr"], + render_scope_name="RenderCamera_0", ) assert render_var_configs == [ ("/RenderCamera_0/Vars/LdrColor", "LdrColor", "LdrColor"), ("/RenderCamera_0/Vars/depth", "depth", "DistanceToImagePlaneSD"), + ("/RenderCamera_0/Vars/HdrColor", "HdrColor", "HdrColor"), ] @@ -307,20 +310,30 @@ def test_render_product_pins_device_ids_to_the_requested_cuda_device(camera_spec assert list(device_ids.default) == [1] -@pytest.mark.parametrize("data_types", [["rgb"], ["rgb", "rgb_hdr"], []]) -def test_render_product_isp_requests_hdr_without_mutating_camera_outputs(camera_spec, render_data, data_types): - """ISP receives one HDR source while the camera's requested outputs remain unchanged.""" +@pytest.mark.parametrize( + ("data_types", "routes_hdr"), + [(["rgb"], False), (["rgb", "rgb_hdr"], True)], +) +def test_render_product_routes_gaussian_hdr_only_for_hdr_outputs(camera_spec, render_data, data_types, routes_hdr): + """Route Gaussian HDR when an HDR output is requested, without changing the camera outputs.""" camera_spec.cfg.data_types = data_types.copy() - camera_spec.cfg.isp_cfg = object() render_product = build_render_product_as_string(camera_spec, render_data, device_id=0) layer = Sdf.Layer.CreateAnonymous(".usda") assert layer.ImportFromString(render_product) - ordered_vars = layer.GetRelationshipAtPath("/RenderCamera_0/RenderProduct.orderedVars") - expected_vars = ("LdrColor", "HdrColor") if data_types else ("HdrColor",) + product_path = "/RenderCamera_0/RenderProduct" + ordered_vars = layer.GetRelationshipAtPath(product_path + ".orderedVars") + expected_vars = ["LdrColor", "HdrColor"] if routes_hdr else ["LdrColor"] assert list(ordered_vars.targetPathList.explicitItems) == [ Sdf.Path(f"/RenderCamera_0/Vars/{name}") for name in expected_vars ] assert camera_spec.cfg.data_types == data_types + schemas = layer.GetPrimAtPath(product_path).GetInfo("apiSchemas").prependedItems + gaussian_setting = layer.GetAttributeAtPath(product_path + ".omni:rtx:rtpt:gaussian:skipTonemapping:enabled") + assert ("OmniRtxSettingsParticleFieldAPI_1" in schemas) is routes_hdr + if routes_hdr: + assert gaussian_setting.default is False + else: + assert gaussian_setting is None def test_ovrtx_rgb_and_rgb_hdr_author_both_render_vars(camera_spec, render_data): diff --git a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py index 83e4f140cfb9..02eeb87b6512 100644 --- a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py +++ b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py @@ -222,10 +222,7 @@ def drain(self): writer = Writer() renderer._visual_material_writer_ref = lambda: writer renderer.render_batch( - [ - SimpleNamespace(render_product_path=path, ppisp_pipeline=None, warp_buffers={}) - for path in renderer._render_product_paths - ] + [SimpleNamespace(render_product_path=path, warp_buffers={}) for path in renderer._render_product_paths] ) assert events == expected_events @@ -258,7 +255,7 @@ def advance_write_floor(**_kwargs): renderer.backend.stage.advance_write_floor = advance_write_floor with pytest.raises(ValueError, match=failure): - renderer.render(SimpleNamespace(render_product_path="/RenderCamera_0/Product", ppisp_pipeline=None)) + renderer.render(SimpleNamespace(render_product_path="/RenderCamera_0/Product")) assert events == expected_events diff --git a/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..5832bdc80d5d --- /dev/null +++ b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst @@ -0,0 +1,3 @@ +* **Breaking:** Replaced renderer-owned PPISP with dictionary processors. + Replace ``CameraCfg.isp_cfg`` with ``PpispProcessorCfg`` and configure + ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.major b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index 02c9335bb20f..1b26eefd53b7 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -13,7 +13,7 @@ import uuid from collections.abc import Sequence from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, NoReturn +from typing import TYPE_CHECKING, Any import numpy as np import warp as wp @@ -42,8 +42,6 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: - from isaaclab_ppisp import PpispPipeline - from omni.replicator.core.scripts.utils.viewport_manager import HydraTexture from isaaclab.sensors.camera.camera_data import CameraData @@ -51,20 +49,8 @@ from .isaac_rtx_renderer_cfg import SIMPLE_SHADING_MODES, IsaacRtxRendererCfg -_PPISP_IMPORT_ERROR_MESSAGE = ( - "isaaclab_ppisp is required when CameraCfg.isp_cfg is set. " - "Run `uv sync` from the Isaac Lab source checkout, or install the Isaac Lab wheel " - "with `uv pip install isaaclab`." -) - - -def _raise_missing_ppisp_error(exc: ModuleNotFoundError) -> NoReturn: - # Only translate missing isaaclab_ppisp imports into the optional-dependency hint; - # unrelated missing modules should surface unchanged for easier debugging. - if exc.name != "isaaclab_ppisp" and not (exc.name and exc.name.startswith("isaaclab_ppisp.")): - raise exc - raise ModuleNotFoundError(_PPISP_IMPORT_ERROR_MESSAGE, name="isaaclab_ppisp") from exc - +# Outputs read from the HdrColor AOV. +_HDR_DATA_TYPES = frozenset({str(RenderBufferKind.RGB_HDR)}) # RTX simple-shading constants. # @@ -136,12 +122,6 @@ class IsaacRtxRenderData: intrinsic_selection: Any = None intrinsic_row_attribute: str = "" intrinsic_stage: Any = None - ppisp_pipeline: PpispPipeline | None = None - """Post-render PPISP pipeline composed when ``spec.cfg.isp_cfg`` is set.""" - _hdr_scratch_wp: wp.array | None = None - """Internal HDR scratch buffer allocated when the user did not request - ``"rgb_hdr"`` in ``data_types`` but the PPISP pipeline still needs - somewhere to receive the HDR AOV before LDR conversion.""" def __post_init__(self): """Compile the stage-bound calibration columns owned by this render product.""" @@ -195,8 +175,11 @@ def __init__(self, cfg: IsaacRtxRendererCfg, *, sim: SimulationContext): apply_isaac_rtx_global_settings(self.cfg.global_settings, settings) if settings.get("/isaaclab/render/deterministic", False): apply_isaac_rtx_determinism_settings(settings) + # Cameras create this renderer during scene construction, before the simulation and + # environments read whether RTX sensors are rendered. + settings.set("/isaaclab/render/rtx_sensors", True) + settings.set("/physics/fabricUpdateTransformations", True) ensure_rtx_hydra_engine_attached() - # ``/isaaclab/render/rtx_sensors`` is owned by ``Camera.__init__`` (must be set pre-``sim.reset()``). def initialize(self) -> None: """Bind shared Fabric destinations after scene creation.""" @@ -209,30 +192,10 @@ def visual_material_writer(self): return FabricVisualMaterialWriter def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: - """Resolve the camera's PPISP cfg and apply RTX-specific USD overrides. - - When ``spec.cfg.isp_cfg`` is set, resolves it (sentinel discovery + - normalization) via :func:`isaaclab_ppisp.resolve_and_normalize` so - :mod:`isaaclab` does not need to know about PPISP. Then pins - ``exposure:*`` to neutral and applies ``OmniRtxCameraExposureAPI_1`` so - RTX's physical-camera exposure model does not compound on top of the - ISP. Without an ISP, the camera prim's authored exposure is left alone. - - :attr:`~isaaclab.sensors.camera.CameraCfg.background_color` is applied - per-render-product in :meth:`create_render_data` via USD attributes. - """ - if spec.cfg.isp_cfg is None: - return - try: - from isaaclab_ppisp import apply_rtx_exposure_overrides, resolve_and_normalize - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - camera_prim_path = spec.camera_prim_paths[0] if spec.camera_prim_paths else None - spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) - if spec.cfg.isp_cfg is None or not spec.camera_prim_paths: - return - apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) + """Route Gaussian HDR through the camera HDR output.""" + if not _HDR_DATA_TYPES.isdisjoint(spec.cfg.data_types): + # Gaussian splats otherwise skip tonemapping and never reach the HdrColor AOV. + get_settings_manager().set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: """Publish the per-output Replicator layout this RTX backend writes. @@ -398,10 +361,7 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: aov=SIMPLE_SHADING_AOV, output_data_type=np.uint8, output_channels=4 ) - needs_hdr_color = str(RenderBufferKind.RGB_HDR) in spec.cfg.data_types or ( - spec.cfg.isp_cfg is not None and any(data_type in ("rgb", "rgba") for data_type in spec.cfg.data_types) - ) - if needs_hdr_color: + if not _HDR_DATA_TYPES.isdisjoint(spec.cfg.data_types): rep.AnnotatorRegistry.register_annotator_from_aov( aov="HdrColor", output_data_type=np.float32, output_channels=4 ) @@ -410,16 +370,10 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: annotators = {} for annotator_type in spec.cfg.data_types: if annotator_type == "rgba" or annotator_type == "rgb": - if spec.cfg.isp_cfg is not None: - if str(RenderBufferKind.RGB_HDR) not in annotators: - annotator = rep.AnnotatorRegistry.get_annotator( - "HdrColor", device=spec.device, do_array_copy=False - ) - annotators[str(RenderBufferKind.RGB_HDR)] = annotator - else: - annotator = rep.AnnotatorRegistry.get_annotator("rgb", device=spec.device, do_array_copy=False) - annotators["rgba"] = annotator - elif annotator_type == str(RenderBufferKind.RGB_HDR): + annotator = rep.AnnotatorRegistry.get_annotator("rgb", device=spec.device, do_array_copy=False) + annotators["rgba"] = annotator + elif annotator_type in _HDR_DATA_TYPES: + # HdrColor reflects the exposure configured on the camera prim. if str(RenderBufferKind.RGB_HDR) not in annotators: annotator = rep.AnnotatorRegistry.get_annotator("HdrColor", device=spec.device, do_array_copy=False) annotators[str(RenderBufferKind.RGB_HDR)] = annotator @@ -482,20 +436,10 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: enable_minimal_render_mode=not needs_color_render, ) - ppisp_pipeline = None - if spec.cfg.isp_cfg is not None: - try: - from isaaclab_ppisp import PpispPipeline - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - ppisp_pipeline = PpispPipeline(spec.cfg.isp_cfg) - return IsaacRtxRenderData( annotators=annotators, render_product=rp, spec=spec, - ppisp_pipeline=ppisp_pipeline, ) def _apply_simple_shading_settings( @@ -556,27 +500,7 @@ def _resolve_simple_shading_mode(self, spec: CameraRenderSpec) -> int | None: def set_outputs(self, render_data: IsaacRtxRenderData, output_data: dict[str, ProxyArray]): """Store reference to output buffers for writing during render. See :meth:`~isaaclab.renderers.base_renderer.BaseRenderer.set_outputs`.""" - if render_data.ppisp_pipeline is not None and str(RenderBufferKind.RGBA) not in output_data: - raise ValueError( - "Isaac RTX renderer ISP requires 'rgba' (or 'rgb', which aliases into rgba) as the" - " LDR output destination, but neither was provided. Add 'rgb' or 'rgba' to" - " Camera.cfg.data_types when isp_cfg is set." - ) render_data.output_data = output_data - # Allocate an internal HDR scratch buffer when PPISP is composed but - # the user did not request the raw HDR AOV in ``data_types`` — the - # PPISP kernel still needs somewhere to receive the HDR annotator - # output before LDR conversion. - if render_data.ppisp_pipeline is not None and str(RenderBufferKind.RGB_HDR) not in output_data: - spec = render_data.spec - assert spec is not None - hdr_spec = self.supported_output_types()[RenderBufferKind.RGB_HDR] - assert hdr_spec.dtype is wp.float32 - render_data._hdr_scratch_wp = wp.zeros( - (spec.num_instances, spec.cfg.height, spec.cfg.width, hdr_spec.channels), - dtype=wp.float32, - device=spec.device, - ) def update_transforms(self) -> None: """Update shared Fabric transforms and propagate the visual hierarchy.""" @@ -664,11 +588,7 @@ def tiling_grid_shape(): # The RTX annotator may return an empty frame while its render product is warming up. # Clear the destination so callers do not observe stale data, then wait for the next frame. - if data_type == str(RenderBufferKind.RGB_HDR) and data_type not in output_data: - assert render_data._hdr_scratch_wp is not None - buf_wp = render_data._hdr_scratch_wp - else: - buf_wp = output_data[data_type].warp + buf_wp = output_data[data_type].warp if tiled_data_buffer.size == 0: buf_wp.zero_() continue @@ -750,14 +670,6 @@ def tiling_grid_shape(): replacement = 0.0 if self.cfg.depth_clipping_behavior == "zero" else cfg.spawn.clipping_range[1] replace_inf_depth_wp(buf_wp, replacement, device=device) - # Post-render PPISP: HDR scene-linear → LDR RGBA. The camera enforces - # that ``rgba`` (or ``rgb`` aliasing into it) is present when an ISP is - # configured, so writing to ``output_data["rgba"]`` is safe. - if render_data.ppisp_pipeline is not None: - hdr_proxy = output_data.get(str(RenderBufferKind.RGB_HDR)) - hdr_source = hdr_proxy.warp if hdr_proxy is not None else render_data._hdr_scratch_wp - render_data.ppisp_pipeline.apply(hdr_source, output_data[str(RenderBufferKind.RGBA)].warp) - def read_output(self, render_data: IsaacRtxRenderData, camera_data: CameraData) -> None: """Populate per-output metadata collected during render(). Pixel data already written in render(). @@ -796,5 +708,3 @@ def cleanup(self, render_data: IsaacRtxRenderData | None): render_data.output_data = None render_data.spec = None render_data.renderer_info.clear() - render_data.ppisp_pipeline = None - render_data._hdr_scratch_wp = None diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py index 4c6511fe85c8..7c2346408365 100644 --- a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py @@ -86,6 +86,99 @@ def test_native_fabric_geometry_needs_no_separate_publication(monkeypatch): provider.get_geometry_points.assert_not_called() +def test_renderer_marks_rtx_sensor_rendering_when_created(monkeypatch): + """Creating the renderer, which cameras do during scene construction, marks RTX sensor rendering.""" + _install_omni_stubs(monkeypatch) + import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + settings = MagicMock() + settings.get.return_value = False + with ( + patch.object(rtx_renderer, "get_settings_manager", return_value=settings), + patch.object(rtx_renderer, "enable_extension"), + patch.object(rtx_renderer, "apply_isaac_rtx_global_settings"), + patch.object(rtx_renderer, "ensure_rtx_hydra_engine_attached"), + ): + rtx_renderer.IsaacRtxRenderer(IsaacRtxRendererCfg(), sim=SimulationContext.instance()) + + settings.set.assert_any_call("/isaaclab/render/rtx_sensors", True) + settings.set.assert_any_call("/physics/fabricUpdateTransformations", True) + + +@pytest.mark.parametrize(("data_types", "routes_hdr"), [(["rgb"], False), (["rgb_hdr"], True)]) +def test_prepare_cameras_routes_gaussian_hdr_and_preserves_exposure(monkeypatch, data_types, routes_hdr): + """Only HDR cameras enable Gaussian HDR routing, and both preserve authored exposure.""" + _install_omni_stubs(monkeypatch) + import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer + + from pxr import Sdf, Usd, UsdGeom + + stage = Usd.Stage.CreateInMemory() + camera = UsdGeom.Camera.Define(stage, "/World/Camera").GetPrim() + camera.CreateAttribute("exposure:iso", Sdf.ValueTypeNames.Float).Set(100.0) + settings = MagicMock() + spec = SimpleNamespace(cfg=SimpleNamespace(data_types=data_types), camera_prim_paths=("/World/Camera",)) + with patch.object(rtx_renderer, "get_settings_manager", return_value=settings): + rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer).prepare_cameras(stage, spec) + + hdr_call = call("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) + assert (hdr_call in settings.set.call_args_list) is routes_hdr + assert camera.GetAttribute("exposure:iso").Get() == 100.0 + + +@pytest.mark.parametrize("data_types", [["rgb_hdr"]]) +def test_hdr_reads_the_hdr_annotator_into_its_output(monkeypatch, data_types): + """HDR attaches one HdrColor annotator and extract it into the camera's own buffers.""" + replicator, syntheticdata = _install_omni_stubs(monkeypatch) + monkeypatch.setattr(syntheticdata, "SyntheticData", MagicMock(), raising=False) + + import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + from pxr import UsdGeom + + import isaaclab.sim.utils.stage as stage_utils + from isaaclab.sensors.camera.camera_data import CameraData + + settings = MagicMock() + settings.get.return_value = False + stage = MagicMock() + stage.SelectPrims.return_value.GetCount.return_value = 1 + stage.GetPrimAtPath.return_value.IsA.side_effect = lambda typ: typ is UsdGeom.Camera + product = SimpleNamespace(path="/Render/Test") + replicator.create = SimpleNamespace(render_product_tiled=MagicMock(return_value=product)) + annotator = MagicMock() + registry = MagicMock() + registry.get_annotator.return_value = annotator + replicator.AnnotatorRegistry = registry + spec = SimpleNamespace( + camera_prim_paths=("/World/Camera",), + device="cpu", + view_count=1, + cfg=SimpleNamespace(data_types=data_types, width=3, height=2), + ) + renderer = rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg() + with ( + patch.object(rtx_renderer, "get_settings_manager", return_value=settings), + patch.object(rtx_renderer, "get_isaac_sim_version", return_value=version.parse("6.0")), + patch.object(stage_utils, "get_current_stage", return_value=stage), + ): + render_data = renderer.create_render_data(spec) + registry.get_annotator.assert_called_once_with("HdrColor", device="cpu", do_array_copy=False) + annotator.attach.assert_called_once_with([product.path]) + + camera_data = CameraData.allocate(data_types, 2, 3, 1, "cpu", supported_specs=renderer.cfg.supported_output_types()) + renderer.set_outputs(render_data, camera_data.output) + annotator.get_data.return_value = np.full((2, 3, 4), 2.0, dtype=np.float32) + renderer._read_annotator_output(render_data) + + assert set(camera_data.output) == set(data_types) + for name in data_types: + np.testing.assert_array_equal(camera_data.output[name].warp.numpy(), 2.0) + + def test_create_render_data_uses_unique_sdf_safe_render_product_name(monkeypatch): """Each tiled render product gets a fresh ``rp_`` name. @@ -132,7 +225,6 @@ def test_create_render_data_uses_unique_sdf_safe_render_product_name(monkeypatch data_types=["rgb"], width=64, height=64, - isp_cfg=None, colorize_semantic_segmentation=False, colorize_instance_segmentation=False, colorize_instance_id_segmentation=False, @@ -248,7 +340,6 @@ def _create_attribute(name, value_type): data_types=data_types, width=64, height=64, - isp_cfg=None, colorize_semantic_segmentation=False, colorize_instance_segmentation=False, colorize_instance_id_segmentation=False, @@ -466,8 +557,6 @@ def test_render_treats_empty_annotator_frame_as_not_ready(monkeypatch, data_type cfg=SimpleNamespace(width=64, height=64), ), renderer_info={}, - ppisp_pipeline=None, - _hdr_scratch_wp=None, ) renderer = rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer) renderer.cfg = IsaacRtxRendererCfg() @@ -507,8 +596,6 @@ def test_render_keeps_leading_channels_of_padded_annotator_tiles(monkeypatch, da output_data={data_type: SimpleNamespace(warp=output_buffer)}, spec=SimpleNamespace(view_count=view_count, device="cpu", cfg=SimpleNamespace(width=width, height=height)), renderer_info={}, - ppisp_pipeline=None, - _hdr_scratch_wp=None, ) renderer = rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer) renderer.cfg = IsaacRtxRendererCfg() @@ -560,8 +647,6 @@ def test_render_batch_updates_once_before_extracting_ready_cameras(monkeypatch, else SimpleNamespace(view_count=1, device="cpu", cfg=SimpleNamespace(width=1, height=1)) ), renderer_info={}, - ppisp_pipeline=None, - _hdr_scratch_wp=None, ) ) if state == "ready": diff --git a/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..5832bdc80d5d --- /dev/null +++ b/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst @@ -0,0 +1,3 @@ +* **Breaking:** Replaced renderer-owned PPISP with dictionary processors. + Replace ``CameraCfg.isp_cfg`` with ``PpispProcessorCfg`` and configure + ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.major b/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ppisp/docs/README.md b/source/isaaclab_ppisp/docs/README.md index 351571fa30db..b049db7fe996 100644 --- a/source/isaaclab_ppisp/docs/README.md +++ b/source/isaaclab_ppisp/docs/README.md @@ -1,8 +1,36 @@ # Isaac Lab PPISP -This extension provides a renderer-backend-agnostic PPISP (Physically Plausible -Image Signal Processing) pipeline for Isaac Lab camera outputs. +This extension provides PPISP (Physically Plausible Image Signal Processing) as +a `DictTensorProcessor` for camera pipelines and directly supplied tensors. -PPISP consumes HDR scene-linear color from renderer backends and writes LDR -`rgb` / `rgba` camera output after each render tick when `CameraCfg.isp_cfg` is -set. +`PpispProcessor` reads scene-linear `rgb_hdr` from a float32 `ProxyArray` with +shape `(N, H, W, 3)` and writes uint8 `rgb`. For camera processing, configure +`CameraCfg.pipeline=[PpispProcessorCfg(params={"cfg": ppisp_cfg})]`. The camera requests +required HDR internally; only the requested public outputs are published. Set +`CameraCfg.exposure=CameraExposureCfg()` (from `isaaclab.sensors.camera`) before initialization +for neutral RTX HDR input. Output selection does not alter exposure. +Set PPISP exposure in stops through `PpispCfg.inputs["exposureOffset"]`. + +Resolve authored settings explicitly before constructing the pipeline: + +```python +from isaaclab_ppisp import PpispProcessorCfg, ppisp_cfg_from_usd_camera + +ppisp_cfg = ppisp_cfg_from_usd_camera(source_camera_prim) +processor_cfg = PpispProcessorCfg(params={"cfg": ppisp_cfg}) +``` + +For RGBA, use `outputs=["rgba"]` and `params={"cfg": ppisp_cfg, "channels": 4}`. +Inputs must contain scene-linear color before camera exposure and response, including when +supplied by another processor; float32 display-encoded RGB is not radiance. + +Each processor owns its controller scratch and output buffers. Optional +`params["controller_weights"]=PpispControllerWeights(device_weights)` can supply +shared, read-only device weights; configuration copies preserve this resource's identity. The +caller owns upload synchronization, and mutable controller workspace stays private. +The controller requires CUDA; static PPISP also runs on CPU. `PpispPipeline` remains +available for direct numerical use with explicitly supplied Warp buffers. + +For direct use, construct `processor = PpispProcessor(processor_cfg, {"rgb_hdr": hdr_tensor})`, +then call `processor({"rgb_hdr": hdr_tensor})` to obtain `{"rgb": color_tensor}`. Copy results +that must survive another call, and call `processor.close()` when finished. diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py index 416f7d9e134e..2b8ae2540871 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py @@ -5,13 +5,9 @@ """Post-render PPISP (Physically Plausible Image Signal Processing) for IsaacLab. -Provides the renderer-backend-agnostic ISP pipeline that converts the renderer's -HDR scene-linear AOV to LDR RGBA at the end of a render tick. Renderer backends -(:class:`~isaaclab_physx.renderers.IsaacRtxRenderer`, -:class:`~isaaclab_ov.renderers.OVRTXRenderer`, -:class:`~isaaclab_newton.renderers.NewtonWarpRenderer`) compose -:class:`PpispPipeline` internally when the camera's -:attr:`~isaaclab.sensors.camera.CameraCfg.isp_cfg` is set. +Provides a dictionary processor that converts scene-linear radiance to RGB or RGBA. +Camera pipelines use :class:`PpispProcessorCfg` with concrete settings. USD discovery +helpers remain explicit, and renderer backends do not own PPISP execution or controller workspace. """ import importlib.metadata diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi index 2d69e60325e9..646c40c9387e 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi +++ b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi @@ -7,10 +7,12 @@ __all__ = [ "PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN", "PPISP_DEFAULT_INPUTS", "PpispCfg", + "PpispControllerWeights", + "PpispDiscoveryMode", "PpispPipeline", + "PpispProcessorCfg", "apply_ppisp_to_rgba", "apply_ppisp_to_rgba_with_controller_params", - "apply_rtx_exposure_overrides", "auto_any_ppisp_cfg", "auto_camera_ppisp_cfg", "compute_ppisp_controller_params", @@ -19,6 +21,7 @@ __all__ = [ "normalize_ppisp_cfg", "ppisp_cfg_from_usd_camera", "ppisp_cfg_from_usd_stage", + "PpispProcessor", "resolve_and_normalize", ] @@ -26,6 +29,7 @@ from .cfg import ( PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN, PPISP_DEFAULT_INPUTS, PpispCfg, + PpispDiscoveryMode, auto_any_ppisp_cfg, auto_camera_ppisp_cfg, default_ppisp_inputs, @@ -36,5 +40,6 @@ from .cfg import ( resolve_and_normalize, ) from .kernels import apply_ppisp_to_rgba, apply_ppisp_to_rgba_with_controller_params, compute_ppisp_controller_params -from .pipeline import PpispPipeline -from .rtx_camera_overrides import apply_rtx_exposure_overrides +from .pipeline import PpispControllerWeights, PpispPipeline +from .processor import PpispProcessor +from .processor_cfg import PpispProcessorCfg diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py b/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py index 37859bda6d86..d9416602171b 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py @@ -12,6 +12,7 @@ from __future__ import annotations from dataclasses import field +from enum import StrEnum from typing import Any from isaaclab.utils import configclass @@ -79,6 +80,16 @@ def default_ppisp_inputs() -> dict[str, float | tuple[float, float]]: return dict(PPISP_DEFAULT_INPUTS) +class PpispDiscoveryMode(StrEnum): + """Explicit USD configuration discovery strategies used before preparing a PPISP stage.""" + + AUTO_CAMERA = "auto_camera" + """Read the specified camera, or the first PPISP camera when no path is supplied.""" + + AUTO_ANY = "auto_any" + """Read the specified camera, then fall back to the first PPISP camera on the stage.""" + + @configclass class PpispCfg: """Configuration for PPISP post-processing. @@ -321,29 +332,17 @@ def has_ppisp_camera_attrs(camera_prim: Any | None) -> bool: return _has_ppisp_camera_attrs(camera_prim) -def resolve_and_normalize(isp_cfg: Any, stage: Any, camera_prim_path: str | None = None) -> PpispCfg | None: - """Resolve a Camera sensor batch's ``isp_cfg`` to a normalised cfg or ``None``. - - Handles all three legal forms of :attr:`~isaaclab.sensors.camera.CameraCfg.isp_cfg`: - - * ``None`` → returns ``None``. - * :class:`~isaaclab.sensors.camera.CameraISPMode` sentinel — checks the - target camera via :func:`auto_camera_ppisp_cfg` (and uses - :func:`auto_any_ppisp_cfg` for ``AUTO_ANY`` or when no camera path is - supplied) to discover a PPISP camera. Returns the parsed + normalised - :class:`PpispCfg`, or ``None`` if no PPISP camera matched. - * Concrete :class:`PpispCfg` — normalises in place (validates input keys, - fills defaults, and merges camera-authored USD values when - ``camera_prim_path`` is set). +def resolve_and_normalize( + isp_cfg: PpispCfg | PpispDiscoveryMode | None, stage: Any, camera_prim_path: str | None = None +) -> PpispCfg | None: + """Resolve explicit settings or a discovery mode using the caller's USD stage. - This is the single entry point renderer backends call inside their - ``prepare_cameras`` hook so :mod:`isaaclab.sensors.camera` does not need - to know about PPISP types at all. The returned cfg applies to the whole - Camera sensor batch; callers pass the first matched camera prim path for - the camera-local discovery phase. + Call this before constructing a PPISP processor. The processor receives only concrete + settings and never looks up a camera or stage. Legacy ``CameraISPMode`` string-enum values + remain accepted for migration without importing the camera package. Args: - isp_cfg: The Camera sensor's :attr:`isp_cfg` value (``None``, ``CameraISPMode``, or :class:`PpispCfg`). + isp_cfg: Explicit settings, a discovery mode, or ``None`` to disable processing. stage: USD stage used for sentinel discovery and camera-path resolution. camera_prim_path: Optional absolute path of the first matched camera prim in the Camera sensor batch. When omitted, discovery uses the @@ -352,14 +351,12 @@ def resolve_and_normalize(isp_cfg: Any, stage: Any, camera_prim_path: str | None Returns: A fully-normalised :class:`PpispCfg`, or ``None`` if the batch has no ISP. """ - # Local import avoids a top-of-module dep on isaaclab.sensors. - from isaaclab.sensors.camera.camera_isp import CameraISPMode - if isp_cfg is None: return None - if isinstance(isp_cfg, CameraISPMode): + if isinstance(isp_cfg, str): + mode = PpispDiscoveryMode(isp_cfg) resolved = auto_camera_ppisp_cfg(stage, camera_prim_path) if camera_prim_path else None - if resolved is None and (isp_cfg == CameraISPMode.AUTO_ANY or not camera_prim_path): + if resolved is None and (mode == PpispDiscoveryMode.AUTO_ANY or not camera_prim_path): resolved = auto_any_ppisp_cfg(stage) if resolved is None: return None diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py b/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py index 02a3ed36cd7c..010761850428 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py @@ -508,7 +508,8 @@ def _ppisp_controller_mlp_native_kernel( _ppisp_controller_mlp_native(features, weights, controller_params, prior_exposure, camera_id, lane) -@wp.kernel(enable_backward=False) +# Keep image transforms compilable on CPU without the CUDA-only native controller snippets. +@wp.kernel(enable_backward=False, module="unique") def _apply_ppisp_kernel( hdr_color: wp.array4d(dtype=wp.float32), out_rgba: wp.array4d(dtype=wp.uint8), @@ -549,7 +550,7 @@ def _apply_ppisp_kernel( For each pixel, the model applies HDR responsivity scaling, exposure scaling, per-channel vignetting, color homography correction, CRF tone - mapping, and uint8 RGBA packing. The first tensor dimension is the + mapping, and uint8 RGB or RGBA packing. The first tensor dimension is the camera/environment index, so image-space effects use local ``height_id`` and ``width_id`` coordinates per camera. """ @@ -591,10 +592,11 @@ def _apply_ppisp_kernel( out_rgba[camera_id, height_id, width_id, 0] = wp.uint8(wp.clamp(out_rgb[0], 0.0, 1.0) * 255.0) out_rgba[camera_id, height_id, width_id, 1] = wp.uint8(wp.clamp(out_rgb[1], 0.0, 1.0) * 255.0) out_rgba[camera_id, height_id, width_id, 2] = wp.uint8(wp.clamp(out_rgb[2], 0.0, 1.0) * 255.0) - out_rgba[camera_id, height_id, width_id, 3] = wp.uint8(255) + if out_rgba.shape[3] == 4: + out_rgba[camera_id, height_id, width_id, 3] = wp.uint8(255) -@wp.kernel(enable_backward=False) +@wp.kernel(enable_backward=False, module="unique") def _apply_ppisp_controller_kernel( hdr_color: wp.array4d(dtype=wp.float32), out_rgba: wp.array4d(dtype=wp.uint8), @@ -670,23 +672,38 @@ def _apply_ppisp_controller_kernel( out_rgba[camera_id, height_id, width_id, 0] = wp.uint8(wp.clamp(out_rgb[0], 0.0, 1.0) * 255.0) out_rgba[camera_id, height_id, width_id, 1] = wp.uint8(wp.clamp(out_rgb[1], 0.0, 1.0) * 255.0) out_rgba[camera_id, height_id, width_id, 2] = wp.uint8(wp.clamp(out_rgb[2], 0.0, 1.0) * 255.0) - out_rgba[camera_id, height_id, width_id, 3] = wp.uint8(255) + if out_rgba.shape[3] == 4: + out_rgba[camera_id, height_id, width_id, 3] = wp.uint8(255) + + +def _validate_ppisp_image_buffers(hdr_color: wp.array, out_rgba: wp.array) -> None: + """Check the shapes, scalar types, and device required by both image kernels.""" + if hdr_color.dtype is not wp.float32: + raise ValueError(f"Camera PPISP HDR input must be wp.float32, got {hdr_color.dtype}.") + if out_rgba.dtype is not wp.uint8: + raise ValueError(f"Camera PPISP RGB/RGBA output must be wp.uint8, got {out_rgba.dtype}.") + if len(hdr_color.shape) != 4 or hdr_color.shape[-1] != 3: + raise ValueError("PPISP input must have shape (N, H, W, 3).") + if ( + len(out_rgba.shape) != 4 + or out_rgba.shape[-1] not in (3, 4) + or out_rgba.shape[:3] != hdr_color.shape[:3] + or out_rgba.device != hdr_color.device + ): + raise ValueError("PPISP output must match the input batch, image dimensions, and device, with 3 or 4 channels.") def apply_ppisp_to_rgba(hdr_color: wp.array, out_rgba: wp.array, cfg: PpispCfg) -> None: - """Apply PPISP to ``hdr_color`` and write LDR RGBA into ``out_rgba``. + """Apply PPISP to ``hdr_color`` and write LDR RGB or RGBA into ``out_rgba``. Args: hdr_color: HDR scene-linear input. ``wp.array4d(dtype=wp.float32)`` of shape ``(N, H, W, 3)``. - out_rgba: LDR RGBA output. ``wp.array4d(dtype=wp.uint8)`` of shape - ``(N, H, W, 4)``. Written in place. + out_rgba: LDR output. ``wp.array4d(dtype=wp.uint8)`` of shape + ``(N, H, W, 3)`` or ``(N, H, W, 4)``. Written in place; alpha is 255 when present. cfg: PPISP configuration. """ - if hdr_color.dtype is not wp.float32: - raise ValueError(f"Camera PPISP HDR input must be wp.float32, got {hdr_color.dtype}.") - if out_rgba.dtype is not wp.uint8: - raise ValueError(f"Camera PPISP RGBA output must be wp.uint8, got {out_rgba.dtype}.") + _validate_ppisp_image_buffers(hdr_color, out_rgba) inputs = cfg.inputs wp.launch( @@ -812,17 +829,14 @@ def apply_ppisp_to_rgba_with_controller_params( Args: hdr_color: HDR scene-linear input, shape ``(N, H, W, 3)``. - out_rgba: LDR RGBA output, shape ``(N, H, W, 4)``. + out_rgba: LDR RGB/RGBA output, shape ``(N, H, W, 3)`` or ``(N, H, W, 4)``. cfg: PPISP configuration. Static inputs provide responsivity, vignetting and CRF. controller_params: ``wp.array2d(float32)`` of shape ``(N, 9)`` holding ``[exposureOffset, colorLatentBlue.xy, colorLatentRed.xy, colorLatentGreen.xy, colorLatentNeutral.xy]``. """ - if hdr_color.dtype is not wp.float32: - raise ValueError(f"Camera PPISP HDR input must be wp.float32, got {hdr_color.dtype}.") - if out_rgba.dtype is not wp.uint8: - raise ValueError(f"Camera PPISP RGBA output must be wp.uint8, got {out_rgba.dtype}.") + _validate_ppisp_image_buffers(hdr_color, out_rgba) if controller_params.dtype is not wp.float32: raise ValueError(f"Camera PPISP controller params must be wp.float32, got {controller_params.dtype}.") if controller_params.shape[0] != out_rgba.shape[0] or controller_params.shape[1] != 9: diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py b/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py index 1fe7a3228e25..04bdea699a9f 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py @@ -3,20 +3,15 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Post-render PPISP pipeline composed into renderer backends. - -:class:`PpispPipeline` runs *after* a renderer fills its HDR scene-linear AOV, -converting HDR → LDR via a single Warp kernel. Renderer backends instantiate -this class when their cfg/spec carries a :class:`PpispCfg` and dispatch -:meth:`apply` once per render tick. The HDR scratch buffer is owned by the -renderer backend, not by this class. -""" +"""PPISP numerical execution with owned controller workspace and optional shared weights.""" from __future__ import annotations +from dataclasses import dataclass + import warp as wp -from .cfg import PpispCfg, normalize_ppisp_cfg +from .cfg import PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN, PpispCfg, normalize_ppisp_cfg from .kernels import ( PPISP_CONTROLLER_FEATURE_LEN, PPISP_CONTROLLER_PARAM_COUNT, @@ -26,89 +21,105 @@ ) -class PpispPipeline: - """Post-render PPISP kernel applier. +@dataclass(frozen=True, eq=False, slots=True) +class PpispControllerWeights: + """Shared, read-only device weights with identity preserved when configurations are copied. - Constructed by renderer backends when ``spec.cfg.isp_cfg`` is set. Owns the - normalised :class:`PpispCfg` and dispatches the PPISP Warp kernel once per - render tick via :meth:`apply`. + The caller supplies the allocation and orders its upload before processing. The wrapper retains + that allocation for every consumer; neither processors nor configuration copies mutate it. + """ - One pipeline instance applies to the whole Camera sensor batch. The PPISP - Warp kernel takes scalar coefficients, so every cloned view in a tiled - batch shares the same ISP configuration — there is no per-view ISP today. - Per-view support would require packing the cfg into GPU arrays and indexing - by ``camera_id`` inside the kernel. + values: wp.array - Today only :class:`PpispCfg` is accepted; future ISP implementations can - either subclass or be selected by cfg type without changes to the backend - renderers. - """ + def __post_init__(self) -> None: + if self.values.shape != (PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN,) or self.values.dtype != wp.float32: + raise ValueError("PPISP controller weights require the expected one-dimensional float32 buffer.") - def __init__(self, cfg: PpispCfg): - """Initialize the PPISP pipeline. + def __deepcopy__(self, memo: dict) -> PpispControllerWeights: + # Config cloning must share model resources, while each processor creates its own workspace. + return self - Normalises ``cfg`` on construction (validates input keys, fills - defaults). - :class:`~isaaclab.sensors.camera.Camera` already normalises ``isp_cfg`` - before passing the :class:`~isaaclab.renderers.CameraRenderSpec` to - the backend, so renderer backends pass a concrete, resolved config here. - Args: - cfg: The PPISP configuration. - """ +class PpispPipeline: + """Apply concrete PPISP settings to explicitly supplied image buffers. + + This numerical object performs no camera, scene, or renderer discovery. Resolve USD settings + with :func:`~isaaclab_ppisp.ppisp_cfg_from_usd_camera` before construction. Each instance owns + its controller workspace and is not reentrant; coefficients apply to its whole image batch. + :class:`~isaaclab_ppisp.PpispProcessor` owns an independent instance for each consumer. + + Args: + cfg: Concrete PPISP settings, normalized in place for direct numerical use. + controller_weights: Optional device-resident controller weights to borrow read-only. + Callers may share these across instances, but must keep them unchanged and order their + upload before execution. Workspace and outputs are never shared automatically. + """ + + def __init__(self, cfg: PpispCfg, *, controller_weights: PpispControllerWeights | None = None): normalized_cfg = normalize_ppisp_cfg(cfg) if normalized_cfg is None: raise ValueError("PpispPipeline requires a concrete PpispCfg.") self.cfg = normalized_cfg + self._shared_controller_weights = controller_weights self._controller_weights_by_device: dict[str, wp.array] = {} self._controller_buffers_by_shape: dict[tuple[str, int, int, int], tuple[wp.array, ...]] = {} + self._closed = False + + def initialize(self, shape: tuple[int, ...], device: str | wp.Device) -> None: + """Prepare controller weights and scratch before the first image is processed.""" + if self._closed: + raise RuntimeError("Cannot initialize a closed PPISP pipeline.") + if self.cfg.controller_weights is None and self._shared_controller_weights is None: + return + if not wp.get_device(device).is_cuda: + raise ValueError("PPISP controller requires a CUDA device.") + device_name = str(wp.get_device(device)) + if device_name not in self._controller_weights_by_device: + if self._shared_controller_weights is None: + if len(self.cfg.controller_weights) != PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN: + raise ValueError(f"PPISP controller requires {PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN} weights.") + weights = wp.array(self.cfg.controller_weights, dtype=wp.float32, device=device_name) + else: + weights = self._shared_controller_weights.values + if weights.device != wp.get_device(device_name): + raise ValueError("Shared PPISP controller weights must match the image device.") + self._controller_weights_by_device[device_name] = weights + self._controller_buffers(shape, device_name) def apply(self, hdr: wp.array, rgba: wp.array) -> None: - """Run the PPISP kernel: HDR scene-linear → LDR RGBA, in place on ``rgba``.""" - if self.cfg.controller_weights is None: + """Write RGB or RGBA to the supplied destination on the current Warp stream.""" + if self._closed: + raise RuntimeError("Cannot apply a closed PPISP pipeline.") + if self.cfg.controller_weights is None and self._shared_controller_weights is None: apply_ppisp_to_rgba(hdr, rgba, self.cfg) return - controller_params = self._compute_controller_params(hdr) - apply_ppisp_to_rgba_with_controller_params(hdr, rgba, self.cfg, controller_params) - - def _compute_controller_params(self, hdr: wp.array) -> wp.array: - """Run the exported PPISP controller and return a Warp view of ``(N, 9)`` params.""" - controller_weights = self.cfg.controller_weights - assert controller_weights is not None - + self.initialize(hdr.shape, hdr.device) device = str(hdr.device) - weights = self._controller_weights_by_device.get(device) - if weights is None: - weights = wp.array(controller_weights, dtype=wp.float32, device=device) - self._controller_weights_by_device[device] = weights - - features, controller_params = self._controller_buffers(hdr) + features, controller_params = self._controller_buffers(hdr.shape, device) compute_ppisp_controller_params( hdr, - weights, + self._controller_weights_by_device[device], features, controller_params, self.cfg.controller_prior_exposure, float(self.cfg.controller_responsivity), ) - return controller_params + apply_ppisp_to_rgba_with_controller_params(hdr, rgba, self.cfg, controller_params) - def _controller_buffers(self, hdr: wp.array) -> tuple[wp.array, ...]: - """Return cached controller scratch buffers matching ``hdr`` shape/device.""" - num_cameras = int(hdr.shape[0]) - image_height = int(hdr.shape[1]) - image_width = int(hdr.shape[2]) - device = str(hdr.device) - key = (device, num_cameras, image_height, image_width) + def close(self) -> None: + """Release references to owned workspace and borrowed weights. Repeated calls are safe.""" + self._controller_weights_by_device.clear() + self._controller_buffers_by_shape.clear() + self._shared_controller_weights = None + self._closed = True + + def _controller_buffers(self, shape: tuple[int, ...], device: str) -> tuple[wp.array, ...]: + key = (device, *shape[:3]) buffers = self._controller_buffers_by_shape.get(key) - if buffers is not None: - return buffers - - # Camera output shape is fixed for a pipeline lifetime; cache buffers so - # controller execution does not allocate every frame. - buffers = ( - wp.empty((num_cameras, PPISP_CONTROLLER_FEATURE_LEN), dtype=wp.float32, device=device), - wp.empty((num_cameras, PPISP_CONTROLLER_PARAM_COUNT), dtype=wp.float32, device=device), - ) - self._controller_buffers_by_shape[key] = buffers + if buffers is None: + buffers = ( + wp.empty((shape[0], PPISP_CONTROLLER_FEATURE_LEN), dtype=wp.float32, device=device), + wp.empty((shape[0], PPISP_CONTROLLER_PARAM_COUNT), dtype=wp.float32, device=device), + ) + self._controller_buffers_by_shape[key] = buffers return buffers diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/processor.py b/source/isaaclab_ppisp/isaaclab_ppisp/processor.py new file mode 100644 index 000000000000..34080bf4f0f3 --- /dev/null +++ b/source/isaaclab_ppisp/isaaclab_ppisp/processor.py @@ -0,0 +1,74 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""PPISP dictionary processor shared by cameras and observations.""" + +from __future__ import annotations + +from copy import deepcopy + +import warp as wp + +from isaaclab.utils.tensor_processors import DictTensorProcessor +from isaaclab.utils.warp import ProxyArray + +from .cfg import PpispCfg +from .pipeline import PpispPipeline +from .processor_cfg import PpispProcessorCfg + + +class PpispProcessor(DictTensorProcessor): + """Transform scene-linear HDR into uint8 color using persistent PPISP storage. + + Input is float32 NHWC with three channels. It must represent color before exposure and + camera response; array storage cannot establish that semantic contract. Output has three + RGB or four RGBA channels. Controllers may share immutable weights via configuration, + while each processor owns its controller workspace and output buffer. + """ + + def __init__(self, cfg: PpispProcessorCfg, data: dict[str, ProxyArray]) -> None: + super().__init__(cfg, data) + if len(cfg.inputs) != 1 or len(cfg.outputs) != 1: + raise ValueError("PPISP requires exactly one input and one output name.") + self._input_name, self._output_name = cfg.inputs[0], cfg.outputs[0] + source = data[self._input_name] + if not isinstance(source, ProxyArray): + raise TypeError("PPISP input must be a ProxyArray.") + if len(source.shape) != 4 or source.shape[-1] != 3 or any(size <= 0 for size in source.shape): + raise ValueError("PPISP input must have shape (N, H, W, 3) with positive dimensions.") + if source.dtype is not wp.float32: + raise ValueError("PPISP input must have dtype wp.float32.") + channels = cfg.params.get("channels", 3) + if channels not in (3, 4): + raise ValueError("PPISP color output requires three RGB or four RGBA channels.") + ppisp_cfg = cfg.params["cfg"] + if not isinstance(ppisp_cfg, PpispCfg) or ppisp_cfg.camera_prim_path is not None: + raise ValueError("PPISP requires a concrete PpispCfg; resolve USD camera attributes before construction.") + self._shape, self._device = source.shape, source.device + self._closed = False + self._pipeline = PpispPipeline(deepcopy(ppisp_cfg), controller_weights=cfg.params.get("controller_weights")) + try: + self._pipeline.initialize(source.shape, source.device) + self.outputs[self._output_name] = ProxyArray( + wp.empty((*source.shape[:3], channels), dtype=wp.uint8, device=source.device) + ) + except Exception: + self.close() + raise + + def __call__(self, data: dict[str, ProxyArray]) -> dict[str, ProxyArray]: + """Read the current input and return the persistent named color tensor.""" + if self._closed: + raise RuntimeError("Cannot execute a closed PPISP processor.") + source = data[self._input_name] + if source.shape != self._shape or source.dtype is not wp.float32 or source.device != self._device: + raise ValueError("PPISP input must match the shape, dtype and device used during construction.") + self._pipeline.apply(source.warp, self.outputs[self._output_name].warp) + return dict(self.outputs) + + def close(self) -> None: + if not self._closed: + self._closed = True + self._pipeline.close() diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/processor_cfg.py b/source/isaaclab_ppisp/isaaclab_ppisp/processor_cfg.py new file mode 100644 index 000000000000..876ebd243a98 --- /dev/null +++ b/source/isaaclab_ppisp/isaaclab_ppisp/processor_cfg.py @@ -0,0 +1,26 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from typing import Any + +from isaaclab.utils import configclass +from isaaclab.utils.tensor_processors import DictTensorProcessorCfg + +from .cfg import PpispCfg + + +@configclass +class PpispProcessorCfg(DictTensorProcessorCfg): + """Process one scene-linear HDR tensor into RGB or RGBA. + + ``inputs`` and ``outputs`` each contain one actual dictionary key. Supply concrete PPISP + settings in ``params={"cfg": ppisp_cfg}``. Add ``"channels": 4`` for RGBA. The source camera + needs explicit neutral exposure; this processor never changes renderer settings. + """ + + class_type = "{DIR}.processor:PpispProcessor" + inputs: list[str] = ["rgb_hdr"] + outputs: list[str] = ["rgb"] + params: dict[str, Any] = {"cfg": PpispCfg()} diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py b/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py deleted file mode 100644 index 5ef5c6bfebf1..000000000000 --- a/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""RTX-specific per-camera-prim USD overrides authored when PPISP is active. - -Kept in a Kit-free module (no ``omni.*`` imports) so it can be unit-tested -without booting Isaac Sim. Both Kit-driven Isaac RTX and Kit-less OVRTX embed -the same RTX core and author the same exposure schema, so the helper lives -here in :mod:`isaaclab_ppisp` and is called from both backends' -``prepare_cameras`` hook. -""" - -from __future__ import annotations - -from typing import Any - -# Neutral RTX-side values authored on the camera prim when PPISP is the ISP -# authority. RTX otherwise compounds its own physical-camera exposure model -# on top of PPISP, leaving the HDR AOV under- or over-exposed. PPISP supplies -# its own ``responsivity`` / ``exposureOffset``; pinning these stage-side -# values (``iso=0`` plus ``autoExposure:enabled=False``) keeps RTX's exposure -# stage a no-op. -# -# Each entry is ``(attribute_name, sdf_type_name, value)`` where -# ``sdf_type_name`` is the suffix of ``Sdf.ValueTypeNames``; values are -# coerced inside :func:`apply_rtx_exposure_overrides`. -_NEUTRAL_CAMERA_EXPOSURE: tuple[tuple[str, str, Any], ...] = ( - ("exposure", "Float", 0.0), - ("exposure:fStop", "Float", 1.0), - ("exposure:iso", "Float", 0.0), - ("exposure:responsivity", "Float", 1.0), - ("exposure:time", "Float", 1.0), - ("omni:rtx:autoExposure:enabled", "Bool", False), -) - -# API schemas applied (prepended) on each camera prim so Kit's USD watcher -# routes the camera's ``exposure:*`` and ``omni:rtx:autoExposure:*`` -# attributes into the per-camera carb settings RTX consumes. -_PPISP_CAMERA_API_SCHEMAS: tuple[str, ...] = ( - "OmniRtxCameraAutoExposureAPI_1", - "OmniRtxCameraExposureAPI_1", -) - - -def apply_rtx_exposure_overrides(stage: Any, prim_paths: list[str]) -> None: - """Pin RTX-side exposure to neutral and apply the API schemas on each camera prim. - - On every prim in ``prim_paths``: - - 1. Author the neutral values from :data:`_NEUTRAL_CAMERA_EXPOSURE` — - ``exposure:*`` knobs disabling RTX's physical-camera exposure stage - plus ``omni:rtx:autoExposure:enabled=False``. - 2. Apply the :data:`_PPISP_CAMERA_API_SCHEMAS` API schemas via - ``SetMetadata("apiSchemas", ...)``. In Kit, ``omni.kit``'s USD - watcher pumps the authored attributes into the per-camera carb - settings RTX consumes. ``Sdf.TokenListOp`` metadata is used rather - than ``Usd.Prim.ApplyAPI`` so this does not require the USD - ``PlugRegistry`` to have discovered the RTX-settings schema plugin - yet. - - Existing API schemas on the prim are preserved — the RTX schemas are - prepended, not replaced. - - Args: - stage: USD stage containing the camera prim(s). - prim_paths: Resolved camera prim paths (typically one per env). - """ - from pxr import Sdf - - sdf_value_types = { - "Bool": Sdf.ValueTypeNames.Bool, - "Float": Sdf.ValueTypeNames.Float, - } - - def coerce(value: Any, sdf_type_name: str) -> Any: - if sdf_type_name == "Float": - return float(value) - return value - - for prim_path in prim_paths: - prim = stage.GetPrimAtPath(prim_path) - if not prim or not prim.IsValid(): - continue - current = prim.GetMetadata("apiSchemas") or Sdf.TokenListOp() - existing = list(current.prependedItems) + list(current.explicitItems) + list(current.appendedItems) - missing = [s for s in _PPISP_CAMERA_API_SCHEMAS if s not in existing] - if missing: - merged = Sdf.TokenListOp.Create(prependedItems=[*missing, *existing]) - prim.SetMetadata("apiSchemas", merged) - for attr_name, sdf_type_name, value in _NEUTRAL_CAMERA_EXPOSURE: - attr = prim.GetAttribute(attr_name) - if not attr: - attr = prim.CreateAttribute(attr_name, sdf_value_types[sdf_type_name], custom=False) - attr.Set(coerce(value, sdf_type_name)) diff --git a/source/isaaclab_ppisp/pyproject.toml b/source/isaaclab_ppisp/pyproject.toml index ade02233204e..52b8d53c60b9 100644 --- a/source/isaaclab_ppisp/pyproject.toml +++ b/source/isaaclab_ppisp/pyproject.toml @@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta" [project] name = "isaaclab_ppisp" version = "0.3.1" -description = "Extension providing a renderer-backend-agnostic PPISP pipeline that converts HDR scene-linear color to LDR RGBA at the end of a render tick." +description = "PPISP processors that convert scene-linear HDR to LDR RGB or RGBA for cameras and observations." license = {text = "BSD-3-Clause"} authors = [{name = "Isaac Lab Project Developers"}] maintainers = [{name = "Isaac Lab Project Developers"}] diff --git a/source/isaaclab_ppisp/test/test_ppisp.py b/source/isaaclab_ppisp/test/test_ppisp.py index 453595d849d0..c60841320a31 100644 --- a/source/isaaclab_ppisp/test/test_ppisp.py +++ b/source/isaaclab_ppisp/test/test_ppisp.py @@ -10,6 +10,7 @@ import pytest from isaaclab_ppisp import ( PpispCfg, + PpispDiscoveryMode, auto_any_ppisp_cfg, auto_camera_ppisp_cfg, default_ppisp_inputs, @@ -226,13 +227,11 @@ def test_auto_any_ppisp_cfg_reads_first_camera_with_ppisp_attrs(): def test_resolve_and_normalize_without_camera_uses_first_ppisp_camera(): - from isaaclab.sensors.camera.camera_isp import CameraISPMode - stage = Usd.Stage.CreateInMemory() _author_camera(stage, "/World/CameraWithoutPpisp") _author_ppisp_camera(stage, "/World/Camera_ppisp", inherits=None, attrs={"exposureOffset": 2.0}) - cfg = resolve_and_normalize(CameraISPMode.AUTO_CAMERA, stage) + cfg = resolve_and_normalize(PpispDiscoveryMode.AUTO_CAMERA, stage) assert cfg is not None assert cfg.camera_prim_path is None diff --git a/source/isaaclab_ppisp/test/test_ppisp_kernels.py b/source/isaaclab_ppisp/test/test_ppisp_kernels.py index 6b3ac48d7ea7..fe196466064a 100644 --- a/source/isaaclab_ppisp/test/test_ppisp_kernels.py +++ b/source/isaaclab_ppisp/test/test_ppisp_kernels.py @@ -4,9 +4,13 @@ # SPDX-License-Identifier: BSD-3-Clause import pytest +import torch import warp as wp from isaaclab_ppisp import ( PpispCfg, + PpispControllerWeights, + PpispProcessor, + PpispProcessorCfg, apply_ppisp_to_rgba, apply_ppisp_to_rgba_with_controller_params, compute_ppisp_controller_params, @@ -37,6 +41,9 @@ PPISP_CONTROLLER_PARAM_COUNT, ) +from isaaclab.test.utils import DeviceScope, test_devices +from isaaclab.utils.warp import ProxyArray + wp.init() @@ -282,15 +289,14 @@ def bounded_softplus(raw: float, min_value: float) -> float: return np.maximum(y, 0.0) ** gamma -def test_ppisp_warp_crf_extreme_centers_match_reference(): - """Saturated CRF centers produce the reference curve instead of NaN-cast bytes.""" +def test_ppisp_processor_rgb_crf_extreme_centers_match_reference(): + """A CPU RGB processor preserves the reference curve, including saturated CRF centers.""" import numpy as np # a gray ramp passes the default exposure, vignetting, and color stages unchanged ramp = np.linspace(0.0, 1.0, 16, dtype=np.float32) hdr_data = np.repeat(ramp[None, None, :, None], 3, axis=3) - hdr_color = wp.from_numpy(hdr_data, dtype=wp.float32) - rgba = _rgba((1, 1, 16, 4)) + hdr_color = wp.from_numpy(hdr_data, dtype=wp.float32, device="cpu") cfg = normalize_ppisp_cfg( PpispCfg( inputs={ @@ -307,10 +313,9 @@ def test_ppisp_warp_crf_extreme_centers_match_reference(): ) ) - apply_ppisp_to_rgba(hdr_color, rgba, cfg) - - rgba_np = rgba.numpy()[0, 0] - assert (rgba_np[:, 3] == 255).all() + processor = PpispProcessor(PpispProcessorCfg(params={"cfg": cfg}), {"rgb_hdr": ProxyArray(hdr_color)}) + rgb_np = processor({"rgb_hdr": ProxyArray(hdr_color)})["rgb"].warp.numpy()[0, 0] + processor.close() for channel, suffix in enumerate("RGB"): expected = _crf_reference( ramp, @@ -321,4 +326,63 @@ def test_ppisp_warp_crf_extreme_centers_match_reference(): ) expected_bytes = np.floor(np.clip(expected, 0.0, 1.0) * 255.0) # float32 kernel math may land one byte away from the float64 reference at a floor boundary - np.testing.assert_allclose(rgba_np[:, channel].astype(float), expected_bytes, atol=1.0, err_msg=suffix) + np.testing.assert_allclose(rgb_np[:, channel].astype(float), expected_bytes, atol=1.0, err_msg=suffix) + np.testing.assert_array_equal(hdr_color.numpy(), hdr_data) + + +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) +def test_ppisp_controller_processors_share_weights_for_rgb_and_rgba(device): + """Both controller output layouts match a known response; closing one consumer preserves another.""" + import numpy as np + + weights_np = np.zeros(PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN, dtype=np.float32) + weights_np[PPISP_CONTROLLER_OFF_EXP_B] = 1.0 # A constant one-stop controller exposure. + weights = wp.from_numpy(weights_np, dtype=wp.float32, device=device) + ramp = np.array([0.0, 0.125, 0.25, 0.5], dtype=np.float32) + hdr_data = np.repeat(ramp[None, None, :, None], 3, axis=3) + hdr = wp.from_numpy(hdr_data, dtype=wp.float32, device=device) + cfg = PpispCfg() + shared_weights = PpispControllerWeights(weights) + processors = [ + PpispProcessor( + PpispProcessorCfg(params={"cfg": cfg, "channels": channels, "controller_weights": shared_weights}), + {"rgb_hdr": ProxyArray(hdr)}, + ) + for channels in (3, 4) + ] + # Use different pixels from preparation and honor prior work on a non-default Torch stream. + with torch.cuda.stream(torch.cuda.Stream(device=device)): + incoming = torch.from_numpy(hdr_data * 0.5).to(device) * 3.0 + with wp.ScopedStream(wp.stream_from_torch(torch.cuda.current_stream(device)), sync_enter=True, sync_exit=True): + output = processors[0]({"rgb_hdr": ProxyArray(wp.from_torch(incoming))})["rgb"] + results = [output.torch.cpu().numpy()] + results.append(processors[1]({"rgb_hdr": ProxyArray(hdr)})["rgb"].warp.numpy()) + for result, input_scale in zip(results, (1.5, 1.0), strict=True): + expected = np.floor(_crf_reference(ramp * input_scale * 2.0, 0.013659, 0.013659, 0.378165, 0.0) * 255.0) + np.testing.assert_allclose(result[0, 0, :, :3].astype(float), np.repeat(expected[:, None], 3, axis=1), atol=1.0) + assert (results[1][..., 3] == 255).all() + processors[0].close() + hdr.zero_() + np.testing.assert_array_equal(processors[1]({"rgb_hdr": ProxyArray(hdr)})["rgb"].warp.numpy()[..., :3], 0) + np.testing.assert_array_equal(weights.numpy(), weights_np) + processors[1].close() + + +@pytest.mark.parametrize( + ("shape", "dtype", "message"), + [((1, 2, 2, 4), wp.float32, "must have shape"), ((1, 2, 2, 3), wp.uint8, "must have dtype")], +) +def test_ppisp_processor_rejects_incompatible_input_storage(shape, dtype, message): + """Reject invalid image channels and scalar types before preparing numerical execution.""" + source = ProxyArray(wp.empty(shape, dtype=dtype, device="cpu")) + with pytest.raises(ValueError, match=message): + PpispProcessor(PpispProcessorCfg(), {"rgb_hdr": source}) + + +def test_ppisp_processor_requires_resolved_configuration(): + """An image operation must not discover its settings from a camera during preparation.""" + source = ProxyArray(wp.empty((1, 2, 2, 3), dtype=wp.float32, device="cpu")) + with pytest.raises(ValueError, match="resolve USD camera attributes"): + PpispProcessor( + PpispProcessorCfg(params={"cfg": PpispCfg(camera_prim_path="/World/Camera")}), {"rgb_hdr": source} + )