diff --git a/docs/source/api/lab/isaaclab.envs.mdp.rst b/docs/source/api/lab/isaaclab.envs.mdp.rst index 9775f0593305..4cffb72ae505 100644 --- a/docs/source/api/lab/isaaclab.envs.mdp.rst +++ b/docs/source/api/lab/isaaclab.envs.mdp.rst @@ -9,6 +9,9 @@ Observations .. automodule:: isaaclab.envs.mdp.observations :members: +.. automodule:: isaaclab.envs.mdp.sensor_observations + :members: + Actions ------- diff --git a/docs/source/api/lab/isaaclab.managers.rst b/docs/source/api/lab/isaaclab.managers.rst index d378386b86de..b4cd0fc7dfcc 100644 --- a/docs/source/api/lab/isaaclab.managers.rst +++ b/docs/source/api/lab/isaaclab.managers.rst @@ -103,7 +103,7 @@ On each recorded sample, each environment retains its lag with probability 0.8 a one. Zero redraws on every sample. Holding the lag keeps the latency constant while frames continue to advance; it does not freeze the returned frame. Reset always redraws the selected environments' lags. -The processing order is observation function, modifiers, noise, clipping, scaling, delay, then history. +The processing order is observation function, tensor processors, noise, clipping, scaling, delay, then history. The ``delay_min_lag`` and ``delay_max_lag`` field names and delay placement follow `mjlab's observation configuration `_. ``delay_hold_prob`` follows mjlab's lag-retention semantics, with a default of 1.0 to preserve Isaac Lab's @@ -115,7 +115,7 @@ For a delay buffer with no recorded sample after initialization or reset, the cu without recording it. Once recording starts, delays exceeding the available history return the oldest sample. Partial resets invalidate only the selected environments' histories. -Observation outputs are independent snapshots: later simulation steps, modifier updates, and resets +Observation outputs are independent snapshots: later simulation steps, processor updates, and resets do not overwrite previously returned tensors. The manager automatically copies borrowed storage before mutation or retention. Term and custom callback outputs are treated conservatively, without decorators or task-level copy settings. Clipping and scaling create independent storage when needed, which later @@ -131,7 +131,7 @@ The manager checks the signature once during initialization. Only an explicitly ``out=None`` enables destination writes; accepting arbitrary ``**kwargs`` does not. The manager probes the ordinary call to infer shape, dtype and device, which must remain constant, then allocates a fresh contiguous destination for each computation. ``out`` is reserved and must not appear in term ``params``. -Custom modifiers, noise and history retain their usual snapshot protections. Built-in +Custom tensor processors, noise and history retain their usual snapshot protections. Built-in :class:`~isaaclab.envs.mdp.observations.image_rgb` terms use this interface automatically to avoid a second full-image copy for normalized uint8 images. Existing observation configurations require no changes. diff --git a/docs/source/api/lab/isaaclab.sensors.camera.rst b/docs/source/api/lab/isaaclab.sensors.camera.rst index e86b1ada9795..89ece542a6c0 100644 --- a/docs/source/api/lab/isaaclab.sensors.camera.rst +++ b/docs/source/api/lab/isaaclab.sensors.camera.rst @@ -13,7 +13,12 @@ The following classes are part of the public :mod:`isaaclab.sensors.camera` API. .. autosummary:: :nosignatures: + CameraExposureCfg CameraISPMode .. autoclass:: CameraISPMode :show-inheritance: + +.. autoclass:: CameraExposureCfg + :members: + :exclude-members: __init__ diff --git a/docs/source/api/lab/isaaclab.utils.rst b/docs/source/api/lab/isaaclab.utils.rst index d9cdd0e5cfa2..d6c9d85756ff 100644 --- a/docs/source/api/lab/isaaclab.utils.rst +++ b/docs/source/api/lab/isaaclab.utils.rst @@ -17,7 +17,7 @@ logger math mesh - modifiers + tensor_processors noise seed sensors @@ -124,10 +124,19 @@ Mesh operations :imported-members: :show-inheritance: -Modifier operations -~~~~~~~~~~~~~~~~~~~ +Tensor processors +~~~~~~~~~~~~~~~~~ + +The processor interfaces live together in :mod:`isaaclab.utils.tensor_processors`: + +* ``TensorProcessor``: tensor to tensor. +* ``DictTensorProcessor``: dictionary of named tensors to dictionary of named tensors. + +Dictionary tensors use ``ProxyArray`` for Warp and Torch access without copying. Processors own +any persistent outputs and state. ``DictTensorProcessorCfg`` declares input/output keys for camera +setup. Camera and observation consumers compose processors and preserve untouched keys. -.. automodule:: isaaclab.utils.modifiers +.. automodule:: isaaclab.utils.tensor_processors :members: :imported-members: :special-members: __call__ diff --git a/docs/source/api/lab_ppisp/isaaclab_ppisp.rst b/docs/source/api/lab_ppisp/isaaclab_ppisp.rst index 79b9565d1d77..ed2117d16a9c 100644 --- a/docs/source/api/lab_ppisp/isaaclab_ppisp.rst +++ b/docs/source/api/lab_ppisp/isaaclab_ppisp.rst @@ -14,10 +14,33 @@ The following classes are part of the public :mod:`isaaclab_ppisp` API. :nosignatures: PpispCfg + PpispProcessorCfg + PpispProcessor + PpispControllerWeights + PpispDiscoveryMode PpispPipeline .. autoclass:: PpispCfg :show-inheritance: +.. autoclass:: PpispProcessorCfg + :members: + :inherited-members: + :exclude-members: __init__ + :show-inheritance: + +.. autoclass:: PpispProcessor + :members: + :inherited-members: + :special-members: __call__ + :show-inheritance: + +.. autoclass:: PpispControllerWeights + :members: + +.. autoclass:: PpispDiscoveryMode + :members: + :show-inheritance: + .. autoclass:: PpispPipeline :show-inheritance: diff --git a/docs/source/concepts/sensors/camera.rst b/docs/source/concepts/sensors/camera.rst index c07d317e6b67..6a35e36cfc3a 100644 --- a/docs/source/concepts/sensors/camera.rst +++ b/docs/source/concepts/sensors/camera.rst @@ -153,8 +153,9 @@ updating it adds frame-query overhead. Output types ------------ -The active renderer validates ``data_types`` and allocates the channel count and data type declared -by its :class:`~isaaclab.renderers.RenderBufferSpec`. +For raw outputs, the active renderer validates the requested names and allocates the channel count +and data type declared by its :class:`~isaaclab.renderers.RenderBufferSpec`. A camera pipeline may +produce additional output names; its processors allocate those buffers. .. list-table:: Common output contracts :header-rows: 1 @@ -168,7 +169,7 @@ by its :class:`~isaaclab.renderers.RenderBufferSpec`. - Low-dynamic-range color * - ``rgb_hdr`` - 3, ``float32`` - - Scene-linear high-dynamic-range color + - High-dynamic-range RGB using the active camera settings * - ``albedo`` - 4, ``uint8`` - Material base color @@ -198,6 +199,12 @@ by its :class:`~isaaclab.renderers.RenderBufferSpec`. non-colorized segmentation uses one ``int32`` ID channel. Label and prim-path mappings are stored in ``camera_data.info[output_name]``. +``rgb_hdr`` uses the camera's active exposure settings. PPISP requires scene-linear HDR; +set ``CameraCfg.exposure=CameraExposureCfg()`` explicitly to disable RTX automatic exposure and +use neutral exposure. Leaving ``exposure=None`` preserves authored camera settings. Selecting an +output never changes exposure. PPISP requests ``rgb_hdr`` privately; include it in ``data_types`` +only to publish the raw input as well. + .. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/camera-renderer-isaac-rtx.webp :align: center :figwidth: 100% @@ -263,6 +270,12 @@ current support matrix is: - Yes - No +.. note:: + + Explicit exposure settings affect every output from the camera prim, including other sensors + sharing it. Use separate camera prims when different exposures are required. Newton's HDR + output is already scene-linear and does not consume RTX exposure attributes. + Querying an unsupported output fails during camera initialization. Renderer configuration controls semantic filters, segmentation colorization, and depth clipping where those options are backend-specific. @@ -305,47 +318,168 @@ background. Set a normalized RGB tuple to use a solid color for pixels that miss The setting is per camera. Cameras with renderer-default and solid backgrounds can coexist in one scene. -Post-render image signal processing ------------------------------------ - -:attr:`~sensors.CameraCfg.isp_cfg` optionally applies an image signal processing (ISP) pass to the -renderer's scene-linear HDR output. The shipped implementation is PPISP (Physically Plausible Image -Signal Processing), which applies responsivity, exposure, vignetting, color correction, and a camera -response function before writing ``rgb`` or ``rgba``. +Post-render image processing +---------------------------- -The field accepts: +:attr:`~sensors.CameraCfg.pipeline` composes dictionary processors after rendering. +:class:`~isaaclab.utils.tensor_processors.DictTensorProcessor` takes a dictionary of named tensors +and returns a dictionary of named tensors. It lives beside +:class:`~isaaclab.utils.tensor_processors.TensorProcessor`, whose call takes and returns one tensor. +:class:`~isaaclab.utils.tensor_processors.DictTensorProcessorCfg` declares the actual input and output +keys needed for renderer setup. The shared dictionary interface also accepts nonvisual sensor data; +each processor defines its layout and value semantics. -* ``None`` to disable post-render ISP. -* :class:`~isaaclab_ppisp.PpispCfg` for explicit coefficients or coefficients imported from a USD - camera. -* :class:`~sensors.CameraISPMode` to discover ``ppisp:*`` attributes on a camera prim. +The camera resolves external inputs before renderer initialization, including inputs needed only by the +pipeline. It publishes the names in ``data_types`` after processing. For example, this camera requests +private ``rgb_hdr`` from its renderer and publishes PPISP's processed ``rgb``: .. code-block:: python - from isaaclab.sensors.camera import CameraCfg, CameraISPMode + from isaaclab.sensors.camera import CameraExposureCfg from isaaclab.utils import replace - from isaaclab_ppisp import PpispCfg + from isaaclab_ppisp import PpispCfg, PpispProcessorCfg + + processed_camera = replace( + front_camera, + data_types=["rgb"], + exposure=CameraExposureCfg(), + pipeline=[PpispProcessorCfg(params={"cfg": PpispCfg(inputs={"exposureOffset": 1.5})})], + ) + +PPISP (Physically Plausible Image Signal Processing) applies responsivity, exposure, vignetting, color +correction, and a camera response function. It consumes float32 scene-linear radiance and produces +uint8 color. Set ``outputs=["rgba"]`` and ``params={"cfg": ppisp_cfg, "channels": 4}`` on the +processor configuration for RGBA output. A pipeline-produced RGBA buffer also supplies a zero-copy +RGB view when ``rgb`` is requested. A pipeline producing only RGB cannot satisfy an explicit RGBA request. + +For custom dictionary processing, subclass ``DictTensorProcessor``. Its constructor receives +``(cfg, data)`` with actual ``ProxyArray`` tensors. Allocate persistent output tensors in +``self.outputs`` without reading input values or advancing state. Implement ``__call__(data)`` +to fill and return those named outputs, and override ``reset`` or ``close`` only when needed. +The pipeline passes the current dictionary to each processor and merges its returned outputs; +untouched keys pass through. Shape, dtype and device come from the arrays directly. + +``PpispProcessor`` subclasses ``DictTensorProcessor`` and can also be constructed and called directly: + +.. code-block:: python + + from isaaclab.utils import instantiate + + cfg = PpispProcessorCfg() + processor = instantiate(cfg, {"rgb_hdr": hdr_tensor}) + color = processor({"rgb_hdr": hdr_tensor})["rgb"] + processor.close() + +Here ``hdr_tensor`` is a scene-linear float32 ``ProxyArray`` with shape ``(N, H, W, 3)``. +The returned color is uint8; copy it before another call if its values must be retained. + +Published camera outputs must keep the camera's batch size, height, width, and NHWC layout so pose +and calibration metadata remain valid. Intermediate buffers may change shape. Use ``processed_sensor`` +below when the final observation should have a different image shape or layout. + +``camera.render_outputs`` exposes borrowed raw renderer buffers, including private inputs. Their +``ProxyArray`` properties provide shape, dtype, and device. Reading raw outputs updates the camera +when due. Neither a processor nor a caller may mutate borrowed inputs. Public processed +buffers and raw buffers retain their storage until camera reinitialization, but later captures replace +their contents; clone values that must survive another capture. Each camera owns its pipeline and processor +workspace. Processors may share explicitly supplied immutable model resources. +Configuration copying must preserve the identity of borrowed resources: PPISP provides +``PpispControllerWeights`` for device weights. Custom processors can use a resource wrapper with +identity-preserving ``__deepcopy__``; raw Warp allocations are not configuration values. - explicit_isp = replace(front_camera, data_types=["rgb"], isp_cfg=PpispCfg(inputs={"exposureOffset": 1.5})) +Pipelines run after scalar and batched captures. When an asynchronous renderer republishes a capture, +the camera retains its processed result and matching capture metadata. Camera reset forwards only the +selected environment indices to processor reset callbacks; teardown closes the processors. For an +observation-only pipeline, declare raw dependencies with ``camera.request_render_inputs(names)`` before +simulation initialization. Requests for new inputs after initialization fail because renderer setup is +already complete. Observation consumers use their own pipeline and snapshot results needed across captures. - discovered_isp = replace(front_camera, data_types=["rgb"], isp_cfg=CameraISPMode.AUTO_CAMERA) +Processing sensor observations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``AUTO_CAMERA`` checks the first matched camera prim. ``AUTO_ANY`` falls back to the first PPISP -camera anywhere on the stage. Discovery happens once during camera construction. +Use :class:`~isaaclab.envs.mdp.processed_sensor` for observation-only processing, including processors +that combine several camera outputs or change image dimensions. The term currently binds sensors +through their ``request_render_inputs()`` and ``render_outputs`` interface: -PPISP is composed by Isaac RTX, OVRTX, and Newton Warp. It requires ``rgb`` or ``rgba`` output. A -static configuration is shared by all cloned views in one camera batch; controller weights may -predict per-view exposure and color parameters, while the remaining coefficients stay shared. ISP -configuration and discovered USD attributes are fixed for the camera lifetime. +.. code-block:: python + + from isaaclab.envs import mdp + from isaaclab.managers import ObservationTermCfg, SceneEntityCfg + from isaaclab_ppisp import PpispProcessorCfg + + front_camera.exposure = CameraExposureCfg() + policy_image = ObservationTermCfg( + func=mdp.processed_sensor, + params={ + "sensor_cfg": SceneEntityCfg("front_camera"), + "pipeline": [PpispProcessorCfg()], + "output_name": "rgb", + }, + ) + +Manager-based environments collect the term's private renderer inputs before simulation reset. A +standalone observation manager must call ``ObservationManager.prepare_scene(cfg, env)`` before reset. +The term processes raw renderer outputs even when the camera has its own pipeline. To read the +camera's published processed output, use an existing image term such as ``mdp.image_rgb`` instead. +The observation pipeline executes on every term evaluation and returns a snapshot; its state belongs to +that term rather than the camera's capture cadence. + +For a single incoming tensor, :class:`~isaaclab.utils.tensor_processors.DictTensorProcessorAdapterCfg` uses the +same dictionary processors while preserving preceding observation transformations. This includes nonvisual +observation tensors from sensors without the render-input interface. This form must preserve shape; +use ``processed_sensor`` for multiple render inputs or changes to render-output shapes, or construct +a ``DictTensorProcessor`` directly for other named buffers. The first incoming tensor constructs the +dictionary processors; later calls reuse their output storage. Its values must meet each processor's +documented input requirements, including any transformations from preceding tensor processors: + +.. code-block:: python + + from isaaclab.utils import tensor_processors + + # This camera publishes radiance for the ordinary image term to read. + front_camera = replace(front_camera, data_types=["rgb_hdr"], exposure=CameraExposureCfg(), pipeline=[]) + policy_image = ObservationTermCfg( + func=mdp.image_rgb, + params={"sensor_cfg": SceneEntityCfg("front_camera"), "data_type": "rgb_hdr", "normalize": False}, + tensor_processors=[ + tensor_processors.DictTensorProcessorAdapterCfg( + pipeline=[PpispProcessorCfg()], + input_name="rgb_hdr", + output_name="rgb", + ) + ], + ) -.. important:: +Migrating renderer-owned ISP configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - With Isaac RTX and OVRTX, enabling ``isp_cfg`` makes PPISP the ISP authority. The renderer - disables RTX auto-exposure, authors neutral ``exposure:*`` values, and applies the - ``OmniRtxCameraAutoExposureAPI_1`` and ``OmniRtxCameraExposureAPI_1`` schemas on every matched - camera prim so RTX does not process the image a second time. Do not combine ``isp_cfg`` with - separately authored RTX exposure or tonemapping settings. When ``isp_cfg`` is ``None``, the - renderer leaves authored camera exposure unchanged. +``CameraCfg.isp_cfg`` no longer executes image processing. A non-None value raises a migration error; +replace it with ``CameraCfg.pipeline`` as above. ``CameraISPMode`` remains importable for legacy +configuration values, but new code should resolve USD attributes explicitly before constructing the +processor: + +.. code-block:: python + + from isaaclab_ppisp import PpispDiscoveryMode, PpispProcessorCfg, resolve_and_normalize + + ppisp_cfg = resolve_and_normalize(PpispDiscoveryMode.AUTO_CAMERA, stage, "/World/Camera") + discovered_camera = replace( + front_camera, + data_types=["rgb"], + exposure=CameraExposureCfg(), + pipeline=[PpispProcessorCfg(params={"cfg": ppisp_cfg})] if ppisp_cfg is not None else [], + ) + +``AUTO_CAMERA`` checks the supplied camera prim; ``AUTO_ANY`` can fall back to the first PPISP camera +on the supplied stage. Alternatively, pass an already resolved camera prim to +``ppisp_cfg_from_usd_camera(camera_prim)``. Discovery has no implicit simulation or stage lookup inside +the processor. Static coefficients apply to the whole camera batch; controller weights may predict +per-view exposure and color parameters. + +For either processing attachment, configure the source camera with +``exposure=CameraExposureCfg()`` before simulation initialization. PPISP's own exposure is set +through ``PpispCfg.inputs["exposureOffset"]`` (stops). Processing does not discover or change +renderer exposure settings. Run the ``ppisp-camera`` example for a complete PPISP workflow. It supports ``--renderer newton_renderer`` (Newton Warp, the default) and ``--renderer isaac_rtx`` (Isaac RTX). @@ -414,3 +548,9 @@ A runnable camera example is available as ``camera``: For saving output to disk, see :doc:`/source/how-to/save_camera_output`. For renderer selection and customization, see :doc:`/source/how-to/configure_rendering`. + +The observation processor API is ``isaaclab.utils.tensor_processors``. Custom stateful processors +inherit ``TensorProcessor`` and use ``TensorProcessorCfg``; attach configurations through +``ObservationTermCfg.tensor_processors``. These replace ``ModifierBase``, ``ModifierCfg``, and +``ObservationTermCfg.modifiers`` respectively. ``DictTensorProcessorAdapterCfg`` adapts a dictionary +processor pipeline to that single-tensor interface. diff --git a/examples/sensors/ppisp_camera.py b/examples/sensors/ppisp_camera.py index 79e95688eb37..ad3a8da4a8db 100644 --- a/examples/sensors/ppisp_camera.py +++ b/examples/sensors/ppisp_camera.py @@ -120,12 +120,13 @@ import matplotlib.pyplot as plt import numpy as np import torch -from isaaclab_ppisp.cfg import PpispCfg, ppisp_cfg_from_usd_camera +from isaaclab_ppisp import PpispCfg, PpispProcessorCfg, ppisp_cfg_from_usd_camera import isaaclab.sim as sim_utils from isaaclab.assets import AssetBaseCfg, RigidObjectCfg from isaaclab.scene import InteractiveSceneCfg from isaaclab.sensors import CameraCfg +from isaaclab.sensors.camera import CameraExposureCfg from isaaclab.sim.spawners.materials import UsdPhysicsRigidBodyMaterialCfg from isaaclab.utils import configclass, instantiate @@ -374,7 +375,8 @@ def make_camera(camera_prim_path: str, *, ppisp_cfg: PpispCfg | None, width: int width=width, data_types=["rgb"], spawn=None, - isp_cfg=ppisp_cfg, + exposure=CameraExposureCfg() if ppisp_cfg is not None else None, + pipeline=[PpispProcessorCfg(params={"cfg": ppisp_cfg})] if ppisp_cfg is not None else [], renderer_cfg=make_renderer_cfg(), ) return instantiate(camera_cfg) diff --git a/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..393b39b971a2 --- /dev/null +++ b/source/isaaclab/changelog.d/feat-render-buffer-pipe.changed.rst @@ -0,0 +1,7 @@ +* **Breaking:** Added ``DictTensorProcessor`` for named buffers and renamed the modifiers API to + ``tensor_processors``, ``TensorProcessor``, and ``TensorProcessorCfg``. Replace observation + ``modifiers`` with ``tensor_processors``; use ``DictTensorProcessorAdapterCfg`` for dictionary operations. +* Replaced renderer-owned PPISP with camera and observation processors consuming ``rgb_hdr``. + Set ``CameraCfg.exposure=CameraExposureCfg()`` explicitly for neutral RTX HDR before PPISP. +* Colocated dictionary and single-tensor processors and removed preparation wrappers and role mappings. + Dictionary processors now own their output tensors and expose a direct dictionary-to-dictionary call. diff --git a/source/isaaclab/changelog.d/feat-render-buffer-pipe.fixed.rst b/source/isaaclab/changelog.d/feat-render-buffer-pipe.fixed.rst new file mode 100644 index 000000000000..db061e515934 --- /dev/null +++ b/source/isaaclab/changelog.d/feat-render-buffer-pipe.fixed.rst @@ -0,0 +1,2 @@ +* Avoided unnecessary CUDA synchronization during masked camera resets when image pipeline + operations had no reset callbacks. diff --git a/source/isaaclab/changelog.d/feat-render-buffer-pipe.major b/source/isaaclab/changelog.d/feat-render-buffer-pipe.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab/isaaclab/envs/manager_based_env.py b/source/isaaclab/isaaclab/envs/manager_based_env.py index f4026af3bd43..51fdbbb4fd8c 100644 --- a/source/isaaclab/isaaclab/envs/manager_based_env.py +++ b/source/isaaclab/isaaclab/envs/manager_based_env.py @@ -187,6 +187,8 @@ def _init_sim(self): if "prestartup" in self.event_manager.available_modes: self.event_manager.apply(mode="prestartup") + ObservationManager.prepare_scene(self.cfg.observations, self) + # play the simulator to activate physics handles # note: this activates the physics simulation view that exposes TensorAPIs # note: when started in extension mode, first call sim.reset_async() and then initialize the managers @@ -609,6 +611,12 @@ def close(self): # destructor is order-sensitive del self.action_manager + observation_error = None + try: + self.observation_manager.close() + except Exception as error: + # A user operation must not prevent scene and simulation teardown. + observation_error = error del self.observation_manager del self.event_manager del self.recorder_manager @@ -622,6 +630,8 @@ def close(self): self._window = None # update closing status self._is_closed = True + if observation_error is not None: + raise observation_error """ Helper functions. diff --git a/source/isaaclab/isaaclab/envs/mdp/__init__.pyi b/source/isaaclab/isaaclab/envs/mdp/__init__.pyi index 4bcfd3c0d824..500a12b82456 100644 --- a/source/isaaclab/isaaclab/envs/mdp/__init__.pyi +++ b/source/isaaclab/isaaclab/envs/mdp/__init__.pyi @@ -89,6 +89,7 @@ __all__ = [ "image_normals", "image_rgb", "image_segmentation", + "processed_sensor", "imu_ang_vel", "imu_lin_acc", "pva_orientation", @@ -270,6 +271,7 @@ from .observations import ( root_pos_w, root_quat_w, ) +from .sensor_observations import processed_sensor from .recorders import ( ActionStateRecorderManagerCfg, InitialStateRecorder, diff --git a/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py b/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py new file mode 100644 index 000000000000..8e7b1e478f6f --- /dev/null +++ b/source/isaaclab/isaaclab/envs/mdp/sensor_observations.py @@ -0,0 +1,83 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Observation adapters for processing named sensor buffers.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING + +import torch +import warp as wp + +from isaaclab.managers import ManagerTermBase, ObservationTermCfg, SceneEntityCfg +from isaaclab.utils.tensor_processors import DictTensorProcessorCfg +from isaaclab.utils.tensor_processors._pipeline import _DictTensorProcessorPipeline + +if TYPE_CHECKING: + from isaaclab.envs import ManagerBasedEnv + + +class processed_sensor(ManagerTermBase): + """Apply dictionary processors to named sensor render outputs. + + The sensor must expose ``request_render_inputs`` and ``render_outputs``, as + :class:`~isaaclab.sensors.Camera` does. Processors define the supported tensor layouts and + value semantics; outputs need not be visual images. + + Dependencies are requested before simulation initialization through + :meth:`~isaaclab.managers.ObservationManager.prepare_scene`. They remain private to the + renderer buffer bundle and need not appear in ``CameraCfg.data_types``. Each term owns + its own processor state and output storage. The pipeline runs on every observation evaluation; + returned tensors are snapshots, so later evaluations cannot overwrite retained history. + + Use this term for multiple render inputs or changed buffer dimensions. To process other + sensor data or the result of earlier observation tensor processors, use + :class:`~isaaclab.utils.tensor_processors.DictTensorProcessorAdapter` on an observation tensor. + To read a camera's already processed public output, use the existing image observation terms. + """ + + @classmethod + def prepare_scene(cls, cfg: ObservationTermCfg, env: ManagerBasedEnv) -> None: + sensor_cfg = cfg.params.get("sensor_cfg", SceneEntityCfg("camera")) + output_name = cfg.params.get("output_name", "rgb") + requirements = _DictTensorProcessorPipeline.required_source_names(cfg.params["pipeline"], [output_name]) + env.scene.sensors[sensor_cfg.name].request_render_inputs(requirements) + + def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedEnv): + super().__init__(cfg, env) + self._sensor = env.scene.sensors[cfg.params["sensor_cfg"].name] + self._output_name = cfg.params["output_name"] + buffers = self._sensor.render_outputs + device = wp.get_device(self.device) + stream = wp.stream_from_torch(torch.cuda.current_stream(str(device))) if device.is_cuda else None + with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): + self._pipe = _DictTensorProcessorPipeline(buffers, cfg.params["pipeline"]) + if self._output_name not in self._pipe.outputs: + self._pipe.close() + raise ValueError(f"Buffer processing does not provide output {self._output_name!r}.") + + def __call__( + self, + env: ManagerBasedEnv, + pipeline: Sequence[DictTensorProcessorCfg], + sensor_cfg: SceneEntityCfg = SceneEntityCfg("camera"), + output_name: str = "rgb", + *, + out: torch.Tensor | None = None, + ) -> torch.Tensor: + """Return a snapshot of the selected processor output. + + The observation manager supplies ``out`` to receive the snapshot without another clone. + Direct callers may omit it to allocate a new snapshot. + """ + return self._pipe.as_tensor(self._sensor.render_outputs, output_name=self._output_name, out=out) + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + self._pipe.reset(env_ids) + + def close(self) -> None: + self._pipe.close() diff --git a/source/isaaclab/isaaclab/managers/manager_base.py b/source/isaaclab/isaaclab/managers/manager_base.py index 1ca1435f6c76..c010af425cc0 100644 --- a/source/isaaclab/isaaclab/managers/manager_base.py +++ b/source/isaaclab/isaaclab/managers/manager_base.py @@ -16,7 +16,7 @@ from ..physics import PhysicsEvent, PhysicsManager from ..utils import string as string_utils from ..utils import string_to_callable, to_dict -from ..utils.modifiers import ModifierCfg +from ..utils.tensor_processors import TensorProcessorCfg from .manager_term_cfg import ManagerTermBaseCfg from .scene_entity_cfg import SceneEntityCfg @@ -90,6 +90,19 @@ def __name__(self) -> str: Operations. """ + @classmethod + def prepare_scene(cls, cfg: ManagerTermBaseCfg, env: ManagerBasedEnv) -> None: + """Declare scene requirements before simulation initialization. + + This hook may request sensor outputs but must not read runtime data or allocate + processing buffers. Observation terms are visited by :meth:`ObservationManager.prepare_scene`. + """ + pass + + def close(self) -> None: + """Release resources owned by the term before the scene closes.""" + pass + def reset(self, env_ids: Sequence[int] | None = None) -> None: """Resets the manager term. @@ -434,7 +447,7 @@ def _resolve_param_value( self._scene_entity_cfgs.append(value) elif isinstance(value, ManagerTermBaseCfg): self._process_term_cfg_at_play(f"{term_name}.{key}", value) - elif isinstance(value, ModifierCfg): + elif isinstance(value, TensorProcessorCfg): for field in fields(value): field_value = getattr(value, field.name) resolved_value = self._resolve_param_value( diff --git a/source/isaaclab/isaaclab/managers/manager_term_cfg.py b/source/isaaclab/isaaclab/managers/manager_term_cfg.py index 01df0af92391..985339b64e8a 100644 --- a/source/isaaclab/isaaclab/managers/manager_term_cfg.py +++ b/source/isaaclab/isaaclab/managers/manager_term_cfg.py @@ -14,8 +14,8 @@ import torch from ..utils import configclass -from ..utils.modifiers import ModifierCfg from ..utils.noise import NoiseCfg, NoiseModelCfg +from ..utils.tensor_processors import TensorProcessorCfg from .scene_entity_cfg import SceneEntityCfg if TYPE_CHECKING: @@ -167,15 +167,15 @@ class ObservationTermCfg(ManagerTermBaseCfg): shape (num_envs, obs_term_dim). """ - modifiers: list[ModifierCfg] | None = None - """The list of data modifiers to apply to the observation in order. Defaults to None, + tensor_processors: list[TensorProcessorCfg] | None = None + """The list of tensor processors to apply to the observation in order. Defaults to None, in which case no modifications will be applied. - Modifiers are applied in the order they are specified in the list. They can be stateless + Processors are applied in the order they are specified in the list. They can be stateless or stateful, and can be used to apply transformations to the observation data. For example, - a modifier can be used to normalize the observation data or to apply a rolling average. + a processor can be used to normalize the observation data or to apply a rolling average. - For more information on modifiers, see the :class:`~isaaclab.utils.modifiers.ModifierCfg` class. + For more information, see :class:`~isaaclab.utils.tensor_processors.TensorProcessorCfg`. """ noise: NoiseCfg | NoiseModelCfg | None = None @@ -206,7 +206,7 @@ class ObservationTermCfg(ManagerTermBaseCfg): delay_max_lag: int = 0 """Maximum observation delay, counted in recorded samples. Zero disables delay. - Set both lag bounds equal for a constant delay. Delay is applied after modifiers, noise, clipping, and + Set both lag bounds equal for a constant delay. Delay is applied after tensor processors, noise, clipping, and scaling, before observation history. Until enough samples have been recorded, the oldest available sample is returned. After reset, no data from the previous episode is returned. """ diff --git a/source/isaaclab/isaaclab/managers/observation_manager.py b/source/isaaclab/isaaclab/managers/observation_manager.py index a4ab4fefd3f4..51094dafeea6 100644 --- a/source/isaaclab/isaaclab/managers/observation_manager.py +++ b/source/isaaclab/isaaclab/managers/observation_manager.py @@ -8,7 +8,9 @@ from __future__ import annotations import inspect -from collections.abc import Sequence +from builtins import ExceptionGroup +from collections.abc import Iterator, Sequence +from dataclasses import fields from typing import TYPE_CHECKING import numpy as np @@ -16,10 +18,10 @@ from prettytable import PrettyTable from ..envs.utils.io_descriptors import _warn_io_descriptors_deprecated -from ..utils import instantiate, modifiers, noise, to_dict +from ..utils import instantiate, noise, string_to_callable, tensor_processors, to_dict from ..utils.buffers import CircularBuffer, DelayBuffer from .manager_base import ManagerBase, ManagerTermBase -from .manager_term_cfg import ObservationGroupCfg, ObservationTermCfg +from .manager_term_cfg import ManagerTermBaseCfg, ObservationGroupCfg, ObservationTermCfg if TYPE_CHECKING: from ..envs import ManagerBasedEnv @@ -58,8 +60,8 @@ class ObservationManager(ManagerBase): called in the order of the terms in the group. The functions are expected to return a tensor with shape (num_envs, ...). - If a noise model or custom modifier is registered for a term, the function is called to corrupt - the observation. The corruption function is expected to return a tensor with the same shape as the observation. + If a noise model or custom tensor processor is registered for a term, it transforms + the observation and returns a tensor with the same shape as the observation. The observations are clipped and scaled as per the configuration settings. Returned observations are independent snapshots, including dictionary entries and history. @@ -85,6 +87,7 @@ def __init__(self, cfg: object, env: ManagerBasedEnv): if cfg is None: raise ValueError("Observation manager configuration is None. Please provide a valid configuration.") + self._is_closed = False # call the base class constructor (this will parse the terms config) super().__init__(cfg, env) @@ -120,6 +123,54 @@ def __init__(self, cfg: object, env: ManagerBasedEnv): # Stores the latest observations. self._obs_buffer: dict[str, torch.Tensor | dict[str, torch.Tensor]] | None = None + @classmethod + def prepare_scene(cls, cfg: object, env: ManagerBasedEnv) -> None: + """Collect observation input requirements before renderer initialization. + + Manager-based environments call this after creating the scene and before resetting + the simulation. Standalone users must call it at the same point when terms require + private sensor outputs. Term instances and processing storage are created later. + """ + if cfg is None: + return + groups = cfg.values() if isinstance(cfg, dict) else vars(cfg).values() + for group in groups: + if not isinstance(group, ObservationGroupCfg): + continue + for term in _iter_term_configs(group): + if not isinstance(term, ManagerTermBaseCfg): + continue + func = string_to_callable(term.func) if isinstance(term.func, str) else term.func + if inspect.isclass(func) and issubclass(func, ManagerTermBase): + func.prepare_scene(term, env) + + def close(self) -> None: + """Close every owned term and tensor processor once, including nested configurations. + + All resources are closed before any callback errors are raised. Repeated calls are safe. + """ + if self._is_closed: + return + self._is_closed = True + resources = [] + seen = set() + for cfg in _iter_term_configs(self._group_obs_term_cfgs): + resource = cfg.func + if isinstance(resource, (ManagerTermBase, tensor_processors.TensorProcessor)) and id(resource) not in seen: + seen.add(id(resource)) + resources.append(resource) + errors = [] + for resource in reversed(resources): + try: + resource.close() + except Exception as error: + errors.append(error) + self._obs_buffer = None + if len(errors) == 1: + raise errors[0] + if errors: + raise ExceptionGroup("Failed to close observation resources.", errors) + def __str__(self) -> str: """Returns: A string representation for the observation manager.""" msg = f" contains {len(self._group_obs_term_names)} groups.\n" @@ -277,7 +328,7 @@ def _collect_io_descriptors(self, group_names_to_export: list[str] = ["policy"]) for k, v in term_cfg.__dict__.items(): # For now we do not add the noise modifier if k in [ - "modifiers", + "tensor_processors", "clip", "scale", "delay_min_lag", @@ -315,7 +366,7 @@ def _collect_io_descriptors(self, group_names_to_export: list[str] = ["policy"]) "flatten_history_dim", ]: formatted_item["overloads"][k] = v - elif k in ["modifiers", "description", "units"]: + elif k in ["tensor_processors", "description", "units"]: formatted_item["extras"][k] = v else: formatted_item[k] = v @@ -338,7 +389,7 @@ def reset(self, env_ids: Sequence[int] | None = None) -> dict[str, float]: self._group_obs_term_delay_buffer[group_name][term_name].reset(env_ids) if term_name in self._group_obs_term_history_buffer[group_name]: self._group_obs_term_history_buffer[group_name][term_name].reset(batch_ids=env_ids) - # call all modifiers that are classes + # call all tensor_processors that are classes for mod in self._group_obs_class_instances: mod.reset(env_ids=env_ids) @@ -382,7 +433,7 @@ def compute_group(self, group_name: str, update_history: bool = False) -> torch. The following steps are performed for each observation term: 1. Compute observation term by calling the function - 2. Apply custom modifiers in the order specified in :attr:`ObservationTermCfg.modifiers` + 2. Apply custom tensor processors in the order specified in :attr:`ObservationTermCfg.tensor_processors` 3. Apply corruption/noise model based on :attr:`ObservationTermCfg.noise` 4. Apply clipping based on :attr:`ObservationTermCfg.clip` 5. Apply scaling based on :attr:`ObservationTermCfg.scale` @@ -419,7 +470,7 @@ def compute_group(self, group_name: str, update_history: bool = False) -> torch. term_cfgs = self._group_obs_term_cfgs[group_name] obs_terms = zip(group_term_names, term_cfgs) - # evaluate terms: compute, add noise, clip, scale, custom modifiers + # evaluate terms: compute, add noise, clip, scale, custom tensor_processors for term_name, term_cfg in obs_terms: output_spec = self._term_output_specs.get((group_name, term_name)) owned = output_spec is not None @@ -433,12 +484,12 @@ def compute_group(self, group_name: str, update_history: bool = False) -> torch. else: obs = term_cfg.func(self._env, **term_cfg.params) # apply post-processing - if term_cfg.modifiers is not None: - for modifier in term_cfg.modifiers: + if term_cfg.tensor_processors is not None: + for modifier in term_cfg.tensor_processors: owned = False # Custom callbacks may modify or retain their input. obs = obs.clone() - if isinstance(modifier.func, modifiers.ModifierBase): + if isinstance(modifier.func, tensor_processors.TensorProcessor): obs = modifier.func(obs) else: obs = modifier.func(obs, **modifier.params) @@ -564,9 +615,9 @@ def _prepare_terms(self): self._group_obs_term_delay_buffer: dict[str, dict[str, DelayBuffer]] = {} self._group_obs_term_history_buffer: dict[str, dict] = {} - # create a list to store classes instances, e.g., for modifiers and noise models + # create a list to store classes instances, e.g., for tensor_processors and noise models # we store it as a separate list to only call reset on them and prevent unnecessary calls - self._group_obs_class_instances: list[modifiers.ModifierBase | noise.NoiseModel] = [] + self._group_obs_class_instances: list[tensor_processors.TensorProcessor | noise.NoiseModel] = [] # make sure the simulation is playing since we compute obs dims which needs asset quantities if not self._env.sim.is_playing(): @@ -676,23 +727,23 @@ def _prepare_terms(self): # cast the scale into torch tensor term_cfg.scale = torch.tensor(term_cfg.scale, dtype=torch.float, device=self._env.device) - # prepare modifiers for each observation - if term_cfg.modifiers is not None: - # initialize list of modifiers for term - for mod_cfg in term_cfg.modifiers: - if not isinstance(mod_cfg, modifiers.ModifierCfg): + # prepare tensor_processors for each observation + if term_cfg.tensor_processors is not None: + # initialize list of tensor_processors for term + for mod_cfg in term_cfg.tensor_processors: + if not isinstance(mod_cfg, tensor_processors.TensorProcessorCfg): raise TypeError( f"Modifier configuration '{mod_cfg}' of observation term '{term_name}' is not of" - f" required type ModifierCfg, Received: '{type(mod_cfg)}'" + f" required type TensorProcessorCfg, Received: '{type(mod_cfg)}'" ) - # construct stateful modifiers with the observation size + # construct stateful tensor_processors with the observation size if inspect.isclass(mod_cfg.func): mod_cfg.func = mod_cfg.func(cfg=mod_cfg, data_dim=obs_dims, device=self._env.device) - if not isinstance(mod_cfg.func, modifiers.ModifierBase): + if not isinstance(mod_cfg.func, tensor_processors.TensorProcessor): raise TypeError( f"Modifier function '{mod_cfg.func}' for observation term '{term_name}'" - f" is not an instance of 'ModifierBase'. Received: '{type(mod_cfg.func)}'." + f" is not an instance of 'TensorProcessor'. Received: '{type(mod_cfg.func)}'." ) self._group_obs_class_instances.append(mod_cfg.func) @@ -770,3 +821,19 @@ def _prepare_terms(self): # add history buffers for each group self._group_obs_term_delay_buffer[group_name] = group_entry_delay_buffer self._group_obs_term_history_buffer[group_name] = group_entry_history_buffer + + +def _iter_term_configs(value: object) -> Iterator[ManagerTermBaseCfg | tensor_processors.TensorProcessorCfg]: + """Visit manager-owned configurations without following term instances back into their configs.""" + if isinstance(value, (ManagerTermBaseCfg, tensor_processors.TensorProcessorCfg, ObservationGroupCfg)): + if isinstance(value, (ManagerTermBaseCfg, tensor_processors.TensorProcessorCfg)): + yield value + for field in fields(value): + if field.name != "func": + yield from _iter_term_configs(getattr(value, field.name)) + elif isinstance(value, dict): + for child in value.values(): + yield from _iter_term_configs(child) + elif isinstance(value, (list, tuple)): + for child in value: + yield from _iter_term_configs(child) diff --git a/source/isaaclab/isaaclab/renderers/output_contract.py b/source/isaaclab/isaaclab/renderers/output_contract.py index fdd7d9e9f248..7ee8025b6693 100644 --- a/source/isaaclab/isaaclab/renderers/output_contract.py +++ b/source/isaaclab/isaaclab/renderers/output_contract.py @@ -26,6 +26,7 @@ class RenderBufferKind(StrEnum): RGB = "rgb" RGBA = "rgba" RGB_HDR = "rgb_hdr" + """HDR RGB using the camera's active rendering settings.""" ALBEDO = "albedo" DEPTH = "depth" DISTANCE_TO_IMAGE_PLANE = "distance_to_image_plane" diff --git a/source/isaaclab/isaaclab/renderers/render_context.py b/source/isaaclab/isaaclab/renderers/render_context.py index cdd1515532b6..2ae854ae9bfa 100644 --- a/source/isaaclab/isaaclab/renderers/render_context.py +++ b/source/isaaclab/isaaclab/renderers/render_context.py @@ -9,8 +9,9 @@ import logging import warnings +import weakref from collections.abc import Sequence -from typing import Any +from typing import TYPE_CHECKING, Any import torch import warp as wp @@ -19,6 +20,9 @@ from .base_renderer import BaseRenderer, VisualMaterialBatch from .renderer_cfg import RendererCfg +if TYPE_CHECKING: + from ..sensors.camera.camera import Camera + logger = logging.getLogger(__name__) @@ -65,6 +69,7 @@ class RenderContext: __slots__ = ( "clone_contexts", "_backend_registry", + "_cameras", "_physics_initialized", "_prepared_renderer_ids", "_prepared_num_envs", @@ -82,6 +87,7 @@ def __init__(self, backend_registry: list[tuple[Any, Any]]) -> None: self.clone_contexts: set[type | str] = set() """Scene representations declared by camera renderers and visualizers before cloning.""" self._backend_registry = backend_registry + self._cameras: weakref.WeakValueDictionary[int, Camera] = weakref.WeakValueDictionary() self._physics_initialized: bool = False # Set to True after the first PHYSICS_READY callback fires. self._prepared_renderer_ids: set[int] = set() self._prepared_num_envs: int | None = None @@ -133,6 +139,19 @@ def ensure_initialize(self) -> None: for _cfg, renderer in self._renderer_entries: renderer.initialize() + def register_camera(self, camera: Camera) -> None: + """Include a camera in stage preparation without extending its lifetime.""" + self._cameras[id(camera)] = camera + + def unregister_camera(self, camera: Camera) -> None: + """Stop preparing a camera whose sensor callbacks have been cleared.""" + self._cameras.pop(id(camera), None) + + def prepare_cameras(self) -> None: + """Prepare every live camera before any sensor can export the shared renderer stage.""" + for camera in tuple(self._cameras.values()): + camera._prepare_renderer() + def register_visual_material(self, material: Any) -> None: """Register one initialized material asset for flat channel composition.""" if any(registered is material for registered in self._visual_materials): @@ -370,6 +389,8 @@ def reset_stage_prepare_flag(self) -> None: def close(self) -> None: """Release material writers and lifecycle bookkeeping, not registry-owned renderers. + Live camera registrations survive a stop so their stage preparation runs again on reset. + Raises: RuntimeError: If a material writer failed to close. """ diff --git a/source/isaaclab/isaaclab/sensors/camera/__init__.pyi b/source/isaaclab/isaaclab/sensors/camera/__init__.pyi index afff9e01fe01..15454b8c29ef 100644 --- a/source/isaaclab/isaaclab/sensors/camera/__init__.pyi +++ b/source/isaaclab/isaaclab/sensors/camera/__init__.pyi @@ -6,6 +6,7 @@ __all__ = [ "Camera", "CameraCfg", + "CameraExposureCfg", "CameraData", "CameraISPMode", "RenderBufferKind", @@ -20,6 +21,7 @@ __all__ = [ from .camera import Camera from .camera_cfg import CameraCfg +from .camera_exposure_cfg import CameraExposureCfg from .camera_data import CameraData, RenderBufferKind, RenderBufferSpec from .camera_isp import CameraISPMode from .tiled_camera import TiledCamera diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 98b6063ae0fb..f81ffe898512 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -7,7 +7,10 @@ import logging import sys -from collections.abc import Sequence +from collections.abc import Mapping, Sequence +from copy import copy +from dataclasses import replace +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Literal, cast import numpy as np @@ -25,11 +28,13 @@ create_rotation_matrix_from_view, quat_from_matrix, ) +from ...utils.tensor_processors._pipeline import _DictTensorProcessorPipeline from ...utils.warp import ProxyArray from ..sensor_base import SensorBase from .camera_data import CameraData, RenderBufferKind if TYPE_CHECKING: + from ...renderers.render_context import RenderContext from .camera_cfg import CameraCfg logger = logging.getLogger(__name__) @@ -206,6 +211,7 @@ def __init__(self, cfg: CameraCfg): RuntimeError: If no camera prim is found at the given path. ValueError: If the provided data types are not supported by the camera or active renderer. """ + self._render_context: RenderContext | None = None # perform check on supported data types self._check_supported_data_types(cfg) # initialize base class @@ -245,41 +251,18 @@ def __init__(self, cfg: CameraCfg): if sim_ctx is not None: sim_ctx.require_visual_shapes() - # An ISP (any ``isp_cfg`` other than ``None``) requires the HDR AOV; - # an explicit ``"rgb_hdr"`` in ``data_types`` also requires the - # HDR-routing flag flipped on the RTX-bearing backends. - require_hdr_output = "rgb_hdr" in self.cfg.data_types or self.cfg.isp_cfg is not None - - # TODO(follow-up PR): move this flag flip out of Camera. The cleanest path is - # an apply_pre_reset_settings() hook on RendererCfg (default no-op) that - # IsaacRtxRendererCfg overrides to flip /isaaclab/render/rtx_sensors. The - # flag must be set pre-sim.reset() because SimulationContext.is_rendering - # and several env classes read it before the renderer's __init__ runs. - renderer_type = getattr(self.cfg.renderer_cfg, "renderer_type", None) - if renderer_type == "isaac_rtx": - from ...app.settings_manager import get_settings_manager - - settings = get_settings_manager() - settings.set("/isaaclab/render/rtx_sensors", True) - settings.set("/physics/fabricUpdateTransformations", True) - if require_hdr_output: - settings.set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) - elif renderer_type == "ovrtx" and require_hdr_output: - from ...app.settings_manager import get_settings_manager - - get_settings_manager().set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) - # FIXME: settings.set is a no-op for ovrtx - # warning only since it affects only ParticleField3DGaussianSplat scene - logger.warning( - "OVRTX backend with PPISP/HDR requires /rtx/rtpt/gaussian/skipTonemapping/enabled to be false." - ) - + # Renderer requirements include private pipeline inputs and exclude produced public outputs. + self._render_cfg = replace(self.cfg, data_types=self.cfg.render_data_types(), pipeline=[]) + self._buffer_pipe: _DictTensorProcessorPipeline | None = None + self._pipeline_capture: object | None = None + self._render_camera_data: CameraData | None = None + self._render_outputs: dict[str, ProxyArray] = {} # UsdGeom Camera prim for the sensor self._sensor_prims: list[UsdGeom.Camera] = [] # Allocated in :meth:`_create_buffers` once the renderer's output contract is known. self._data: CameraData | None = None # The backend's ``__init__`` is its pre-physics phase, so it has to exist before - # ``sim.reset()``; sensor initialization only runs on ``PhysicsEvent.PHYSICS_READY``, which is + # ``sim.reset()``; sensor initialization only runs once physics is ready, which is # too late. Backends are shared per renderer config, so this stays cheap for many cameras. self._renderer: BaseRenderer | None = None if sim_ctx is not None: @@ -299,16 +282,19 @@ def __del__(self, _sys=sys): """ if _sys.is_finalizing() or _sys.meta_path is None: return - # unsubscribe callbacks - super().__del__() - - # release the frame view's backend state, getattr covers partial initialization - if getattr(self, "_view", None) is not None: - self._view.close() - self._view = None - # cleanup render resources (renderer may be None if never initialized) - if getattr(self, "_renderer", None) is not None: - self._renderer.cleanup(getattr(self, "_render_data", None)) + try: + # unsubscribe callbacks + super().__del__() + + # release the frame view's backend state, getattr covers partial initialization + if getattr(self, "_view", None) is not None: + self._view.close() + self._view = None + # cleanup render resources (renderer may be None if never initialized) + if getattr(self, "_renderer", None) is not None: + self._renderer.cleanup(getattr(self, "_render_data", None)) + finally: + self._close_pipeline() def __str__(self) -> str: """Returns: A string containing information about the instance.""" @@ -348,6 +334,37 @@ def data(self) -> CameraData: # return the data return self._data + @property + def render_outputs(self) -> Mapping[str, ProxyArray]: + """Borrowed renderer buffers before the camera pipeline, including private pipeline inputs. + + Access updates the camera when due. Arrays must not be modified by callers, and their contents + are replaced by later captures. Copy any values that must outlive the next camera update. + """ + self._update_outdated_buffers() + return MappingProxyType(self._render_outputs) + + def request_render_inputs(self, names: Sequence[str]) -> None: + """Request private renderer buffers before camera initialization. + + Observation pipelines use this during scene preparation. Inputs already requested are a no-op; + adding outputs after initialization raises because renderer setup and USD export are complete. + These requests do not change the camera's public :attr:`CameraData.output` selection. + """ + additional = [ + name for name in names if name not in self._render_cfg.data_types and name not in self._render_outputs + ] + if not additional: + return + if self._is_initialized: + raise RuntimeError("Camera renderer inputs must be requested before initialization.") + render_cfg = replace( + self._render_cfg, data_types=list(dict.fromkeys([*self._render_cfg.data_types, *additional])) + ) + self._check_supported_data_types(render_cfg) + render_cfg.validate_config() + self._render_cfg = render_cfg + @property def frame(self) -> ProxyArray: """Frame number when the measurement took place.""" @@ -635,6 +652,9 @@ def reset(self, env_ids: Sequence[int] | slice | None = None, env_mask: wp.array self._renderer.reset(self._render_data, env_ids) # reset the timestamps super().reset(env_ids, env_mask) + if self._buffer_pipe is not None and self._buffer_pipe.has_reset_callbacks: + pipeline_env_ids = wp.to_torch(env_mask).nonzero().squeeze(-1) if env_mask is not None else env_ids + self._buffer_pipe.reset(pipeline_env_ids) # reset the data # note: this recomputation is useful if one performs events such as randomizations on the camera poses. if env_mask is not None: @@ -671,22 +691,22 @@ def _initialize_impl(self): if self._renderer is None: self._renderer = sim_ctx.get_or_create_backend(self.cfg.renderer_cfg) - # Build the render spec early — both the wrapper ISP (which delegates - # any renderer-side per-camera setup) and ``create_render_data`` consume - # it, and the prims are already authored at this point. + # Per-camera USD setup must run **before** ``ensure_prepare_stage`` so renderers that snapshot + # the stage (ovrtx's ``stage.Export``) capture the overrides in their exported USD. Every camera + # already did this in :meth:`_prepare_renderer`; repeating it is harmless and covers + # cameras initialized outside that preparation pass. cam_paths = tuple(str(p.GetPath()) for p in sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) render_spec = CameraRenderSpec( - cfg=self.cfg, - device=str(self._device), + cfg=self._render_cfg, + device=str(sim_ctx.device), num_instances=self._num_envs, camera_prim_paths=cam_paths, view_count=self._num_envs, ) + if self.cfg.exposure is not None: + from .camera_exposure import apply_camera_exposure - # Delegate per-camera USD setup to the renderer — must run **before** - # ``ensure_prepare_stage`` so renderers that snapshot the stage - # (ovrtx's ``stage.Export``) capture the resulting overrides in their - # exported USD. + apply_camera_exposure(self.stage, list(cam_paths), self.cfg.exposure) self._renderer.prepare_cameras(self.stage, render_spec) # Stage preprocessing must happen before creating the view because the view keeps @@ -731,7 +751,7 @@ def _prepare_camera(self, env_mask: wp.array) -> None: self._update_poses(env_mask=env_mask, frame_op=1) else: self._update_camera_state(env_mask=env_mask, frame_op=1) - self._renderer.prepare_capture(self._render_data, self._data, self._frame) + self._renderer.prepare_capture(self._render_data, self._render_camera_data, self._frame) def _update_buffers_impl(self, env_mask: wp.array): if not self._env_mask_has_any(env_mask): @@ -745,12 +765,13 @@ def _update_buffers_impl(self, env_mask: wp.array): sim_ctx.render_context.render_into_camera( renderer, self._render_data, - self._data, + self._render_camera_data, sim_ctx.get_physics_step_count(), ) else: renderer.render(self._render_data) - renderer.read_output(self._render_data, self._data) + renderer.read_output(self._render_data, self._render_camera_data) + self._apply_pipeline() @staticmethod def _update_buffers_batch_impl(sensors: Sequence[SensorBase]) -> None: @@ -772,9 +793,41 @@ def _update_buffers_batch_impl(sensors: Sequence[SensorBase]) -> None: return sim_ctx.render_context.render_into_cameras( - [(camera._renderer, camera._render_data, camera._data) for camera in ready], + [(camera._renderer, camera._render_data, camera._render_camera_data) for camera in ready], sim_ctx.get_physics_step_count(), ) + for camera in ready: + camera._apply_pipeline() + + def _close_pipeline(self) -> None: + """Release pipeline resources, including partially initialized cameras.""" + pipe = getattr(self, "_buffer_pipe", None) + self._buffer_pipe = None + if pipe is not None: + pipe.close() + + def _apply_pipeline(self) -> None: + """Process a new capture once and publish metadata matching its processed pixels.""" + if self._buffer_pipe is None: + if self._data is not self._render_camera_data: + for name in self._data.output: + self._data.info[name] = self._render_camera_data.info.get(name) + return + info = self._render_camera_data.info + capture = next( + (value["capture"] for value in info.values() if isinstance(value, dict) and "capture" in value), None + ) + if capture is not None and capture is self._pipeline_capture: + return + self._buffer_pipe(self._render_outputs) + self._pipeline_capture = capture + for name in self._data.output: + if name in self._buffer_pipe.produced_outputs or ( + name == "rgb" and "rgba" in self._buffer_pipe.produced_outputs + ): + self._data.info[name] = {"capture": capture} if capture is not None else None + else: + self._data.info[name] = info.get(name) """ Private Helpers @@ -790,7 +843,7 @@ def _check_supported_data_types(self, cfg: CameraCfg): ) # check if there is any intersection in unsupported types # reason: these use np structured data types which are not compatible with the camera buffer contract - common_elements = set(cfg.data_types) & Camera.UNSUPPORTED_TYPES + common_elements = set(cfg.render_data_types()) & Camera.UNSUPPORTED_TYPES if common_elements: # provide alternative fast counterparts fast_common_elements = [] @@ -813,7 +866,7 @@ def _create_buffers(self): known: list[str] = [] unknown: list[str] = [] unsupported: list[str] = [] - for name in self.cfg.data_types: + for name in self._render_cfg.data_types: try: if RenderBufferKind(name) in specs: known.append(name) @@ -845,7 +898,58 @@ def _create_buffers(self): self._data.create_buffers(self._view.count, str(self._device)) self._initialize_intrinsics() self._update_poses() - self._renderer.set_outputs(self._render_data, self._data.output) + self._render_camera_data = self._data + self._render_outputs = self._render_camera_data.output + self._renderer.set_outputs(self._render_data, self._render_outputs) + self._close_pipeline() + self._pipeline_capture = None + if self.cfg.pipeline: + self._buffer_pipe = _DictTensorProcessorPipeline(self._render_outputs, self.cfg.pipeline) + try: + outputs = dict(self._buffer_pipe.outputs) + alias_rgb = ( + "rgba" in self._buffer_pipe.produced_outputs and "rgb" not in self._buffer_pipe.produced_outputs + ) + expected_shape = (self._view.count, self.cfg.height, self.cfg.width) + for name in self.cfg.data_types: + buffer = outputs["rgba" if alias_rgb and name == "rgb" else name] + if len(buffer.shape) != 4 or buffer.shape[:3] != expected_shape: + raise ValueError( + f"Camera pipeline output {name!r} must use NHWC with batch and spatial shape" + f" {expected_shape}; received {buffer.shape}. Use an observation pipe" + " for shape-changing results, because camera calibration describes the rendered image." + ) + # Preserve RGB's view semantics when a processor produces RGBA, without publishing stale raw RGB. + if alias_rgb: + rgba = outputs["rgba"].warp + if len(rgba.shape) != 4 or rgba.shape[-1] != 4: + raise ValueError("A pipeline output named rgba must have shape (N, H, W, 4).") + rgb = wp.array( + ptr=rgba.ptr, + shape=(*rgba.shape[:-1], 3), + strides=rgba.strides, + dtype=rgba.dtype, + device=rgba.device, + copy=False, + ) + outputs["rgb"] = ProxyArray(rgb) + self._data = copy(self._render_camera_data) + self._data._output = {name: outputs[name] for name in self.cfg.data_types} + self._data.info = dict.fromkeys(self._data.output) + except Exception as error: + try: + self._close_pipeline() + except Exception as cleanup_error: + error.add_note(f"Failed to close camera pipeline after initialization failed: {cleanup_error}") + raise + elif set(self._render_cfg.data_types) != set(self.cfg.data_types): + # Observation-only dependencies remain private even when this camera has no own pipeline. + public = set(self.cfg.data_types) + if not public.isdisjoint({"rgb", "rgba"}): + public.update({"rgb", "rgba"}) + self._data = copy(self._render_camera_data) + self._data._output = {name: buffer for name, buffer in self._render_outputs.items() if name in public} + self._data.info = dict.fromkeys(self._data.output) def _read_authored_opencv_intrinsics( self, prim: Usd.Prim, width: int, height: int, env_id: int @@ -1057,15 +1161,62 @@ def _env_mask_has_any(self, env_mask: wp.array) -> bool: Internal simulation callbacks. """ + def _register_callbacks(self): + """Register the sensor lifecycle and shared renderer stage preparation.""" + super()._register_callbacks() + self._render_context = sim_utils.SimulationContext.instance().render_context + self._render_context.register_camera(self) + + def _clear_callbacks(self) -> None: + """Release stage preparation along with the ordinary sensor subscriptions.""" + if self._render_context is not None: + self._render_context.unregister_camera(self) + self._render_context = None + super()._clear_callbacks() + + def _prepare_renderer(self): + """Apply this camera's renderer USD overrides before any camera initializes. + + A renderer that exports the stage when the first camera initializes, such as OVRTX, then sees + the overrides of every camera. + """ + if self._is_initialized: + return + sim_ctx = sim_utils.SimulationContext.instance() + if self._renderer is None: + self._renderer = sim_ctx.get_or_create_backend(self.cfg.renderer_cfg) + # Matches the environment count that sensor initialization derives next. + clone_plan = sim_ctx.get_clone_plan() + if clone_plan is not None: + num_views = len(clone_plan.topology.world_prototype_layout) + else: + num_views = len(sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) + cam_paths = tuple(str(p.GetPath()) for p in sim_utils.find_matching_prims(self.cfg.prim_path, self.stage)) + render_spec = CameraRenderSpec( + cfg=self._render_cfg, + device=str(sim_ctx.device), + num_instances=num_views, + camera_prim_paths=cam_paths, + view_count=num_views, + ) + if self.cfg.exposure is not None: + from .camera_exposure import apply_camera_exposure + + apply_camera_exposure(self.stage, list(cam_paths), self.cfg.exposure) + self._renderer.prepare_cameras(self.stage, render_spec) + def _invalidate_initialize_callback(self, event): """Invalidates the scene elements.""" - if self._renderer is not None and self._render_data is not None: - self._renderer.cleanup(self._render_data) - self._render_data = None - self._renderer = None - # call parent - super()._invalidate_initialize_callback(event) - # release backend state deterministically, then invalidate the view - if self._view is not None: - self._view.close() - self._view = None + try: + self._close_pipeline() + finally: + if self._renderer is not None and self._render_data is not None: + self._renderer.cleanup(self._render_data) + self._render_data = None + self._renderer = None + # call parent + super()._invalidate_initialize_callback(event) + # release backend state deterministically, then invalidate the view + if self._view is not None: + self._view.close() + self._view = None diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py index e52396484d34..e540e8915702 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py @@ -13,9 +13,10 @@ from ...sim import FisheyeCameraCfg, PinholeCameraCfg from ...utils import configclass from ..sensor_base_cfg import SensorBaseCfg -from .camera_isp import CameraISPMode +from .camera_exposure_cfg import CameraExposureCfg if TYPE_CHECKING: + from ...utils.tensor_processors import DictTensorProcessorCfg from .camera import Camera # Default values for the RTX-flavored fields kept on :class:`CameraCfg` for @@ -199,29 +200,47 @@ class OffsetCfg: renderer_cfg: RendererCfg = field(default_factory=RendererCfg) """Renderer configuration for camera sensor.""" - isp_cfg: Any | CameraISPMode | None = None - """Post-render ISP cfg applied by the renderer backend after it produces HDR output. + exposure: CameraExposureCfg | None = None + """Explicit camera exposure, or None to preserve authored USD settings. - Defaults to ``None`` (ISP disabled). Auto-discovery is opt-in via a - :class:`CameraISPMode` sentinel — see below. + Use ``CameraExposureCfg()`` for neutral HDR input to PPISP through either a camera + dictionary processor or an observation tensor processor. Applied before stage export; + changes affect every output from this camera prim. Newton ignores RTX exposure settings. + """ - Accepted values: + pipeline: list[DictTensorProcessorCfg] = [] + """Ordered dictionary processors applied after each newly published capture. - * ``None`` — ISP disabled. No HDR AOV is requested and no RTX-side - tonemapping flags are flipped. - * A :class:`CameraISPMode` sentinel — the renderer backend walks the USD stage to - discover an ISP shader (e.g. via the :mod:`isaaclab_ppisp` package). - * A concrete ISP cfg dataclass (e.g. :class:`isaaclab_ppisp.PpispCfg`) — used directly. + Processors consume and return dictionaries of named tensors. They may combine several render + outputs or preceding results. The renderer receives the pipeline's external dependencies + before initialization. + Only :attr:`data_types` are published in :attr:`CameraData.output`; unprocessed inputs remain + available through :attr:`Camera.render_outputs`. Processors borrow their inputs read-only and + own persistent result buffers. Camera reset and teardown reset and close the processors. + """ - The cfg applies once per Camera sensor batch. The PPISP Warp kernel takes - scalar coefficients, so every cloned view in a tiled batch shares the same - ISP configuration — there is no per-view ISP today. + isp_cfg: Any | None = None + """Retired renderer-owned ISP setting, retained to report a migration error. - :mod:`isaaclab.sensors.camera` does not depend on any ISP implementation; the - annotation is intentionally loose (``Any``) so the sensor layer can carry the - cfg through to a renderer that knows what to do with it. + Non-None values raise :class:`ValueError`. Configure :attr:`pipeline` with a PPISP step instead. """ + def render_data_types(self) -> list[str]: + """Resolve raw renderer inputs, including requested outputs not produced by the pipeline.""" + if self.isp_cfg is not None: + raise ValueError( + "CameraCfg.isp_cfg has been retired. Configure CameraCfg.pipeline with a PPISP step instead." + ) + if not self.pipeline: + return list(self.data_types) + from ...utils.tensor_processors._pipeline import _DictTensorProcessorPipeline + + produced = {name for step in self.pipeline for name in step.outputs} + if "rgb" in produced and "rgba" not in produced and "rgba" in self.data_types: + raise ValueError("A pipeline producing rgb must also produce rgba when rgba is requested.") + published = [name for name in self.data_types if not (name == "rgb" and "rgba" in produced)] + return sorted(_DictTensorProcessorPipeline.required_source_names(self.pipeline, published)) + def __post_init__(self): """Forward deprecated RTX-flavored fields onto :attr:`renderer_cfg`. @@ -229,6 +248,10 @@ def __post_init__(self): :class:`DeprecationWarning` and is copied onto ``self.renderer_cfg`` when that cfg defines the same-named field. """ + if self.isp_cfg is not None: + raise ValueError( + "CameraCfg.isp_cfg has been retired. Configure CameraCfg.pipeline with a PPISP step instead." + ) renderer_type = getattr(self.renderer_cfg, "renderer_type", None) if renderer_type == "default": from ...utils.backend_utils import get_default_renderer_cfg @@ -253,13 +276,14 @@ def __post_init__(self): def validate_config(self) -> None: """Validate the requested data types against the selected renderer contract.""" + raw_names = self.render_data_types() if self.renderer_cfg is None: return supported_specs = self.renderer_cfg.supported_output_types() if supported_specs is None: return supported = {str(kind) for kind in supported_specs} - unsupported = sorted(set(self.data_types) - supported) + unsupported = sorted(set(raw_names) - supported) if unsupported: raise ValueError( f"Renderer {type(self.renderer_cfg).__name__} only supports data types {sorted(supported)}, " diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_data.py b/source/isaaclab/isaaclab/sensors/camera/camera_data.py index 78933793a57e..09b122afc94c 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_data.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_data.py @@ -104,6 +104,12 @@ def output(self) -> dict[str, ProxyArray] | None: and C is the number of channels. Use ``.torch`` for a ``torch.Tensor`` view or ``.warp`` for the underlying ``wp.array``. + When :attr:`CameraCfg.pipeline` is configured, values are the pipeline results selected by + :attr:`CameraCfg.data_types`. Results preserve NHWC layout, batch size and image dimensions + so camera calibration remains applicable; channel count and dtype may change. Use an observation + pipe for results with different dimensions. Buffers are reused at the next capture. Unprocessed + inputs are available from :attr:`Camera.render_outputs`. + The format of the data is available in the `Replicator Documentation`_. For semantic-based data, this corresponds to the ``"data"`` key in the output of the sensor. diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_exposure.py b/source/isaaclab/isaaclab/sensors/camera/camera_exposure.py new file mode 100644 index 000000000000..f1d5ec0de55c --- /dev/null +++ b/source/isaaclab/isaaclab/sensors/camera/camera_exposure.py @@ -0,0 +1,53 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Author explicitly configured exposure before renderer stage export.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .camera_exposure_cfg import CameraExposureCfg + +_CAMERA_EXPOSURE_SCHEMAS = ("OmniRtxCameraAutoExposureAPI_1", "OmniRtxCameraExposureAPI_1") + + +def apply_camera_exposure(stage: Any, prim_paths: list[str], cfg: CameraExposureCfg) -> None: + """Apply camera exposure independently of requested render outputs. + + Existing API schemas are preserved. Schema metadata is authored directly so the RTX schema + plugin does not need to be loaded yet. + + Args: + stage: USD stage containing the camera prims. + prim_paths: Resolved camera prim paths. + cfg: Explicit exposure settings shared by these camera prims. + """ + from pxr import Sdf + + settings = ( + ("exposure", "Float", cfg.exposure), + ("exposure:fStop", "Float", cfg.f_stop), + ("exposure:iso", "Float", cfg.iso), + ("exposure:responsivity", "Float", cfg.responsivity), + ("exposure:time", "Float", cfg.time), + ("omni:rtx:autoExposure:enabled", "Bool", cfg.auto_exposure), + ) + value_types = {"Bool": Sdf.ValueTypeNames.Bool, "Float": Sdf.ValueTypeNames.Float} + for prim_path in prim_paths: + prim = stage.GetPrimAtPath(prim_path) + if not prim or not prim.IsValid(): + continue + current = prim.GetMetadata("apiSchemas") or Sdf.TokenListOp() + existing = list(current.prependedItems) + list(current.explicitItems) + list(current.appendedItems) + missing = [schema for schema in _CAMERA_EXPOSURE_SCHEMAS if schema not in existing] + if missing: + prim.SetMetadata("apiSchemas", Sdf.TokenListOp.Create(prependedItems=[*missing, *existing])) + for attr_name, sdf_type_name, value in settings: + attr = prim.GetAttribute(attr_name) + if not attr: + attr = prim.CreateAttribute(attr_name, value_types[sdf_type_name], custom=False) + attr.Set(value) diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_exposure_cfg.py b/source/isaaclab/isaaclab/sensors/camera/camera_exposure_cfg.py new file mode 100644 index 000000000000..d9e20aa9e44b --- /dev/null +++ b/source/isaaclab/isaaclab/sensors/camera/camera_exposure_cfg.py @@ -0,0 +1,34 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from isaaclab.utils import configclass + + +@configclass +class CameraExposureCfg: + """Explicit RTX camera exposure settings, applied before renderer stage export. + + Defaults disable automatic exposure and make HDR neutral for PPISP. Settings affect every + output from the camera prim. Newton does not consume RTX exposure attributes; its HDR output + is already scene-linear. Leave ``CameraCfg.exposure`` unset to preserve authored settings. + """ + + exposure: float = 0.0 + """Exposure compensation in stops.""" + + f_stop: float = 1.0 + """Exposure aperture ratio (separate from the lens depth-of-field aperture).""" + + iso: float = 0.0 + """RTX sensor ISO; zero selects neutral exposure.""" + + responsivity: float = 1.0 + """Sensor response multiplier.""" + + time: float = 1.0 + """Exposure duration in seconds.""" + + auto_exposure: bool = False + """Whether RTX automatic exposure is enabled.""" diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_isp.py b/source/isaaclab/isaaclab/sensors/camera/camera_isp.py index 8934198bbcf8..17a2256f6ffe 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_isp.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_isp.py @@ -3,12 +3,10 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Sentinel enum for :attr:`~isaaclab.sensors.camera.CameraCfg.isp_cfg`. +"""Legacy ISP discovery names retained for migration of saved configurations. -The renderer backend (which depends on the appropriate ISP implementation -package) is responsible for resolving these sentinels to a concrete cfg and -applying the ISP pipeline. :mod:`isaaclab.sensors.camera` carries the sentinel -through to the renderer without knowing about any specific ISP implementation. +``CameraCfg.isp_cfg`` no longer executes processing. Use ``CameraCfg.pipeline`` and +the discovery configuration provided by the image-processing package instead. """ from __future__ import annotations @@ -17,14 +15,7 @@ class CameraISPMode(StrEnum): - """Sentinel modes for :attr:`~isaaclab.sensors.camera.CameraCfg.isp_cfg`. - - Selects how the renderer backend discovers the ISP configuration from the - USD stage at camera initialization time. Discovery resolves one cfg for - the whole Camera sensor batch, using the first matched camera prim as the - camera-bound lookup target. ``isp_cfg=None`` disables ISP entirely; this - enum carries only the *active* discovery strategies. - """ + """Retired camera ISP discovery modes; use the processing package's discovery configuration.""" AUTO_CAMERA = "auto_camera" """Discover an ISP cfg bound to the batch's first matched camera prim. diff --git a/source/isaaclab/isaaclab/sensors/sensor_base.py b/source/isaaclab/isaaclab/sensors/sensor_base.py index 65a72c169d37..b23b1fc43036 100644 --- a/source/isaaclab/isaaclab/sensors/sensor_base.py +++ b/source/isaaclab/isaaclab/sensors/sensor_base.py @@ -343,7 +343,7 @@ def _register_callbacks(self): def _invoke(callback_name, event): getattr(obj_ref, callback_name)(event) - # Backend-agnostic: PHYSICS_READY (init) and STOP (invalidate) + # Backend-agnostic: PHYSICS_READY initializes sensors; STOP invalidates them. self._initialize_handle = physics_mgr_cls.register_callback( lambda payload: PhysicsManager.safe_callback_invoke( _invoke, "_initialize_callback", payload, physics_manager=physics_mgr_cls diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index a35f849a9925..b3ba99a5b94e 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -263,6 +263,15 @@ def __init__(self, cfg: SimulationCfg | None = None): PhysicsEvent.PHYSICS_READY, order=5, ) + # Prepare every camera before SensorBase initializes any sensor at order 10. Renderers + # that export the stage during the first camera's initialization then see every override. + self.physics_manager.register_callback( + lambda _payload: PhysicsManager.safe_callback_invoke( + self._render_context.prepare_cameras, physics_manager=self.physics_manager + ), + PhysicsEvent.PHYSICS_READY, + order=9, + ) self.physics_manager.register_callback( lambda _payload: self._render_context.close(), PhysicsEvent.STOP, order=100 ) diff --git a/source/isaaclab/isaaclab/utils/__init__.pyi b/source/isaaclab/isaaclab/utils/__init__.pyi index 76dd49752ed0..99f7ef906c6b 100644 --- a/source/isaaclab/isaaclab/utils/__init__.pyi +++ b/source/isaaclab/isaaclab/utils/__init__.pyi @@ -31,8 +31,10 @@ __all__ = [ "create_trimesh_from_geom_shape", "convert_faces_to_triangles", "PRIMITIVE_MESH_TYPES", - "ModifierCfg", - "ModifierBase", + "TensorProcessorCfg", + "TensorProcessor", + "DictTensorProcessor", + "DictTensorProcessorCfg", "DigitalFilter", "DigitalFilterCfg", "Integrator", @@ -87,9 +89,11 @@ from .mesh import ( convert_faces_to_triangles, PRIMITIVE_MESH_TYPES, ) -from .modifiers import ( - ModifierCfg, - ModifierBase, +from .tensor_processors import ( + DictTensorProcessor, + DictTensorProcessorCfg, + TensorProcessorCfg, + TensorProcessor, DigitalFilter, DigitalFilterCfg, Integrator, diff --git a/source/isaaclab/isaaclab/utils/modifiers/__init__.py b/source/isaaclab/isaaclab/utils/modifiers/__init__.py deleted file mode 100644 index d351b42e3875..000000000000 --- a/source/isaaclab/isaaclab/utils/modifiers/__init__.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Sub-module containing different modifiers implementations. - -Modifiers are used to apply stateful or stateless modifications to tensor data. They take -in a tensor and a configuration and return a tensor with the modification applied. This way users -can define custom operations to apply to a tensor. For instance, a modifier can be used to normalize -the input data or to apply a rolling average. - -They are primarily used to apply custom operations in the :class:`~isaaclab.managers.ObservationManager` -as an alternative to the built-in noise, clip and scale post-processing operations. For more details, see -the :class:`~isaaclab.managers.ObservationTermCfg` class. - -Usage with a function modifier: - -.. code-block:: python - - import torch - from isaaclab.utils import modifiers - - # create a random tensor - my_tensor = torch.rand(256, 128, device="cuda") - - # create a modifier configuration - cfg = modifiers.ModifierCfg(func=modifiers.clip, params={"bounds": (0.0, torch.inf)}) - - # apply the modifier - my_modified_tensor = cfg.func(my_tensor, cfg) - - -Usage with a class modifier: - -.. code-block:: python - - import torch - from isaaclab.utils import modifiers - - # create a random tensor - my_tensor = torch.rand(256, 128, device="cuda") - - # create a modifier configuration - # a digital filter with a simple delay of 1 timestep - cfg = modifiers.DigitalFilterCfg(A=[0.0], B=[0.0, 1.0]) - - # create the modifier instance - my_modifier = modifiers.DigitalFilter(cfg, my_tensor.shape, "cuda") - - # apply the modifier as a callable object - my_modified_tensor = my_modifier(my_tensor) - -""" - -from ..module import lazy_export - -lazy_export() diff --git a/source/isaaclab/isaaclab/utils/modifiers/__init__.pyi b/source/isaaclab/isaaclab/utils/modifiers/__init__.pyi deleted file mode 100644 index e15f1284085c..000000000000 --- a/source/isaaclab/isaaclab/utils/modifiers/__init__.pyi +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -__all__ = [ - "ModifierCfg", - "DigitalFilterCfg", - "IntegratorCfg", - "ModifierBase", - "DigitalFilter", - "Integrator", - "bias", - "clip", - "scale", -] - -from .modifier_cfg import ModifierCfg, DigitalFilterCfg, IntegratorCfg -from .modifier_base import ModifierBase -from .modifier import DigitalFilter, Integrator, bias, clip, scale diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/__init__.py b/source/isaaclab/isaaclab/utils/tensor_processors/__init__.py new file mode 100644 index 000000000000..f2f85320e7cf --- /dev/null +++ b/source/isaaclab/isaaclab/utils/tensor_processors/__init__.py @@ -0,0 +1,59 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Stateful and stateless processing for tensors and dictionaries of named tensors. + +Single-tensor functions and :class:`TensorProcessor` subclasses take and return one tensor. +They apply custom operations in the :class:`~isaaclab.managers.ObservationManager`, such as +normalization or rolling averages. Configure them through +:attr:`~isaaclab.managers.ObservationTermCfg.tensor_processors`. + +:class:`DictTensorProcessor` subclasses take and return dictionaries of named +:class:`~isaaclab.utils.warp.ProxyArray` tensors. They own persistent outputs and state for camera +pipelines and processed sensor observation terms. :class:`DictTensorProcessorAdapter` applies +these dictionary processors to an incoming observation tensor. + +Usage with a function processor: + +.. code-block:: python + + import torch + from isaaclab.utils import tensor_processors + + # create a random tensor + my_tensor = torch.rand(256, 128, device="cuda") + + # create a processor configuration + cfg = tensor_processors.TensorProcessorCfg(func=tensor_processors.clip, params={"bounds": (0.0, torch.inf)}) + + # apply the processor + my_modified_tensor = cfg.func(my_tensor, **cfg.params) + + +Usage with a class processor: + +.. code-block:: python + + import torch + from isaaclab.utils import tensor_processors + + # create a random tensor + my_tensor = torch.rand(256, 128, device="cuda") + + # create a processor configuration + # a digital filter with a simple delay of 1 timestep + cfg = tensor_processors.DigitalFilterCfg(A=[0.0], B=[0.0, 1.0]) + + # create the processor instance + my_processor = tensor_processors.DigitalFilter(cfg, my_tensor.shape, "cuda") + + # apply the processor as a callable object + my_modified_tensor = my_processor(my_tensor) + +""" + +from ..module import lazy_export + +lazy_export() diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/__init__.pyi b/source/isaaclab/isaaclab/utils/tensor_processors/__init__.pyi new file mode 100644 index 000000000000..50ca97dd9829 --- /dev/null +++ b/source/isaaclab/isaaclab/utils/tensor_processors/__init__.pyi @@ -0,0 +1,28 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "DictTensorProcessor", + "DictTensorProcessorCfg", + "TensorProcessorCfg", + "DictTensorProcessorAdapterCfg", + "DictTensorProcessorAdapter", + "DigitalFilterCfg", + "IntegratorCfg", + "TensorProcessor", + "DigitalFilter", + "Integrator", + "bias", + "clip", + "scale", +] + +from .tensor_processor_cfg import TensorProcessorCfg, DigitalFilterCfg, IntegratorCfg, DictTensorProcessorAdapterCfg +from .dict_tensor_processor_adapter import DictTensorProcessorAdapter +from .tensor_processor import TensorProcessor +from .processors import DigitalFilter, Integrator, bias, clip, scale + +from .dict_tensor_processor import DictTensorProcessor +from .dict_tensor_processor_cfg import DictTensorProcessorCfg diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/_pipeline.py b/source/isaaclab/isaaclab/utils/tensor_processors/_pipeline.py new file mode 100644 index 000000000000..b5cd8c927f62 --- /dev/null +++ b/source/isaaclab/isaaclab/utils/tensor_processors/_pipeline.py @@ -0,0 +1,170 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from builtins import ExceptionGroup +from collections.abc import Mapping, Sequence + +import torch +import warp as wp + +from ..configclass import instantiate +from ..warp import ProxyArray +from .dict_tensor_processor import DictTensorProcessor +from .dict_tensor_processor_cfg import DictTensorProcessorCfg + + +class _DictTensorProcessorPipeline: + """Compose dictionary processors in order, preserving untouched input keys. + + Each processor receives current named tensors and returns its named outputs. Preparation + constructs processors once; calls reuse their output storage. Each consumer owns its own + pipeline. Input buffers are borrowed, and returned mappings are independent snapshots of + references, not tensor copies. Use ``as_tensor`` when an observation must retain values. + """ + + def __init__(self, source_buffers: Mapping[str, ProxyArray], processors: Sequence[DictTensorProcessorCfg]): + self._source_buffers = dict(source_buffers) + self.required_inputs = self.required_source_names(processors) + self._processors: list[tuple[DictTensorProcessor, dict[str, ProxyArray]]] = [] + self._closed = False + self.outputs = dict(source_buffers) + self.produced_outputs = frozenset(name for cfg in processors for name in cfg.outputs) + try: + for cfg in processors: + if not cfg.outputs or len(set(cfg.outputs)) != len(cfg.outputs): + raise ValueError("A processor must declare distinct output names.") + missing = set(cfg.inputs) - self.outputs.keys() + if missing: + raise ValueError(f"Missing input buffers for preparation: {sorted(missing)}.") + processor = instantiate(cfg, dict(self.outputs)) + if not isinstance(processor, DictTensorProcessor): + raise TypeError("Dictionary processors must inherit DictTensorProcessor.") + self._processors.append((processor, dict(processor.outputs))) + if set(processor.outputs) != set(cfg.outputs): + raise ValueError("Processor outputs must match its declared output names.") + if any(not isinstance(buffer, ProxyArray) for buffer in processor.outputs.values()): + raise TypeError("Processor outputs must be ProxyArray objects.") + devices = {str(self.outputs[name].device) for name in cfg.inputs} + devices.update(str(buffer.device) for buffer in processor.outputs.values()) + if len(devices) > 1: + raise ValueError("A processor must use one device for its inputs and outputs.") + self.outputs.update(processor.outputs) + except Exception as error: + try: + self.close() + except Exception as cleanup_error: + raise error from cleanup_error + raise + + @property + def has_reset_callbacks(self) -> bool: + """Whether any processor owns resettable state.""" + return any(type(processor).reset is not DictTensorProcessor.reset for processor, _ in self._processors) + + @staticmethod + def _validate_buffer(name: str, value: ProxyArray, reference: ProxyArray) -> None: + if not isinstance(value, ProxyArray): + raise TypeError(f"Buffer {name!r} must be a ProxyArray.") + if value.shape != reference.shape or value.dtype != reference.dtype or value.device != reference.device: + raise ValueError( + f"Buffer {name!r} must match shape, dtype and device" + f" {reference.shape}, {reference.dtype}, {reference.device}." + ) + + @staticmethod + def required_source_names( + processors: Sequence[DictTensorProcessorCfg], published_names: Sequence[str] = () + ) -> frozenset[str]: + """Resolve external keys before allocating buffers or initializing a renderer.""" + required: set[str] = set() + produced: set[str] = set() + for cfg in processors: + required.update(set(cfg.inputs) - produced) + produced.update(cfg.outputs) + required.update(set(published_names) - produced) + return frozenset(required) + + def __call__(self, data: dict[str, ProxyArray]) -> dict[str, ProxyArray]: + """Apply processors and return their results with untouched inputs passed through.""" + if self._closed: + raise RuntimeError("Cannot execute a closed processor pipeline.") + missing = self.required_inputs - data.keys() + if missing: + raise ValueError(f"Missing required buffers: {sorted(missing)}.") + for name in self._source_buffers.keys() & data.keys(): + self._validate_buffer(name, data[name], self._source_buffers[name]) + current = dict(data) + for processor, outputs in self._processors: + result = processor(dict(current)) + if result.keys() != outputs.keys() or any(result[name] is not output for name, output in outputs.items()): + raise ValueError("Processors must return their persistent named output buffers.") + current.update(result) + self._source_buffers.update({name: data[name] for name in self._source_buffers.keys() & data.keys()}) + self.outputs = dict(current) + return current + + def apply_to_tensor( + self, + data: torch.Tensor, + *, + input_name: str, + output_name: str, + auxiliary_buffers: Mapping[str, ProxyArray] | None = None, + ) -> torch.Tensor: + """Process an incoming observation tensor and return an independently owned snapshot. + + The supplied tensor is authoritative: earlier observation transformations are preserved. + Auxiliary inputs must already be ready on the current Warp/Torch stream. CUDA work is + ordered with the caller's Torch stream; no host/device transfer is performed. + """ + if auxiliary_buffers is not None and input_name in auxiliary_buffers: + raise ValueError("Auxiliary buffers must not replace the incoming observation tensor.") + buffers = dict(auxiliary_buffers or {}) + buffers[input_name] = ProxyArray(wp.from_torch(data)) + return self.as_tensor(buffers, output_name=output_name) + + def as_tensor( + self, buffers: Mapping[str, ProxyArray], *, output_name: str, out: torch.Tensor | None = None + ) -> torch.Tensor: + """Execute on the current Torch stream and snapshot one output for an observation. + + Source buffers must be ready on the current Warp or Torch stream. Stream ordering + is established without a device-wide synchronization; no host/device copy is made. + A supplied ``out`` must match the prepared output's shape, dtype and device and is overwritten in place. + """ + if self._closed: + raise RuntimeError("Cannot execute a closed buffer pipe.") + reference = self.outputs[output_name] if output_name in self.produced_outputs else buffers[output_name] + if out is not None: + self._validate_buffer("out", ProxyArray(wp.from_torch(out)), reference) + device = wp.get_device(reference.device) + stream = wp.stream_from_torch(torch.cuda.current_stream(str(device))) if device.is_cuda else None + with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): + result = self(buffers)[output_name].torch + return result.clone() if out is None else out.copy_(result) + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + """Reset processor-owned state for selected batch entries.""" + for processor, _ in self._processors: + processor.reset(env_ids) + + def close(self) -> None: + """Close processors in reverse order, including after failed construction.""" + if self._closed: + return + self._closed = True + processors, self._processors = self._processors, [] + self._source_buffers.clear() + self.outputs = {} + errors = [] + for processor, _ in reversed(processors): + try: + processor.close() + except Exception as error: + errors.append(error) + if errors: + raise ExceptionGroup("Failed to close dictionary processors.", errors) diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor.py b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor.py new file mode 100644 index 000000000000..1ee875cdabe0 --- /dev/null +++ b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor.py @@ -0,0 +1,47 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Sequence +from typing import TYPE_CHECKING + +from ..warp import ProxyArray + +if TYPE_CHECKING: + from .dict_tensor_processor_cfg import DictTensorProcessorCfg + + +class DictTensorProcessor(ABC): + """Transform a dictionary of named tensors into a dictionary of named tensors. + + This is the dictionary counterpart of :class:`TensorProcessor`. Tensor values use + :class:`ProxyArray` so processors can use Warp or Torch without copying. Inputs are + borrowed read-only. Constructors inspect their shape, dtype and device and allocate + persistent outputs without reading input values or advancing processor state. + + Subclasses populate ``outputs`` with the buffers they produce, implement ``__call__``, + and override ``reset`` or ``close`` when they own state or external resources. Calls + return those same output buffers; values remain valid until the next call. Consumers + that retain results must copy them. GPU work uses the caller's current Warp stream. + """ + + def __init__(self, cfg: DictTensorProcessorCfg, data: dict[str, ProxyArray]) -> None: + self.cfg = cfg + self.outputs: dict[str, ProxyArray] = {} + + @abstractmethod + def __call__(self, data: dict[str, ProxyArray]) -> dict[str, ProxyArray]: + """Process current inputs and return the named outputs without mutating inputs.""" + raise NotImplementedError + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + """Reset state for selected batch entries, or all entries when None.""" + pass + + def close(self) -> None: + """Release owned resources; repeated calls must have no effect.""" + pass diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_adapter.py b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_adapter.py new file mode 100644 index 000000000000..4987652e9a79 --- /dev/null +++ b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_adapter.py @@ -0,0 +1,75 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Dictionary processing for an incoming observation tensor.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING + +import torch +import warp as wp + +from ..warp import ProxyArray +from ._pipeline import _DictTensorProcessorPipeline +from .tensor_processor import TensorProcessor + +if TYPE_CHECKING: + from .tensor_processor_cfg import DictTensorProcessorAdapterCfg + + +class DictTensorProcessorAdapter(TensorProcessor): + """Apply dictionary processors to the tensor received from the preceding processor. + + The adapter owns a processor pipeline and returns a snapshot. Its output must preserve the input + shape, as required by :class:`TensorProcessor`. The first incoming tensor initializes the pipeline; + subsequent calls reuse its output storage. Inputs may represent images or nonvisual sensor + observations. For multiple render inputs or shape changes to render outputs, use + :class:`~isaaclab.envs.mdp.processed_sensor` instead. + """ + + def __init__(self, cfg: DictTensorProcessorAdapterCfg, data_dim: tuple[int, ...], device: str): + super().__init__(cfg, data_dim, device) + self._pipe: _DictTensorProcessorPipeline | None = None + self._closed = False + + def __call__(self, data: torch.Tensor) -> torch.Tensor: + if self._closed: + raise RuntimeError("The buffer pipe processor is closed.") + if tuple(data.shape) != tuple(self._data_dim): + raise ValueError( + f"Observation processor input shape must be {tuple(self._data_dim)}, got {tuple(data.shape)}." + ) + if self._pipe is None: + stream = wp.stream_from_torch(torch.cuda.current_stream(data.device)) if data.is_cuda else None + with wp.ScopedStream(stream, sync_enter=True, sync_exit=True): + pipe = _DictTensorProcessorPipeline( + {self._cfg.input_name: ProxyArray(wp.from_torch(data))}, self._cfg.pipeline + ) + try: + if self._cfg.output_name not in pipe.outputs: + raise ValueError(f"Buffer processing does not provide output {self._cfg.output_name!r}.") + if pipe.outputs[self._cfg.output_name].shape != tuple(self._data_dim): + raise ValueError( + "Observation tensor_processors must preserve shape;" + " use a processing observation term for shape changes." + ) + except Exception: + pipe.close() + raise + self._pipe = pipe + return self._pipe.apply_to_tensor(data, input_name=self._cfg.input_name, output_name=self._cfg.output_name) + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + if self._pipe is not None: + self._pipe.reset(env_ids) + + def close(self) -> None: + if self._closed: + return + self._closed = True + if self._pipe is not None: + self._pipe.close() diff --git a/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_cfg.py b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_cfg.py new file mode 100644 index 000000000000..119afafac7d5 --- /dev/null +++ b/source/isaaclab/isaaclab/utils/tensor_processors/dict_tensor_processor_cfg.py @@ -0,0 +1,40 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from dataclasses import MISSING +from typing import TYPE_CHECKING, Any + +from ..configclass import configclass + +if TYPE_CHECKING: + from .dict_tensor_processor import DictTensorProcessor + + +@configclass +class DictTensorProcessorCfg: + """Configure one dictionary processor and declare its producer dependencies. + + Names are actual dictionary keys, with no separate operation roles. Cameras use these + declarations before renderer initialization. The class receives ``(cfg, data)`` and + owns its output storage, state, and lifecycle, just like a single-tensor processor. + """ + + class_type: type[DictTensorProcessor] | str = MISSING + """Processor class, or a resolvable import string.""" + + inputs: list[str] = MISSING + """Dictionary keys required from the producer or preceding processors.""" + + outputs: list[str] = MISSING + """Dictionary keys produced or replaced by this processor.""" + + params: dict[str, Any] = {} + """Processor-specific settings, accessible through ``cfg.params``. + + Configurations are copied. Explicitly shared device resources must preserve identity + through ``__deepcopy__``; do not place raw Warp allocations in this dictionary. + """ diff --git a/source/isaaclab/isaaclab/utils/modifiers/modifier.py b/source/isaaclab/isaaclab/utils/tensor_processors/processors.py similarity index 92% rename from source/isaaclab/isaaclab/utils/modifiers/modifier.py rename to source/isaaclab/isaaclab/utils/tensor_processors/processors.py index 2a6f456981b1..ff9a4a333f97 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/modifier.py +++ b/source/isaaclab/isaaclab/utils/tensor_processors/processors.py @@ -11,13 +11,13 @@ import torch from ..array import index_fill_ -from .modifier_base import ModifierBase +from .tensor_processor import TensorProcessor if TYPE_CHECKING: - from . import modifier_cfg + from . import tensor_processor_cfg ## -# Modifiers as functions +# Tensor processors as functions ## @@ -62,12 +62,12 @@ def bias(data: torch.Tensor, value: float) -> torch.Tensor: ## -# Sample of class based modifiers +# Sample of class based tensor_processors ## -class DigitalFilter(ModifierBase): - r"""Modifier used to apply digital filtering to the input data. +class DigitalFilter(TensorProcessor): + r"""Processor used to apply digital filtering to the input data. `Digital filters `_ are used to process discrete-time signals to extract useful parts of the signal, such as smoothing, noise reduction, or frequency separation. @@ -129,14 +129,14 @@ class DigitalFilter(ModifierBase): :math:`B = [1 - \alpha]`. """ - def __init__(self, cfg: modifier_cfg.DigitalFilterCfg, data_dim: tuple[int, ...], device: str): + def __init__(self, cfg: tensor_processor_cfg.DigitalFilterCfg, data_dim: tuple[int, ...], device: str): """Initializes digital filter. Args: cfg: Configuration parameters. data_dim: The dimensions of the data to be modified. First element is the batch size which usually corresponds to number of environments in the simulation. - device: The device to run the modifier on. + device: The device to run the processor on. Raises: ValueError: If filter coefficients are None. @@ -191,8 +191,8 @@ def __call__(self, data: torch.Tensor) -> torch.Tensor: return y_i -class Integrator(ModifierBase): - r"""Modifier that applies a numerical forward integration based on a middle Reimann sum. +class Integrator(TensorProcessor): + r"""Processor that applies a numerical forward integration based on a middle Reimann sum. An integrator is used to calculate the integral of a signal over time. The integral of a signal is the area under the curve of the signal. The integral can be approximated using numerical methods @@ -213,14 +213,14 @@ class Integrator(ModifierBase): :math:`\Delta t` is the time step between samples. """ - def __init__(self, cfg: modifier_cfg.IntegratorCfg, data_dim: tuple[int, ...], device: str): + def __init__(self, cfg: tensor_processor_cfg.IntegratorCfg, data_dim: tuple[int, ...], device: str): """Initializes the integrator configuration and state. Args: cfg: Integral parameters. data_dim: The dimensions of the data to be modified. First element is the batch size which usually corresponds to number of environments in the simulation. - device: The device to run the modifier on. + device: The device to run the processor on. """ # initialize parent class super().__init__(cfg, data_dim, device) diff --git a/source/isaaclab/isaaclab/utils/modifiers/modifier_base.py b/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor.py similarity index 51% rename from source/isaaclab/isaaclab/utils/modifiers/modifier_base.py rename to source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor.py index 65a7fe0bb8a2..cdf0d681c0ab 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/modifier_base.py +++ b/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor.py @@ -12,53 +12,38 @@ import torch if TYPE_CHECKING: - from .modifier_cfg import ModifierCfg + from .tensor_processor_cfg import TensorProcessorCfg -class ModifierBase(ABC): - """Base class for modifiers implemented as classes. - - Modifiers implementations can be functions or classes. If a modifier is a class, it should - inherit from this class and implement the required methods. - - A class implementation of a modifier can be used to store state information between calls. - This is useful for modifiers that require stateful operations, such as rolling averages - or delays or decaying filters. - - Example pseudo-code to create and use the class: - - .. code-block:: python - - from isaaclab.utils import modifiers - - # define custom keyword arguments to pass to ModifierCfg - kwarg_dict = {"arg_1": VAL_1, "arg_2": VAL_2} - - # create modifier configuration object - # func is the class name of the modifier and params is the dictionary of arguments - modifier_config = modifiers.ModifierCfg(func=modifiers.ModifierBase, params=kwarg_dict) - - # define modifier instance - my_modifier = modifiers.ModifierBase(cfg=modifier_config) +class TensorProcessor(ABC): + """Transform one tensor into one tensor, optionally retaining state between calls. + This is the single-tensor counterpart of :class:`DictTensorProcessor`. Subclasses + implement ``__call__`` and ``reset``; override ``close`` when external resources need + cleanup. Observation processors preserve the input shape. Configuration and dimensions + are supplied by the observation manager once during construction. """ - def __init__(self, cfg: ModifierCfg, data_dim: tuple[int, ...], device: str) -> None: - """Initializes the modifier class. + def __init__(self, cfg: TensorProcessorCfg, data_dim: tuple[int, ...], device: str) -> None: + """Initializes the processor class. Args: cfg: Configuration parameters. data_dim: The dimensions of the data to be modified. First element is the batch size which usually corresponds to number of environments in the simulation. - device: The device to run the modifier on. + device: The device to run the processor on. """ self._cfg = cfg self._data_dim = data_dim self._device = device + def close(self) -> None: + """Release resources owned by this processor.""" + pass + @abstractmethod def reset(self, env_ids: Sequence[int] | None = None): - """Resets the Modifier. + """Resets the Processor. Args: env_ids: The environment ids. Defaults to None, in which case diff --git a/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py b/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor_cfg.py similarity index 51% rename from source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py rename to source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor_cfg.py index d279fc62f2ab..5b48601a0b07 100644 --- a/source/isaaclab/isaaclab/utils/modifiers/modifier_cfg.py +++ b/source/isaaclab/isaaclab/utils/tensor_processors/tensor_processor_cfg.py @@ -14,37 +14,60 @@ from .. import configclass if TYPE_CHECKING: - from .modifier import DigitalFilter, Integrator - from .modifier_base import ModifierBase + from .dict_tensor_processor_adapter import DictTensorProcessorAdapter + from .dict_tensor_processor_cfg import DictTensorProcessorCfg + from .processors import DigitalFilter, Integrator + from .tensor_processor import TensorProcessor @configclass -class ModifierCfg: - """Configuration parameters for function and class modifiers.""" +class TensorProcessorCfg: + """Configuration parameters for function and class tensor processors.""" - func: Callable[..., torch.Tensor] | type[ModifierBase] | str = MISSING - """Function or :class:`ModifierBase` class used by the modifier. + func: Callable[..., torch.Tensor] | type[TensorProcessor] | str = MISSING + """Function or :class:`TensorProcessor` class used by the processor. - Functions must take a tensor as their first argument. Classes must inherit from :class:`ModifierBase`; the + Functions must take a tensor as their first argument. Classes must inherit from :class:`TensorProcessor`; the observation manager constructs them with the configuration, observation dimensions, and device. """ params: dict[str, Any] = {} - """Parameters used by the modifier. Defaults to an empty dictionary. + """Parameters used by the processor. Defaults to an empty dictionary. - Function modifiers receive them as keyword arguments on each call. Class modifiers access them through this + Functions receive them as keyword arguments on each call. Classes access them through this configuration during construction; they are not forwarded to the constructed instance. """ @configclass -class DigitalFilterCfg(ModifierCfg): - """Configuration parameters for a digital filter modifier. +class DictTensorProcessorAdapterCfg(TensorProcessorCfg): + """Bind an incoming observation tensor to a dictionary processor pipeline. + + Dictionary processors are constructed from the first tensor received after preceding processors. + Their documented input requirements define the expected array layout and value semantics. + """ + + func: type[DictTensorProcessorAdapter] | str = "{DIR}.dict_tensor_processor_adapter:DictTensorProcessorAdapter" + """Processor class consuming the incoming tensor.""" + + pipeline: list[DictTensorProcessorCfg] = MISSING + """Ordered dictionary processor configurations.""" + + input_name: str = "rgb" + """Dictionary key assigned to the incoming tensor.""" + + output_name: str = "rgb" + """Dictionary key selected from the processed outputs.""" + + +@configclass +class DigitalFilterCfg(TensorProcessorCfg): + """Configuration parameters for a digital filter processor. For more information, please check the :class:`DigitalFilter` class. """ - func: type[DigitalFilter] | str = "{DIR}.modifier:DigitalFilter" + func: type[DigitalFilter] | str = "{DIR}.processors:DigitalFilter" """The digital filter function to be called for applying the filter.""" A: list[float] = MISSING @@ -69,13 +92,13 @@ class DigitalFilterCfg(ModifierCfg): @configclass -class IntegratorCfg(ModifierCfg): - """Configuration parameters for an integrator modifier. +class IntegratorCfg(TensorProcessorCfg): + """Configuration parameters for an integrator processor. For more information, please check the :class:`Integrator` class. """ - func: type[Integrator] | str = "{DIR}.modifier:Integrator" + func: type[Integrator] | str = "{DIR}.processors:Integrator" """The integrator function to be called for applying the integrator.""" dt: float = MISSING diff --git a/source/isaaclab/test/envs/test_buffer_observations.py b/source/isaaclab/test/envs/test_buffer_observations.py new file mode 100644 index 000000000000..f74856539b3b --- /dev/null +++ b/source/isaaclab/test/envs/test_buffer_observations.py @@ -0,0 +1,287 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared pipe behavior at the observation manager boundary.""" + +from builtins import ExceptionGroup +from types import SimpleNamespace + +import pytest +import torch +import warp as wp + +from isaaclab.envs import ManagerBasedEnv +from isaaclab.managers import ( + ManagerTermBase, + ManagerTermBaseCfg, + ObservationGroupCfg, + ObservationManager, + ObservationTermCfg, +) +from isaaclab.utils import configclass, tensor_processors +from isaaclab.utils.tensor_processors import DictTensorProcessor, DictTensorProcessorCfg +from isaaclab.utils.warp import ProxyArray + +pytestmark = pytest.mark.unit + + +def test_environment_releases_simulation_when_observation_cleanup_fails(): + """An operation teardown error must not strand the environment or its simulation singleton.""" + events = [] + + def close_observations(): + events.append("observations") + raise RuntimeError("Operation cleanup failed") + + env = ManagerBasedEnv.__new__(ManagerBasedEnv) + env._is_closed = False + env._window = None + env.sim = SimpleNamespace(stop=lambda: events.append("stop"), clear_instance=lambda: events.append("clear")) + env.action_manager = object() + env.observation_manager = SimpleNamespace(close=close_observations) + env.event_manager = object() + env.recorder_manager = object() + env.scene = object() + env.obs_buf = {"policy": torch.ones(1)} + with pytest.raises(RuntimeError, match="Operation cleanup failed"): + env.close() + assert events == ["stop", "observations", "clear"] + assert env.obs_buf == {} + env.close() + assert events == ["stop", "observations", "clear"] + + +@wp.kernel +def _gain(source: wp.array4d(dtype=wp.float32), gain: float, result: wp.array4d(dtype=wp.float32)): + n, y, x, c = wp.tid() + result[n, y, x, c] = source[n, y, x, c] * gain + + +@wp.kernel +def _depth_weighted_half( + color: wp.array4d(dtype=wp.float32), + depth: wp.array4d(dtype=wp.float32), + result: wp.array4d(dtype=wp.float32), +): + n, y, x, c = wp.tid() + result[n, y, x, c] = color[n, y, 2 * x, c] * depth[n, y, 2 * x, 0] + + +class _Gain(DictTensorProcessor): + def __init__(self, cfg, data): + super().__init__(cfg, data) + self.outputs[cfg.outputs[0]] = ProxyArray(wp.empty_like(data[cfg.inputs[0]].warp)) + + def __call__(self, data): + output = self.outputs[self.cfg.outputs[0]] + wp.launch( + _gain, + dim=output.shape, + inputs=[data[self.cfg.inputs[0]].warp, self.cfg.params.get("gain", 2.0), output.warp], + device=output.device, + ) + return dict(self.outputs) + + +def _read_ranges(env): + return env.ranges + + +def _env(**kwargs): + return SimpleNamespace(num_envs=2, device="cpu", sim=SimpleNamespace(is_playing=lambda: True), **kwargs) + + +class _NestedImage(ManagerTermBase): + """Compose an image term through the manager's supported nested configuration containers.""" + + def __call__(self, env, terms, *, out=None): + term = terms[0]["image"] + params = {name: value for name, value in term.params.items() if name != "out"} + return term.func(env, **params, out=out) + + def reset(self, env_ids=None): + self.cfg.params["terms"][0]["image"].func.reset(env_ids) + + def close(self): + self._env.lifecycle.append(("outer_close",)) + + +def test_pipe_modifier_uses_preceding_modifier_output(): + """A pipe consumes the current non-image observation after preceding tensor_processors.""" + ranges = torch.ones((2, 4)) + preparations = [] + + class RangeGain(DictTensorProcessor): + def __init__(self, cfg, data): + super().__init__(cfg, data) + preparations.append(data["ranges"].shape) + self.outputs["ranges"] = ProxyArray(wp.empty_like(data["ranges"].warp)) + + def __call__(self, data): + torch.mul(data["ranges"].torch, self.cfg.params["gain"], out=self.outputs["ranges"].torch) + return dict(self.outputs) + + @configclass + class Policy(ObservationGroupCfg): + concatenate_terms = False + ranges = ObservationTermCfg( + func=_read_ranges, + tensor_processors=[ + tensor_processors.TensorProcessorCfg(func=tensor_processors.bias, params={"value": 3.0}), + tensor_processors.DictTensorProcessorAdapterCfg( + pipeline=[ + DictTensorProcessorCfg( + class_type=RangeGain, + inputs=["ranges"], + outputs=["ranges"], + params={"gain": 2.0}, + ) + ], + input_name="ranges", + output_name="ranges", + ), + ], + ) + + env = _env(ranges=ranges) + manager = ObservationManager({"policy": Policy()}, env) + manager.reset([1]) + assert preparations == [] + first = manager.compute()["policy"]["ranges"] + torch.testing.assert_close(first, torch.full_like(ranges, 8.0)) + torch.testing.assert_close(ranges, torch.ones_like(ranges)) + env.ranges = torch.full_like(ranges, 2.0) + torch.testing.assert_close(manager.compute()["policy"]["ranges"], torch.full_like(ranges, 10.0)) + torch.testing.assert_close(first, torch.full_like(ranges, 8.0)) + assert preparations == [tuple(ranges.shape)] + manager.close() + + +def test_pipe_modifier_retries_failed_preparation_and_stays_closed(): + """A rejected output is released; a later valid preparation works only before close.""" + image = torch.ones((2, 2, 4, 3)) + lifecycle = [] + valid_shape = False + + class TrackingGain(_Gain): + def __init__(self, cfg, data): + super().__init__(cfg, data) + lifecycle.append("prepare") + if not valid_shape: + self.outputs["rgb"] = ProxyArray(wp.empty((2, 2, 2, 3), dtype=wp.float32, device="cpu")) + + def close(self): + lifecycle.append("close") + + cfg = tensor_processors.DictTensorProcessorAdapterCfg( + pipeline=[DictTensorProcessorCfg(class_type=TrackingGain, inputs=["rgb"], outputs=["rgb"])] + ) + modifier = tensor_processors.DictTensorProcessorAdapter(cfg, tuple(image.shape), "cpu") + modifier.reset([1]) + with pytest.raises(ValueError, match="input shape"): + modifier(image[:, :, ::2]) + assert lifecycle == [] + with pytest.raises(ValueError, match="must preserve shape"): + modifier(image) + assert lifecycle == ["prepare", "close"] + valid_shape = True + torch.testing.assert_close(modifier(image), image * 2) + modifier.close() + modifier.close() + with pytest.raises(RuntimeError, match="closed"): + modifier(image) + assert lifecycle == ["prepare", "close", "prepare", "close"] + + unused_modifier = tensor_processors.DictTensorProcessorAdapter(cfg, tuple(image.shape), "cpu") + unused_modifier.close() + with pytest.raises(RuntimeError, match="closed"): + unused_modifier(image) + assert lifecycle == ["prepare", "close", "prepare", "close"] + + +def test_processed_sensor_declares_private_inputs_and_reports_transformed_shape(): + """Preparation resolves private AOVs before binding; manager dimensions describe the processed output.""" + lifecycle = [] + + class DepthWeighted(DictTensorProcessor): + def __init__(self, cfg, data): + super().__init__(cfg, data) + color = data["rgb_hdr"] + n, h, w, c = color.shape + self.outputs["rgb"] = ProxyArray(wp.empty((n, h, w // 2, c), dtype=wp.float32, device=color.device)) + + def __call__(self, data): + output = self.outputs["rgb"] + wp.launch( + _depth_weighted_half, + dim=output.shape, + inputs=[data["rgb_hdr"].warp, data["depth"].warp, output.warp], + device=output.device, + ) + return dict(self.outputs) + + def reset(self, env_ids=None): + lifecycle.append(("reset", env_ids)) + + def close(self): + lifecycle.append(("close",)) + raise ValueError("operation close failed") + + radiance = torch.full((2, 2, 4, 3), 2.0) + depth = torch.tensor([1.0, 9.0, 3.0, 9.0]).reshape(1, 1, 4, 1).expand(2, 2, 4, 1).contiguous() + requests = [] + camera = SimpleNamespace( + request_render_inputs=lambda names: requests.append(set(names)), + render_outputs={"rgb_hdr": ProxyArray(wp.from_torch(radiance)), "depth": ProxyArray(wp.from_torch(depth))}, + ) + sensors = {"camera": camera} + env = _env(scene=SimpleNamespace(sensors=sensors, keys=sensors.keys), lifecycle=lifecycle) + + @configclass + class Policy(ObservationGroupCfg): + concatenate_terms = False + image = ObservationTermCfg( + func=_NestedImage, + params={ + "terms": [ + { + "image": ManagerTermBaseCfg( + func="isaaclab.envs.mdp:processed_sensor", + params={ + "pipeline": [ + DictTensorProcessorCfg( + class_type=DepthWeighted, + inputs=["rgb_hdr", "depth"], + outputs=["rgb"], + ) + ] + }, + ) + } + ] + }, + ) + + @configclass + class Observations: + policy = Policy() + + cfg = Observations() + ObservationManager.prepare_scene(cfg, env) + assert requests == [{"rgb_hdr", "depth"}] + manager = ObservationManager(cfg, env) + assert manager.group_obs_dim["policy"] == [(2, 2, 3)] + result = manager.compute()["policy"]["image"] + expected = torch.tensor([2.0, 6.0]).reshape(1, 1, 2, 1).expand(2, 2, 2, 3) + torch.testing.assert_close(result, expected) + radiance.fill_(4.0) + torch.testing.assert_close(manager.compute()["policy"]["image"], expected * 2) + torch.testing.assert_close(result, expected) + manager.reset([1]) + with pytest.raises(ExceptionGroup, match="Failed to close dictionary processors") as error: + manager.close() + assert str(error.value.exceptions[0]) == "operation close failed" + manager.close() + assert lifecycle[-3:] == [("reset", [1]), ("close",), ("outer_close",)] diff --git a/source/isaaclab/test/managers/test_manager_base.py b/source/isaaclab/test/managers/test_manager_base.py index bd11ccfcfc27..a123273d3791 100644 --- a/source/isaaclab/test/managers/test_manager_base.py +++ b/source/isaaclab/test/managers/test_manager_base.py @@ -24,7 +24,7 @@ from isaaclab.managers import ManagerTermBase, ManagerTermBaseCfg, SceneEntityCfg from isaaclab.managers.manager_base import ManagerBase from isaaclab.physics import PhysicsEvent -from isaaclab.utils import configclass, modifiers, to_dict, update_from_dict +from isaaclab.utils import configclass, tensor_processors, to_dict, update_from_dict pytestmark = pytest.mark.unit @@ -84,7 +84,7 @@ class NestedFieldTermCfg(ManagerTermBaseCfg): """Manager term with owned and opaque fields outside ``params``.""" nested_term: ManagerTermBaseCfg = ManagerTermBaseCfg(func=increment_dummy1_by_one) - modifier: modifiers.ModifierCfg = modifiers.ModifierCfg(func=increment_dummy1_by_one) + modifier: tensor_processors.TensorProcessorCfg = tensor_processors.TensorProcessorCfg(func=increment_dummy1_by_one) metadata: dict[str, str] = {"func": f"{__name__}:reset_dummy2_to_zero"} opaque: OpaqueCfg = OpaqueCfg() diff --git a/source/isaaclab/test/managers/test_observation_manager.py b/source/isaaclab/test/managers/test_observation_manager.py index fe00e34d64fd..000f56e3b55f 100644 --- a/source/isaaclab/test/managers/test_observation_manager.py +++ b/source/isaaclab/test/managers/test_observation_manager.py @@ -23,7 +23,7 @@ ObservationTermCfg, RewardTermCfg, ) -from isaaclab.utils import DelayBuffer, configclass, modifiers, noise, to_dict, update_from_dict, validate +from isaaclab.utils import DelayBuffer, configclass, noise, tensor_processors, to_dict, update_from_dict, validate pytestmark = pytest.mark.unit @@ -580,12 +580,12 @@ class PolicyCfg(ObservationGroupCfg): @pytest.mark.parametrize("position_term", [pos_w_data, destination_position]) def test_modifier_compute(setup_env, position_term): env = setup_env - """Test the observation computation with modifiers.""" + """Test the observation computation with tensor_processors.""" - modifier_1 = modifiers.ModifierCfg(func=modifiers.bias, params={"value": 1.0}) - modifier_2 = modifiers.ModifierCfg(func=modifiers.scale, params={"multiplier": 2.0}) - modifier_3 = modifiers.ModifierCfg(func=modifiers.clip, params={"bounds": (-0.5, 0.5)}) - modifier_4 = modifiers.IntegratorCfg(dt=env.dt) + modifier_1 = tensor_processors.TensorProcessorCfg(func=tensor_processors.bias, params={"value": 1.0}) + modifier_2 = tensor_processors.TensorProcessorCfg(func=tensor_processors.scale, params={"multiplier": 2.0}) + modifier_3 = tensor_processors.TensorProcessorCfg(func=tensor_processors.clip, params={"bounds": (-0.5, 0.5)}) + modifier_4 = tensor_processors.IntegratorCfg(dt=env.dt) @configclass class MyObservationManagerCfg: @@ -596,19 +596,19 @@ class PolicyCfg(ObservationGroupCfg): """Test config class for policy observation group.""" concatenate_terms = False - term_1 = ObservationTermCfg(func=position_term, modifiers=[]) - term_2 = ObservationTermCfg(func=position_term, modifiers=[modifier_1]) - term_3 = ObservationTermCfg(func=position_term, modifiers=[modifier_1, modifier_4]) + term_1 = ObservationTermCfg(func=position_term, tensor_processors=[]) + term_2 = ObservationTermCfg(func=position_term, tensor_processors=[modifier_1]) + term_3 = ObservationTermCfg(func=position_term, tensor_processors=[modifier_1, modifier_4]) @configclass class CriticCfg(ObservationGroupCfg): """Test config class for critic observation group""" concatenate_terms = False - term_1 = ObservationTermCfg(func=position_term, modifiers=[]) - term_2 = ObservationTermCfg(func=position_term, modifiers=[modifier_1]) - term_3 = ObservationTermCfg(func=position_term, modifiers=[modifier_1, modifier_2]) - term_4 = ObservationTermCfg(func=position_term, modifiers=[modifier_1, modifier_2, modifier_3]) + term_1 = ObservationTermCfg(func=position_term, tensor_processors=[]) + term_2 = ObservationTermCfg(func=position_term, tensor_processors=[modifier_1]) + term_3 = ObservationTermCfg(func=position_term, tensor_processors=[modifier_1, modifier_2]) + term_4 = ObservationTermCfg(func=position_term, tensor_processors=[modifier_1, modifier_2, modifier_3]) policy: ObservationGroupCfg = PolicyCfg() critic: ObservationGroupCfg = CriticCfg() @@ -695,7 +695,7 @@ def test_modifier_invalid_config(setup_env): env = setup_env """Test modifier initialization with invalid config.""" - modifier = modifiers.ModifierCfg(func=modifiers.clip, params={"min": -0.5, "max": 0.5}) + modifier = tensor_processors.TensorProcessorCfg(func=tensor_processors.clip, params={"min": -0.5, "max": 0.5}) @configclass class MyObservationManagerCfg: @@ -706,7 +706,7 @@ class PolicyCfg(ObservationGroupCfg): """Test config class for policy observation group.""" concatenate_terms = False - term_1 = ObservationTermCfg(func=pos_w_data, modifiers=[modifier]) + term_1 = ObservationTermCfg(func=pos_w_data, tensor_processors=[modifier]) policy: ObservationGroupCfg = PolicyCfg() @@ -800,10 +800,10 @@ def __init__(self, num_envs: int = 2) -> None: self.observation = torch.arange(num_envs, dtype=torch.float32).unsqueeze(-1) -class StatefulBiasModifier(modifiers.ModifierBase): +class StatefulBiasModifier(tensor_processors.TensorProcessor): """Stateful modifier used to verify lazy callable resolution.""" - def __init__(self, cfg: modifiers.ModifierCfg, data_dim: tuple[int, ...], device: str) -> None: + def __init__(self, cfg: tensor_processors.TensorProcessorCfg, data_dim: tuple[int, ...], device: str) -> None: super().__init__(cfg, data_dim, device) self.value = cfg.params["value"] self.reset_count = 0 @@ -842,33 +842,37 @@ def test_class_modifier_roundtrip_preserves_func_and_params(): """Reproduce #6067 with a class modifier and non-empty parameters.""" cfg = HistoryObservationsCfg() cfg.policy.history_length = None - cfg.policy.dummy.modifiers = [modifiers.ModifierCfg(func=StatefulBiasModifier, params={"value": 2.0})] + cfg.policy.dummy.tensor_processors = [ + tensor_processors.TensorProcessorCfg(func=StatefulBiasModifier, params={"value": 2.0}) + ] update_from_dict(cfg, to_dict(cfg)) term_cfg = cfg.policy.dummy - assert term_cfg.modifiers is not None - modifier_cfg = term_cfg.modifiers[0] - assert isinstance(modifier_cfg, modifiers.ModifierCfg) - assert isinstance(modifier_cfg.func, str) - assert modifier_cfg.params == {"value": 2.0} + assert term_cfg.tensor_processors is not None + tensor_processor_cfg = term_cfg.tensor_processors[0] + assert isinstance(tensor_processor_cfg, tensor_processors.TensorProcessorCfg) + assert isinstance(tensor_processor_cfg.func, str) + assert tensor_processor_cfg.params == {"value": 2.0} env = DummyEnv() manager = ObservationManager(cfg, cast("ManagerBasedEnv", env)) prepared_term_cfg = manager.cfg.policy.dummy - assert prepared_term_cfg.modifiers is not None - prepared_modifier_cfg = prepared_term_cfg.modifiers[0] - assert isinstance(prepared_modifier_cfg.func, StatefulBiasModifier) + assert prepared_term_cfg.tensor_processors is not None + prepared_tensor_processor_cfg = prepared_term_cfg.tensor_processors[0] + assert isinstance(prepared_tensor_processor_cfg.func, StatefulBiasModifier) observations = manager.compute()["policy"] torch.testing.assert_close(observations, env.observation + 2.0) manager.reset() - assert prepared_modifier_cfg.func.reset_count == 1 + assert prepared_tensor_processor_cfg.func.reset_count == 1 -def test_stateless_modifier_cfg_roundtrip_preserves_signature_validation(): +def test_stateless_tensor_processor_cfg_roundtrip_preserves_signature_validation(): """A stateless modifier remains callable and inspectable after a configuration round-trip.""" cfg = HistoryObservationsCfg() cfg.policy.history_length = None - cfg.policy.dummy.modifiers = [modifiers.ModifierCfg(func=modifiers.bias, params={"value": 2.0})] + cfg.policy.dummy.tensor_processors = [ + tensor_processors.TensorProcessorCfg(func=tensor_processors.bias, params={"value": 2.0}) + ] update_from_dict(cfg, to_dict(cfg)) env = DummyEnv() @@ -881,10 +885,10 @@ def test_class_modifier_validates_constructed_instance(): """Class modifier validation checks the constructed object.""" cfg = HistoryObservationsCfg() cfg.policy.history_length = None - cfg.policy.dummy.modifiers = [modifiers.ModifierCfg(func=InvalidModifier)] + cfg.policy.dummy.tensor_processors = [tensor_processors.TensorProcessorCfg(func=InvalidModifier)] update_from_dict(cfg, to_dict(cfg)) - with pytest.raises(TypeError, match="is not an instance of 'ModifierBase'"): + with pytest.raises(TypeError, match="is not an instance of 'TensorProcessor'"): ObservationManager(cfg, cast("ManagerBasedEnv", DummyEnv())) diff --git a/source/isaaclab_ppisp/test/test_rtx_camera_overrides.py b/source/isaaclab/test/renderers/test_camera_exposure.py similarity index 68% rename from source/isaaclab_ppisp/test/test_rtx_camera_overrides.py rename to source/isaaclab/test/renderers/test_camera_exposure.py index 77aafda82ef7..2167edf3edb7 100644 --- a/source/isaaclab_ppisp/test/test_rtx_camera_overrides.py +++ b/source/isaaclab/test/renderers/test_camera_exposure.py @@ -3,15 +3,19 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Tests for the RTX-side camera-exposure overrides applied when PPISP is active.""" +"""Tests for the RTX camera-exposure overrides used for scene-linear color outputs.""" -from isaaclab_ppisp import apply_rtx_exposure_overrides +import pytest from pxr import Sdf, Usd, UsdGeom +from isaaclab.sensors.camera import CameraExposureCfg +from isaaclab.sensors.camera.camera_exposure import apply_camera_exposure -def test_apply_rtx_exposure_overrides_resets_authored_exposure_values(): - """When PPISP is the ISP authority, the camera prim's RTX-side exposure must be neutral.""" + +@pytest.mark.parametrize("exposure, iso", [(0.0, 0.0), (2.0, 200.0)]) +def test_camera_exposure_applies_explicit_settings(exposure, iso): + """Scene-linear outputs require neutral RTX-side exposure on the camera prim.""" stage = Usd.Stage.CreateInMemory() cam = UsdGeom.Camera.Define(stage, "/World/Camera") prim = cam.GetPrim() @@ -23,15 +27,15 @@ def test_apply_rtx_exposure_overrides_resets_authored_exposure_values(): prim.CreateAttribute("exposure:time", Sdf.ValueTypeNames.Float).Set(1.0 / 60.0) prim.CreateAttribute("omni:rtx:autoExposure:enabled", Sdf.ValueTypeNames.Bool).Set(True) - apply_rtx_exposure_overrides(stage, ["/World/Camera"]) + apply_camera_exposure(stage, ["/World/Camera"], CameraExposureCfg(exposure=exposure, iso=iso)) - assert prim.GetAttribute("exposure").Get() == 0.0 + assert prim.GetAttribute("exposure").Get() == exposure assert prim.GetAttribute("exposure:fStop").Get() == 1.0 - assert prim.GetAttribute("exposure:iso").Get() == 0.0 + assert prim.GetAttribute("exposure:iso").Get() == iso assert prim.GetAttribute("exposure:responsivity").Get() == 1.0 assert prim.GetAttribute("exposure:time").Get() == 1.0 assert prim.GetAttribute("omni:rtx:autoExposure:enabled").Get() is False - # The PPISP API schemas must also be applied — that is what makes Kit's + # The RTX exposure API schemas must also be applied — that is what makes Kit's # USD watcher route the camera prim's ``exposure:*`` and # ``omni:rtx:autoExposure:*`` attributes into the per-camera carb # settings RTX consumes. diff --git a/source/isaaclab/test/renderers/test_camera_output_contract.py b/source/isaaclab/test/renderers/test_camera_output_contract.py index c13430483768..34f133010b75 100644 --- a/source/isaaclab/test/renderers/test_camera_output_contract.py +++ b/source/isaaclab/test/renderers/test_camera_output_contract.py @@ -6,10 +6,13 @@ """Tests for the renderer→camera output contract.""" import warnings +from builtins import ExceptionGroup +from dataclasses import replace from types import SimpleNamespace import numpy as np import pytest +import torch import warp as wp pytest.importorskip("isaaclab_physx") @@ -22,6 +25,8 @@ from isaaclab.sim import PinholeCameraCfg from isaaclab.test.utils import DeviceScope, test_devices from isaaclab.utils import clone, validate +from isaaclab.utils.tensor_processors import DictTensorProcessor, DictTensorProcessorCfg +from isaaclab.utils.warp import ProxyArray pytestmark = [pytest.mark.integration, pytest.mark.rendering] @@ -241,8 +246,9 @@ def test_newton_warp_segmentation_spec_follows_colorize_flags(colorize): assert specs[kind] == expected -def test_newton_warp_wraps_requested_rgb_hdr_output(): - """NewtonWarpRenderer wires requested RGB_HDR proxies to the Newton HDR output slot.""" +@pytest.mark.parametrize("data_type", [RenderBufferKind.RGB_HDR]) +def test_newton_warp_wraps_requested_hdr_output(data_type): + """Newton wires both linear color outputs to its HDR output slot.""" pytest.importorskip("isaaclab_newton") pytest.importorskip("newton") wp.init() @@ -255,10 +261,10 @@ def test_newton_warp_wraps_requested_rgb_hdr_output(): render_data = RenderData(fake_sensor, SimpleNamespace(cfg=camera_cfg)) hdr_proxy = ProxyArray(wp.zeros((2, 3, 4, 3), dtype=wp.float32, device="cpu")) - render_data.set_outputs({str(RenderBufferKind.RGB_HDR): hdr_proxy}) + render_data.set_outputs({str(data_type): hdr_proxy}) assert render_data.outputs.hdr_color_image is not None - assert render_data.get_output(RenderBufferKind.RGB_HDR) is render_data.outputs.hdr_color_image + assert render_data.get_output(data_type) is render_data.outputs.hdr_color_image def _make_camera_cfg(data_types: list[str]) -> CameraCfg: @@ -271,12 +277,13 @@ def _make_camera_cfg(data_types: list[str]) -> CameraCfg: ) -def test_camera_data_allocates_supported_subset_and_aliases_rgb(): - """CameraData allocates the intersection of requested + supported and aliases rgb into rgba.""" - cfg = _make_camera_cfg(["rgb", "rgba", "depth", "instance_segmentation"]) +def test_camera_data_allocates_supported_subset_and_aliases_color(): + """CameraData allocates requested + supported outputs, aliasing rgb into rgba.""" + cfg = _make_camera_cfg(["rgb", "rgba", "rgb_hdr", "depth", "instance_segmentation"]) specs = { RenderBufferKind.RGBA: RenderBufferSpec(4, wp.uint8), RenderBufferKind.RGB: RenderBufferSpec(3, wp.uint8), + RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), RenderBufferKind.DEPTH: RenderBufferSpec(1, wp.float32), RenderBufferKind.NORMALS: RenderBufferSpec(3, wp.float32), } @@ -284,14 +291,16 @@ def test_camera_data_allocates_supported_subset_and_aliases_rgb(): data_types=cfg.data_types, height=8, width=16, num_views=2, device="cpu", supported_specs=specs ) - assert set(data.output.keys()) == {"rgba", "rgb", "depth"} + assert set(data.output.keys()) == {"rgba", "rgb", "rgb_hdr", "depth"} assert data.output["rgba"].shape == (2, 8, 16, 4) assert data.output["rgba"].dtype == wp.uint8 assert data.output["depth"].shape == (2, 8, 16, 1) assert data.output["depth"].dtype == wp.float32 assert data.output["rgb"].warp.ptr == data.output["rgba"].warp.ptr + assert data.output["rgb_hdr"].shape == (2, 8, 16, 3) + assert data.output["rgb_hdr"].dtype == wp.float32 assert data.image_shape == (8, 16) - assert data.info == {"rgba": None, "rgb": None, "depth": None} + assert data.info == dict.fromkeys(data.output) def test_camera_data_allocate_raises_on_unknown_name(): @@ -308,3 +317,274 @@ def test_camera_data_allocate_raises_on_unknown_name(): ) assert "not_a_real_type" in str(exc_info.value) assert "RenderBufferKind" in str(exc_info.value) + + +def test_every_camera_prepares_before_the_shared_stage_export(monkeypatch): + """Every live camera prepares before stage export, including after stop and with callback error forwarding.""" + import weakref + + pytest.importorskip("isaaclab_newton") + pytest.importorskip("newton") + from isaaclab_newton.physics import NewtonCfg, NewtonManager + + from pxr import Sdf, UsdGeom + + from isaaclab.physics import PhysicsEvent + from isaaclab.renderers import RendererCfg + from isaaclab.sensors.camera import Camera, CameraExposureCfg + from isaaclab.sensors.camera import camera as camera_module + from isaaclab.sensors.sensor_base import SensorBase + + specs = { + RenderBufferKind.RGB: RenderBufferSpec(3, wp.uint8), + RenderBufferKind.RGBA: RenderBufferSpec(4, wp.uint8), + RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), + } + exported_iso = [] + prepare_error = None + + def prepare_cameras(stage, spec): + if prepare_error is not None: + raise prepare_error + + def export_stage(stage, num_envs): + exported_iso.append(prims[1].GetAttribute("exposure:iso").Get()) + + renderer = SimpleNamespace( + initialize=lambda: None, + supported_output_types=lambda: specs, + prepare_cameras=prepare_cameras, + prepare_stage=export_stage, + create_render_data=lambda spec: SimpleNamespace(spec=spec), + set_outputs=lambda data, outputs: None, + cleanup=lambda data: None, + close=lambda: None, + ) + renderer_cfg = RendererCfg(class_type=lambda cfg: renderer, renderer_type="contract") + monkeypatch.setattr(SensorBase, "_initialize_impl", lambda self: None) + monkeypatch.setattr(Camera, "_initialize_intrinsics", lambda self: None) + monkeypatch.setattr(Camera, "_update_poses", lambda self: None) + monkeypatch.setattr( + camera_module, + "FrameView", + lambda path, **kwargs: SimpleNamespace(count=2, prims=[stage.GetPrimAtPath(path)] * 2, close=lambda: None), + ) + + class Identity(DictTensorProcessor): + def __init__(self, cfg, data): + super().__init__(cfg, data) + self.outputs["processed_radiance"] = ProxyArray(wp.empty_like(data["rgb_hdr"].warp)) + + def __call__(self, data): + wp.copy(self.outputs["processed_radiance"].warp, data["rgb_hdr"].warp) + return dict(self.outputs) + + def make_camera(prim, data_types): + camera = Camera.__new__(Camera) + pipeline = ( + [DictTensorProcessorCfg(Identity, ["rgb_hdr"], ["processed_radiance"])] + if data_types == ["processed_radiance"] + else [] + ) + camera.cfg = CameraCfg( + prim_path=str(prim.GetPath()), + data_types=data_types, + height=2, + width=3, + spawn=None, + pipeline=pipeline, + exposure=CameraExposureCfg() if pipeline else None, + renderer_cfg=renderer_cfg, + ) + camera._render_cfg = replace(camera.cfg, data_types=camera.cfg.render_data_types(), pipeline=[]) + camera.stage = stage + camera._device = "cpu" + camera._num_envs = 2 + camera._is_initialized = False + camera._sensor_prims = [] + camera._renderer = renderer + camera._render_data = None + camera._view = None + camera._register_callbacks() + return camera + + sim_cfg = sim_utils.SimulationCfg(device="cpu", physics=NewtonCfg()) + with sim_utils.build_simulation_context(sim_cfg=sim_cfg) as sim: + stage = sim.stage + prims = [UsdGeom.Camera.Define(stage, path).GetPrim() for path in ("/World/First", "/World/Second")] + iso = prims[1].CreateAttribute("exposure:iso", Sdf.ValueTypeNames.Float) + monkeypatch.setattr( + sim, "get_clone_plan", lambda: SimpleNamespace(topology=SimpleNamespace(world_prototype_layout=np.zeros(2))) + ) + sim.get_or_create_backend(renderer_cfg) + cameras = [make_camera(prim, names) for prim, names in zip(prims, (["rgb"], ["processed_radiance"]))] + discarded = make_camera(prims[0], ["rgb"]) + discarded_ref = weakref.ref(discarded) + del discarded + assert discarded_ref() is None + + try: + for _ in range(2): + iso.Set(100.0) + sim.physics_manager.dispatch_event(PhysicsEvent.PHYSICS_READY) + assert all(camera.is_initialized for camera in cameras) + sim.physics_manager.dispatch_event(PhysicsEvent.STOP) + assert exported_iso == [0.0, 0.0] + + # External event buses store callback failures instead of propagating them directly. + errors = [] + monkeypatch.setattr(NewtonManager, "store_callback_exception", errors.append, raising=False) + prepare_error = ValueError("camera preparation failed") + sim.physics_manager.dispatch_event(PhysicsEvent.PHYSICS_READY) + assert errors and errors[0] is prepare_error + assert exported_iso == [0.0, 0.0] + finally: + for camera in cameras: + camera.__del__() + + +def test_camera_pipeline_keeps_raw_inputs_and_processes_only_new_captures(monkeypatch): + """Camera publication preserves raw pixels, capture metadata and calibration-compatible geometry.""" + from isaaclab.sensors.camera import Camera + from isaaclab.sensors.sensor_base import SensorBase + from isaaclab.sim import SimulationContext + + events = [] + output_shape = (2, 1, 1, 4) + close_fails = False + + class Color(DictTensorProcessor): + def __init__(self, cfg, data): + super().__init__(cfg, data) + self.outputs["rgba"] = ProxyArray(wp.empty(output_shape, dtype=wp.uint8, device=data["rgb_hdr"].device)) + + def __call__(self, data): + events.append("apply") + output = self.outputs["rgba"].torch + output[..., :3].copy_(100 * (data["rgb_hdr"].torch + data["depth"].torch)) + output[..., 3] = 255 + return dict(self.outputs) + + def reset(self, env_ids=None): + events.append(tuple(env_ids)) + + def close(self): + events.append("close") + if close_fails: + raise RuntimeError("processor close failed") + + cfg = CameraCfg( + prim_path="/World/Camera", + spawn=None, + height=1, + width=1, + data_types=["rgb", "depth"], + pipeline=[DictTensorProcessorCfg(Color, ["rgb_hdr", "depth"], ["rgba"])], + ) + specs = { + RenderBufferKind.RGB_HDR: RenderBufferSpec(3, wp.float32), + RenderBufferKind.DEPTH: RenderBufferSpec(1, wp.float32), + } + capture = {"frame": 1} + radiance = 0.2 + + def read_output(render_data, camera_data): + camera_data.output["rgb_hdr"].torch.fill_(radiance) + camera_data.output["depth"].torch.fill_(0.5) + for name in camera_data.info: + camera_data.info[name] = {"capture": capture} + + camera = Camera.__new__(Camera) + camera.cfg = cfg + camera._render_cfg = replace(cfg, data_types=cfg.render_data_types(), pipeline=[]) + camera._view = SimpleNamespace(count=2, close=lambda: None) + camera._device = "cpu" + camera._num_envs = 2 + camera._is_initialized = True + camera._render_data = None + camera._clear_callbacks = lambda: None + camera._renderer = SimpleNamespace( + supported_output_types=lambda: specs, + set_outputs=lambda data, outputs: None, + render=lambda data: None, + read_output=read_output, + reset=lambda data, ids: None, + cleanup=lambda data: None, + ) + monkeypatch.setattr(SimulationContext, "instance", staticmethod(lambda: None)) + monkeypatch.setattr(SensorBase, "reset", lambda self, ids, mask: None) + monkeypatch.setattr(camera, "_initialize_intrinsics", lambda: None) + monkeypatch.setattr(camera, "_update_poses", lambda **kwargs: None) + monkeypatch.setattr(camera, "_prepare_camera", lambda mask: None) + monkeypatch.setattr(camera, "_update_outdated_buffers", lambda: None) + camera._create_buffers() + mask = wp.ones(2, dtype=wp.bool, device="cpu") + camera._update_buffers_impl(mask) + + assert set(camera.data.output) == {"rgb", "depth"} + assert set(camera.render_outputs) == {"rgb_hdr", "depth"} + torch.testing.assert_close(camera.data.output["rgb"].torch, torch.full((2, 1, 1, 3), 70, dtype=torch.uint8)) + torch.testing.assert_close(camera.render_outputs["rgb_hdr"].torch, torch.full((2, 1, 1, 3), 0.2)) + assert camera.data.output["depth"] is camera.render_outputs["depth"] + assert camera.data.info["rgb"]["capture"] is capture + + camera._update_buffers_impl(mask) + assert events == ["apply"] + capture = {"frame": 2} + radiance = 0.4 + camera._update_buffers_impl(mask) + torch.testing.assert_close(camera.data.output["rgb"].torch, torch.full((2, 1, 1, 3), 90, dtype=torch.uint8)) + assert events == ["apply", "apply"] + reset_mask = wp.array([False, True], dtype=wp.bool, device="cpu") + camera.reset(env_mask=reset_mask) + assert events[-1] == (1,) + # A rebuilt pipe cannot publish resized pixels against unchanged camera calibration. + output_shape = (2, 2, 1, 4) + with pytest.raises(ValueError, match="camera calibration"): + camera._create_buffers() + assert camera._buffer_pipe is None + assert events[-2:] == ["close", "close"] + output_shape = (2, 1, 1, 4) + monkeypatch.setattr(Color, "reset", DictTensorProcessor.reset) + camera._create_buffers() + # Stateless operations must not materialize indices: CUDA nonzero would synchronize the host. + with monkeypatch.context() as patch: + patch.setattr( + torch.Tensor, "nonzero", lambda self: pytest.fail("Stateless camera reset materialized mask indices") + ) + camera.reset(env_mask=reset_mask) + camera._render_data = object() + camera._renderer.cleanup = lambda data: events.append("renderer-close") + camera._view.close = lambda: events.append("view-close") + close_fails = True + with pytest.raises(ExceptionGroup, match="close dictionary processors"): + camera._invalidate_initialize_callback(None) + assert events[-3:] == ["close", "renderer-close", "view-close"] + assert not camera.is_initialized + assert camera._renderer is camera._render_data is camera._view is camera._buffer_pipe is None + camera.__del__() + + +def test_camera_requests_private_observation_inputs_before_initialization(): + """An observation may request raw inputs before reset, while late additions fail clearly.""" + from isaaclab.sensors.camera import Camera + + camera = Camera.__new__(Camera) + camera.cfg = _make_camera_cfg(["rgb"]) + camera._render_cfg = replace(camera.cfg, pipeline=[]) + camera._render_outputs = {} + camera._is_initialized = False + camera._clear_callbacks = lambda: None + camera.request_render_inputs(["depth", "rgb_hdr"]) + assert camera.cfg.data_types == ["rgb"] + assert set(camera._render_cfg.data_types) == {"rgb", "depth", "rgb_hdr"} + camera._is_initialized = True + camera.request_render_inputs(["depth"]) + with pytest.raises(RuntimeError, match="before initialization"): + camera.request_render_inputs(["normals"]) + + +def test_camera_retired_isp_setting_reports_pipeline_migration(): + """Legacy camera ISP configuration fails before it can silently skip image processing.""" + with pytest.raises(ValueError, match="CameraCfg.pipeline"): + CameraCfg(isp_cfg="auto_any") diff --git a/source/isaaclab/test/renderers/test_simulation_render_context.py b/source/isaaclab/test/renderers/test_simulation_render_context.py index 7256e7e7a015..ab6beb2bf63c 100644 --- a/source/isaaclab/test/renderers/test_simulation_render_context.py +++ b/source/isaaclab/test/renderers/test_simulation_render_context.py @@ -294,6 +294,8 @@ def __init__(self, renderer, name, update_period=0.0): self._data = CameraData() self._data.create_buffers(2, "cpu") self._data.info = {} + self._render_camera_data = self._data + self._buffer_pipe = None self._frame = ProxyArray(wp.zeros(2, dtype=wp.int64, device="cpu")) self._ALL_INDICES = wp.array([0, 1], dtype=wp.int32, device="cpu") self._ALL_ENV_MASK = wp.ones(2, dtype=wp.bool, device="cpu") diff --git a/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py b/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py index 0756f2b72d26..16aa58f8754e 100644 --- a/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py +++ b/source/isaaclab/test/sensors/generate_synthetic_gaussian_asset.py @@ -24,15 +24,14 @@ from typing import TYPE_CHECKING import torch -from isaaclab_ppisp import PpispCfg, normalize_ppisp_cfg +from isaaclab_ppisp import PpispCfg, PpispProcessorCfg, normalize_ppisp_cfg, ppisp_cfg_from_usd_camera from pxr import Gf, Sdf, Usd, UsdGeom, Vt import isaaclab.sim as sim_utils from isaaclab.assets import AssetBaseCfg, RigidObjectCfg from isaaclab.scene import InteractiveScene, InteractiveSceneCfg -from isaaclab.sensors.camera import Camera, CameraCfg -from isaaclab.sensors.camera.camera_isp import CameraISPMode +from isaaclab.sensors.camera import Camera, CameraCfg, CameraExposureCfg from isaaclab.terrains import TerrainImporterCfg from isaaclab.utils import configclass @@ -728,7 +727,8 @@ def render_synthetic_gaussian_scene( width=width, data_types=data_types, spawn=None, - isp_cfg=isp_cfg, + exposure=CameraExposureCfg(), + pipeline=[PpispProcessorCfg(params={"cfg": isp_cfg})], renderer_cfg=renderer_cfg, ) camera = Camera(cfg) @@ -736,7 +736,9 @@ def render_synthetic_gaussian_scene( for _ in range(stabilisation_steps): sim.step() camera.update(sim_dt) - outputs = {name: tensor.clone().detach().cpu().to(torch.float32) for name, tensor in camera.data.output.items()} + outputs = { + name: tensor.torch.clone().detach().cpu().to(torch.float32) for name, tensor in camera.data.output.items() + } del camera return outputs @@ -756,9 +758,7 @@ def render_synthetic_gaussian_scene_with_static_ppisp_attrs( ) -> dict[str, torch.Tensor]: """Render the synthesised gaussian asset through authored static PPISP camera attributes. - The camera uses :class:`CameraISPMode.AUTO_CAMERA`; renderer backends must - discover the camera-authored PPISP attributes and route them through their - PPISP workflow. + Resolve the camera-authored attributes before constructing the shared PPISP image operation. """ with fresh_synthetic_gaussian_interactive_scene(usd_path, sim_cfg, num_envs=num_envs) as sim: author_static_ppisp_camera_attrs(sim.stage, ppisp_cfg=ppisp_cfg) @@ -769,7 +769,7 @@ def render_synthetic_gaussian_scene_with_static_ppisp_attrs( width=width, sim_dt=sim_dt, stabilisation_steps=stabilisation_steps, - isp_cfg=CameraISPMode.AUTO_CAMERA, + ppisp_cfg=ppisp_cfg_from_usd_camera(sim.stage.GetPrimAtPath(_camera_path_for_env(0))), sim=sim, ) @@ -797,7 +797,7 @@ def render_synthetic_gaussian_scene_with_controller_ppisp_attrs( width=width, sim_dt=sim_dt, stabilisation_steps=stabilisation_steps, - isp_cfg=CameraISPMode.AUTO_CAMERA, + ppisp_cfg=ppisp_cfg_from_usd_camera(sim.stage.GetPrimAtPath(_camera_path_for_env(0))), sim=sim, ) @@ -810,7 +810,7 @@ def _render_synthetic_gaussian_camera( width: int, sim_dt: float, stabilisation_steps: int, - isp_cfg: PpispCfg | CameraISPMode | None, + ppisp_cfg: PpispCfg, sim: SimulationContext, ) -> dict[str, torch.Tensor]: cfg = CameraCfg( @@ -820,7 +820,8 @@ def _render_synthetic_gaussian_camera( width=width, data_types=data_types, spawn=None, - isp_cfg=isp_cfg, + exposure=CameraExposureCfg(), + pipeline=[PpispProcessorCfg(params={"cfg": ppisp_cfg})], renderer_cfg=renderer_cfg, ) camera = Camera(cfg) @@ -828,6 +829,8 @@ def _render_synthetic_gaussian_camera( for _ in range(stabilisation_steps): sim.step() camera.update(sim_dt) - outputs = {name: tensor.clone().detach().cpu().to(torch.float32) for name, tensor in camera.data.output.items()} + outputs = { + name: tensor.torch.clone().detach().cpu().to(torch.float32) for name, tensor in camera.data.output.items() + } del camera return outputs diff --git a/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py b/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py index 1371bebe5b0e..c40a7e79cbbb 100644 --- a/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py +++ b/source/isaaclab/test/sensors/test_camera_ppisp_gaussian_ovrtx.py @@ -3,12 +3,12 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Validate the camera PPISP wrapper applied to a 3D Gaussian (NuRec / +"""Validate the camera PPISP pipeline applied to a 3D Gaussian (NuRec / ParticleField) scene through the ``ovrtx`` renderer. The asset is synthesised at test time by :mod:`generate_synthetic_gaussian_asset` -and rendered via :func:`generate_synthetic_gaussian_asset.render_synthetic_gaussian_scene`. -The aggressive wrapper PPISP cfg +and rendered through the camera's shared render-buffer pipeline. +The aggressive PPISP cfg (:func:`generate_synthetic_gaussian_asset.make_aggressive_ppisp_cfg`) intentionally engages every feature past its subtle-correction defaults so each can be asserted independently: @@ -38,8 +38,8 @@ * Uses Newton physics because ``ovrtx`` is incompatible with Kit/Isaac Sim and the PhysX backend requires Kit (``carb``) to bootstrap. * Requests ``"rgb_hdr"`` in ``data_types`` because the test asserts the raw - HDR source with :func:`assert_ppisp_lifts_exposure`. The PPISP render path - itself also allocates an internal HDR buffer when ``isp_cfg`` is set, so + HDR source with :func:`assert_ppisp_lifts_exposure`. The PPISP camera pipeline + requests its private ``"rgb_hdr"`` input automatically, so ``"rgb_hdr"`` is not required just to enable PPISP. """ @@ -57,10 +57,10 @@ make_aggressive_ppisp_cfg, make_neutral_ppisp_cfg, make_synthetic_gaussian_usd, - render_synthetic_gaussian_scene, render_synthetic_gaussian_scene_with_controller_ppisp_attrs, render_synthetic_gaussian_scene_with_static_ppisp_attrs, ) +from isaaclab_ppisp import PpispCfg from isaaclab.sim import SimulationCfg @@ -87,17 +87,14 @@ SIM_DT = 1.0 / 60.0 MULTI_TILE_COUNT = 4 -# Mark the gaussian-on-OVRTX tests xfail by default: the wrapper-side -# ``carb.settings.set_bool`` call that disables RTX-side tonemapping is a -# no-op for the kit-less ovrtx backend, and the equivalent must be applied -# externally before launching pytest. -_XFAIL_OVRTX_GAUSSIAN_PPISP = pytest.mark.xfail( - reason=( - "kit-less ovrtx cannot toggle RTX-side tonemapping at runtime; the equivalent must be " - "applied externally before launching pytest" - ), - strict=False, -) + +def _ovrtx_ppisp_cfg() -> PpispCfg: + """Expose the unexposed scene-linear Gaussian signal into the CRF's useful range.""" + cfg = make_aggressive_ppisp_cfg() + # rgb_hdr has a center near 0.12; +2 EV places it near 0.48 before the + # response curve. The shared fixture's -5 EV targets legacy exposed HDR. + cfg.inputs["exposureOffset"] = 2.0 + return cfg def _ovrtx_sim_cfg(device: str) -> SimulationCfg: @@ -110,12 +107,11 @@ def _ovrtx_sim_cfg(device: str) -> SimulationCfg: @pytest.mark.parametrize("device", ["cuda:0"]) @_SKIP_MISSING_OVRTX -@_XFAIL_OVRTX_GAUSSIAN_PPISP def test_camera_ppisp_authored_static_attrs_are_applied_on_synthetic_gaussians_ovrtx(device): """OVRTX must apply camera-authored static PPISP attributes.""" with tempfile.TemporaryDirectory(prefix="isaaclab-synth-gauss-") as tmpdir: asset_path = make_synthetic_gaussian_usd(f"{tmpdir}/synthetic_gaussians.usda") - aggressive_cfg = make_aggressive_ppisp_cfg() + aggressive_cfg = _ovrtx_ppisp_cfg() neutral = render_synthetic_gaussian_scene_with_static_ppisp_attrs( asset_path, @@ -141,12 +137,11 @@ def test_camera_ppisp_authored_static_attrs_are_applied_on_synthetic_gaussians_o @pytest.mark.parametrize("device", ["cuda:0"]) @_SKIP_MISSING_OVRTX -@_XFAIL_OVRTX_GAUSSIAN_PPISP def test_camera_ppisp_controller_matches_static_attrs_on_synthetic_gaussians_ovrtx(device): """OVRTX controller output must match the equivalent static PPISP cfg.""" with tempfile.TemporaryDirectory(prefix="isaaclab-synth-gauss-") as tmpdir: asset_path = make_synthetic_gaussian_usd(f"{tmpdir}/synthetic_gaussians.usda") - ppisp_cfg = make_aggressive_ppisp_cfg() + ppisp_cfg = _ovrtx_ppisp_cfg() static = render_synthetic_gaussian_scene_with_static_ppisp_attrs( asset_path, @@ -171,7 +166,10 @@ def test_camera_ppisp_controller_matches_static_attrs_on_synthetic_gaussians_ovr @pytest.mark.parametrize("device", ["cuda:0"]) @_SKIP_MISSING_OVRTX -@_XFAIL_OVRTX_GAUSSIAN_PPISP +@pytest.mark.xfail( + reason="OVRTX renders cloned Gaussian tiles black in native RGB and radiance, including without a camera pipeline", + strict=False, +) def test_camera_ppisp_wrapper_signatures_on_synthetic_gaussians_ovrtx_multitile(device): """Multi-tile wrapper PPISP via ``ovrtx`` must hold the same invariants independently for every tile. @@ -184,10 +182,11 @@ def test_camera_ppisp_wrapper_signatures_on_synthetic_gaussians_ovrtx_multitile( """ with tempfile.TemporaryDirectory(prefix="isaaclab-synth-gauss-") as tmpdir: asset_path = make_synthetic_gaussian_usd(f"{tmpdir}/synthetic_gaussians.usda") - output = render_synthetic_gaussian_scene( + output = render_synthetic_gaussian_scene_with_static_ppisp_attrs( asset_path, sim_cfg=_ovrtx_sim_cfg(device), renderer_cfg=OVRTXRendererCfg(), + ppisp_cfg=_ovrtx_ppisp_cfg(), data_types=["rgb", "rgb_hdr"], num_envs=MULTI_TILE_COUNT, sim_dt=SIM_DT, diff --git a/source/isaaclab/test/sensors/test_opencv_distortion.py b/source/isaaclab/test/sensors/test_opencv_distortion.py index 6d276bc0f451..bb7e6dc8caf3 100644 --- a/source/isaaclab/test/sensors/test_opencv_distortion.py +++ b/source/isaaclab/test/sensors/test_opencv_distortion.py @@ -192,6 +192,7 @@ def _camera_for_prims(prims, width=640, height=480, device="cpu"): fake._ALL_INDICES = wp.array(np.arange(len(prims)), dtype=wp.int32, device=device) fake._ALL_ENV_MASK = wp.ones(len(prims), dtype=wp.bool, device=device) # attributes touched by ``__del__``/``_clear_callbacks`` when the fake object is garbage collected + fake._render_context = None fake._initialize_handle = None fake._invalidate_initialize_handle = None fake._prim_deletion_handle = None diff --git a/source/isaaclab/test/sensors/test_sensor_destructors.py b/source/isaaclab/test/sensors/test_sensor_destructors.py index a5d057d9b8f3..b4bbb7245d64 100644 --- a/source/isaaclab/test/sensors/test_sensor_destructors.py +++ b/source/isaaclab/test/sensors/test_sensor_destructors.py @@ -77,6 +77,7 @@ def cleanup(self, _render_data): view = _View() renderer = _Renderer() camera = object.__new__(Camera) + camera._render_context = None camera._view = view camera._renderer = renderer camera._render_data = None diff --git a/source/isaaclab/test/utils/test_configclass.py b/source/isaaclab/test/utils/test_configclass.py index 7d53871de683..a8fbfb09ad13 100644 --- a/source/isaaclab/test/utils/test_configclass.py +++ b/source/isaaclab/test/utils/test_configclass.py @@ -79,7 +79,7 @@ def double(x): @configclass -class ModifierCfg: +class TensorProcessorCfg: params: dict[str, Any] = {"A": 1, "B": 2} @@ -111,7 +111,7 @@ class BasicDemoCfg: device_id: int = 0 env: EnvCfg = EnvCfg() robot_default_state: RobotDefaultStateCfg = RobotDefaultStateCfg() - list_config = [ModifierCfg(), ModifierCfg(params={"A": 3, "B": 4})] + list_config = [TensorProcessorCfg(), TensorProcessorCfg(params={"A": 3, "B": 4})] @configclass diff --git a/source/isaaclab/test/utils/test_dict_tensor_processor.py b/source/isaaclab/test/utils/test_dict_tensor_processor.py new file mode 100644 index 000000000000..afcdf3145240 --- /dev/null +++ b/source/isaaclab/test/utils/test_dict_tensor_processor.py @@ -0,0 +1,267 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Contracts for buffer routing, ownership and lifecycle independent of camera rendering.""" + +from builtins import ExceptionGroup + +import pytest +import torch +import warp as wp + +from isaaclab.utils.tensor_processors import DictTensorProcessor, DictTensorProcessorCfg +from isaaclab.utils.tensor_processors._pipeline import _DictTensorProcessorPipeline +from isaaclab.utils.warp import ProxyArray + +pytestmark = pytest.mark.unit + + +class _Scale(DictTensorProcessor): + def __init__(self, cfg, data): + super().__init__(cfg, data) + self.outputs[cfg.outputs[0]] = ProxyArray(wp.empty_like(data[cfg.inputs[0]].warp)) + + def __call__(self, data): + torch.mul( + data[self.cfg.inputs[0]].torch, + self.cfg.params.get("factor", 2.0), + out=self.outputs[self.cfg.outputs[0]].torch, + ) + return dict(self.outputs) + + +class _Composite(DictTensorProcessor): + def __init__(self, cfg, data): + super().__init__(cfg, data) + color = data["color"] + self.outputs["color"] = ProxyArray(wp.empty((color.shape[0], 1, 1, 3), dtype=color.dtype, device=color.device)) + + def __call__(self, data): + torch.sub(data["color"].torch[:, :1, :1], data["range"].torch[:, :1, :1], out=self.outputs["color"].torch) + return dict(self.outputs) + + +@pytest.fixture +def composite_pipe(): + sources = { + "raw_color": ProxyArray(wp.zeros((2, 2, 2, 3), dtype=wp.float32, device="cpu")), + "range": ProxyArray(wp.zeros((2, 2, 2, 1), dtype=wp.float32, device="cpu")), + } + steps = [ + DictTensorProcessorCfg(class_type=_Scale, inputs=["raw_color"], outputs=["color"]), + DictTensorProcessorCfg( + class_type=_Composite, + inputs=["color", "range"], + outputs=["color"], + ), + ] + assert _DictTensorProcessorPipeline.required_source_names(steps, ["color", "labels"]) == { + "raw_color", + "range", + "labels", + } + pipe = _DictTensorProcessorPipeline(sources, steps) + yield pipe + pipe.close() + + +def test_ordered_multi_input_operations_preserve_sources_and_reuse_output_storage(composite_pipe): + """Bindings consume earlier replacements; shape changes and strided inputs preserve ownership.""" + backing = torch.full((2, 2, 2, 4), 3.0) + color = ProxyArray(wp.from_torch(backing[..., :3])) + depth = ProxyArray(wp.full((2, 2, 2, 1), 2.0, dtype=wp.float32, device="cpu")) + labels = ProxyArray(wp.ones((2, 2, 2, 1), dtype=wp.int32, device="cpu")) + sources = {"raw_color": color, "range": depth, "labels": labels} + output = composite_pipe(sources) + snapshot = output["color"].torch.clone() + destination = output["color"] + + torch.testing.assert_close(snapshot, torch.full((2, 1, 1, 3), 4.0)) + torch.testing.assert_close(backing, torch.full_like(backing, 3.0)) + torch.testing.assert_close(depth.torch, torch.full_like(depth.torch, 2.0)) + assert output["labels"] is labels + assert output["raw_color"] is color + assert set(sources) == {"raw_color", "range", "labels"} + assert composite_pipe.outputs["color"].shape == (2, 1, 1, 3) + + backing.fill_(5.0) + updated = composite_pipe(sources) + assert updated["color"] is destination + torch.testing.assert_close(destination.torch, torch.full_like(snapshot, 8.0)) + torch.testing.assert_close(snapshot, torch.full_like(snapshot, 4.0)) + updated.pop("color") + assert composite_pipe.outputs["color"] is destination + + +def test_tensor_adapter_preserves_incoming_transformations_and_returns_snapshots(composite_pipe): + """The current observation supplies color, auxiliary depth stays read-only, and outputs are independent.""" + original = torch.full((2, 2, 2, 3), 3.0) + incoming = original * 2.0 + depth = ProxyArray(wp.full((2, 2, 2, 1), 2.0, dtype=wp.float32, device="cpu")) + kwargs = {"input_name": "raw_color", "output_name": "color", "auxiliary_buffers": {"range": depth}} + first = composite_pipe.apply_to_tensor(incoming, **kwargs) + second = composite_pipe.apply_to_tensor(torch.full_like(incoming, 9.0), **kwargs) + torch.testing.assert_close(first, torch.full((2, 1, 1, 3), 10.0)) + torch.testing.assert_close(second, torch.full_like(first, 16.0)) + torch.testing.assert_close(incoming, torch.full_like(incoming, 6.0)) + torch.testing.assert_close(depth.torch, torch.full_like(depth.torch, 2.0)) + assert first.data_ptr() != second.data_ptr() + + with pytest.raises(ValueError, match="must not replace"): + composite_pipe.apply_to_tensor( + incoming, + input_name="raw_color", + output_name="color", + auxiliary_buffers={"raw_color": ProxyArray(wp.from_torch(original)), "range": depth}, + ) + + +def test_tensor_snapshot_accepts_passthrough_buffer_omitted_from_previous_call(): + """Optional pass-through buffers can return after a call that did not supply them.""" + original = ProxyArray(wp.zeros((2,), dtype=wp.float32, device="cpu")) + pipe = _DictTensorProcessorPipeline({"value": original}, []) + try: + pipe({}) + current = ProxyArray(wp.full((2,), 3.0, dtype=wp.float32, device="cpu")) + snapshot = pipe.as_tensor({"value": current}, output_name="value") + current.torch.fill_(5.0) + torch.testing.assert_close(snapshot, torch.full((2,), 3.0)) + finally: + pipe.close() + + +@pytest.mark.parametrize("invalid", ["missing", "shape", "dtype"]) +def test_invalid_source_buffers_are_rejected_before_processing(invalid): + """Missing or incompatible external storage must fail before a callback reads it.""" + calls = [] + source = ProxyArray(wp.zeros((2, 1, 1, 1), dtype=wp.float32, device="cpu")) + + class CountCalls(_Scale): + def __call__(self, data): + calls.append(True) + return super().__call__(data) + + pipe = _DictTensorProcessorPipeline( + {"source": source}, + [DictTensorProcessorCfg(class_type=CountCalls, inputs=["source"], outputs=["result"])], + ) + try: + shape = (1, 1, 1, 1) if invalid == "shape" else source.shape + dtype = wp.uint8 if invalid == "dtype" else wp.float32 + inputs = {} if invalid == "missing" else {"source": ProxyArray(wp.zeros(shape, dtype=dtype, device="cpu"))} + with pytest.raises(ValueError, match="Missing required|shape, dtype and device"): + pipe(inputs) + assert not calls + finally: + pipe.close() + + +@pytest.mark.parametrize("invalid", ["missing_dependency", "output_names", "duplicate_outputs", "output_type"]) +def test_invalid_declarations_release_processors(invalid): + """Construction failure closes each successfully constructed processor in reverse order.""" + closed = [] + source = ProxyArray(wp.zeros((2, 1, 1, 1), dtype=wp.float32, device="cpu")) + + class Tracked(_Scale): + def __init__(self, cfg, data): + super().__init__(cfg, data) + if cfg.params["name"] == "second": + if invalid == "output_names": + self.outputs = {"wrong": next(iter(self.outputs.values()))} + elif invalid == "output_type": + self.outputs[cfg.outputs[0]] = self.outputs[cfg.outputs[0]].warp + + def close(self): + closed.append(self.cfg.params["name"]) + + first = DictTensorProcessorCfg(Tracked, ["source"], ["middle"], {"name": "first"}) + second = DictTensorProcessorCfg(Tracked, ["middle"], ["result"], {"name": "second"}) + if invalid == "missing_dependency": + second.inputs = ["absent"] + message = "Missing input buffers for preparation" + elif invalid == "output_names": + message = "declared output names" + elif invalid == "duplicate_outputs": + second.outputs = ["result", "result"] + message = "distinct output names" + else: + message = "outputs must be ProxyArray" + with pytest.raises(TypeError if invalid == "output_type" else ValueError, match=message): + _DictTensorProcessorPipeline({"source": source}, [first, second]) + assert closed == (["second", "first"] if invalid in ("output_names", "output_type") else ["first"]) + + +@pytest.mark.parametrize("close_fails", [False, True], ids=["normal_close", "failing_close"]) +def test_reset_selects_batch_entries_and_close_releases_operations_in_reverse_order(close_fails): + """State belongs to each processor; selected resets and idempotent teardown preserve its lifecycle.""" + closed = [] + source = ProxyArray(wp.zeros((2, 1, 1, 1), dtype=wp.float32, device="cpu")) + + class History(_Scale): + def __init__(self, cfg, data): + super().__init__(cfg, data) + self.history = torch.zeros(data["value"].shape) + + def __call__(self, data): + self.history.add_(data["value"].torch) + self.outputs["value"].torch.copy_(self.history) + return dict(self.outputs) + + def reset(self, env_ids=None): + self.history[slice(None) if env_ids is None else env_ids] = 0 + + def close(self): + name = self.cfg.params["name"] + closed.append(name) + if close_fails and name == "second": + raise RuntimeError("Processor cleanup failed") + + pipe = _DictTensorProcessorPipeline( + {"value": source}, + [DictTensorProcessorCfg(History, ["value"], ["value"], {"name": name}) for name in ("first", "second")], + ) + source = {"value": ProxyArray(wp.ones(source.shape, dtype=wp.float32, device="cpu"))} + try: + torch.testing.assert_close(pipe(source)["value"].torch.flatten(), torch.tensor([1.0, 1.0])) + torch.testing.assert_close(pipe(source)["value"].torch.flatten(), torch.tensor([3.0, 3.0])) + pipe.reset([0]) + torch.testing.assert_close(pipe(source)["value"].torch.flatten(), torch.tensor([1.0, 6.0])) + pipe.reset() + torch.testing.assert_close(pipe(source)["value"].torch.flatten(), torch.tensor([1.0, 1.0])) + finally: + if close_fails: + with pytest.raises(ExceptionGroup, match="Failed to close"): + pipe.close() + else: + pipe.close() + pipe.close() + assert closed == ["second", "first"] + with pytest.raises(RuntimeError, match="closed"): + pipe(source) + with pytest.raises(RuntimeError, match="closed"): + pipe.as_tensor(source, output_name="value") + + +@pytest.mark.parametrize("replacement", ["missing", "new_storage"]) +def test_processor_cannot_replace_published_output_storage(replacement): + """Camera consumers retain prepared tensors, so missing or replaced outputs must fail.""" + + class ReplacesOutput(_Scale): + def __call__(self, data): + super().__call__(data) + if replacement == "missing": + return {} + self.outputs["result"] = ProxyArray(wp.empty_like(data["source"].warp)) + return dict(self.outputs) + + source = ProxyArray(wp.ones((2,), dtype=wp.float32, device="cpu")) + pipe = _DictTensorProcessorPipeline( + {"source": source}, [DictTensorProcessorCfg(ReplacesOutput, ["source"], ["result"])] + ) + try: + with pytest.raises(ValueError, match="persistent named output buffers"): + pipe({"source": source}) + finally: + pipe.close() diff --git a/source/isaaclab/test/utils/test_modifiers.py b/source/isaaclab/test/utils/test_modifiers.py index c131ce76addf..7ac0700b15a5 100644 --- a/source/isaaclab/test/utils/test_modifiers.py +++ b/source/isaaclab/test/utils/test_modifiers.py @@ -8,7 +8,7 @@ import pytest import torch -import isaaclab.utils.modifiers as modifiers +import isaaclab.utils.tensor_processors as tensor_processors from isaaclab.test.utils import test_devices from isaaclab.utils import configclass @@ -17,9 +17,9 @@ @configclass class ModifierTestCfg: - """Configuration for testing modifiers.""" + """Configuration for testing tensor_processors.""" - cfg: modifiers.ModifierCfg = MISSING + cfg: tensor_processors.TensorProcessorCfg = MISSING init_data: torch.Tensor = MISSING result: torch.Tensor = MISSING num_iter: int = 10 @@ -28,17 +28,17 @@ class ModifierTestCfg: @pytest.mark.parametrize( "func, params, result", [ - (modifiers.scale, {"multiplier": 2.0}, [2.0, 4.0, 6.0]), - (modifiers.bias, {"value": 1.0}, [2.0, 3.0, 4.0]), - (modifiers.clip, {"bounds": (1.5, 2.5)}, [1.5, 2.0, 2.5]), - (modifiers.clip, {"bounds": (1.5, None)}, [1.5, 2.0, 3.0]), - (modifiers.clip, {"bounds": (None, 2.5)}, [1.0, 2.0, 2.5]), + (tensor_processors.scale, {"multiplier": 2.0}, [2.0, 4.0, 6.0]), + (tensor_processors.bias, {"value": 1.0}, [2.0, 3.0, 4.0]), + (tensor_processors.clip, {"bounds": (1.5, 2.5)}, [1.5, 2.0, 2.5]), + (tensor_processors.clip, {"bounds": (1.5, None)}, [1.5, 2.0, 3.0]), + (tensor_processors.clip, {"bounds": (None, 2.5)}, [1.0, 2.0, 2.5]), ], ids=["scale", "bias", "clip", "clip_no_upper_bound", "clip_no_lower_bound"], ) -def test_stateless_modifiers(func, params, result): - """Test the stateless scale, bias, and clip modifiers.""" - cfg = modifiers.ModifierCfg(func=func, params=params) +def test_stateless_tensor_processors(func, params, result): + """Test the stateless scale, bias, and clip tensor_processors.""" + cfg = tensor_processors.TensorProcessorCfg(func=func, params=params) output = cfg.func(torch.tensor([1.0, 2.0, 3.0]), **cfg.params) assert torch.allclose(output, torch.tensor(result)) @@ -54,7 +54,7 @@ def test_digital_filter(device): # create test config test_cfg = ModifierTestCfg( - cfg=modifiers.DigitalFilterCfg(A=A, B=B), init_data=init_data, result=result, num_iter=16 + cfg=tensor_processors.DigitalFilterCfg(A=A, B=B), init_data=init_data, result=result, num_iter=16 ) # create a modifier instance @@ -89,7 +89,7 @@ def test_integral(device): # create test config test_cfg = ModifierTestCfg( - cfg=modifiers.IntegratorCfg(dt=dt), + cfg=tensor_processors.IntegratorCfg(dt=dt), init_data=init_data, result=result, num_iter=6, diff --git a/source/isaaclab_experimental/changelog.d/dict-tensor-processors.changed.rst b/source/isaaclab_experimental/changelog.d/dict-tensor-processors.changed.rst new file mode 100644 index 000000000000..be55076a3cac --- /dev/null +++ b/source/isaaclab_experimental/changelog.d/dict-tensor-processors.changed.rst @@ -0,0 +1,2 @@ +* **Breaking:** Renamed the modifiers API to ``tensor_processors``, ``TensorProcessor``, and + ``TensorProcessorCfg``. Use ``ObservationTermCfg.tensor_processors`` for observation processing. diff --git a/source/isaaclab_experimental/changelog.d/dict-tensor-processors.major b/source/isaaclab_experimental/changelog.d/dict-tensor-processors.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_experimental/isaaclab_experimental/managers/observation_manager.py b/source/isaaclab_experimental/isaaclab_experimental/managers/observation_manager.py index 051aaa8a6ff6..5fed28b70c63 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/managers/observation_manager.py +++ b/source/isaaclab_experimental/isaaclab_experimental/managers/observation_manager.py @@ -32,11 +32,11 @@ The observation manager can be used to compute observations for all the groups or for a specific group. The observations are computed by calling the registered functions for each term in the group. The functions are -called in the order of the terms in the group. The functions are expected to return a tensor with shape -(num_envs, ...). +called in the order of the terms in the group. The functions write into pre-allocated Warp buffers +with shape (num_envs, ...) and return None. -If a noise model or custom modifier is registered for a term, the function is called to corrupt -the observation. The corruption function is expected to return a tensor with the same shape as the observation. +If a tensor processor or noise model is registered for a term, it updates the observation buffer +in place, preserving its shape. Tensor processors run before noise models. The observations are clipped and scaled as per the configuration settings. Experimental (Warp-first) note: @@ -59,7 +59,7 @@ from isaaclab.managers.manager_term_cfg import ObservationGroupCfg, ObservationTermCfg from isaaclab.utils import instantiate, to_dict -from isaaclab_experimental.utils import modifiers, noise +from isaaclab_experimental.utils import noise, tensor_processors from isaaclab_experimental.utils.buffers import CircularBuffer from isaaclab_experimental.utils.torch_utils import clone_obs_buffer @@ -133,11 +133,11 @@ class ObservationManager(ManagerBase): The observation manager can be used to compute observations for all the groups or for a specific group. The observations are computed by calling the registered functions for each term in the group. The functions are - called in the order of the terms in the group. The functions are expected to return a tensor with shape - (num_envs, ...). + called in the order of the terms in the group. The functions write into pre-allocated Warp buffers + with shape (num_envs, ...) and return None. - If a noise model or custom modifier is registered for a term, the function is called to corrupt - the observation. The corruption function is expected to return a tensor with the same shape as the observation. + If a tensor processor or noise model is registered for a term, it updates the observation buffer + in place, preserving its shape. Tensor processors run before noise models. The observations are clipped and scaled as per the configuration settings. Experimental (Warp-first) note: @@ -350,7 +350,7 @@ def _collect_io_descriptors(self, group_names_to_export: list[str] = ["policy"]) func(self._env, **term_cfg.params, inspect=True) desc = func._descriptor.__dict__.copy() overloads = {} - for k in ["modifiers", "clip", "scale", "history_length", "flatten_history_dim"]: + for k in ["tensor_processors", "clip", "scale", "history_length", "flatten_history_dim"]: if hasattr(term_cfg, k): overloads[k] = getattr(term_cfg, k) desc.update(overloads) @@ -374,7 +374,7 @@ def _collect_io_descriptors(self, group_names_to_export: list[str] = ["policy"]) v = v.detach().cpu().numpy().tolist() if k in ["scale", "clip", "history_length", "flatten_history_dim"]: formatted_item["overloads"][k] = v - elif k in ["modifiers", "description", "units"]: + elif k in ["tensor_processors", "description", "units"]: formatted_item["extras"][k] = v else: formatted_item[k] = v @@ -409,7 +409,7 @@ def reset( for term_name in self._group_obs_term_names[group_name]: if term_name in self._group_obs_term_history_buffer[group_name]: self._group_obs_term_history_buffer[group_name][term_name].reset(env_mask=env_mask) - # call all modifiers/noise models that are classes + # call all tensor processors/noise models that are classes for mod in self._group_obs_class_instances: mod.reset(env_mask=env_mask) @@ -457,13 +457,13 @@ def compute_group(self, group_name: str, update_history: bool = False) -> torch. """Computes the observations for a given group. The observations for a given group are computed by calling the registered functions for each - term in the group. The functions are called in the order of the terms in the group. The functions - are expected to return a tensor with shape (num_envs, ...). + term in the group. The functions are called in the order of the terms in the group and write into + pre-allocated Warp buffers with shape (num_envs, ...). The following steps are performed for each observation term: 1. Compute observation term by calling the function - 2. Apply custom modifiers in the order specified in :attr:`ObservationTermCfg.modifiers` + 2. Apply custom tensor processors in the order specified in :attr:`ObservationTermCfg.tensor_processors` 3. Apply corruption/noise model based on :attr:`ObservationTermCfg.noise` 4. Apply clipping based on :attr:`ObservationTermCfg.clip` 5. Apply scaling based on :attr:`ObservationTermCfg.scale` @@ -500,14 +500,14 @@ def compute_group(self, group_name: str, update_history: bool = False) -> torch. # Persistent per-term obs dict (pre-allocated in _prepare_terms). group_obs = self._group_obs_dict[group_name] - # evaluate terms: compute, add noise, clip, scale, custom modifiers + # evaluate terms: compute, apply tensor processors, add noise, clip, scale for term_name, term_cfg in zip(group_term_names, self._group_obs_term_cfgs[group_name]): # compute term's value into pre-allocated Warp output term_cfg.func(self._env, term_cfg.out_wp, **term_cfg.params) - # apply custom modifiers (in-place on out_wp) - if term_cfg.modifiers is not None: - for modifier in term_cfg.modifiers: + # apply custom tensor processors (in-place on out_wp) + if term_cfg.tensor_processors is not None: + for modifier in term_cfg.tensor_processors: modifier.func(term_cfg.out_wp, **modifier.params) # apply noise (Warp in-place on out_wp) @@ -605,9 +605,9 @@ def _prepare_terms(self): # noqa: C901 self._group_obs_concatenate_dim: dict[str, int] = dict() self._group_obs_term_history_buffer: dict[str, dict] = dict() - # create a list to store classes instances, e.g., for modifiers and noise models + # create a list to store class instances, e.g., for tensor processors and noise models # we store it as a separate list to only call reset on them and prevent unnecessary calls - self._group_obs_class_instances: list[modifiers.ModifierBase | noise.NoiseModel] = list() + self._group_obs_class_instances: list[tensor_processors.TensorProcessor | noise.NoiseModel] = list() # Persistent Warp output buffers for concatenated 2D groups (optional fast-path). # For other cases (non-concat groups, history outputs, non-2D concat dims), we allocate per-term outputs. @@ -735,27 +735,27 @@ def _prepare_terms(self): # noqa: C901 ) term_cfg.scale_wp = wp.array(scale_vals, dtype=wp.float32, device=self._env.device) - # prepare modifiers for each observation - if term_cfg.modifiers is not None: - # initialize list of modifiers for term - for mod_cfg in term_cfg.modifiers: - # check if class modifier and initialize with observation size when adding - if isinstance(mod_cfg, modifiers.ModifierCfg): - # to list of modifiers + # prepare tensor processors for each observation + if term_cfg.tensor_processors is not None: + # initialize list of tensor processors for term + for mod_cfg in term_cfg.tensor_processors: + # check if class processor and initialize with observation size when adding + if isinstance(mod_cfg, tensor_processors.TensorProcessorCfg): + # to list of tensor processors if inspect.isclass(mod_cfg.func): - if not issubclass(mod_cfg.func, modifiers.ModifierBase): + if not issubclass(mod_cfg.func, tensor_processors.TensorProcessor): raise TypeError( f"Modifier function '{mod_cfg.func}' for observation term '{term_name}'" - f" is not a subclass of 'ModifierBase'. Received: '{type(mod_cfg.func)}'." + f" is not a subclass of 'TensorProcessor'. Received: '{type(mod_cfg.func)}'." ) mod_cfg.func = mod_cfg.func(cfg=mod_cfg, data_dim=obs_dims, device=self._env.device) - # add to list of class modifiers + # add to list of class tensor processors self._group_obs_class_instances.append(mod_cfg.func) else: raise TypeError( f"Modifier configuration '{mod_cfg}' of observation term '{term_name}' is not of" - f" required type ModifierCfg, Received: '{type(mod_cfg)}'" + f" required type TensorProcessorCfg, Received: '{type(mod_cfg)}'" ) # check if function is callable diff --git a/source/isaaclab_experimental/isaaclab_experimental/utils/__init__.pyi b/source/isaaclab_experimental/isaaclab_experimental/utils/__init__.pyi index de2860732514..f4aaa2e64681 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/utils/__init__.pyi +++ b/source/isaaclab_experimental/isaaclab_experimental/utils/__init__.pyi @@ -9,7 +9,7 @@ __all__ = [ "WarpGraphCache", "clone_obs_buffer", "buffers", - "modifiers", + "tensor_processors", "noise", "warp", ] @@ -17,4 +17,4 @@ __all__ = [ from .manager_call_switch import ManagerCallMode, ManagerCallSwitch from .torch_utils import clone_obs_buffer from .warp_graph_cache import WarpGraphCache -from . import buffers, modifiers, noise, warp +from . import buffers, tensor_processors, noise, warp diff --git a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/__init__.py b/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/__init__.py deleted file mode 100644 index 3563b166642a..000000000000 --- a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Warp-native modifier implementations (experimental). - -Re-exports stable configs and base classes, then overrides the function-based -modifiers (``scale``, ``bias``, ``clip``) with Warp-native versions that -operate in-place on ``wp.array``. - -Calling convention (matches Warp MDP terms):: - - modifier.func(data_wp, **params) -> None # in-place on wp.array -""" - -from .modifier import bias, clip, scale # noqa: F401 -from .modifier_base import ModifierBase # noqa: F401 - -# Override with Warp-native implementations -from .modifier_cfg import ModifierCfg # noqa: F401 diff --git a/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/__init__.py b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/__init__.py new file mode 100644 index 000000000000..fa6f870c2a67 --- /dev/null +++ b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Warp-native processor implementations (experimental). + +Provides Warp-native configuration and base classes, plus function-based tensor +processors (``scale``, ``bias``, ``clip``) that operate in-place on ``wp.array``. + +Calling convention (matches Warp MDP terms):: + + processor.func(data_wp, **params) -> None # in-place on wp.array +""" + +from .processors import bias, clip, scale # noqa: F401 +from .tensor_processor import TensorProcessor # noqa: F401 +from .tensor_processor_cfg import TensorProcessorCfg # noqa: F401 diff --git a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier.py b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/processors.py similarity index 78% rename from source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier.py rename to source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/processors.py index 887db5e4ee06..6406befb1dc4 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier.py +++ b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/processors.py @@ -3,12 +3,12 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Warp-native function-based modifiers (experimental). +"""Warp-native function-based tensor processors (experimental). -Each modifier takes a ``wp.array`` as its first argument and operates **in-place** +Each processor takes a ``wp.array`` as its first argument and operates **in-place** via ``wp.launch``. The calling convention mirrors Warp MDP terms:: - modifier.func(data_wp, **params) -> None + processor.func(data_wp, **params) -> None """ from __future__ import annotations @@ -27,7 +27,8 @@ def _scale_kernel(data: wp.array(dtype=wp.float32, ndim=2), multiplier: wp.float def scale(data: wp.array, multiplier: float) -> None: """Scale all elements of *data* by *multiplier* in-place. - Warp-native drop-in replacement for :func:`isaaclab.utils.modifiers.scale`. + Warp-native counterpart of :func:`isaaclab.utils.tensor_processors.scale`, updating the supplied + array in place and returning None. Args: data: The observation buffer to modify. Shape ``(num_envs, D)``. @@ -48,7 +49,8 @@ def _bias_kernel(data: wp.array(dtype=wp.float32, ndim=2), value: wp.float32): def bias(data: wp.array, value: float) -> None: """Add a uniform *value* to all elements of *data* in-place. - Warp-native drop-in replacement for :func:`isaaclab.utils.modifiers.bias`. + Warp-native counterpart of :func:`isaaclab.utils.tensor_processors.bias`, updating the supplied + array in place and returning None. Args: data: The observation buffer to modify. Shape ``(num_envs, D)``. @@ -69,7 +71,8 @@ def _clip_kernel(data: wp.array(dtype=wp.float32, ndim=2), lo: wp.float32, hi: w def clip(data: wp.array, bounds: tuple[float | None, float | None]) -> None: """Clamp all elements of *data* to [lo, hi] in-place. - Warp-native drop-in replacement for :func:`isaaclab.utils.modifiers.clip`. + Warp-native counterpart of :func:`isaaclab.utils.tensor_processors.clip`, updating the supplied + array in place and returning None. Args: data: The observation buffer to modify. Shape ``(num_envs, D)``. diff --git a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier_base.py b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor.py similarity index 67% rename from source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier_base.py rename to source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor.py index 083e1576e891..ccfb87ab6fc9 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier_base.py +++ b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Warp-native modifier base class (experimental).""" +"""Warp-native processor base class (experimental).""" from __future__ import annotations @@ -13,29 +13,29 @@ import warp as wp if TYPE_CHECKING: - from .modifier_cfg import ModifierCfg + from .tensor_processor_cfg import TensorProcessorCfg -class ModifierBase(ABC): - """Base class for Warp-native class-based modifiers. +class TensorProcessor(ABC): + """Base class for Warp-native class-based tensor processors. - Experimental fork of :class:`isaaclab.utils.modifiers.ModifierBase` adapted for the + Experimental fork of :class:`isaaclab.utils.tensor_processors.TensorProcessor` adapted for the Warp-first calling convention. Subclasses operate **in-place** on ``wp.array`` buffers and return ``None``. - A class implementation of a modifier can be used to store state information between - calls. This is useful for modifiers that require stateful operations, such as + A class implementation of a processor can be used to store state information between + calls. This is useful for tensor processors that require stateful operations, such as rolling averages, delays, or decaying filters. """ - def __init__(self, cfg: ModifierCfg, data_dim: tuple[int, ...], device: str) -> None: - """Initializes the modifier class. + def __init__(self, cfg: TensorProcessorCfg, data_dim: tuple[int, ...], device: str) -> None: + """Initializes the processor class. Args: cfg: Configuration parameters. data_dim: The dimensions of the data to be modified. First element is the batch size (number of environments). - device: The device to run the modifier on. + device: The device to run the processor on. """ self._cfg = cfg self._data_dim = data_dim @@ -43,7 +43,7 @@ def __init__(self, cfg: ModifierCfg, data_dim: tuple[int, ...], device: str) -> @abstractmethod def reset(self, env_mask: wp.array | None = None): - """Resets the modifier. + """Resets the processor. Args: env_mask: Boolean env mask of shape ``(num_envs,)`` selecting environments diff --git a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier_cfg.py b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor_cfg.py similarity index 65% rename from source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier_cfg.py rename to source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor_cfg.py index 7cc22eaee6a9..26bb86e33e25 100644 --- a/source/isaaclab_experimental/isaaclab_experimental/utils/modifiers/modifier_cfg.py +++ b/source/isaaclab_experimental/isaaclab_experimental/utils/tensor_processors/tensor_processor_cfg.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Warp-native modifier configuration (experimental).""" +"""Warp-native processor configuration (experimental).""" from collections.abc import Callable from dataclasses import MISSING @@ -13,23 +13,23 @@ @configclass -class ModifierCfg: - """Configuration parameters for Warp-native modifiers. +class TensorProcessorCfg: + """Configuration parameters for Warp-native tensor processors. - Experimental fork of :class:`isaaclab.utils.modifiers.ModifierCfg` adapted for the - Warp-first calling convention where modifier functions operate **in-place** on a + Experimental fork of :class:`isaaclab.utils.tensor_processors.TensorProcessorCfg` adapted for the + Warp-first calling convention where processor functions operate **in-place** on a ``wp.array`` buffer and return ``None``. """ func: Callable[..., None] = MISSING - """Function or callable class used by modifier. + """Function or callable class used by processor. The function must take a ``wp.array`` as the first argument and operate on it **in-place** (no return value). The remaining arguments are specified in the :attr:`params` attribute. It also supports callable classes that implement ``__call__()``. In this case the - class should inherit from :class:`ModifierBase` and implement the required methods. + class should inherit from :class:`TensorProcessor` and implement the required methods. """ params: dict[str, Any] = dict() diff --git a/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..5832bdc80d5d --- /dev/null +++ b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.changed.rst @@ -0,0 +1,3 @@ +* **Breaking:** Replaced renderer-owned PPISP with dictionary processors. + Replace ``CameraCfg.isp_cfg`` with ``PpispProcessorCfg`` and configure + ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.major b/source/isaaclab_newton/changelog.d/feat-render-buffer-pipe.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py index ae06432e1ac9..6e26ae05b3ce 100644 --- a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py +++ b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py @@ -11,7 +11,7 @@ import warnings from dataclasses import dataclass from functools import partial -from typing import TYPE_CHECKING, Any, NoReturn +from typing import TYPE_CHECKING, Any import newton import warp as wp @@ -27,8 +27,6 @@ from .segmentation import NewtonSegmentationMapper, NewtonSegmentationMapping if TYPE_CHECKING: - from isaaclab_ppisp import PpispPipeline - from isaaclab.sensors.camera.camera_data import CameraData from isaaclab.utils.warp import ProxyArray @@ -68,21 +66,6 @@ def _update_camera_rays(intrinsics: wp.array(dtype=wp.mat33f), rays: wp.array4d( rays[0, y, x, 1] = wp.normalize(direction) -_PPISP_IMPORT_ERROR_MESSAGE = ( - "isaaclab_ppisp is required when CameraCfg.isp_cfg is set. " - "Run `uv sync` from the Isaac Lab source checkout, or install the Isaac Lab wheel " - "with `uv pip install isaaclab`." -) - - -def _raise_missing_ppisp_error(exc: ModuleNotFoundError) -> NoReturn: - # Only translate missing isaaclab_ppisp imports into the optional-dependency hint; - # unrelated missing modules should surface unchanged for easier debugging. - if exc.name != "isaaclab_ppisp" and not (exc.name and exc.name.startswith("isaaclab_ppisp.")): - raise exc - raise ModuleNotFoundError(_PPISP_IMPORT_ERROR_MESSAGE, name="isaaclab_ppisp") from exc - - class RenderData: # Back-compat alias for callers of ``RenderData.OutputNames``. OutputNames = RenderBufferKind @@ -100,6 +83,7 @@ class RenderData: # :class:`~isaaclab_newton.renderers.segmentation.NewtonSegmentationMapper`. _OUTPUT_MAP: dict[str, tuple[str, type]] = { str(RenderBufferKind.RGBA): ("color_image", wp.uint32), + # Newton applies no camera exposure, so its HDR color is already scene-linear. str(RenderBufferKind.RGB_HDR): ("hdr_color_image", wp.vec3f), str(RenderBufferKind.ALBEDO): ("albedo_image", wp.uint32), str(RenderBufferKind.NORMALS): ("normals_image", wp.vec3f), @@ -183,27 +167,6 @@ def __init__( # OpenCV lens-distortion model (``spawn.distortion``), consumed by :meth:`_build_distortion_rays` # to trace distorted per-pixel rays instead of the centered, square-pixel pinhole field. self._distortion = spawn.distortion if spawn is not None else None - # Post-render PPISP pipeline composed when ``spec.cfg.isp_cfg`` is set. - # ``isp_cfg`` is already fully normalized by ``prepare_cameras`` by the time it reaches here. - self.ppisp_pipeline: PpispPipeline | None = None - if spec.cfg.isp_cfg is not None: - try: - from isaaclab_ppisp import PpispPipeline - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - self.ppisp_pipeline = PpispPipeline(spec.cfg.isp_cfg) - self._hdr_scratch_wp: wp.array | None = None - """Internal HDR scratch buffer allocated when PPISP is composed but the - user did not request ``"rgb_hdr"`` in ``data_types``. Also exposed to - the Newton sensor through :attr:`CameraOutputs.hdr_color_image` as a - vec3f reinterpretation of this same backing storage.""" - self._ppisp_hdr_source: wp.array | None = None - """PPISP HDR source bound once in :meth:`set_outputs` from the caller's - ``rgb_hdr`` output or :attr:`_hdr_scratch_wp`.""" - self._ppisp_rgba_dest: wp.array | None = None - """PPISP LDR destination bound once in :meth:`set_outputs` from the - caller's ``rgba`` output.""" def _view(self, proxy: ProxyArray, dtype: type, shape: tuple[int, ...]) -> wp.array: """Alias the caller's output buffer as a ``(world_count, 1, H, W)`` warp array of ``dtype``. @@ -262,24 +225,6 @@ def set_outputs(self, output_data: dict[str, ProxyArray]): # requested segmentation outputs are remapped from this single buffer in :meth:`_convert_segmentation`. if self._seg_dests: self.outputs.shape_index_image = wp.zeros(shape, dtype=wp.uint32, device=self.newton_sensor.model.device) - # When PPISP is composed but the user did not request the raw HDR AOV, - # allocate an internal HDR scratch buffer and route a vec3f-shaped view - # of it as the Newton sensor's ``hdr_color_image`` so the renderer - # fills it directly. - if self.ppisp_pipeline is not None and self.outputs.hdr_color_image is None: - ref_proxy = next(iter(output_data.values())) - self._hdr_scratch_wp = wp.zeros( - (self.newton_sensor.model.world_count, self.height, self.width, 3), - dtype=wp.float32, - device=ref_proxy.device, - ) - self.outputs.hdr_color_image = wp.array( - ptr=self._hdr_scratch_wp.ptr, - dtype=wp.vec3f, - shape=shape, - device=self._hdr_scratch_wp.device, - copy=False, - ) # Newton clears HDR to black regardless of clear_color. A hit mask preserves dark geometry. if ( self.outputs.hdr_color_image is not None @@ -287,17 +232,6 @@ def set_outputs(self, output_data: dict[str, ProxyArray]): and self.outputs.shape_index_image is None ): self.outputs.shape_index_image = wp.zeros(shape, dtype=wp.uint32, device=self.newton_sensor.model.device) - # Bind the two warp arrays the per-frame PPISP dispatch needs. - if self.ppisp_pipeline is not None: - if str(RenderBufferKind.RGBA) not in output_data: - raise ValueError( - "Newton renderer ISP requires 'rgba' (or 'rgb', which aliases into rgba) as the" - " LDR output destination, but neither was provided. Add 'rgb' or 'rgba' to" - " Camera.cfg.data_types when isp_cfg is set." - ) - hdr_proxy = output_data.get(str(RenderBufferKind.RGB_HDR)) - self._ppisp_hdr_source = hdr_proxy.warp if hdr_proxy is not None else self._hdr_scratch_wp - self._ppisp_rgba_dest = output_data[str(RenderBufferKind.RGBA)].warp def get_output(self, output_name: str) -> wp.array: if output_name in self._depth_dests: @@ -512,15 +446,10 @@ def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: return self.cfg.supported_output_types() def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: - """Resolve the camera's PPISP cfg before rendering. - - :mod:`isaaclab.sensors.camera` does not depend on PPISP; the renderer - owns the sentinel-resolution + cfg-normalization step. Newton has no - USD-side overrides to author beyond this. - - Also captures the USD ``stage`` so the segmentation mapper can read the scene's - :class:`UsdSemantics.LabelsAPI` labels when a segmentation output is requested. + """Capture the USD stage for the segmentation mapper. + The mapper reads the scene's :class:`UsdSemantics.LabelsAPI` labels when a segmentation + output is requested. Newton authors no camera overrides. """ self._stage = stage if spec.cfg.spawn is None or spec.cfg.spawn.distortion is None: @@ -539,15 +468,6 @@ def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: stacklevel=2, ) break - if spec.cfg.isp_cfg is None: - return - try: - from isaaclab_ppisp import resolve_and_normalize - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - camera_prim_path = spec.camera_prim_paths[0] if spec.camera_prim_paths else None - spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) def prepare_stage(self, stage: Any, num_envs: int) -> None: """No-op for Newton Warp - uses Newton scene directly without stage export. @@ -642,14 +562,6 @@ def render(self, render_data: RenderData): use_cuda_graph=self.cfg.use_cuda_graph and backend.model.tri_count == 0, ) - # Post-render PPISP: HDR scene-linear → LDR RGBA. Source/destination - # tensors were bound once in ``set_outputs``. - if render_data.ppisp_pipeline is not None: - render_data.ppisp_pipeline.apply( - render_data._ppisp_hdr_source, - render_data._ppisp_rgba_dest, - ) - def _launch_render(self, render_data: RenderData) -> None: """Launch the tiled-camera render kernels for sensor graph capture.""" # default_render_config is shared state across all Newton sensors, so set max_distance diff --git a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py index 201689809193..584ceddcd84a 100644 --- a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py +++ b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py @@ -312,7 +312,7 @@ def read(_data): monkeypatch.setattr(renderer, "_launch_render", read) for _ in range(2): - render_data = SimpleNamespace(graph=None, ppisp_pipeline=None) + render_data = SimpleNamespace(graph=None) renderer.render(render_data) assert render_data.graph is None assert len(refits) == 1 diff --git a/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..5832bdc80d5d --- /dev/null +++ b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.changed.rst @@ -0,0 +1,3 @@ +* **Breaking:** Replaced renderer-owned PPISP with dictionary processors. + Replace ``CameraCfg.isp_cfg`` with ``PpispProcessorCfg`` and configure + ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.major b/source/isaaclab_ov/changelog.d/feat-render-buffer-pipe.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index e3af1f53525f..964865597dc8 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -32,7 +32,7 @@ from collections import deque from collections.abc import Iterable, Iterator, Sequence from pathlib import Path -from typing import TYPE_CHECKING, Any, NoReturn, cast +from typing import TYPE_CHECKING, Any, cast logger = logging.getLogger(__name__) @@ -112,7 +112,6 @@ ) if TYPE_CHECKING: - from isaaclab_ppisp import PpispPipeline from ovrtx import AttributeBinding, Operation, PendingFetch, RenderProductSetOutputs from isaaclab.renderers.base_renderer import VisualMaterialBatch @@ -150,11 +149,6 @@ } -_PPISP_IMPORT_ERROR_MESSAGE = ( - "isaaclab_ppisp is required when CameraCfg.isp_cfg is set. " - "Run `uv sync` from the Isaac Lab source checkout, or install the Isaac Lab wheel " - "with `uv pip install isaaclab`." -) _READ_GPU_TRANSFORMS_ENV = "ISAAC_LAB_OVRTX_READ_GPU_TRANSFORMS" @@ -182,14 +176,6 @@ def ovrtx_use_ovstage_enabled() -> bool: return value == "1" -def _raise_missing_ppisp_error(exc: ModuleNotFoundError) -> NoReturn: - # Only translate missing isaaclab_ppisp imports into the optional-dependency hint; - # unrelated missing modules should surface unchanged for easier debugging. - if exc.name != "isaaclab_ppisp" and not (exc.name and exc.name.startswith("isaaclab_ppisp.")): - raise exc - raise ModuleNotFoundError(_PPISP_IMPORT_ERROR_MESSAGE, name="isaaclab_ppisp") from exc - - def _read_gpu_transforms_enabled() -> bool: """Return whether OVRTX should read GPU transforms from its internal transform cache.""" value = os.environ.get(_READ_GPU_TRANSFORMS_ENV, "1").strip() @@ -340,16 +326,6 @@ def __init__(self, spec: CameraRenderSpec, device, render_scope_name: str): # Populated for "semantic_segmentation" (with an "idToLabels" mapping) and # "instance_segmentation" (with "idToLabels" and "idToSemantics" mappings). self.renderer_info: dict[str, Any] = {} - # Post-render PPISP pipeline composed when ``spec.cfg.isp_cfg`` is set. - # ``isp_cfg`` is already fully normalized by ``prepare_cameras`` by the time it reaches here. - self.ppisp_pipeline: PpispPipeline | None = None - if spec.cfg.isp_cfg is not None: - try: - from isaaclab_ppisp import PpispPipeline - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - self.ppisp_pipeline = PpispPipeline(spec.cfg.isp_cfg) def cleanup(self) -> None: """Release this camera's native resources and buffers. Safe to call repeatedly. @@ -368,7 +344,6 @@ def cleanup(self) -> None: self.intrinsic_bindings.clear() self.warp_buffers.clear() self.renderer_info.clear() - self.ppisp_pipeline = None class OVRTXRenderer(BaseRenderer): @@ -440,29 +415,6 @@ def visual_material_writer(self): """Return a writer factory once this renderer has a scene consumer.""" return self._create_visual_material_writer if self._initialized_scene else None - def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: - """Resolve the camera's PPISP cfg and apply OVRTX-specific USD overrides. - - When ``spec.cfg.isp_cfg`` is set, resolves it (sentinel discovery + - normalization) via :func:`isaaclab_ppisp.resolve_and_normalize` so - :mod:`isaaclab` does not need to know about PPISP. Then pins - ``exposure:*`` to neutral and applies ``OmniRtxCameraExposureAPI_1`` so - the RTX exposure model OVRTX embeds does not compound on top of the - ISP. Without an ISP, the camera prim's authored exposure is left alone. - """ - if spec.cfg.isp_cfg is None: - return - try: - from isaaclab_ppisp import apply_rtx_exposure_overrides, resolve_and_normalize - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - camera_prim_path = spec.camera_prim_paths[0] if spec.camera_prim_paths else None - spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) - if spec.cfg.isp_cfg is None or not spec.camera_prim_paths: - return - apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) - def prepare_stage(self, stage: Any, num_envs: int) -> None: """Prepare the USD stage for OVRTX before :meth:`create_render_data`. @@ -900,23 +852,6 @@ def set_outputs(self, render_data: OVRTXCameraRenderData, output_data: dict[str, render_data.warp_buffers = { name: proxy.warp for name, proxy in output_data.items() if name != str(RenderBufferKind.RGB) } - # When PPISP is composed but the user did not request the raw HDR AOV, - # allocate an internal HDR scratch buffer under "rgb_hdr" so both the - # HdrColor extractor and PPISP dispatch can use the same buffer map. - if render_data.ppisp_pipeline is not None and str(RenderBufferKind.RGB_HDR) not in render_data.warp_buffers: - ref_proxy = next(iter(output_data.values())) - render_data.warp_buffers[str(RenderBufferKind.RGB_HDR)] = wp.zeros( - (render_data.num_envs, render_data.height, render_data.width, 3), - dtype=wp.float32, - device=ref_proxy.device, - ) - if render_data.ppisp_pipeline is not None: - if str(RenderBufferKind.RGBA) not in render_data.warp_buffers: - raise ValueError( - "OVRTX renderer ISP requires 'rgba' (or 'rgb', which aliases into rgba) as the" - " LDR output destination, but neither was provided. Add 'rgb' or 'rgba' to" - " Camera.cfg.data_types when isp_cfg is set." - ) def _update_camera_legacy( self, @@ -1238,24 +1173,12 @@ def _extract_hdr_color_tiles( return if tiled_data.dtype not in (wp.float16, wp.float32): raise TypeError(f"Unsupported OVRTX HdrColor dtype: {tiled_data.dtype}.") - self._launch_extract_all_tiles(render_data, tiled_data, output_buffers["rgb_hdr"]) - - def _prepare_ppisp_hdr_source( - self, render_data: OVRTXCameraRenderData, tiled_data: wp.array, output_buffers: dict - ) -> wp.array: - """Return the PPISP HdrColor source on the output buffer device.""" - if render_data.ppisp_pipeline is None: - return tiled_data - - output_device = str(output_buffers[str(RenderBufferKind.RGB_HDR)].device) - if str(tiled_data.device) == output_device: - return tiled_data - - # The render product pins ``deviceIds`` to this renderer's CUDA device, so the mapping - # normally lands on the output device already. This stays as a fallback for the case OVRTX - # reports as "deviceIds ... not in the active device set" and falls back to automatic - # assignment. - return wp.clone(tiled_data, device=output_device) + output = output_buffers["rgb_hdr"] + if tiled_data.device != output.device: + # OVRTX may fall back to automatic device assignment when the requested deviceIds + # is outside its active device set. Extract on the destination's device in that case. + tiled_data = wp.clone(tiled_data, device=output.device) + self._launch_extract_all_tiles(render_data, tiled_data, output) def _process_render_frame(self, render_data: OVRTXCameraRenderData, frame, output_buffers: dict) -> None: """Extract RGB, depth, albedo, and semantic from a single render frame into output_buffers.""" @@ -1268,7 +1191,7 @@ def _process_render_frame(self, render_data: OVRTXCameraRenderData, frame, outpu if ldr_color is not None: buffer_key = None - if render_data.ppisp_pipeline is None and "rgba" in output_buffers: + if "rgba" in output_buffers: buffer_key = "rgba" else: # The output buffers must contain only one simple shading data type at most after resolution of the data @@ -1303,7 +1226,6 @@ def _process_render_frame(self, render_data: OVRTXCameraRenderData, frame, outpu hdr_color = frame.render_vars.get(render_data.render_var_keys[_HDR_COLOR_VAR]) if hdr_color is not None and "rgb_hdr" in output_buffers: with self._map_render_var_to_dlpack(hdr_color) as tiled_hdr_data: - tiled_hdr_data = self._prepare_ppisp_hdr_source(render_data, tiled_hdr_data, output_buffers) self._extract_hdr_color_tiles(render_data, tiled_hdr_data, output_buffers) self._process_id_segmentation_render_var( @@ -1419,13 +1341,6 @@ def _process_render_products( data.warp_buffers, ) - # Post-render PPISP uses each camera's own HDR source and RGBA destination. - if data.ppisp_pipeline is not None: - data.ppisp_pipeline.apply( - data.warp_buffers[str(RenderBufferKind.RGB_HDR)], - data.warp_buffers[str(RenderBufferKind.RGBA)], - ) - def _close_legacy(self) -> None: """Release the renderer's tensor bindings. See :meth:`close`.""" diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py index 1c740c6547ad..3883f1e9a91f 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py @@ -155,8 +155,7 @@ def build_render_product_as_string( resolved camera path after runtime cloning. Args: - spec: Camera configuration, environment count, and camera paths. ISP configurations - automatically request the HDR render variable in addition to the configured outputs. + spec: Camera configuration, environment count, and camera paths. render_data: Camera's render scope and product identity. device_id: CUDA device index the render product is pinned to, so its render var buffers are allocated on the same device as the Warp kernels that read them. @@ -167,8 +166,6 @@ def build_render_product_as_string( Render product USD layer, including the USDA header and default prim metadata. """ data_types = list(spec.cfg.data_types) - if spec.cfg.isp_cfg is not None and "rgb_hdr" not in data_types: - data_types.append("rgb_hdr") tiled_width, tiled_height = render_data.num_cols * render_data.width, render_data.num_rows * render_data.height render_var_configs = get_render_var_configs(data_types, render_data.render_scope_name) minimal_mode = next( @@ -199,6 +196,12 @@ def build_render_product_as_string( # Workaround: ovrtx 0.5 misses GPU-read transform changes, so eco mode freezes output after 500 frames. render_mode_lines.append("bool omni:rtx:rt:ecoMode:enabled = false") + api_schemas = ["OmniRtxSettingsCommonAdvancedAPI_1"] + if "rgb_hdr" in data_types: + # OVRTX 0.5 reads this per product. Its default bypasses HdrColor for Gaussian pixels. + api_schemas.append("OmniRtxSettingsParticleFieldAPI_1") + render_mode_lines.append("bool omni:rtx:rtpt:gaussian:skipTonemapping:enabled = false") + api_schemas_block = ", ".join(f'"{schema}"' for schema in api_schemas) render_mode_block = "\n ".join(render_mode_lines) ordered_vars = ", ".join(f"<{path}>" for path, _, _ in render_var_configs) @@ -216,7 +219,7 @@ def build_render_product_as_string( def Scope "{render_data.render_scope_name}" {{ def RenderProduct "RenderProduct" ( - prepend apiSchemas = ["OmniRtxSettingsCommonAdvancedAPI_1"] + prepend apiSchemas = [{api_schemas_block}] ) {{ rel camera = [<{render_data.camera_paths[0]}>] uint[] deviceIds = [{device_id}] diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index 7317f8f8b12d..bb473c1815be 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -196,13 +196,11 @@ def test_ovrtx_render_submits_every_product_and_routes_requested_outputs( cameras = [_make_ovrtx_camera_render_data() for _ in range(2 if batch else 1)] products = {} processed = [] - postprocessed = [] submissions = [] published_ordinals = [] for index, camera in enumerate(cameras): camera.render_product_path = f"/Render/Camera{index}" camera.warp_buffers = {str(RenderBufferKind.RGB_HDR): object(), str(RenderBufferKind.RGBA): object()} - camera.ppisp_pipeline = types.SimpleNamespace(apply=lambda *buffers: postprocessed.append(buffers)) products[camera.render_product_path] = types.SimpleNamespace(frames=[object()]) renderer._render_product_paths = [*products, "/Render/UnrequestedCamera"] if missing_output == "product": @@ -229,15 +227,10 @@ def advance_write_floor(*, ordinal): assert processed == [ (camera, products[camera.render_product_path].frames[0], camera.warp_buffers) for camera in cameras ] - assert postprocessed == [ - (camera.warp_buffers[str(RenderBufferKind.RGB_HDR)], camera.warp_buffers[str(RenderBufferKind.RGBA)]) - for camera in cameras - ] else: with pytest.raises(RuntimeError, match=cameras[-1].render_product_path): render(request) assert not processed - assert not postprocessed assert len(submissions) == 1 # Unrequested products are submitted too, and only the requested cameras are read back. @@ -544,6 +537,46 @@ def capture(value, indices): assert renderer._camera_render_data == [] +@pytest.mark.parametrize("data_types", [["rgb_hdr"]]) +def test_ovrtx_hdr_color_fills_requested_hdr_outputs(monkeypatch, data_types): + """Radiance and HDR read one HdrColor var into the camera's own buffer.""" + renderer = _make_ovrtx_renderer_without_backend() + renderer._device = "cpu" + cfg = _make_camera_cfg(data_types) + data = CameraData.allocate( + data_types=cfg.data_types, + height=8, + width=16, + num_views=2, + device="cpu", + supported_specs=renderer.supported_output_types(), + ) + render_data = _make_ovrtx_camera_render_data() + renderer.set_outputs(render_data, data.output) + + monkeypatch.setattr(renderer, "_map_render_var_to_dlpack", lambda source: contextlib.nullcontext(source)) + source = wp.full((8, 32, 4), 2.0, dtype=wp.float32, device="cpu") + frame = types.SimpleNamespace(render_vars={render_data.render_var_keys["HdrColor"]: source}) + renderer._process_render_frame(render_data, frame, render_data.warp_buffers) + + for name in data_types: + np.testing.assert_array_equal(data.output[name].warp.numpy(), 2.0) + + +@pytest.mark.parametrize("data_types", [["rgb"], ["rgb_hdr"]]) +def test_ovrtx_prepare_cameras_preserves_authored_exposure(data_types): + """Selecting render outputs preserves authored camera exposure.""" + from pxr import Sdf, Usd, UsdGeom + + stage = Usd.Stage.CreateInMemory() + camera = UsdGeom.Camera.Define(stage, "/World/Camera").GetPrim() + camera.CreateAttribute("exposure:iso", Sdf.ValueTypeNames.Float).Set(100.0) + spec = types.SimpleNamespace(cfg=types.SimpleNamespace(data_types=data_types), camera_prim_paths=("/World/Camera",)) + _make_ovrtx_renderer_without_backend().prepare_cameras(stage, spec) + + assert camera.GetAttribute("exposure:iso").Get() == 100.0 + + def test_ovrtx_set_outputs_wraps_caller_torch_zero_copy(): """OVRTXRenderer.set_outputs publishes warp views over the caller's warp storage.""" renderer = _make_ovrtx_renderer_without_backend() @@ -577,47 +610,6 @@ def test_ovrtx_set_outputs_wraps_caller_torch_zero_copy(): assert "rgb" not in render_data.warp_buffers -def test_ovrtx_set_outputs_routes_ppisp_buffers_through_warp_buffers(): - """OVRTXRenderer.set_outputs stores PPISP source/destination in warp_buffers.""" - renderer = _make_ovrtx_renderer_without_backend() - - cfg = _make_camera_cfg(["rgb"]) - data = CameraData.allocate( - data_types=cfg.data_types, - height=8, - width=16, - num_views=2, - device="cpu", - supported_specs=renderer.supported_output_types(), - ) - render_data = _make_ovrtx_camera_render_data() - render_data.ppisp_pipeline = object() - renderer.set_outputs(render_data, data.output) - - assert render_data.warp_buffers["rgba"].ptr == data.output["rgba"].warp.ptr - assert "rgb_hdr" in render_data.warp_buffers - assert render_data.warp_buffers["rgb_hdr"].shape == (2, 8, 16, 3) - assert render_data.warp_buffers["rgb_hdr"].dtype is wp.float32 - - -def test_ovrtx_process_frame_skips_ldr_rgba_when_ppisp_is_active(): - """PPISP owns RGBA output, so OVRTX LdrColor should not pre-fill it.""" - - class FailingRenderVar: - def map(self, *args, **kwargs): - raise AssertionError("PPISP RGBA output must not read OVRTX LdrColor") - - renderer = _make_ovrtx_renderer_without_backend() - render_data = _make_ovrtx_camera_render_data() - render_data.ppisp_pipeline = object() - key = "LdrColor" - if ovrtx_renderer_module.uses_prim_path_render_vars(ovrtx_renderer_module.OVRTX_VERSION): - key = f"/{render_data.render_scope_name}/Vars/{key}" - frame = types.SimpleNamespace(render_vars={key: FailingRenderVar()}) - - renderer._process_render_frame(render_data, frame, {"rgba": object()}) - - # Render-var keys depend only on the OVRTX version; ``use_ovstage`` only selects the registration path. @pytest.mark.parametrize(("use_ovstage", "version"), [(False, "0.4"), (True, "0.5")]) def test_ovrtx_process_frame_reads_authored_camera_render_vars(monkeypatch, use_ovstage, version): @@ -707,31 +699,42 @@ def fake_map(self, render_var): render_data.cleanup() -def test_ovrtx_ppisp_hdr_source_is_cloned_to_output_device(monkeypatch): - """PPISP HdrColor source is moved to the HDR output buffer device.""" - - class FakeArray: - device = "cuda:1" - - class OutputArray: - device = "cuda:0" - - cloned = object() +def test_ovrtx_radiance_is_extracted_on_the_output_device(monkeypatch): + """A mapped HDR buffer on another device is copied before radiance tile extraction.""" + renderer = _make_ovrtx_renderer_without_backend() + renderer._device = "cpu" + render_data = _make_ovrtx_camera_render_data() + data = CameraData.allocate( + data_types=["rgb_hdr"], + height=8, + width=16, + num_views=2, + device="cpu", + supported_specs=renderer.supported_output_types(), + ) + renderer.set_outputs(render_data, data.output) + # Only the cross-device transport is mocked; the destination is filled by the real extractor. + source = types.SimpleNamespace(device="cuda:1", dtype=wp.float32) + cloned = wp.full((8, 32, 4), 2.0, dtype=wp.float32, device="cpu") clone_calls = [] def fake_clone(src, *, device): clone_calls.append((src, device)) return cloned - monkeypatch.setattr(wp, "clone", fake_clone) + extract = renderer._launch_extract_all_tiles - renderer = _make_ovrtx_renderer_without_backend() - render_data = _make_ovrtx_camera_render_data() - render_data.ppisp_pipeline = object() - source = FakeArray() + def extract_on_output_device(render_data, source_buffer, output_buffer): + assert clone_calls == [(source, output_buffer.device)], "Copy the mapped HDR buffer before extracting it." + extract(render_data, source_buffer, output_buffer) + + monkeypatch.setattr(wp, "clone", fake_clone) + monkeypatch.setattr(renderer, "_launch_extract_all_tiles", extract_on_output_device) + monkeypatch.setattr(renderer, "_map_render_var_to_dlpack", lambda source: contextlib.nullcontext(source)) + frame = types.SimpleNamespace(render_vars={render_data.render_var_keys["HdrColor"]: source}) + renderer._process_render_frame(render_data, frame, render_data.warp_buffers) - assert renderer._prepare_ppisp_hdr_source(render_data, source, {"rgb_hdr": OutputArray()}) is cloned - assert clone_calls == [(source, "cuda:0")] + np.testing.assert_array_equal(data.output["rgb_hdr"].warp.numpy(), 2.0) def test_launch_extract_all_tiles_rejects_wider_output_channels(): @@ -876,7 +879,6 @@ def test_ovrtx_cleanup_releases_only_the_given_render_data(monkeypatch, cleanup_ renderer._camera_render_data.append(render_data) render_data.warp_buffers = {"rgba": wp.zeros((8, 16, 4), dtype=wp.uint8, device="cpu")} render_data.renderer_info = {"semantic_segmentation": {"idToLabels": {}}} - render_data.ppisp_pipeline = object() if use_ovstage: render_data.camera_xform_query = "to_remove" render_data.resources.callback(renderer.backend.paths.destroy_path_list, "to_remove") @@ -923,7 +925,6 @@ def test_ovrtx_cleanup_releases_only_the_given_render_data(monkeypatch, cleanup_ assert render_data.intrinsic_bindings == [] assert render_data.warp_buffers == {} assert render_data.renderer_info == {} - assert render_data.ppisp_pipeline is None assert render_data.pending is render_data.ready is None assert renderer._render_product_paths == ["/RenderCamera_0/RenderProduct_camera"] assert renderer._initialized_scene is True diff --git a/source/isaaclab_ov/test/test_ovrtx_usd.py b/source/isaaclab_ov/test/test_ovrtx_usd.py index ded8dc4feffb..b4e57277ddd4 100644 --- a/source/isaaclab_ov/test/test_ovrtx_usd.py +++ b/source/isaaclab_ov/test/test_ovrtx_usd.py @@ -219,14 +219,17 @@ def test_ovrtx_authors_one_render_var_per_requested_data_type(): def test_ovrtx_data_types_sharing_a_source_author_one_render_var(): - """``rgb``/``rgba`` and ``depth``/``distance_to_image_plane`` collapse onto one render var each.""" + """``rgb``/``rgba``, ``depth``/``distance_to_image_plane``, and ``rgb_hdr`` collapse onto one + render var each.""" render_var_configs = get_render_var_configs( - ["rgb", "rgba", "depth", "distance_to_image_plane"], render_scope_name="RenderCamera_0" + ["rgb", "rgba", "depth", "distance_to_image_plane", "rgb_hdr"], + render_scope_name="RenderCamera_0", ) assert render_var_configs == [ ("/RenderCamera_0/Vars/LdrColor", "LdrColor", "LdrColor"), ("/RenderCamera_0/Vars/depth", "depth", "DistanceToImagePlaneSD"), + ("/RenderCamera_0/Vars/HdrColor", "HdrColor", "HdrColor"), ] @@ -307,20 +310,30 @@ def test_render_product_pins_device_ids_to_the_requested_cuda_device(camera_spec assert list(device_ids.default) == [1] -@pytest.mark.parametrize("data_types", [["rgb"], ["rgb", "rgb_hdr"], []]) -def test_render_product_isp_requests_hdr_without_mutating_camera_outputs(camera_spec, render_data, data_types): - """ISP receives one HDR source while the camera's requested outputs remain unchanged.""" +@pytest.mark.parametrize( + ("data_types", "routes_hdr"), + [(["rgb"], False), (["rgb", "rgb_hdr"], True)], +) +def test_render_product_routes_gaussian_hdr_only_for_hdr_outputs(camera_spec, render_data, data_types, routes_hdr): + """Route Gaussian HDR when an HDR output is requested, without changing the camera outputs.""" camera_spec.cfg.data_types = data_types.copy() - camera_spec.cfg.isp_cfg = object() render_product = build_render_product_as_string(camera_spec, render_data, device_id=0) layer = Sdf.Layer.CreateAnonymous(".usda") assert layer.ImportFromString(render_product) - ordered_vars = layer.GetRelationshipAtPath("/RenderCamera_0/RenderProduct.orderedVars") - expected_vars = ("LdrColor", "HdrColor") if data_types else ("HdrColor",) + product_path = "/RenderCamera_0/RenderProduct" + ordered_vars = layer.GetRelationshipAtPath(product_path + ".orderedVars") + expected_vars = ["LdrColor", "HdrColor"] if routes_hdr else ["LdrColor"] assert list(ordered_vars.targetPathList.explicitItems) == [ Sdf.Path(f"/RenderCamera_0/Vars/{name}") for name in expected_vars ] assert camera_spec.cfg.data_types == data_types + schemas = layer.GetPrimAtPath(product_path).GetInfo("apiSchemas").prependedItems + gaussian_setting = layer.GetAttributeAtPath(product_path + ".omni:rtx:rtpt:gaussian:skipTonemapping:enabled") + assert ("OmniRtxSettingsParticleFieldAPI_1" in schemas) is routes_hdr + if routes_hdr: + assert gaussian_setting.default is False + else: + assert gaussian_setting is None def test_ovrtx_rgb_and_rgb_hdr_author_both_render_vars(camera_spec, render_data): diff --git a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py index 83e4f140cfb9..02eeb87b6512 100644 --- a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py +++ b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py @@ -222,10 +222,7 @@ def drain(self): writer = Writer() renderer._visual_material_writer_ref = lambda: writer renderer.render_batch( - [ - SimpleNamespace(render_product_path=path, ppisp_pipeline=None, warp_buffers={}) - for path in renderer._render_product_paths - ] + [SimpleNamespace(render_product_path=path, warp_buffers={}) for path in renderer._render_product_paths] ) assert events == expected_events @@ -258,7 +255,7 @@ def advance_write_floor(**_kwargs): renderer.backend.stage.advance_write_floor = advance_write_floor with pytest.raises(ValueError, match=failure): - renderer.render(SimpleNamespace(render_product_path="/RenderCamera_0/Product", ppisp_pipeline=None)) + renderer.render(SimpleNamespace(render_product_path="/RenderCamera_0/Product")) assert events == expected_events diff --git a/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..5832bdc80d5d --- /dev/null +++ b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.changed.rst @@ -0,0 +1,3 @@ +* **Breaking:** Replaced renderer-owned PPISP with dictionary processors. + Replace ``CameraCfg.isp_cfg`` with ``PpispProcessorCfg`` and configure + ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.major b/source/isaaclab_physx/changelog.d/feat-render-buffer-pipe.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index d5636fe5e676..8d7c222818af 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -13,7 +13,7 @@ import uuid from collections.abc import Sequence from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, NoReturn +from typing import TYPE_CHECKING, Any import numpy as np import warp as wp @@ -42,8 +42,6 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: - from isaaclab_ppisp import PpispPipeline - from omni.replicator.core.scripts.utils.viewport_manager import HydraTexture from isaaclab.sensors.camera.camera_data import CameraData @@ -51,20 +49,8 @@ from .isaac_rtx_renderer_cfg import SIMPLE_SHADING_MODES, IsaacRtxRendererCfg -_PPISP_IMPORT_ERROR_MESSAGE = ( - "isaaclab_ppisp is required when CameraCfg.isp_cfg is set. " - "Run `uv sync` from the Isaac Lab source checkout, or install the Isaac Lab wheel " - "with `uv pip install isaaclab`." -) - - -def _raise_missing_ppisp_error(exc: ModuleNotFoundError) -> NoReturn: - # Only translate missing isaaclab_ppisp imports into the optional-dependency hint; - # unrelated missing modules should surface unchanged for easier debugging. - if exc.name != "isaaclab_ppisp" and not (exc.name and exc.name.startswith("isaaclab_ppisp.")): - raise exc - raise ModuleNotFoundError(_PPISP_IMPORT_ERROR_MESSAGE, name="isaaclab_ppisp") from exc - +# Outputs read from the HdrColor AOV. +_HDR_DATA_TYPES = frozenset({str(RenderBufferKind.RGB_HDR)}) # RTX simple-shading constants. # @@ -136,12 +122,6 @@ class IsaacRtxRenderData: intrinsic_selection: Any = None intrinsic_row_attribute: str = "" intrinsic_stage: Any = None - ppisp_pipeline: PpispPipeline | None = None - """Post-render PPISP pipeline composed when ``spec.cfg.isp_cfg`` is set.""" - _hdr_scratch_wp: wp.array | None = None - """Internal HDR scratch buffer allocated when the user did not request - ``"rgb_hdr"`` in ``data_types`` but the PPISP pipeline still needs - somewhere to receive the HDR AOV before LDR conversion.""" def __post_init__(self): """Compile the stage-bound calibration columns owned by this render product.""" @@ -195,8 +175,11 @@ def __init__(self, cfg: IsaacRtxRendererCfg): apply_isaac_rtx_global_settings(self.cfg.global_settings, settings) if settings.get("/isaaclab/render/deterministic", False): apply_isaac_rtx_determinism_settings(settings) + # Cameras create this renderer during scene construction, before the simulation and + # environments read whether RTX sensors are rendered. + settings.set("/isaaclab/render/rtx_sensors", True) + settings.set("/physics/fabricUpdateTransformations", True) ensure_rtx_hydra_engine_attached() - # ``/isaaclab/render/rtx_sensors`` is owned by ``Camera.__init__`` (must be set pre-``sim.reset()``). def initialize(self) -> None: """Bind shared Fabric destinations after scene creation.""" @@ -210,30 +193,10 @@ def visual_material_writer(self): return FabricVisualMaterialWriter def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: - """Resolve the camera's PPISP cfg and apply RTX-specific USD overrides. - - When ``spec.cfg.isp_cfg`` is set, resolves it (sentinel discovery + - normalization) via :func:`isaaclab_ppisp.resolve_and_normalize` so - :mod:`isaaclab` does not need to know about PPISP. Then pins - ``exposure:*`` to neutral and applies ``OmniRtxCameraExposureAPI_1`` so - RTX's physical-camera exposure model does not compound on top of the - ISP. Without an ISP, the camera prim's authored exposure is left alone. - - :attr:`~isaaclab.sensors.camera.CameraCfg.background_color` is applied - per-render-product in :meth:`create_render_data` via USD attributes. - """ - if spec.cfg.isp_cfg is None: - return - try: - from isaaclab_ppisp import apply_rtx_exposure_overrides, resolve_and_normalize - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - camera_prim_path = spec.camera_prim_paths[0] if spec.camera_prim_paths else None - spec.cfg.isp_cfg = resolve_and_normalize(spec.cfg.isp_cfg, stage, camera_prim_path) - if spec.cfg.isp_cfg is None or not spec.camera_prim_paths: - return - apply_rtx_exposure_overrides(stage, list(spec.camera_prim_paths)) + """Route Gaussian HDR through the camera HDR output.""" + if not _HDR_DATA_TYPES.isdisjoint(spec.cfg.data_types): + # Gaussian splats otherwise skip tonemapping and never reach the HdrColor AOV. + get_settings_manager().set("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: """Publish the per-output Replicator layout this RTX backend writes. @@ -399,10 +362,7 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: aov=SIMPLE_SHADING_AOV, output_data_type=np.uint8, output_channels=4 ) - needs_hdr_color = str(RenderBufferKind.RGB_HDR) in spec.cfg.data_types or ( - spec.cfg.isp_cfg is not None and any(data_type in ("rgb", "rgba") for data_type in spec.cfg.data_types) - ) - if needs_hdr_color: + if not _HDR_DATA_TYPES.isdisjoint(spec.cfg.data_types): rep.AnnotatorRegistry.register_annotator_from_aov( aov="HdrColor", output_data_type=np.float32, output_channels=4 ) @@ -411,16 +371,10 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: annotators = {} for annotator_type in spec.cfg.data_types: if annotator_type == "rgba" or annotator_type == "rgb": - if spec.cfg.isp_cfg is not None: - if str(RenderBufferKind.RGB_HDR) not in annotators: - annotator = rep.AnnotatorRegistry.get_annotator( - "HdrColor", device=spec.device, do_array_copy=False - ) - annotators[str(RenderBufferKind.RGB_HDR)] = annotator - else: - annotator = rep.AnnotatorRegistry.get_annotator("rgb", device=spec.device, do_array_copy=False) - annotators["rgba"] = annotator - elif annotator_type == str(RenderBufferKind.RGB_HDR): + annotator = rep.AnnotatorRegistry.get_annotator("rgb", device=spec.device, do_array_copy=False) + annotators["rgba"] = annotator + elif annotator_type in _HDR_DATA_TYPES: + # HdrColor reflects the exposure configured on the camera prim. if str(RenderBufferKind.RGB_HDR) not in annotators: annotator = rep.AnnotatorRegistry.get_annotator("HdrColor", device=spec.device, do_array_copy=False) annotators[str(RenderBufferKind.RGB_HDR)] = annotator @@ -483,20 +437,10 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: enable_minimal_render_mode=not needs_color_render, ) - ppisp_pipeline = None - if spec.cfg.isp_cfg is not None: - try: - from isaaclab_ppisp import PpispPipeline - except ModuleNotFoundError as exc: - _raise_missing_ppisp_error(exc) - - ppisp_pipeline = PpispPipeline(spec.cfg.isp_cfg) - return IsaacRtxRenderData( annotators=annotators, render_product=rp, spec=spec, - ppisp_pipeline=ppisp_pipeline, ) def _apply_simple_shading_settings( @@ -557,27 +501,7 @@ def _resolve_simple_shading_mode(self, spec: CameraRenderSpec) -> int | None: def set_outputs(self, render_data: IsaacRtxRenderData, output_data: dict[str, ProxyArray]): """Store reference to output buffers for writing during render. See :meth:`~isaaclab.renderers.base_renderer.BaseRenderer.set_outputs`.""" - if render_data.ppisp_pipeline is not None and str(RenderBufferKind.RGBA) not in output_data: - raise ValueError( - "Isaac RTX renderer ISP requires 'rgba' (or 'rgb', which aliases into rgba) as the" - " LDR output destination, but neither was provided. Add 'rgb' or 'rgba' to" - " Camera.cfg.data_types when isp_cfg is set." - ) render_data.output_data = output_data - # Allocate an internal HDR scratch buffer when PPISP is composed but - # the user did not request the raw HDR AOV in ``data_types`` — the - # PPISP kernel still needs somewhere to receive the HDR annotator - # output before LDR conversion. - if render_data.ppisp_pipeline is not None and str(RenderBufferKind.RGB_HDR) not in output_data: - spec = render_data.spec - assert spec is not None - hdr_spec = self.supported_output_types()[RenderBufferKind.RGB_HDR] - assert hdr_spec.dtype is wp.float32 - render_data._hdr_scratch_wp = wp.zeros( - (spec.num_instances, spec.cfg.height, spec.cfg.width, hdr_spec.channels), - dtype=wp.float32, - device=spec.device, - ) def update_transforms(self) -> None: """Update shared Fabric transforms and propagate the visual hierarchy.""" @@ -666,11 +590,7 @@ def tiling_grid_shape(): # The RTX annotator may return an empty frame while its render product is warming up. # Clear the destination so callers do not observe stale data, then wait for the next frame. - if data_type == str(RenderBufferKind.RGB_HDR) and data_type not in output_data: - assert render_data._hdr_scratch_wp is not None - buf_wp = render_data._hdr_scratch_wp - else: - buf_wp = output_data[data_type].warp + buf_wp = output_data[data_type].warp if tiled_data_buffer.size == 0: buf_wp.zero_() continue @@ -752,14 +672,6 @@ def tiling_grid_shape(): replacement = 0.0 if self.cfg.depth_clipping_behavior == "zero" else cfg.spawn.clipping_range[1] replace_inf_depth_wp(buf_wp, replacement, device=device) - # Post-render PPISP: HDR scene-linear → LDR RGBA. The camera enforces - # that ``rgba`` (or ``rgb`` aliasing into it) is present when an ISP is - # configured, so writing to ``output_data["rgba"]`` is safe. - if render_data.ppisp_pipeline is not None: - hdr_proxy = output_data.get(str(RenderBufferKind.RGB_HDR)) - hdr_source = hdr_proxy.warp if hdr_proxy is not None else render_data._hdr_scratch_wp - render_data.ppisp_pipeline.apply(hdr_source, output_data[str(RenderBufferKind.RGBA)].warp) - def read_output(self, render_data: IsaacRtxRenderData, camera_data: CameraData) -> None: """Populate per-output metadata collected during render(). Pixel data already written in render(). @@ -798,5 +710,3 @@ def cleanup(self, render_data: IsaacRtxRenderData | None): render_data.output_data = None render_data.spec = None render_data.renderer_info.clear() - render_data.ppisp_pipeline = None - render_data._hdr_scratch_wp = None diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py index 0a75262bc7f3..95c37c9bae6b 100644 --- a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py @@ -86,6 +86,110 @@ def test_native_fabric_geometry_needs_no_separate_publication(monkeypatch): provider.get_geometry_points.assert_not_called() +def test_renderer_marks_rtx_sensor_rendering_when_created(monkeypatch): + """Creating the renderer, which cameras do during scene construction, marks RTX sensor rendering.""" + _install_omni_stubs(monkeypatch) + import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + settings = MagicMock() + settings.get.return_value = False + with ( + patch.object(rtx_renderer, "get_settings_manager", return_value=settings), + patch.object(rtx_renderer, "enable_extension"), + patch.object(rtx_renderer, "apply_isaac_rtx_global_settings"), + patch.object(rtx_renderer, "ensure_rtx_hydra_engine_attached"), + ): + rtx_renderer.IsaacRtxRenderer(IsaacRtxRendererCfg()) + + settings.set.assert_any_call("/isaaclab/render/rtx_sensors", True) + settings.set.assert_any_call("/physics/fabricUpdateTransformations", True) + + +@pytest.mark.parametrize(("data_types", "routes_hdr"), [(["rgb"], False), (["rgb_hdr"], True)]) +def test_prepare_cameras_routes_gaussian_hdr_for_hdr_cameras(monkeypatch, data_types, routes_hdr): + """Gaussian HDR routing is enabled only for cameras that read HDR color.""" + _install_omni_stubs(monkeypatch) + import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer + + settings = MagicMock() + spec = SimpleNamespace(cfg=SimpleNamespace(data_types=data_types), camera_prim_paths=()) + with patch.object(rtx_renderer, "get_settings_manager", return_value=settings): + rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer).prepare_cameras(None, spec) + + hdr_call = call("/rtx/rtpt/gaussian/skipTonemapping/enabled", False) + assert (hdr_call in settings.set.call_args_list) is routes_hdr + + +@pytest.mark.parametrize("data_types", [["rgb"], ["rgb_hdr"]]) +def test_prepare_cameras_preserves_authored_exposure(monkeypatch, data_types): + """Selecting render outputs preserves authored camera exposure.""" + _install_omni_stubs(monkeypatch) + from isaaclab_physx.renderers.isaac_rtx_renderer import IsaacRtxRenderer + + from pxr import Sdf, Usd, UsdGeom + + stage = Usd.Stage.CreateInMemory() + camera = UsdGeom.Camera.Define(stage, "/World/Camera").GetPrim() + camera.CreateAttribute("exposure:iso", Sdf.ValueTypeNames.Float).Set(100.0) + spec = SimpleNamespace(cfg=SimpleNamespace(data_types=data_types), camera_prim_paths=("/World/Camera",)) + IsaacRtxRenderer.__new__(IsaacRtxRenderer).prepare_cameras(stage, spec) + + assert camera.GetAttribute("exposure:iso").Get() == 100.0 + + +@pytest.mark.parametrize("data_types", [["rgb_hdr"]]) +def test_hdr_reads_the_hdr_annotator_into_its_output(monkeypatch, data_types): + """HDR attaches one HdrColor annotator and extract it into the camera's own buffers.""" + replicator, syntheticdata = _install_omni_stubs(monkeypatch) + monkeypatch.setattr(syntheticdata, "SyntheticData", MagicMock(), raising=False) + + import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + from pxr import UsdGeom + + import isaaclab.sim.utils.stage as stage_utils + from isaaclab.sensors.camera.camera_data import CameraData + + settings = MagicMock() + settings.get.return_value = False + stage = MagicMock() + stage.SelectPrims.return_value.GetCount.return_value = 1 + stage.GetPrimAtPath.return_value.IsA.side_effect = lambda typ: typ is UsdGeom.Camera + product = SimpleNamespace(path="/Render/Test") + replicator.create = SimpleNamespace(render_product_tiled=MagicMock(return_value=product)) + annotator = MagicMock() + registry = MagicMock() + registry.get_annotator.return_value = annotator + replicator.AnnotatorRegistry = registry + spec = SimpleNamespace( + camera_prim_paths=("/World/Camera",), + device="cpu", + view_count=1, + cfg=SimpleNamespace(data_types=data_types, width=3, height=2), + ) + renderer = rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg() + with ( + patch.object(rtx_renderer, "get_settings_manager", return_value=settings), + patch.object(rtx_renderer, "get_isaac_sim_version", return_value=version.parse("6.0")), + patch.object(stage_utils, "get_current_stage", return_value=stage), + ): + render_data = renderer.create_render_data(spec) + registry.get_annotator.assert_called_once_with("HdrColor", device="cpu", do_array_copy=False) + annotator.attach.assert_called_once_with([product.path]) + + camera_data = CameraData.allocate(data_types, 2, 3, 1, "cpu", supported_specs=renderer.cfg.supported_output_types()) + renderer.set_outputs(render_data, camera_data.output) + annotator.get_data.return_value = np.full((2, 3, 4), 2.0, dtype=np.float32) + renderer._read_annotator_output(render_data) + + assert set(camera_data.output) == set(data_types) + for name in data_types: + np.testing.assert_array_equal(camera_data.output[name].warp.numpy(), 2.0) + + def test_create_render_data_uses_unique_sdf_safe_render_product_name(monkeypatch): """Each tiled render product gets a fresh ``rp_`` name. @@ -132,7 +236,6 @@ def test_create_render_data_uses_unique_sdf_safe_render_product_name(monkeypatch data_types=["rgb"], width=64, height=64, - isp_cfg=None, colorize_semantic_segmentation=False, colorize_instance_segmentation=False, colorize_instance_id_segmentation=False, @@ -184,6 +287,7 @@ def test_create_render_data_uses_unique_sdf_safe_render_product_name(monkeypatch pytest.param(["rgb", "simple_shading_full_mdl"], 3, False, id="rgb_keeps_path_tracing"), pytest.param(["rgba", "simple_shading_full_mdl"], 3, False, id="rgba_keeps_path_tracing"), pytest.param(["rgb_hdr", "simple_shading_full_mdl"], 3, False, id="rgb_hdr_keeps_path_tracing"), + pytest.param(["rgb_hdr", "simple_shading_full_mdl"], 3, False, id="radiance_keeps_path_tracing"), ], ) def test_simple_shading_configures_its_render_product( @@ -248,7 +352,6 @@ def _create_attribute(name, value_type): data_types=data_types, width=64, height=64, - isp_cfg=None, colorize_semantic_segmentation=False, colorize_instance_segmentation=False, colorize_instance_id_segmentation=False, @@ -466,8 +569,6 @@ def test_render_treats_empty_annotator_frame_as_not_ready(monkeypatch, data_type cfg=SimpleNamespace(width=64, height=64), ), renderer_info={}, - ppisp_pipeline=None, - _hdr_scratch_wp=None, ) renderer = rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer) renderer.cfg = IsaacRtxRendererCfg() @@ -507,8 +608,6 @@ def test_render_keeps_leading_channels_of_padded_annotator_tiles(monkeypatch, da output_data={data_type: SimpleNamespace(warp=output_buffer)}, spec=SimpleNamespace(view_count=view_count, device="cpu", cfg=SimpleNamespace(width=width, height=height)), renderer_info={}, - ppisp_pipeline=None, - _hdr_scratch_wp=None, ) renderer = rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer) renderer.cfg = IsaacRtxRendererCfg() @@ -560,8 +659,6 @@ def test_render_batch_updates_once_before_extracting_ready_cameras(monkeypatch, else SimpleNamespace(view_count=1, device="cpu", cfg=SimpleNamespace(width=1, height=1)) ), renderer_info={}, - ppisp_pipeline=None, - _hdr_scratch_wp=None, ) ) if state == "ready": diff --git a/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst b/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst new file mode 100644 index 000000000000..5832bdc80d5d --- /dev/null +++ b/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.changed.rst @@ -0,0 +1,3 @@ +* **Breaking:** Replaced renderer-owned PPISP with dictionary processors. + Replace ``CameraCfg.isp_cfg`` with ``PpispProcessorCfg`` and configure + ``CameraCfg.exposure=CameraExposureCfg()`` for neutral RTX ``rgb_hdr`` input. diff --git a/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.major b/source/isaaclab_ppisp/changelog.d/feat-render-buffer-pipe.major new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ppisp/docs/README.md b/source/isaaclab_ppisp/docs/README.md index 351571fa30db..37ee029329db 100644 --- a/source/isaaclab_ppisp/docs/README.md +++ b/source/isaaclab_ppisp/docs/README.md @@ -1,8 +1,45 @@ # Isaac Lab PPISP -This extension provides a renderer-backend-agnostic PPISP (Physically Plausible -Image Signal Processing) pipeline for Isaac Lab camera outputs. +This extension provides PPISP (Physically Plausible Image Signal Processing) as +a `DictTensorProcessor` used by cameras and observations. `DictTensorProcessor` and +`TensorProcessor` live together in `isaaclab.utils.tensor_processors`; their call signatures are +dictionary-to-dictionary and tensor-to-tensor respectively. -PPISP consumes HDR scene-linear color from renderer backends and writes LDR -`rgb` / `rgba` camera output after each render tick when `CameraCfg.isp_cfg` is -set. +`PpispProcessor` reads scene-linear `rgb_hdr` from a float32 `ProxyArray` with +shape `(N, H, W, 3)` and writes uint8 `rgb`. For camera processing, configure +`CameraCfg.pipeline=[PpispProcessorCfg(params={"cfg": ppisp_cfg})]`. The camera requests +required HDR internally; only the requested public outputs are published. Set +`CameraCfg.exposure=CameraExposureCfg()` (from `isaaclab.sensors.camera`) before initialization +for neutral RTX HDR input. Output selection does not alter exposure. The same setting is required +when PPISP runs only as an observation tensor processor through `DictTensorProcessorAdapterCfg`. +Set PPISP exposure in stops through `PpispCfg.inputs["exposureOffset"]`. + +Resolve authored settings explicitly before constructing the pipeline: + +```python +from isaaclab_ppisp import PpispProcessorCfg, ppisp_cfg_from_usd_camera + +ppisp_cfg = ppisp_cfg_from_usd_camera(source_camera_prim) +processor_cfg = PpispProcessorCfg(params={"cfg": ppisp_cfg}) +``` + +`PpispProcessor` can also run outside a camera. Construct it with its configuration and an +actual dictionary of `ProxyArray` tensors; construction validates shape and dtype and allocates +persistent output once on the same device. Input/output names can be changed independently of PPISP math. For RGBA, +use `outputs=["rgba"]` and `params={"cfg": ppisp_cfg, "channels": 4}`. +PPISP consumes the preceding processor's radiance, so an earlier exposure or scale +processor changes its result. The producer must supply scene-linear color before +camera exposure and response. Array storage cannot establish that semantic contract; +float32 display-encoded RGB must not be supplied as radiance. + +Each processor owns its controller scratch and output buffers. Optional +`params["controller_weights"]=PpispControllerWeights(device_weights)` can supply +shared, read-only device weights; configuration copies preserve this resource's identity. The +caller owns upload synchronization, and mutable controller workspace stays private. +The controller requires CUDA; static PPISP also runs on CPU. `PpispPipeline` remains +available for direct numerical use with explicitly supplied Warp buffers. + +For direct use, construct `PpispProcessor(processor_cfg, {"rgb_hdr": hdr_tensor})`, then call +`processor({"rgb_hdr": hdr_tensor})` to obtain `{"rgb": color_tensor}`. It owns its persistent +output and numerical workspace; copy values that must survive another call, and call +`processor.close()` when finished. diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py index 416f7d9e134e..244159b63550 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.py @@ -5,13 +5,9 @@ """Post-render PPISP (Physically Plausible Image Signal Processing) for IsaacLab. -Provides the renderer-backend-agnostic ISP pipeline that converts the renderer's -HDR scene-linear AOV to LDR RGBA at the end of a render tick. Renderer backends -(:class:`~isaaclab_physx.renderers.IsaacRtxRenderer`, -:class:`~isaaclab_ov.renderers.OVRTXRenderer`, -:class:`~isaaclab_newton.renderers.NewtonWarpRenderer`) compose -:class:`PpispPipeline` internally when the camera's -:attr:`~isaaclab.sensors.camera.CameraCfg.isp_cfg` is set. +Provides a dictionary processor that converts scene-linear radiance to RGB or RGBA. +Camera and observation pipelines use :class:`PpispProcessorCfg` with concrete settings. USD discovery +helpers remain explicit, and renderer backends do not own PPISP execution or controller workspace. """ import importlib.metadata diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi index 2d69e60325e9..646c40c9387e 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi +++ b/source/isaaclab_ppisp/isaaclab_ppisp/__init__.pyi @@ -7,10 +7,12 @@ __all__ = [ "PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN", "PPISP_DEFAULT_INPUTS", "PpispCfg", + "PpispControllerWeights", + "PpispDiscoveryMode", "PpispPipeline", + "PpispProcessorCfg", "apply_ppisp_to_rgba", "apply_ppisp_to_rgba_with_controller_params", - "apply_rtx_exposure_overrides", "auto_any_ppisp_cfg", "auto_camera_ppisp_cfg", "compute_ppisp_controller_params", @@ -19,6 +21,7 @@ __all__ = [ "normalize_ppisp_cfg", "ppisp_cfg_from_usd_camera", "ppisp_cfg_from_usd_stage", + "PpispProcessor", "resolve_and_normalize", ] @@ -26,6 +29,7 @@ from .cfg import ( PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN, PPISP_DEFAULT_INPUTS, PpispCfg, + PpispDiscoveryMode, auto_any_ppisp_cfg, auto_camera_ppisp_cfg, default_ppisp_inputs, @@ -36,5 +40,6 @@ from .cfg import ( resolve_and_normalize, ) from .kernels import apply_ppisp_to_rgba, apply_ppisp_to_rgba_with_controller_params, compute_ppisp_controller_params -from .pipeline import PpispPipeline -from .rtx_camera_overrides import apply_rtx_exposure_overrides +from .pipeline import PpispControllerWeights, PpispPipeline +from .processor import PpispProcessor +from .processor_cfg import PpispProcessorCfg diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py b/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py index 37859bda6d86..d9416602171b 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/cfg.py @@ -12,6 +12,7 @@ from __future__ import annotations from dataclasses import field +from enum import StrEnum from typing import Any from isaaclab.utils import configclass @@ -79,6 +80,16 @@ def default_ppisp_inputs() -> dict[str, float | tuple[float, float]]: return dict(PPISP_DEFAULT_INPUTS) +class PpispDiscoveryMode(StrEnum): + """Explicit USD configuration discovery strategies used before preparing a PPISP stage.""" + + AUTO_CAMERA = "auto_camera" + """Read the specified camera, or the first PPISP camera when no path is supplied.""" + + AUTO_ANY = "auto_any" + """Read the specified camera, then fall back to the first PPISP camera on the stage.""" + + @configclass class PpispCfg: """Configuration for PPISP post-processing. @@ -321,29 +332,17 @@ def has_ppisp_camera_attrs(camera_prim: Any | None) -> bool: return _has_ppisp_camera_attrs(camera_prim) -def resolve_and_normalize(isp_cfg: Any, stage: Any, camera_prim_path: str | None = None) -> PpispCfg | None: - """Resolve a Camera sensor batch's ``isp_cfg`` to a normalised cfg or ``None``. - - Handles all three legal forms of :attr:`~isaaclab.sensors.camera.CameraCfg.isp_cfg`: - - * ``None`` → returns ``None``. - * :class:`~isaaclab.sensors.camera.CameraISPMode` sentinel — checks the - target camera via :func:`auto_camera_ppisp_cfg` (and uses - :func:`auto_any_ppisp_cfg` for ``AUTO_ANY`` or when no camera path is - supplied) to discover a PPISP camera. Returns the parsed + normalised - :class:`PpispCfg`, or ``None`` if no PPISP camera matched. - * Concrete :class:`PpispCfg` — normalises in place (validates input keys, - fills defaults, and merges camera-authored USD values when - ``camera_prim_path`` is set). +def resolve_and_normalize( + isp_cfg: PpispCfg | PpispDiscoveryMode | None, stage: Any, camera_prim_path: str | None = None +) -> PpispCfg | None: + """Resolve explicit settings or a discovery mode using the caller's USD stage. - This is the single entry point renderer backends call inside their - ``prepare_cameras`` hook so :mod:`isaaclab.sensors.camera` does not need - to know about PPISP types at all. The returned cfg applies to the whole - Camera sensor batch; callers pass the first matched camera prim path for - the camera-local discovery phase. + Call this before constructing a PPISP processor. The processor receives only concrete + settings and never looks up a camera or stage. Legacy ``CameraISPMode`` string-enum values + remain accepted for migration without importing the camera package. Args: - isp_cfg: The Camera sensor's :attr:`isp_cfg` value (``None``, ``CameraISPMode``, or :class:`PpispCfg`). + isp_cfg: Explicit settings, a discovery mode, or ``None`` to disable processing. stage: USD stage used for sentinel discovery and camera-path resolution. camera_prim_path: Optional absolute path of the first matched camera prim in the Camera sensor batch. When omitted, discovery uses the @@ -352,14 +351,12 @@ def resolve_and_normalize(isp_cfg: Any, stage: Any, camera_prim_path: str | None Returns: A fully-normalised :class:`PpispCfg`, or ``None`` if the batch has no ISP. """ - # Local import avoids a top-of-module dep on isaaclab.sensors. - from isaaclab.sensors.camera.camera_isp import CameraISPMode - if isp_cfg is None: return None - if isinstance(isp_cfg, CameraISPMode): + if isinstance(isp_cfg, str): + mode = PpispDiscoveryMode(isp_cfg) resolved = auto_camera_ppisp_cfg(stage, camera_prim_path) if camera_prim_path else None - if resolved is None and (isp_cfg == CameraISPMode.AUTO_ANY or not camera_prim_path): + if resolved is None and (mode == PpispDiscoveryMode.AUTO_ANY or not camera_prim_path): resolved = auto_any_ppisp_cfg(stage) if resolved is None: return None diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py b/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py index 02a3ed36cd7c..010761850428 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/kernels.py @@ -508,7 +508,8 @@ def _ppisp_controller_mlp_native_kernel( _ppisp_controller_mlp_native(features, weights, controller_params, prior_exposure, camera_id, lane) -@wp.kernel(enable_backward=False) +# Keep image transforms compilable on CPU without the CUDA-only native controller snippets. +@wp.kernel(enable_backward=False, module="unique") def _apply_ppisp_kernel( hdr_color: wp.array4d(dtype=wp.float32), out_rgba: wp.array4d(dtype=wp.uint8), @@ -549,7 +550,7 @@ def _apply_ppisp_kernel( For each pixel, the model applies HDR responsivity scaling, exposure scaling, per-channel vignetting, color homography correction, CRF tone - mapping, and uint8 RGBA packing. The first tensor dimension is the + mapping, and uint8 RGB or RGBA packing. The first tensor dimension is the camera/environment index, so image-space effects use local ``height_id`` and ``width_id`` coordinates per camera. """ @@ -591,10 +592,11 @@ def _apply_ppisp_kernel( out_rgba[camera_id, height_id, width_id, 0] = wp.uint8(wp.clamp(out_rgb[0], 0.0, 1.0) * 255.0) out_rgba[camera_id, height_id, width_id, 1] = wp.uint8(wp.clamp(out_rgb[1], 0.0, 1.0) * 255.0) out_rgba[camera_id, height_id, width_id, 2] = wp.uint8(wp.clamp(out_rgb[2], 0.0, 1.0) * 255.0) - out_rgba[camera_id, height_id, width_id, 3] = wp.uint8(255) + if out_rgba.shape[3] == 4: + out_rgba[camera_id, height_id, width_id, 3] = wp.uint8(255) -@wp.kernel(enable_backward=False) +@wp.kernel(enable_backward=False, module="unique") def _apply_ppisp_controller_kernel( hdr_color: wp.array4d(dtype=wp.float32), out_rgba: wp.array4d(dtype=wp.uint8), @@ -670,23 +672,38 @@ def _apply_ppisp_controller_kernel( out_rgba[camera_id, height_id, width_id, 0] = wp.uint8(wp.clamp(out_rgb[0], 0.0, 1.0) * 255.0) out_rgba[camera_id, height_id, width_id, 1] = wp.uint8(wp.clamp(out_rgb[1], 0.0, 1.0) * 255.0) out_rgba[camera_id, height_id, width_id, 2] = wp.uint8(wp.clamp(out_rgb[2], 0.0, 1.0) * 255.0) - out_rgba[camera_id, height_id, width_id, 3] = wp.uint8(255) + if out_rgba.shape[3] == 4: + out_rgba[camera_id, height_id, width_id, 3] = wp.uint8(255) + + +def _validate_ppisp_image_buffers(hdr_color: wp.array, out_rgba: wp.array) -> None: + """Check the shapes, scalar types, and device required by both image kernels.""" + if hdr_color.dtype is not wp.float32: + raise ValueError(f"Camera PPISP HDR input must be wp.float32, got {hdr_color.dtype}.") + if out_rgba.dtype is not wp.uint8: + raise ValueError(f"Camera PPISP RGB/RGBA output must be wp.uint8, got {out_rgba.dtype}.") + if len(hdr_color.shape) != 4 or hdr_color.shape[-1] != 3: + raise ValueError("PPISP input must have shape (N, H, W, 3).") + if ( + len(out_rgba.shape) != 4 + or out_rgba.shape[-1] not in (3, 4) + or out_rgba.shape[:3] != hdr_color.shape[:3] + or out_rgba.device != hdr_color.device + ): + raise ValueError("PPISP output must match the input batch, image dimensions, and device, with 3 or 4 channels.") def apply_ppisp_to_rgba(hdr_color: wp.array, out_rgba: wp.array, cfg: PpispCfg) -> None: - """Apply PPISP to ``hdr_color`` and write LDR RGBA into ``out_rgba``. + """Apply PPISP to ``hdr_color`` and write LDR RGB or RGBA into ``out_rgba``. Args: hdr_color: HDR scene-linear input. ``wp.array4d(dtype=wp.float32)`` of shape ``(N, H, W, 3)``. - out_rgba: LDR RGBA output. ``wp.array4d(dtype=wp.uint8)`` of shape - ``(N, H, W, 4)``. Written in place. + out_rgba: LDR output. ``wp.array4d(dtype=wp.uint8)`` of shape + ``(N, H, W, 3)`` or ``(N, H, W, 4)``. Written in place; alpha is 255 when present. cfg: PPISP configuration. """ - if hdr_color.dtype is not wp.float32: - raise ValueError(f"Camera PPISP HDR input must be wp.float32, got {hdr_color.dtype}.") - if out_rgba.dtype is not wp.uint8: - raise ValueError(f"Camera PPISP RGBA output must be wp.uint8, got {out_rgba.dtype}.") + _validate_ppisp_image_buffers(hdr_color, out_rgba) inputs = cfg.inputs wp.launch( @@ -812,17 +829,14 @@ def apply_ppisp_to_rgba_with_controller_params( Args: hdr_color: HDR scene-linear input, shape ``(N, H, W, 3)``. - out_rgba: LDR RGBA output, shape ``(N, H, W, 4)``. + out_rgba: LDR RGB/RGBA output, shape ``(N, H, W, 3)`` or ``(N, H, W, 4)``. cfg: PPISP configuration. Static inputs provide responsivity, vignetting and CRF. controller_params: ``wp.array2d(float32)`` of shape ``(N, 9)`` holding ``[exposureOffset, colorLatentBlue.xy, colorLatentRed.xy, colorLatentGreen.xy, colorLatentNeutral.xy]``. """ - if hdr_color.dtype is not wp.float32: - raise ValueError(f"Camera PPISP HDR input must be wp.float32, got {hdr_color.dtype}.") - if out_rgba.dtype is not wp.uint8: - raise ValueError(f"Camera PPISP RGBA output must be wp.uint8, got {out_rgba.dtype}.") + _validate_ppisp_image_buffers(hdr_color, out_rgba) if controller_params.dtype is not wp.float32: raise ValueError(f"Camera PPISP controller params must be wp.float32, got {controller_params.dtype}.") if controller_params.shape[0] != out_rgba.shape[0] or controller_params.shape[1] != 9: diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py b/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py index 1fe7a3228e25..04bdea699a9f 100644 --- a/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py +++ b/source/isaaclab_ppisp/isaaclab_ppisp/pipeline.py @@ -3,20 +3,15 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Post-render PPISP pipeline composed into renderer backends. - -:class:`PpispPipeline` runs *after* a renderer fills its HDR scene-linear AOV, -converting HDR → LDR via a single Warp kernel. Renderer backends instantiate -this class when their cfg/spec carries a :class:`PpispCfg` and dispatch -:meth:`apply` once per render tick. The HDR scratch buffer is owned by the -renderer backend, not by this class. -""" +"""PPISP numerical execution with owned controller workspace and optional shared weights.""" from __future__ import annotations +from dataclasses import dataclass + import warp as wp -from .cfg import PpispCfg, normalize_ppisp_cfg +from .cfg import PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN, PpispCfg, normalize_ppisp_cfg from .kernels import ( PPISP_CONTROLLER_FEATURE_LEN, PPISP_CONTROLLER_PARAM_COUNT, @@ -26,89 +21,105 @@ ) -class PpispPipeline: - """Post-render PPISP kernel applier. +@dataclass(frozen=True, eq=False, slots=True) +class PpispControllerWeights: + """Shared, read-only device weights with identity preserved when configurations are copied. - Constructed by renderer backends when ``spec.cfg.isp_cfg`` is set. Owns the - normalised :class:`PpispCfg` and dispatches the PPISP Warp kernel once per - render tick via :meth:`apply`. + The caller supplies the allocation and orders its upload before processing. The wrapper retains + that allocation for every consumer; neither processors nor configuration copies mutate it. + """ - One pipeline instance applies to the whole Camera sensor batch. The PPISP - Warp kernel takes scalar coefficients, so every cloned view in a tiled - batch shares the same ISP configuration — there is no per-view ISP today. - Per-view support would require packing the cfg into GPU arrays and indexing - by ``camera_id`` inside the kernel. + values: wp.array - Today only :class:`PpispCfg` is accepted; future ISP implementations can - either subclass or be selected by cfg type without changes to the backend - renderers. - """ + def __post_init__(self) -> None: + if self.values.shape != (PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN,) or self.values.dtype != wp.float32: + raise ValueError("PPISP controller weights require the expected one-dimensional float32 buffer.") - def __init__(self, cfg: PpispCfg): - """Initialize the PPISP pipeline. + def __deepcopy__(self, memo: dict) -> PpispControllerWeights: + # Config cloning must share model resources, while each processor creates its own workspace. + return self - Normalises ``cfg`` on construction (validates input keys, fills - defaults). - :class:`~isaaclab.sensors.camera.Camera` already normalises ``isp_cfg`` - before passing the :class:`~isaaclab.renderers.CameraRenderSpec` to - the backend, so renderer backends pass a concrete, resolved config here. - Args: - cfg: The PPISP configuration. - """ +class PpispPipeline: + """Apply concrete PPISP settings to explicitly supplied image buffers. + + This numerical object performs no camera, scene, or renderer discovery. Resolve USD settings + with :func:`~isaaclab_ppisp.ppisp_cfg_from_usd_camera` before construction. Each instance owns + its controller workspace and is not reentrant; coefficients apply to its whole image batch. + :class:`~isaaclab_ppisp.PpispProcessor` owns an independent instance for each consumer. + + Args: + cfg: Concrete PPISP settings, normalized in place for direct numerical use. + controller_weights: Optional device-resident controller weights to borrow read-only. + Callers may share these across instances, but must keep them unchanged and order their + upload before execution. Workspace and outputs are never shared automatically. + """ + + def __init__(self, cfg: PpispCfg, *, controller_weights: PpispControllerWeights | None = None): normalized_cfg = normalize_ppisp_cfg(cfg) if normalized_cfg is None: raise ValueError("PpispPipeline requires a concrete PpispCfg.") self.cfg = normalized_cfg + self._shared_controller_weights = controller_weights self._controller_weights_by_device: dict[str, wp.array] = {} self._controller_buffers_by_shape: dict[tuple[str, int, int, int], tuple[wp.array, ...]] = {} + self._closed = False + + def initialize(self, shape: tuple[int, ...], device: str | wp.Device) -> None: + """Prepare controller weights and scratch before the first image is processed.""" + if self._closed: + raise RuntimeError("Cannot initialize a closed PPISP pipeline.") + if self.cfg.controller_weights is None and self._shared_controller_weights is None: + return + if not wp.get_device(device).is_cuda: + raise ValueError("PPISP controller requires a CUDA device.") + device_name = str(wp.get_device(device)) + if device_name not in self._controller_weights_by_device: + if self._shared_controller_weights is None: + if len(self.cfg.controller_weights) != PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN: + raise ValueError(f"PPISP controller requires {PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN} weights.") + weights = wp.array(self.cfg.controller_weights, dtype=wp.float32, device=device_name) + else: + weights = self._shared_controller_weights.values + if weights.device != wp.get_device(device_name): + raise ValueError("Shared PPISP controller weights must match the image device.") + self._controller_weights_by_device[device_name] = weights + self._controller_buffers(shape, device_name) def apply(self, hdr: wp.array, rgba: wp.array) -> None: - """Run the PPISP kernel: HDR scene-linear → LDR RGBA, in place on ``rgba``.""" - if self.cfg.controller_weights is None: + """Write RGB or RGBA to the supplied destination on the current Warp stream.""" + if self._closed: + raise RuntimeError("Cannot apply a closed PPISP pipeline.") + if self.cfg.controller_weights is None and self._shared_controller_weights is None: apply_ppisp_to_rgba(hdr, rgba, self.cfg) return - controller_params = self._compute_controller_params(hdr) - apply_ppisp_to_rgba_with_controller_params(hdr, rgba, self.cfg, controller_params) - - def _compute_controller_params(self, hdr: wp.array) -> wp.array: - """Run the exported PPISP controller and return a Warp view of ``(N, 9)`` params.""" - controller_weights = self.cfg.controller_weights - assert controller_weights is not None - + self.initialize(hdr.shape, hdr.device) device = str(hdr.device) - weights = self._controller_weights_by_device.get(device) - if weights is None: - weights = wp.array(controller_weights, dtype=wp.float32, device=device) - self._controller_weights_by_device[device] = weights - - features, controller_params = self._controller_buffers(hdr) + features, controller_params = self._controller_buffers(hdr.shape, device) compute_ppisp_controller_params( hdr, - weights, + self._controller_weights_by_device[device], features, controller_params, self.cfg.controller_prior_exposure, float(self.cfg.controller_responsivity), ) - return controller_params + apply_ppisp_to_rgba_with_controller_params(hdr, rgba, self.cfg, controller_params) - def _controller_buffers(self, hdr: wp.array) -> tuple[wp.array, ...]: - """Return cached controller scratch buffers matching ``hdr`` shape/device.""" - num_cameras = int(hdr.shape[0]) - image_height = int(hdr.shape[1]) - image_width = int(hdr.shape[2]) - device = str(hdr.device) - key = (device, num_cameras, image_height, image_width) + def close(self) -> None: + """Release references to owned workspace and borrowed weights. Repeated calls are safe.""" + self._controller_weights_by_device.clear() + self._controller_buffers_by_shape.clear() + self._shared_controller_weights = None + self._closed = True + + def _controller_buffers(self, shape: tuple[int, ...], device: str) -> tuple[wp.array, ...]: + key = (device, *shape[:3]) buffers = self._controller_buffers_by_shape.get(key) - if buffers is not None: - return buffers - - # Camera output shape is fixed for a pipeline lifetime; cache buffers so - # controller execution does not allocate every frame. - buffers = ( - wp.empty((num_cameras, PPISP_CONTROLLER_FEATURE_LEN), dtype=wp.float32, device=device), - wp.empty((num_cameras, PPISP_CONTROLLER_PARAM_COUNT), dtype=wp.float32, device=device), - ) - self._controller_buffers_by_shape[key] = buffers + if buffers is None: + buffers = ( + wp.empty((shape[0], PPISP_CONTROLLER_FEATURE_LEN), dtype=wp.float32, device=device), + wp.empty((shape[0], PPISP_CONTROLLER_PARAM_COUNT), dtype=wp.float32, device=device), + ) + self._controller_buffers_by_shape[key] = buffers return buffers diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/processor.py b/source/isaaclab_ppisp/isaaclab_ppisp/processor.py new file mode 100644 index 000000000000..34080bf4f0f3 --- /dev/null +++ b/source/isaaclab_ppisp/isaaclab_ppisp/processor.py @@ -0,0 +1,74 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""PPISP dictionary processor shared by cameras and observations.""" + +from __future__ import annotations + +from copy import deepcopy + +import warp as wp + +from isaaclab.utils.tensor_processors import DictTensorProcessor +from isaaclab.utils.warp import ProxyArray + +from .cfg import PpispCfg +from .pipeline import PpispPipeline +from .processor_cfg import PpispProcessorCfg + + +class PpispProcessor(DictTensorProcessor): + """Transform scene-linear HDR into uint8 color using persistent PPISP storage. + + Input is float32 NHWC with three channels. It must represent color before exposure and + camera response; array storage cannot establish that semantic contract. Output has three + RGB or four RGBA channels. Controllers may share immutable weights via configuration, + while each processor owns its controller workspace and output buffer. + """ + + def __init__(self, cfg: PpispProcessorCfg, data: dict[str, ProxyArray]) -> None: + super().__init__(cfg, data) + if len(cfg.inputs) != 1 or len(cfg.outputs) != 1: + raise ValueError("PPISP requires exactly one input and one output name.") + self._input_name, self._output_name = cfg.inputs[0], cfg.outputs[0] + source = data[self._input_name] + if not isinstance(source, ProxyArray): + raise TypeError("PPISP input must be a ProxyArray.") + if len(source.shape) != 4 or source.shape[-1] != 3 or any(size <= 0 for size in source.shape): + raise ValueError("PPISP input must have shape (N, H, W, 3) with positive dimensions.") + if source.dtype is not wp.float32: + raise ValueError("PPISP input must have dtype wp.float32.") + channels = cfg.params.get("channels", 3) + if channels not in (3, 4): + raise ValueError("PPISP color output requires three RGB or four RGBA channels.") + ppisp_cfg = cfg.params["cfg"] + if not isinstance(ppisp_cfg, PpispCfg) or ppisp_cfg.camera_prim_path is not None: + raise ValueError("PPISP requires a concrete PpispCfg; resolve USD camera attributes before construction.") + self._shape, self._device = source.shape, source.device + self._closed = False + self._pipeline = PpispPipeline(deepcopy(ppisp_cfg), controller_weights=cfg.params.get("controller_weights")) + try: + self._pipeline.initialize(source.shape, source.device) + self.outputs[self._output_name] = ProxyArray( + wp.empty((*source.shape[:3], channels), dtype=wp.uint8, device=source.device) + ) + except Exception: + self.close() + raise + + def __call__(self, data: dict[str, ProxyArray]) -> dict[str, ProxyArray]: + """Read the current input and return the persistent named color tensor.""" + if self._closed: + raise RuntimeError("Cannot execute a closed PPISP processor.") + source = data[self._input_name] + if source.shape != self._shape or source.dtype is not wp.float32 or source.device != self._device: + raise ValueError("PPISP input must match the shape, dtype and device used during construction.") + self._pipeline.apply(source.warp, self.outputs[self._output_name].warp) + return dict(self.outputs) + + def close(self) -> None: + if not self._closed: + self._closed = True + self._pipeline.close() diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/processor_cfg.py b/source/isaaclab_ppisp/isaaclab_ppisp/processor_cfg.py new file mode 100644 index 000000000000..876ebd243a98 --- /dev/null +++ b/source/isaaclab_ppisp/isaaclab_ppisp/processor_cfg.py @@ -0,0 +1,26 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from typing import Any + +from isaaclab.utils import configclass +from isaaclab.utils.tensor_processors import DictTensorProcessorCfg + +from .cfg import PpispCfg + + +@configclass +class PpispProcessorCfg(DictTensorProcessorCfg): + """Process one scene-linear HDR tensor into RGB or RGBA. + + ``inputs`` and ``outputs`` each contain one actual dictionary key. Supply concrete PPISP + settings in ``params={"cfg": ppisp_cfg}``. Add ``"channels": 4`` for RGBA. The source camera + needs explicit neutral exposure; this processor never changes renderer settings. + """ + + class_type = "{DIR}.processor:PpispProcessor" + inputs: list[str] = ["rgb_hdr"] + outputs: list[str] = ["rgb"] + params: dict[str, Any] = {"cfg": PpispCfg()} diff --git a/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py b/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py deleted file mode 100644 index 5ef5c6bfebf1..000000000000 --- a/source/isaaclab_ppisp/isaaclab_ppisp/rtx_camera_overrides.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""RTX-specific per-camera-prim USD overrides authored when PPISP is active. - -Kept in a Kit-free module (no ``omni.*`` imports) so it can be unit-tested -without booting Isaac Sim. Both Kit-driven Isaac RTX and Kit-less OVRTX embed -the same RTX core and author the same exposure schema, so the helper lives -here in :mod:`isaaclab_ppisp` and is called from both backends' -``prepare_cameras`` hook. -""" - -from __future__ import annotations - -from typing import Any - -# Neutral RTX-side values authored on the camera prim when PPISP is the ISP -# authority. RTX otherwise compounds its own physical-camera exposure model -# on top of PPISP, leaving the HDR AOV under- or over-exposed. PPISP supplies -# its own ``responsivity`` / ``exposureOffset``; pinning these stage-side -# values (``iso=0`` plus ``autoExposure:enabled=False``) keeps RTX's exposure -# stage a no-op. -# -# Each entry is ``(attribute_name, sdf_type_name, value)`` where -# ``sdf_type_name`` is the suffix of ``Sdf.ValueTypeNames``; values are -# coerced inside :func:`apply_rtx_exposure_overrides`. -_NEUTRAL_CAMERA_EXPOSURE: tuple[tuple[str, str, Any], ...] = ( - ("exposure", "Float", 0.0), - ("exposure:fStop", "Float", 1.0), - ("exposure:iso", "Float", 0.0), - ("exposure:responsivity", "Float", 1.0), - ("exposure:time", "Float", 1.0), - ("omni:rtx:autoExposure:enabled", "Bool", False), -) - -# API schemas applied (prepended) on each camera prim so Kit's USD watcher -# routes the camera's ``exposure:*`` and ``omni:rtx:autoExposure:*`` -# attributes into the per-camera carb settings RTX consumes. -_PPISP_CAMERA_API_SCHEMAS: tuple[str, ...] = ( - "OmniRtxCameraAutoExposureAPI_1", - "OmniRtxCameraExposureAPI_1", -) - - -def apply_rtx_exposure_overrides(stage: Any, prim_paths: list[str]) -> None: - """Pin RTX-side exposure to neutral and apply the API schemas on each camera prim. - - On every prim in ``prim_paths``: - - 1. Author the neutral values from :data:`_NEUTRAL_CAMERA_EXPOSURE` — - ``exposure:*`` knobs disabling RTX's physical-camera exposure stage - plus ``omni:rtx:autoExposure:enabled=False``. - 2. Apply the :data:`_PPISP_CAMERA_API_SCHEMAS` API schemas via - ``SetMetadata("apiSchemas", ...)``. In Kit, ``omni.kit``'s USD - watcher pumps the authored attributes into the per-camera carb - settings RTX consumes. ``Sdf.TokenListOp`` metadata is used rather - than ``Usd.Prim.ApplyAPI`` so this does not require the USD - ``PlugRegistry`` to have discovered the RTX-settings schema plugin - yet. - - Existing API schemas on the prim are preserved — the RTX schemas are - prepended, not replaced. - - Args: - stage: USD stage containing the camera prim(s). - prim_paths: Resolved camera prim paths (typically one per env). - """ - from pxr import Sdf - - sdf_value_types = { - "Bool": Sdf.ValueTypeNames.Bool, - "Float": Sdf.ValueTypeNames.Float, - } - - def coerce(value: Any, sdf_type_name: str) -> Any: - if sdf_type_name == "Float": - return float(value) - return value - - for prim_path in prim_paths: - prim = stage.GetPrimAtPath(prim_path) - if not prim or not prim.IsValid(): - continue - current = prim.GetMetadata("apiSchemas") or Sdf.TokenListOp() - existing = list(current.prependedItems) + list(current.explicitItems) + list(current.appendedItems) - missing = [s for s in _PPISP_CAMERA_API_SCHEMAS if s not in existing] - if missing: - merged = Sdf.TokenListOp.Create(prependedItems=[*missing, *existing]) - prim.SetMetadata("apiSchemas", merged) - for attr_name, sdf_type_name, value in _NEUTRAL_CAMERA_EXPOSURE: - attr = prim.GetAttribute(attr_name) - if not attr: - attr = prim.CreateAttribute(attr_name, sdf_value_types[sdf_type_name], custom=False) - attr.Set(coerce(value, sdf_type_name)) diff --git a/source/isaaclab_ppisp/pyproject.toml b/source/isaaclab_ppisp/pyproject.toml index ade02233204e..52b8d53c60b9 100644 --- a/source/isaaclab_ppisp/pyproject.toml +++ b/source/isaaclab_ppisp/pyproject.toml @@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta" [project] name = "isaaclab_ppisp" version = "0.3.1" -description = "Extension providing a renderer-backend-agnostic PPISP pipeline that converts HDR scene-linear color to LDR RGBA at the end of a render tick." +description = "PPISP processors that convert scene-linear HDR to LDR RGB or RGBA for cameras and observations." license = {text = "BSD-3-Clause"} authors = [{name = "Isaac Lab Project Developers"}] maintainers = [{name = "Isaac Lab Project Developers"}] diff --git a/source/isaaclab_ppisp/test/test_ppisp.py b/source/isaaclab_ppisp/test/test_ppisp.py index 453595d849d0..c60841320a31 100644 --- a/source/isaaclab_ppisp/test/test_ppisp.py +++ b/source/isaaclab_ppisp/test/test_ppisp.py @@ -10,6 +10,7 @@ import pytest from isaaclab_ppisp import ( PpispCfg, + PpispDiscoveryMode, auto_any_ppisp_cfg, auto_camera_ppisp_cfg, default_ppisp_inputs, @@ -226,13 +227,11 @@ def test_auto_any_ppisp_cfg_reads_first_camera_with_ppisp_attrs(): def test_resolve_and_normalize_without_camera_uses_first_ppisp_camera(): - from isaaclab.sensors.camera.camera_isp import CameraISPMode - stage = Usd.Stage.CreateInMemory() _author_camera(stage, "/World/CameraWithoutPpisp") _author_ppisp_camera(stage, "/World/Camera_ppisp", inherits=None, attrs={"exposureOffset": 2.0}) - cfg = resolve_and_normalize(CameraISPMode.AUTO_CAMERA, stage) + cfg = resolve_and_normalize(PpispDiscoveryMode.AUTO_CAMERA, stage) assert cfg is not None assert cfg.camera_prim_path is None diff --git a/source/isaaclab_ppisp/test/test_ppisp_kernels.py b/source/isaaclab_ppisp/test/test_ppisp_kernels.py index 6b3ac48d7ea7..352ea6335c83 100644 --- a/source/isaaclab_ppisp/test/test_ppisp_kernels.py +++ b/source/isaaclab_ppisp/test/test_ppisp_kernels.py @@ -4,9 +4,12 @@ # SPDX-License-Identifier: BSD-3-Clause import pytest +import torch import warp as wp from isaaclab_ppisp import ( PpispCfg, + PpispControllerWeights, + PpispProcessorCfg, apply_ppisp_to_rgba, apply_ppisp_to_rgba_with_controller_params, compute_ppisp_controller_params, @@ -37,6 +40,10 @@ PPISP_CONTROLLER_PARAM_COUNT, ) +from isaaclab.test.utils import DeviceScope, test_devices +from isaaclab.utils.tensor_processors._pipeline import _DictTensorProcessorPipeline +from isaaclab.utils.warp import ProxyArray + wp.init() @@ -282,15 +289,15 @@ def bounded_softplus(raw: float, min_value: float) -> float: return np.maximum(y, 0.0) ** gamma -def test_ppisp_warp_crf_extreme_centers_match_reference(): - """Saturated CRF centers produce the reference curve instead of NaN-cast bytes.""" +@pytest.mark.parametrize("attachment", ["dictionary", "tensor"]) +def test_ppisp_pipe_rgb_crf_extreme_centers_match_reference(attachment): + """A CPU RGB pipe preserves the reference curve, including saturated CRF centers.""" import numpy as np # a gray ramp passes the default exposure, vignetting, and color stages unchanged ramp = np.linspace(0.0, 1.0, 16, dtype=np.float32) hdr_data = np.repeat(ramp[None, None, :, None], 3, axis=3) - hdr_color = wp.from_numpy(hdr_data, dtype=wp.float32) - rgba = _rgba((1, 1, 16, 4)) + hdr_color = wp.from_numpy(hdr_data, dtype=wp.float32, device="cpu") cfg = normalize_ppisp_cfg( PpispCfg( inputs={ @@ -307,10 +314,24 @@ def test_ppisp_warp_crf_extreme_centers_match_reference(): ) ) - apply_ppisp_to_rgba(hdr_color, rgba, cfg) - - rgba_np = rgba.numpy()[0, 0] - assert (rgba_np[:, 3] == 255).all() + if attachment == "dictionary": + from isaaclab_ppisp import PpispProcessor + + processor = PpispProcessor(PpispProcessorCfg(params={"cfg": cfg}), {"rgb_hdr": ProxyArray(hdr_color)}) + rgb_np = processor({"rgb_hdr": ProxyArray(hdr_color)})["rgb"].warp.numpy()[0, 0] + processor.close() + else: + from isaaclab.utils.tensor_processors import DictTensorProcessorAdapter, DictTensorProcessorAdapterCfg + + processor = DictTensorProcessorAdapter( + DictTensorProcessorAdapterCfg( + pipeline=[PpispProcessorCfg(params={"cfg": cfg})], input_name="rgb_hdr", output_name="rgb" + ), + hdr_data.shape, + "cpu", + ) + rgb_np = processor(wp.to_torch(hdr_color)).numpy()[0, 0] + processor.close() for channel, suffix in enumerate("RGB"): expected = _crf_reference( ramp, @@ -321,4 +342,62 @@ def test_ppisp_warp_crf_extreme_centers_match_reference(): ) expected_bytes = np.floor(np.clip(expected, 0.0, 1.0) * 255.0) # float32 kernel math may land one byte away from the float64 reference at a floor boundary - np.testing.assert_allclose(rgba_np[:, channel].astype(float), expected_bytes, atol=1.0, err_msg=suffix) + np.testing.assert_allclose(rgb_np[:, channel].astype(float), expected_bytes, atol=1.0, err_msg=suffix) + np.testing.assert_array_equal(hdr_color.numpy(), hdr_data) + + +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) +def test_ppisp_controller_pipes_share_weights_for_rgb_and_rgba(device): + """Both controller output layouts match a known response; closing one consumer preserves another.""" + import numpy as np + + weights_np = np.zeros(PPISP_CONTROLLER_EXPECTED_WEIGHTS_LEN, dtype=np.float32) + weights_np[PPISP_CONTROLLER_OFF_EXP_B] = 1.0 # A constant one-stop controller exposure. + weights = wp.from_numpy(weights_np, dtype=wp.float32, device=device) + ramp = np.array([0.0, 0.125, 0.25, 0.5], dtype=np.float32) + hdr_data = np.repeat(ramp[None, None, :, None], 3, axis=3) + hdr = wp.from_numpy(hdr_data, dtype=wp.float32, device=device) + cfg = PpispCfg() + shared_weights = PpispControllerWeights(weights) + pipes = [ + _DictTensorProcessorPipeline( + {"rgb_hdr": ProxyArray(hdr)}, + [PpispProcessorCfg(params={"cfg": cfg, "channels": channels, "controller_weights": shared_weights})], + ) + for channels in (3, 4) + ] + # Use different pixels from preparation and honor prior work on a non-default Torch stream. + with torch.cuda.stream(torch.cuda.Stream(device=device)): + incoming = torch.from_numpy(hdr_data * 0.5).to(device) * 3.0 + observation = pipes[0].apply_to_tensor(incoming, input_name="rgb_hdr", output_name="rgb") + results = [observation.cpu().numpy()] + results.append(pipes[1]({"rgb_hdr": ProxyArray(hdr)})["rgb"].warp.numpy()) + for result, input_scale in zip(results, (1.5, 1.0), strict=True): + expected = np.floor(_crf_reference(ramp * input_scale * 2.0, 0.013659, 0.013659, 0.378165, 0.0) * 255.0) + np.testing.assert_allclose(result[0, 0, :, :3].astype(float), np.repeat(expected[:, None], 3, axis=1), atol=1.0) + assert (results[1][..., 3] == 255).all() + pipes[0].close() + hdr.zero_() + np.testing.assert_array_equal(pipes[1]({"rgb_hdr": ProxyArray(hdr)})["rgb"].warp.numpy()[..., :3], 0) + np.testing.assert_array_equal(weights.numpy(), weights_np) + pipes[1].close() + + +@pytest.mark.parametrize( + ("shape", "dtype", "message"), + [((1, 2, 2, 4), wp.float32, "must have shape"), ((1, 2, 2, 3), wp.uint8, "must have dtype")], +) +def test_ppisp_pipe_rejects_incompatible_input_storage(shape, dtype, message): + """Reject invalid image channels and scalar types before preparing numerical execution.""" + source = ProxyArray(wp.empty(shape, dtype=dtype, device="cpu")) + with pytest.raises(ValueError, match=message): + _DictTensorProcessorPipeline({"rgb_hdr": source}, [PpispProcessorCfg()]) + + +def test_ppisp_pipe_requires_resolved_configuration(): + """An image operation must not discover its settings from a camera during preparation.""" + source = ProxyArray(wp.empty((1, 2, 2, 3), dtype=wp.float32, device="cpu")) + with pytest.raises(ValueError, match="resolve USD camera attributes"): + _DictTensorProcessorPipeline( + {"rgb_hdr": source}, [PpispProcessorCfg(params={"cfg": PpispCfg(camera_prim_path="/World/Camera")})] + )