From 4af9e76ad5963bb81256f573c575bef1cd4ad417 Mon Sep 17 00:00:00 2001 From: Octi Zhang Date: Sun, 27 Sep 2026 19:44:36 -0700 Subject: [PATCH] Backport Newton consumer resource ownership to release/3.0.0 Backport #8027 without changes. (cherry picked from commit b05a35a0eedda86662f82e663993c2b7046bf94f) --- .../lab_newton/isaaclab_newton.physics.rst | 8 + docs/source/concepts/backend_architecture.rst | 34 +- .../extending_newton_solvers.rst | 14 +- .../developer-tools/scene_data_providers.rst | 8 +- .../migration/migrating_to_isaaclab_3-0.rst | 20 +- .../demos/newton_viewer_block_and_tackle.py | 5 +- .../newton-consumer-ownership.major.rst | 25 + .../isaaclab/renderers/render_context.py | 7 +- .../isaaclab/scene_data/scene_data_backend.py | 2 +- .../scene_data/scene_data_provider.py | 37 +- .../sensors/ray_caster/ray_caster_cfg.py | 6 + .../isaaclab/isaaclab/sim/simulation_cfg.py | 9 +- .../isaaclab/sim/simulation_context.py | 60 +- .../isaaclab/visualizers/base_visualizer.py | 6 + .../isaaclab/visualizers/visualizer_cfg.py | 14 +- .../markers/test_visualization_markers.py | 224 ++----- .../test_simulation_render_context.py | 4 +- .../scene_data/test_scene_data_transforms.py | 35 +- ...test_newton_manager_visualization_state.py | 275 ++------- .../sim/test_physics_manager_lifecycle.py | 24 + .../test_simulation_context_visualizers.py | 131 ++-- .../test/visualizers/test_visualizer.py | 14 +- .../coupler-builder-preparation.rst | 4 + .../isaaclab_contrib/coupling/coupler.py | 8 +- .../test/coupling/test_coupler.py | 16 +- .../native-query-ownership.major.rst | 30 + .../assets/articulation/articulation.py | 121 ++-- .../isaaclab_newton/cloner/replicate.py | 48 +- .../isaaclab_newton/physics/__init__.pyi | 6 +- .../isaaclab_newton/physics/mpm_manager.py | 4 +- .../isaaclab_newton/physics/newton_manager.py | 584 +++++------------- .../physics/newton_manager_cfg.py | 30 +- .../isaaclab_newton/physics/vbd_manager.py | 10 +- .../renderers/newton_warp_renderer.py | 62 +- .../renderers/newton_warp_renderer_cfg.py | 3 + .../frame_transformer/frame_transformer.py | 2 +- .../ray_caster/newton_raycast_sensor.py | 58 +- .../sim/views/newton_site_frame_view.py | 5 +- .../test/assets/test_articulation.py | 52 +- .../test/assets/test_deformable_object.py | 5 +- .../cloner/test_newton_builder_world_hook.py | 26 +- .../test/cloner/test_rename_builder_labels.py | 55 +- .../physics/test_newton_fabric_body_sync.py | 7 +- .../test_newton_manager_abstraction.py | 272 +++----- .../test/physics/test_vbd_core.py | 58 +- .../test/renderers/test_visual_material.py | 26 +- .../sensors/test_newton_raycast_sensor.py | 78 +-- .../consumer-owned-ovrtx-dependencies.rst | 5 + .../isaaclab_ov/renderers/ovrtx_renderer.py | 6 + .../test/test_ovrtx_renderer_contract.py | 18 +- .../changelog.d/consumer-owned-fabric-cfg.rst | 11 + .../assets/rigid_object/rigid_object.py | 4 + .../renderers/isaac_rtx_renderer.py | 3 +- .../renderers/isaac_rtx_renderer_utils.py | 5 +- .../test_isaac_rtx_renderer_utils.py | 6 +- .../native-query-ownership.major.rst | 18 + .../kit/kit_visualizer.py | 59 +- .../kit/kit_visualizer_cfg.py | 6 + .../newton/newton_visualizer.py | 227 +++---- .../newton/newton_visualizer_cfg.py | 6 + .../isaaclab_visualizers/newton_adapter.py | 2 +- .../rerun/rerun_visualizer.py | 157 ++--- .../rerun/rerun_visualizer_cfg.py | 6 + .../viser/viser_visualizer.py | 137 ++-- .../viser/viser_visualizer_cfg.py | 6 + .../test_kit_visualizer_scene_partitioning.py | 15 +- .../test/test_newton_adapter.py | 175 ++++-- .../test/test_newton_visual_material.py | 7 - .../test_newton_visualizer_viewer_release.py | 6 +- 69 files changed, 1478 insertions(+), 1939 deletions(-) create mode 100644 source/isaaclab/changelog.d/newton-consumer-ownership.major.rst create mode 100644 source/isaaclab_contrib/changelog.d/coupler-builder-preparation.rst create mode 100644 source/isaaclab_newton/changelog.d/native-query-ownership.major.rst create mode 100644 source/isaaclab_ov/changelog.d/consumer-owned-ovrtx-dependencies.rst create mode 100644 source/isaaclab_physx/changelog.d/consumer-owned-fabric-cfg.rst create mode 100644 source/isaaclab_visualizers/changelog.d/native-query-ownership.major.rst diff --git a/docs/source/api/lab_newton/isaaclab_newton.physics.rst b/docs/source/api/lab_newton/isaaclab_newton.physics.rst index 5379728ffba1..69c9f1c1b6e4 100644 --- a/docs/source/api/lab_newton/isaaclab_newton.physics.rst +++ b/docs/source/api/lab_newton/isaaclab_newton.physics.rst @@ -10,6 +10,7 @@ NewtonManager NewtonCfg NewtonBackendCfg + NewtonBuilderCfg NewtonSoftContactCfg NewtonCollisionPipelineCfg NewtonFeatherstoneManager @@ -59,6 +60,13 @@ Physics Configuration :show-inheritance: :exclude-members: __init__ +.. autoclass:: NewtonBuilderCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +.. autofunction:: create_newton_builder + .. autoclass:: NewtonSoftContactCfg :members: :show-inheritance: diff --git a/docs/source/concepts/backend_architecture.rst b/docs/source/concepts/backend_architecture.rst index 8e6ebcee578a..7c31be223d16 100644 --- a/docs/source/concepts/backend_architecture.rst +++ b/docs/source/concepts/backend_architecture.rst @@ -70,21 +70,24 @@ and release resources with ``close()``. The manager exposes :class:`~isaaclab.physics.PhysicsEvent` callbacks for cross-backend lifecycle work. ``MODEL_INIT`` occurs during scene construction, -``PHYSICS_READY`` after physics initialization, and ``STOP`` during shutdown. +``PHYSICS_READY`` after physics initialization, and ``STOP`` before native resources are replaced +or shut down. The concrete ``close()`` implementation dispatches the ``STOP`` event. ``SimulationContext`` owns native resources and renderer instances in one registry. ``get_or_create_backend(backend_cfg)`` reuses one resource for equal configurations of the same concrete type; a cache miss constructs ``instantiate(backend_cfg)``. -:class:`~isaaclab.sim.BackendCfg` describes resource construction inputs, and +:class:`~isaaclab.sim.BackendCfg` describes resource settings and identity, and :class:`~isaaclab.renderers.RendererCfg` extends it for renderer instances. ``PhysicsCfg`` selects a physics manager. Finalize configurations before registration and treat them, including nested values, as read-only afterward. Use a new configuration for different settings. ``close_backend(backend)`` closes the exact registered object after all consumers have released their bindings; -it does not compare or hash configurations. Resources implement ``close()``; -failed release retains the entry for retry. After physics shutdown invalidates camera +it does not compare or hash configurations. Resources declared through ``BackendCfg`` must +implement ``close()``. Plain construction cfgs share Python-owned data without a teardown +operation; removing the registry entry releases its reference. A failed close retains +the entry for retry. After physics shutdown invalidates camera render data, simulation teardown closes material writers, renderer instances, visualizers, and remaining native resources, in that order, before closing the stage. @@ -96,6 +99,29 @@ Exposing native handles does not replace SDP transport. Clone contexts are registered separately as ``sim.clone_contexts[Context] = Context(...)`` before plan dispatch. They apply the plan but do not own native runtime resources. +Newton has two resources with different lifetimes, not two interchangeable backends: + +* ``ModelBuilder`` holds mutable construction data. Cloning populates it and sensors declare + requirements before finalization. It remains available for hard reset. ``NewtonBuilderCfg`` + is a plain construction cfg, not a ``BackendCfg``; the builder needs no native ``close()``. +* ``NewtonBackend`` owns the finalized model and native buffers. Physics and render consumers + borrow those handles. Closing it releases runtime allocations without closing the builder. + +Both resources use the same registry: + +.. code-block:: python + + builder_cfg = NewtonBuilderCfg(physics_cfg=sim.cfg.physics) + builder = sim.get_or_create_backend(builder_cfg) + # Clone/import populates this builder before model allocation. + model_cfg = NewtonBackendCfg(physics_cfg=sim.cfg.physics, device=sim.device) + backend = sim.get_or_create_backend(model_cfg) + +Both configurations use the selected physics cfg; non-Newton physics selects a render-only +representation. ``SimulationContext`` has no backend-specific cfg fields, and consumers do not +access clone contexts. Consumers request body transforms and visual points directly through SDP. +Queries share the native resource's BVHs but keep each consumer's captured work separate. + Portable asset and sensor interfaces ------------------------------------ diff --git a/docs/source/developer-tools/extending_newton_solvers.rst b/docs/source/developer-tools/extending_newton_solvers.rst index 91dda16a4479..26a5d58dfaa1 100644 --- a/docs/source/developer-tools/extending_newton_solvers.rst +++ b/docs/source/developer-tools/extending_newton_solvers.rst @@ -91,14 +91,13 @@ class invokes them. - Stores the simulation context, reads gravity from the simulation configuration, and creates the scene data backend. - none - * - :meth:`~isaaclab_newton.physics.NewtonManager.create_builder` and - :meth:`~isaaclab_newton.physics.NewtonManager.set_builder` - - The cloner imports declared prototypes, composes worlds, and supplies the completed builder. - - ``_register_builder_attributes()`` (not called by ``set_builder()``) + * - ``sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics))`` + - Acquires the shared builder through the active manager's ``create_builder()`` factory. + The cloner imports declared prototypes and composes worlds into it. + - ``_register_builder_attributes()`` * - :meth:`~isaaclab_newton.physics.NewtonManager.start_simulation` - Finalizes the model, then allocates states and reset masks. - - ``_register_builder_attributes()``, - ``_prepare_builder_for_finalize()`` + - ``_prepare_builder_for_finalize()`` * - :meth:`~isaaclab_newton.physics.NewtonManager.initialize_solver` - Builds the solver, checks that it was assigned, and allocates contacts. - ``_build_solver()``, ``_initialize_contacts()`` @@ -106,8 +105,7 @@ class invokes them. - A hard reset re-runs ``start_simulation()`` and ``initialize_solver()`` against a re-finalized model; a soft reset reuses the existing model, solver, contacts, and graph. - - ``_register_builder_attributes()``, - ``_prepare_builder_for_finalize()``, ``_build_solver()``, + - ``_prepare_builder_for_finalize()``, ``_build_solver()``, ``_initialize_contacts()`` (hard reset only) * - :meth:`~isaaclab_newton.physics.NewtonManager.step` - Runs one actuator pass plus ``num_substeps`` solver substeps, then diff --git a/docs/source/developer-tools/scene_data_providers.rst b/docs/source/developer-tools/scene_data_providers.rst index d66f1ec871fd..170ff4085d83 100644 --- a/docs/source/developer-tools/scene_data_providers.rst +++ b/docs/source/developer-tools/scene_data_providers.rst @@ -141,9 +141,11 @@ path. PhysX owns its native Fabric refresh and publishes the resulting matrices through SDP without fetching packed poses. ``isaaclab_physx.renderers.fabric.FabricBackend`` owns the shared native stage and hierarchy handles. Its identity is the stage and device, not the SDP source or attribute type. -``SimulationContext`` declares ``fabric_cfg`` when Kit is available. After physics initializes, Kit, -Isaac RTX, and explicit Fabric synchronization obtain the same resource through -``get_or_create_backend(sim.fabric_cfg)``. Transform bindings are state on that resource, not a +After physics initializes, Kit, Isaac RTX, and explicit Fabric synchronization obtain +the same resource through +``sim.get_or_create_backend(FabricBackendCfg(stage=sim.stage, device=sim.device))``. +The simulation registry does not need to know which backends exist. +Transform bindings are state on that resource, not a separate backend. Consumers pass the simulation's SDP to ``update_transforms(provider)``; for foreign physics it converts directly into Fabric local matrices, then propagates the GPU hierarchy. Core ``RenderContext`` owns no Fabric bindings. diff --git a/docs/source/migration/migrating_to_isaaclab_3-0.rst b/docs/source/migration/migrating_to_isaaclab_3-0.rst index c5d38d7e0965..9f7f8149a357 100644 --- a/docs/source/migration/migrating_to_isaaclab_3-0.rst +++ b/docs/source/migration/migrating_to_isaaclab_3-0.rst @@ -1613,9 +1613,10 @@ For configuration and data access examples, see the :ref:`overview_sensors_joint import torch from isaaclab.scene import InteractiveSceneCfg from isaaclab.sensors import JointWrenchSensorCfg + from isaaclab.utils import replace class MySceneCfg(InteractiveSceneCfg): - robot = ROBOT_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot") + robot = replace(ROBOT_CFG, prim_path="{ENV_REGEX_NS}/Robot") joint_wrench = JointWrenchSensorCfg(prim_path="{ENV_REGEX_NS}/Robot") sensor = env.scene.sensors["joint_wrench"] @@ -1643,9 +1644,10 @@ depend on the joint-wrench sensor instead: from isaaclab.managers import ObservationTermCfg as ObsTerm from isaaclab.scene import InteractiveSceneCfg from isaaclab.sensors import JointWrenchSensorCfg + from isaaclab.utils import replace class MySceneCfg(InteractiveSceneCfg): - robot = ROBOT_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot") + robot = replace(ROBOT_CFG, prim_path="{ENV_REGEX_NS}/Robot") joint_wrench = JointWrenchSensorCfg(prim_path="{ENV_REGEX_NS}/Robot") feet_body_forces = ObsTerm( @@ -2640,8 +2642,9 @@ tracking is handled directly by :class:`~isaaclab_visualizers.kit.KitVisualizer` The ``tiled_cam_*`` configuration fields on visualizer configs (e.g. ``tiled_cam_view``, ``tiled_cam_num``, ``tiled_cam_prim_path``) have been removed and replaced by the unified ``streaming_*`` API available on all four visualizer backends. A one-release deprecation -shim forwards each removed field to its ``streaming_*`` equivalent and emits +shim forwards the old fields except renderer selection to their ``streaming_*`` equivalents and emits :class:`DeprecationWarning`; the shim will be removed in the next major release. +Renderer selection now takes a configuration directly, without a nickname compatibility path. .. list-table:: Field rename reference :header-rows: 1 @@ -2663,8 +2666,15 @@ shim forwards each removed field to its ``streaming_*`` equivalent and emits - ``streaming_cam_eye`` - * - ``tiled_cam_renderer`` - - ``streaming_cam_renderer`` - - Accepts ``"newton_warp"``, ``"ovrtx"``, ``"isaac_rtx"``, or ``None`` + - ``streaming_cam_renderer_cfg`` + - Renderer configuration, not a nickname + +Replace ``streaming_cam_renderer="ovrtx"`` with ``streaming_cam_renderer_cfg=OVRTXRendererCfg()`` +(imported from ``isaaclab_ov.renderers``). Likewise, pass ``NewtonWarpRendererCfg()`` or +``IsaacRtxRendererCfg()`` for Newton Warp or Isaac RTX. Custom configurations use their existing +``class_type`` class or resolvable string. Kit defaults to Isaac RTX; the other visualizers default +to Newton Warp. An unavailable explicitly selected renderer raises its construction error instead +of silently selecting another renderer. .. code-block:: python diff --git a/examples/demos/newton_viewer_block_and_tackle.py b/examples/demos/newton_viewer_block_and_tackle.py index 73fddba18628..817fc8c5a32a 100644 --- a/examples/demos/newton_viewer_block_and_tackle.py +++ b/examples/demos/newton_viewer_block_and_tackle.py @@ -24,7 +24,7 @@ import newton import newton.utils import warp as wp -from isaaclab_newton.physics import NewtonCfg, NewtonManager, NewtonShapeCfg, VBDSolverCfg +from isaaclab_newton.physics import NewtonBuilderCfg, NewtonCfg, NewtonManager, NewtonShapeCfg, VBDSolverCfg import isaaclab.sim as sim_utils from isaaclab.utils import configclass @@ -315,11 +315,10 @@ def main() -> None: sim_cfg = sim_utils.SimulationCfg(dt=1.0 / 60.0, device=args_cli.device, physics=resolved_physics_cfg) sim = sim_utils.SimulationContext(sim_cfg) sim.set_camera_view(eye=(1.35, -2.1, 1.25), target=(0.50, 0.0, 0.72)) - builder = NewtonManager.create_builder() + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics)) builder.rigid_gap = CABLE_GAP cable_body_start, wrapped_xforms = _build_system(builder) builder.color(balance_colors=False) - NewtonManager.set_builder(builder) sim.reset() _initialize_wrapped_cable(cable_body_start, wrapped_xforms) print("[INFO]: Setup complete. Right-drag the yellow cable handle downward to lift the red load.", flush=True) diff --git a/source/isaaclab/changelog.d/newton-consumer-ownership.major.rst b/source/isaaclab/changelog.d/newton-consumer-ownership.major.rst new file mode 100644 index 000000000000..bc54c3b8ad4e --- /dev/null +++ b/source/isaaclab/changelog.d/newton-consumer-ownership.major.rst @@ -0,0 +1,25 @@ +Changed +^^^^^^^ + +* Extended ``SimulationContext.get_or_create_backend(cfg)`` to own Python resources as well as native + backends. Resources declared through ``BackendCfg`` implemented ``close()``; plain construction cfgs + shared Python-owned data released with the registry reference. + Resource construction used ``instantiate(cfg)`` without backend-specific context fields. +* Reused SDP transform mappings for matching source and destination layouts, allowing independent + consumers to share converted buffers without retaining SDP bindings on a native backend. Matching + unique native and consumer paths needed no index-map allocation. Binding rejected duplicate named + paths and destinations without native publications, including when the publication was empty. +* Added ``RayCasterCfg.use_cuda_graph`` for Newton query execution independently of physics capture. + Set it to ``False`` to execute Newton ray casts eagerly. +* **Breaking:** Replaced ``VisualizerCfg.streaming_cam_renderer`` and its deprecated + ``tiled_cam_renderer`` alias with ``streaming_cam_renderer_cfg``. Pass a renderer configuration, + such as ``NewtonWarpRendererCfg()`` or ``OVRTXRendererCfg()``, instead of a renderer nickname. + Its ``class_type`` accepted either an implementation class or a resolvable string. + +Fixed +^^^^^ + +* Released rendering bindings on physics stop so a hard reset reinitialized consumers against + the replacement native resource. +* Preserved each visualizer's default streaming renderer when applying camera settings from + another visualizer configuration. diff --git a/source/isaaclab/isaaclab/renderers/render_context.py b/source/isaaclab/isaaclab/renderers/render_context.py index 3056a2ccbb0c..cdd1515532b6 100644 --- a/source/isaaclab/isaaclab/renderers/render_context.py +++ b/source/isaaclab/isaaclab/renderers/render_context.py @@ -10,7 +10,7 @@ import logging import warnings from collections.abc import Sequence -from typing import TYPE_CHECKING, Any +from typing import Any import torch import warp as wp @@ -19,9 +19,6 @@ from .base_renderer import BaseRenderer, VisualMaterialBatch from .renderer_cfg import RendererCfg -if TYPE_CHECKING: - from ..sim import BackendCfg - logger = logging.getLogger(__name__) @@ -81,7 +78,7 @@ class RenderContext: "_consumers_finalized", ) - def __init__(self, backend_registry: list[tuple[BackendCfg, Any]]) -> None: + def __init__(self, backend_registry: list[tuple[Any, Any]]) -> None: self.clone_contexts: set[type | str] = set() """Scene representations declared by camera renderers and visualizers before cloning.""" self._backend_registry = backend_registry diff --git a/source/isaaclab/isaaclab/scene_data/scene_data_backend.py b/source/isaaclab/isaaclab/scene_data/scene_data_backend.py index 49367dcc148f..2a14892296d1 100644 --- a/source/isaaclab/isaaclab/scene_data/scene_data_backend.py +++ b/source/isaaclab/isaaclab/scene_data/scene_data_backend.py @@ -178,5 +178,5 @@ def transform_count(self) -> int: @property def transform_paths(self) -> list[str]: - """Return the paths for each transform.""" + """Return unique native body paths in publication order, fixed for each initialized layout.""" raise NotImplementedError diff --git a/source/isaaclab/isaaclab/scene_data/scene_data_provider.py b/source/isaaclab/isaaclab/scene_data/scene_data_provider.py index 17d96b0593de..c1b611af6ef6 100644 --- a/source/isaaclab/isaaclab/scene_data/scene_data_provider.py +++ b/source/isaaclab/isaaclab/scene_data/scene_data_provider.py @@ -66,6 +66,7 @@ def __init__(self, backend: SceneDataBackend): self._num_envs_cache: int | None = None self._interactive_scene: Any | None = None self._cache: dict[Any, TimestampedBuffer] = {} + self._transform_mappings: dict[tuple, wp.array | None] = {} def get_transforms( self, @@ -276,26 +277,36 @@ def create_mapping(self, paths: list[str | None]) -> wp.array(dtype=wp.int32) | appear in ``paths`` (or maps to ``None``) receive an index of ``-1`` and are skipped during conversion. + Named paths must be unique in both layouts. Binding validates this once per layout pair; + independent consumers reuse the mapping and share SDP conversions. + Args: paths: Desired output ordering expressed as prim paths. Use ``None`` for slots that should not receive any transform. Returns: A Warp int32 array of length :attr:`transform_count` containing the - remapped indices, or ``None`` if the sim backend provides no transform - paths or if no mapping is needed. + remapped indices, or ``None`` for identical layouts. + + Raises: + ValueError: A source or named destination path is repeated. + KeyError: A named destination has no native publication. """ - if input_paths := self.backend.transform_paths: - # The map keeps resolution linear in the number of paths. For duplicate - # paths the first occurrence wins, matching ``list.index``. - path_to_out: dict[str | None, int] = {} - for out_idx, out_path in enumerate(paths): - if out_path not in path_to_out: - path_to_out[out_path] = out_idx - mapping = [path_to_out.get(path, -1) for path in input_paths] - if len(paths) != len(input_paths) or not np.array_equal(mapping, np.arange(len(input_paths))): - return wp.array(mapping, dtype=wp.int32, device=_publication_device(self.backend.transforms)) - return None + input_paths, paths = tuple(self.backend.transform_paths), tuple(paths) + key = input_paths, paths + if key not in self._transform_mappings: + sources = {path: index for index, path in enumerate(input_paths)} + destinations = {path: index for index, path in enumerate(paths) if path is not None} + if len(sources) != len(input_paths) or len(destinations) != len(paths) - paths.count(None): + raise ValueError("Transform paths must be unique; only destination None slots may repeat.") + result = None + if input_paths != paths: + mapping = np.full(len(input_paths), -1, dtype=np.int32) + for path, index in destinations.items(): + mapping[sources[path]] = index + result = wp.array(mapping, dtype=wp.int32, device=_publication_device(self.backend.transforms)) + self._transform_mappings[key] = result + return self._transform_mappings[key] def get_geometry_points( self, *, output: wp.array | SceneDataFormat.FabricPoints | None = None, offsets: dict[str, int] | None = None diff --git a/source/isaaclab/isaaclab/sensors/ray_caster/ray_caster_cfg.py b/source/isaaclab/isaaclab/sensors/ray_caster/ray_caster_cfg.py index ebdc5f7dc2f7..96d943ccb79a 100644 --- a/source/isaaclab/isaaclab/sensors/ray_caster/ray_caster_cfg.py +++ b/source/isaaclab/isaaclab/sensors/ray_caster/ray_caster_cfg.py @@ -91,6 +91,12 @@ class OffsetCfg: rays outside the carrier's geometry in that case. """ + use_cuda_graph: bool = True + """Capture Newton BVH queries independently of the physics solver's graph setting. + + CPU queries and the static Warp-mesh implementations run eagerly. + """ + drift_range: tuple[float, float] = (0.0, 0.0) """The range of drift (in meters) to add to the ray starting positions (xyz) in world frame. Defaults to (0.0, 0.0). diff --git a/source/isaaclab/isaaclab/sim/simulation_cfg.py b/source/isaaclab/isaaclab/sim/simulation_cfg.py index d7331ada9f49..a1da0f12a5c5 100644 --- a/source/isaaclab/isaaclab/sim/simulation_cfg.py +++ b/source/isaaclab/isaaclab/sim/simulation_cfg.py @@ -11,8 +11,9 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import MISSING -from typing import Literal +from typing import Any, Literal from ..physics import PhysicsCfg from ..utils import configclass @@ -22,13 +23,13 @@ @configclass class BackendCfg: - """Construction inputs and value identity for a simulation-owned resource. + """Declarative settings and value identity for a simulation-owned resource. Finalize all fields before registration and treat them as read-only afterward. """ - class_type: type = MISSING - """Resource class constructed as ``instantiate(cfg)``; must implement ``close()``.""" + class_type: Callable[[BackendCfg], Any] = MISSING + """Constructor called through ``instantiate(cfg)``; the returned resource must implement ``close()``.""" @configclass diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index ec88104a8714..ff13c270653f 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -151,7 +151,7 @@ def __init__(self, cfg: SimulationCfg | None = None): # Store config self.cfg = SimulationCfg() if cfg is None else cfg - self._backend_registry: list[tuple[BackendCfg, Any]] = [] + self._backend_registry: list[tuple[Any, Any]] = [] self.clone_contexts: dict[type, Any] = {} """Clone-context instances registered by type before plan dispatch; not native resource owners.""" @@ -219,12 +219,6 @@ def __init__(self, cfg: SimulationCfg | None = None): # Construct visualizers before cloning; initialize their runtime bindings after physics is ready. self._scene_data_provider = SceneDataProvider(self.physics_manager.get_scene_data_backend()) - self.fabric_cfg: BackendCfg | None = None - """Native Fabric stage/device configuration, or None without Kit.""" - if use_isaac_sim: - from isaaclab_physx.renderers.fabric import FabricBackendCfg # noqa: PLC0415 - - self.fabric_cfg = FabricBackendCfg(stage=self.stage, device=self.device) self._visualizers: list[BaseVisualizer] = [] self._pending_visualizers: list[BaseVisualizer] = [] self._reset_requested: bool = False @@ -276,6 +270,9 @@ def __init__(self, cfg: SimulationCfg | None = None): PhysicsEvent.PHYSICS_READY, order=5, ) + self.physics_manager.register_callback( + lambda _payload: self._render_context.close(), PhysicsEvent.STOP, order=100 + ) # Publish the context before configured consumers register their clone requirements. type(self)._instance = self @@ -470,43 +467,25 @@ def _apply_default_visualizer_cfg(self, cfg: Any) -> None: """Apply shared default visualizer settings to a backend-specific config. Only propagates fields that were **explicitly set** in ``default_visualizer_cfg`` - (i.e. differ from the base :class:`~isaaclab.visualizers.VisualizerCfg` defaults) - AND are still at the backend cfg's own class default (i.e. not already - customised by the caller). This prevents base-class defaults such as - ``streaming_view=False`` from stomping backend-specific defaults like - ``NewtonGLVisualizerCfg.streaming_view=True``. + (i.e. differ from its own class defaults) and are still at the target cfg's + class defaults. Backend-specific defaults, such as the streaming renderer, + do not transfer between visualizer types. """ - from ..visualizers.visualizer_cfg import VisualizerCfg - default_cfg = getattr(self.cfg, "default_visualizer_cfg", None) if default_cfg is None: return - # Base VisualizerCfg defaults — used to detect which fields on default_cfg - # were explicitly set by the env vs. left at the base-class default. try: - base_defaults = VisualizerCfg() + source_defaults, target_defaults = type(default_cfg)(), type(cfg)() except Exception: - base_defaults = None - # Backend-specific class defaults — used to detect which fields on cfg - # the caller has already customised beyond the class defaults. - try: - factory_defaults = type(cfg)() - except Exception: - factory_defaults = None + # Without factory defaults, explicit choices cannot be distinguished. + return for field in fields(default_cfg): if field.name in ("class_type", "visualizer_type") or not hasattr(cfg, field.name): continue default_val = getattr(default_cfg, field.name) - # Skip fields that were not explicitly set in default_cfg (still at base default). - if base_defaults is not None and hasattr(base_defaults, field.name): - if default_val == getattr(base_defaults, field.name): - continue - # Preserve explicitly customised fields on cfg. When factory_defaults is None - # (backend cfg constructor raised), skip the field rather than overwriting it - # unconditionally — we cannot tell whether the caller customised it. - if factory_defaults is None: + if default_val == getattr(source_defaults, field.name): continue - if getattr(cfg, field.name) != getattr(factory_defaults, field.name): + if getattr(cfg, field.name) != getattr(target_defaults, field.name): continue setattr(cfg, field.name, default_val) @@ -971,11 +950,12 @@ def get_setting(self, name: str) -> Any: """Get a setting value.""" return self._settings_helper.get(name) - def get_or_create_backend(self, cfg: BackendCfg) -> Any: - """Return the simulation-owned resource or renderer for a configuration. + def get_or_create_backend(self, cfg: Any) -> Any: + """Return the simulation-owned object for a construction configuration. Equal configurations of the same concrete type share a resource. Finalize configurations before registration and treat them as read-only afterward; use a new cfg for new settings. + ``BackendCfg`` declares a resource requiring ``close()``; other cfgs declare Python-owned data. Args: cfg: Construction inputs. A cache miss constructs ``instantiate(cfg)``. @@ -995,8 +975,9 @@ def get_or_create_backend(self, cfg: BackendCfg) -> Any: return resource def close_backend(self, backend: Any) -> None: - """Close one registered resource by object identity after all consumers release their bindings. + """Release one registered object by identity. + ``BackendCfg`` resources are closed; plain construction data only loses its registry reference. A failed release retains the registry entry so teardown can be retried. Args: @@ -1007,7 +988,8 @@ def close_backend(self, backend: Any) -> None: """ for index, (cfg, resource) in enumerate(self._backend_registry): if resource is backend: - resource.close() + if isinstance(cfg, BackendCfg): + resource.close() self._backend_registry.pop(index) if isinstance(cfg, RendererCfg): self._render_context._prepared_renderer_ids.discard(id(resource)) @@ -1046,8 +1028,8 @@ def run_cleanup(callback: Callable[[], Any]) -> None: instance.clone_contexts.clear() for cfg, resource in instance._backend_registry: - if not isinstance(cfg, RendererCfg): - run_cleanup(resource.close) + if isinstance(cfg, BackendCfg) and not isinstance(cfg, RendererCfg): + run_cleanup(lambda resource=resource: resource.close()) instance._backend_registry.clear() # Tear down the stage. We skip clear_stage() (prim-by-prim deletion) since diff --git a/source/isaaclab/isaaclab/visualizers/base_visualizer.py b/source/isaaclab/isaaclab/visualizers/base_visualizer.py index ea41de14e8d1..06fdc69d5711 100644 --- a/source/isaaclab/isaaclab/visualizers/base_visualizer.py +++ b/source/isaaclab/isaaclab/visualizers/base_visualizer.py @@ -16,6 +16,8 @@ from typing import TYPE_CHECKING, Any from urllib.parse import urlparse +from .. import sim as sim_utils + if TYPE_CHECKING: from collections.abc import Callable @@ -53,6 +55,10 @@ def __init__(self, cfg: VisualizerCfg): self._live_plot_env_idx: int = 0 self._live_plots_step_counter: int = 0 self._reset_requested: bool = False + # Declare the camera renderer before cloning gathers consumer requirements. + owns_camera = cfg.streaming_sensor_prim_path is None and cfg.streaming_cam_target_prim_path is not None + if cfg.streaming_view and owns_camera: + sim_utils.SimulationContext.instance().get_or_create_backend(cfg.streaming_cam_renderer_cfg) @property def visual_material_writer(self) -> Callable[[tuple[VisualMaterialBatch, ...]], Any] | None: diff --git a/source/isaaclab/isaaclab/visualizers/visualizer_cfg.py b/source/isaaclab/isaaclab/visualizers/visualizer_cfg.py index ccf2bbbd0412..429ab54854ac 100644 --- a/source/isaaclab/isaaclab/visualizers/visualizer_cfg.py +++ b/source/isaaclab/isaaclab/visualizers/visualizer_cfg.py @@ -12,6 +12,7 @@ from ..utils import configclass if TYPE_CHECKING: + from ..renderers import RendererCfg from .base_visualizer import BaseVisualizer @@ -95,13 +96,12 @@ class VisualizerCfg: streaming_cam_eye: tuple[float, float, float] = (4.0, -4.0, 3.0) """Eye offset [m] for the auto-created streaming camera relative to the target prim.""" - streaming_cam_renderer: str | None = None + streaming_cam_renderer_cfg: RendererCfg | None = None """Renderer for the auto-created streaming camera. - One of ``"newton_warp"``, ``"ovrtx"``, or ``None`` (let each backend - choose its own default). Defaults to ``None`` so each backend selects - an appropriate renderer automatically. Ignored when - :attr:`streaming_sensor_prim_path` is set. + Concrete visualizer configs declare their default renderer configuration. + Its ``class_type`` selects the implementation, including custom renderers. + Ignored when :attr:`streaming_sensor_prim_path` is set. """ # Shared settings @@ -184,9 +184,6 @@ class VisualizerCfg: tiled_cam_target_prim_path: str | None = None """Deprecated. Use :attr:`streaming_cam_target_prim_path` instead.""" - tiled_cam_renderer: str | None = None - """Deprecated. Use :attr:`streaming_cam_renderer` instead.""" - def __post_init__(self) -> None: import warnings @@ -200,7 +197,6 @@ def __post_init__(self) -> None: ("tiled_cam_prim_path", "streaming_sensor_prim_path"), ("tiled_cam_eye", "streaming_cam_eye"), ("tiled_cam_target_prim_path", "streaming_cam_target_prim_path"), - ("tiled_cam_renderer", "streaming_cam_renderer"), ] for old, new in _simple: val = getattr(self, old) diff --git a/source/isaaclab/test/markers/test_visualization_markers.py b/source/isaaclab/test/markers/test_visualization_markers.py index d79e81d6c631..01a4b9635dc5 100644 --- a/source/isaaclab/test/markers/test_visualization_markers.py +++ b/source/isaaclab/test/markers/test_visualization_markers.py @@ -6,13 +6,15 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher -from isaaclab.utils import clone +from isaaclab.utils import clone, instantiate # launch omniverse app simulation_app = AppLauncher(headless=True).app """Rest everything follows.""" +from types import SimpleNamespace + import isaaclab_visualizers.newton.newton_visualization_markers as newton_markers import isaaclab_visualizers.newton.newton_visualizer as newton_visualizer import isaaclab_visualizers.rerun.rerun_visualizer as rerun_visualizer @@ -339,16 +341,24 @@ def test_newton_marker_backend_registers_and_updates_state_without_frame_capture assert newton_backend.count == 2 -def test_newton_visualizer_step_renders_markers(monkeypatch: pytest.MonkeyPatch): - """NewtonVisualizer.step should ask active Newton marker groups to render.""" - marker_calls = [] +@pytest.mark.parametrize( + "module, cfg_type, marker_error", + [ + (newton_visualizer, NewtonGLVisualizerCfg, False), + (viser_visualizer, ViserVisualizerCfg, True), + (rerun_visualizer, RerunVisualizerCfg, True), + ], + ids=["newton", "viser", "rerun"], +) +def test_visualizer_step_renders_markers_and_closes_frame(monkeypatch, caplog, module, cfg_type, marker_error): + """Markers use the current native state; overlay failures still close the frame.""" + calls, marker_calls = [], [] + state = SimpleNamespace(body_q=None) + backend = SimpleNamespace(model=SimpleNamespace(num_envs=4, body_count=0), state_0=state, geometry_offsets={}) - class _FakeViewer: + class Viewer: _update_frequency = 1 - - def __init__(self): - self.calls = [] - self.show_contacts = False + show_contacts = False def is_paused(self): return False @@ -357,187 +367,49 @@ def is_running(self): return True def begin_frame(self, sim_time): - self.calls.append(("begin_frame", sim_time)) + calls.append(("begin_frame", sim_time)) - def log_state(self, state): - self.calls.append(("log_state", state)) + def log_state(self, value): + calls.append(("log_state", value)) def log_arrows(self, name, starts, ends, colors): pass def end_frame(self): - self.calls.append(("end_frame",)) + calls.append(("end_frame",)) - class _FakeNewtonManager: - @staticmethod - def get_state(scene_data_provider=None): - assert scene_data_provider == "provider" - return {"state": "ok"} - - @staticmethod - def get_num_envs() -> int: - return 4 - - @staticmethod - def get_contacts(): - return None - - def _fake_render_markers(viewer, visible_env_ids, num_envs): + def render_markers(viewer, visible_env_ids, num_envs): marker_calls.append((viewer, visible_env_ids, num_envs)) + if marker_error: + raise RuntimeError("marker overlay failed") - import isaaclab_newton.physics as newton_physics - - monkeypatch.setattr(newton_physics, "NewtonManager", _FakeNewtonManager) - # newton_visualizer now imports NewtonManager at module level, so patch the - # module-level name directly (not just the origin module attribute). - monkeypatch.setattr(newton_visualizer, "NewtonManager", _FakeNewtonManager) - monkeypatch.setattr(newton_visualizer, "render_newton_visualization_markers", _fake_render_markers) - - viewer = _FakeViewer() - visualizer = newton_visualizer.NewtonVisualizer(NewtonGLVisualizerCfg(enable_markers=True)) + provider = SimpleNamespace( + get_transforms=lambda output, **kwargs: False, + get_camera_transforms=lambda: {}, + get_contact_sensors=lambda: {}, + ) + monkeypatch.setattr(module, "render_newton_visualization_markers", render_markers) + monkeypatch.setattr(newton_visualizer.NewtonManager, "get_contacts", lambda: None) + cfg = cfg_type() + visualizer = instantiate(cfg) + visualizer.backend = backend visualizer._is_initialized = True - visualizer._is_closed = False - visualizer._viewer = viewer - visualizer._scene_data_provider = "provider" + visualizer._viewer = viewer = Viewer() + visualizer._scene_data_provider = provider + visualizer._transform_mapping = None visualizer._resolved_visible_env_ids = [1, 3] - visualizer.step(0.25) - - assert viewer.calls == [("begin_frame", pytest.approx(0.25)), ("log_state", {"state": "ok"}), ("end_frame",)] - assert marker_calls == [(viewer, [1, 3], 4)] - - -def test_viser_visualizer_marker_render_failure_does_not_interrupt_state_updates( - monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture -): - """Viser marker failures should be logged without dropping body state frames.""" - marker_calls = [] - - class _FakeViewer: - def __init__(self): - self.calls = [] - - def begin_frame(self, sim_time: float) -> None: - self.calls.append(("begin_frame", sim_time)) - - def log_state(self, state) -> None: - self.calls.append(("log_state", state)) - - def end_frame(self) -> None: - self.calls.append(("end_frame",)) - - class _FakeProvider: - num_envs = 4 - usd_stage = None - - def get_camera_transforms(self): - return {} - - class _FakeNewtonManager: - @staticmethod - def get_model(): - return "dummy-model" - - @staticmethod - def get_state(scene_data_provider=None): - assert scene_data_provider is provider - return {"state": "ok"} - - @staticmethod - def get_num_envs() -> int: - return provider.num_envs - - def _fake_create_viewer(self, record_to_viser: str | None, metadata: dict | None = None): - self._viewer = viewer - - def _raise_marker_render(*args, **kwargs): - marker_calls.append((args, kwargs)) - raise RuntimeError("marker overlay failed") - - import isaaclab_newton.physics as newton_physics - - provider = _FakeProvider() - viewer = _FakeViewer() - monkeypatch.setattr(newton_physics, "NewtonManager", _FakeNewtonManager) - monkeypatch.setattr(viser_visualizer.ViserVisualizer, "_create_viewer", _fake_create_viewer) - monkeypatch.setattr(viser_visualizer, "render_newton_visualization_markers", _raise_marker_render) - - visualizer = viser_visualizer.ViserVisualizer(ViserVisualizerCfg()) - visualizer.initialize(provider) - with caplog.at_level("WARNING"): - visualizer.step(0.25) - - assert marker_calls - assert viewer.calls == [("begin_frame", pytest.approx(0.25)), ("log_state", {"state": "ok"}), ("end_frame",)] - assert "Marker rendering failed; continuing body updates" in caplog.text - - -def test_rerun_visualizer_marker_failure_still_ends_frame(monkeypatch: pytest.MonkeyPatch): - """Rerun should close the frame even if marker rendering raises.""" - captured = {} - - class _FakeViewer: - def __init__(self): - self.calls = [] - - def is_paused(self): - return False - - def begin_frame(self, sim_time): - self.calls.append(("begin_frame", sim_time)) - - def log_state(self, state): - self.calls.append(("log_state", state)) + if module is rerun_visualizer: + with pytest.raises(RuntimeError, match="marker overlay failed"): + visualizer.step(0.25) + else: + visualizer.step(0.25) - def end_frame(self): - self.calls.append(("end_frame",)) - - class _FakeProvider: - def get_metadata(self) -> dict: - return {"num_envs": 4} - - def get_newton_state(self): - return {"ok": True} - - def get_camera_transforms(self): - return {} - - class _FakeNewtonManager: - @staticmethod - def get_model(): - return "dummy-model" - - @staticmethod - def get_state(scene_data_provider=None): - captured["state_provider"] = scene_data_provider - return {"ok": True} - - @staticmethod - def get_num_envs() -> int: - return 4 - - def _raise_marker_render(*args, **kwargs): - raise RuntimeError("marker render failed") - - import isaaclab_newton.physics as newton_physics - - monkeypatch.setattr(newton_physics, "NewtonManager", _FakeNewtonManager) - monkeypatch.setattr(rerun_visualizer, "render_newton_visualization_markers", _raise_marker_render) - - visualizer = rerun_visualizer.RerunVisualizer(RerunVisualizerCfg()) - viewer = _FakeViewer() - visualizer._is_initialized = True - visualizer._is_closed = False - visualizer._viewer = viewer - visualizer._scene_data_provider = _FakeProvider() - visualizer._resolved_visible_env_ids = None - - with pytest.raises(RuntimeError, match="marker render failed"): - visualizer.step(0.25) - - assert captured["state_provider"] is visualizer._scene_data_provider - assert [call[0] for call in viewer.calls] == ["begin_frame", "log_state", "end_frame"] + assert calls == [("begin_frame", pytest.approx(0.25)), ("log_state", state), ("end_frame",)] + assert marker_calls == [(viewer, [1, 3], 4)] + if module is viser_visualizer: + assert "Marker rendering failed; continuing body updates" in caplog.text def test_newton_marker_mesh_registration_is_per_viewer(monkeypatch: pytest.MonkeyPatch): diff --git a/source/isaaclab/test/renderers/test_simulation_render_context.py b/source/isaaclab/test/renderers/test_simulation_render_context.py index 14946d089610..daa1accc268a 100644 --- a/source/isaaclab/test/renderers/test_simulation_render_context.py +++ b/source/isaaclab/test/renderers/test_simulation_render_context.py @@ -64,14 +64,14 @@ def test_renderer_registry_sharing_and_early_clone_requirements(sim): assert sim.get_or_create_backend(replace(cfg, semantic_filter="class:robot")) is not renderer assert sim.get_or_create_backend(NewtonWarpRendererCfg(class_type=_renderer)) is not renderer - sim.get_or_create_backend(BackendCfg(class_type=lambda cfg: object())) + sim.get_or_create_backend(BackendCfg(class_type=lambda cfg: SimpleNamespace(close=Mock()))) assert sim.render_context.renderer_types == ("isaac_rtx", "isaac_rtx", "newton_warp") def test_renderer_initializes_once_before_or_after_physics_ready(sim): cfg = RendererCfg(class_type=_renderer) first = sim.get_or_create_backend(cfg) - sim.get_or_create_backend(BackendCfg(class_type=lambda cfg: object())) + sim.get_or_create_backend(BackendCfg(class_type=lambda cfg: SimpleNamespace(close=Mock()))) sim.render_context.ensure_initialize() sim.render_context.ensure_initialize() first.initialize.assert_called_once_with() diff --git a/source/isaaclab/test/scene_data/test_scene_data_transforms.py b/source/isaaclab/test/scene_data/test_scene_data_transforms.py index b0179b10346b..5ad2bef21f75 100644 --- a/source/isaaclab/test/scene_data/test_scene_data_transforms.py +++ b/source/isaaclab/test/scene_data/test_scene_data_transforms.py @@ -47,9 +47,11 @@ def test_get_transforms_matches_backend_device_when_warp_default_is_cuda(): ) with wp.ScopedDevice("cuda:0"): + assert provider.create_mapping(["/World/a", "/World/b", "/World/c"]) is None mapping = provider.create_mapping(["/World/c", "/World/a", "/World/b"]) assert mapping is not None assert str(mapping.device) == "cpu" + assert provider.create_mapping(["/World/c", "/World/a", "/World/b"]) is mapping output = SceneDataFormat.Vec3_Quat() assert provider.get_transforms(output, mapping=mapping, allow_passthrough=False) @@ -141,18 +143,37 @@ def test_owned_transform_buffers_are_written_directly_and_do_not_alias_cache(for np.testing.assert_array_equal(array.numpy(), getattr(shared, name).numpy()) -def test_mapping_preserves_unmapped_destination_slots(): +def test_mapping_binds_unique_paths_and_preserves_unmapped_slots(): data = SceneDataFormat.Transform() data.transforms = wp.array([[1, 2, 3, 0, 0, 0, 1], [4, 5, 6, 0, 0, 0, 1]], dtype=wp.transformf, device="cpu") provider = SceneDataProvider( _Backend(transforms=data, transforms_timestamp=0, transform_count=2, transform_paths=["/a", "/b"]) ) - mapping = provider.create_mapping(["/a", "/b", None]) - output = SceneDataFormat.Transform() - output.transforms = wp.zeros(3, dtype=wp.transformf, device="cpu") - assert provider.get_transforms(output, mapping, allow_passthrough=False, count=3) - np.testing.assert_array_equal(output.transforms.numpy()[:2], data.transforms.numpy()) - np.testing.assert_array_equal(output.transforms.numpy()[2], np.zeros(7)) + expected = np.concatenate((data.transforms.numpy(), np.zeros((1, 7)))) + for paths, indices in ( + (["/a", "/b"], [0, 1]), + (["/b", "/a"], [1, 0]), + (["/b"], [1]), + (["/a", None, None, "/b"], [0, -1, -1, 1]), + ): + mapping = provider.create_mapping(paths) + assert provider.create_mapping(paths.copy()) is mapping + assert (mapping is None) == (paths == provider.backend.transform_paths) + output = SceneDataFormat.Transform() + output.transforms = wp.zeros(len(paths), dtype=wp.transformf, device="cpu") + assert provider.get_transforms(output, mapping, allow_passthrough=False, count=len(paths)) + np.testing.assert_array_equal(output.transforms.numpy(), expected[indices]) + with pytest.raises(KeyError, match="/missing"): + provider.create_mapping(["/a", "/missing"]) + with pytest.raises(ValueError, match="unique"): + provider.create_mapping(["/a", "/a"]) + provider.backend.transform_paths = ["/a", "/a"] + with pytest.raises(ValueError, match="unique"): + provider.create_mapping(["/a", "/a"]) + provider.backend.transform_paths = [] + assert provider.create_mapping([]) is None + with pytest.raises(KeyError, match="/missing"): + provider.create_mapping(["/missing"]) # Scale is applied by one shared helper, so it rotates across the formats instead of doubling them. diff --git a/source/isaaclab/test/sim/test_newton_manager_visualization_state.py b/source/isaaclab/test/sim/test_newton_manager_visualization_state.py index b37d07a5b5f4..af0910e10a2a 100644 --- a/source/isaaclab/test/sim/test_newton_manager_visualization_state.py +++ b/source/isaaclab/test/sim/test_newton_manager_visualization_state.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Clone-owned Newton visualization models and SDP state synchronization under foreign physics.""" +"""Clone-built, registry-owned Newton resources and native SDP publication.""" from __future__ import annotations @@ -16,6 +16,7 @@ from isaaclab.assets import AssetBaseCfg from isaaclab.cloner import make_clone_plan from isaaclab.sim import SpawnerCfg +from isaaclab.utils import replace pytestmark = pytest.mark.integration @@ -51,95 +52,72 @@ def _make_surface_cloth_stage(path: str = "/World/envs/env_0/Cloth"): return stage -@pytest.mark.parametrize(("body_count", "particle_count"), [(0, 0), (3, 0), (0, 4)]) -def test_visualization_model_is_built_during_clone_and_allocated_on_physics_ready( - monkeypatch, body_count, particle_count -): - """Cloning owns parsing; READY owns native allocation; getters never discover or allocate.""" - import warp as wp +def test_clone_inputs_create_one_registry_resource_until_closed(monkeypatch): + """Consumers acquire a completed resource, including late consumers and replacement after close.""" from isaaclab_newton.cloner import NewtonReplicateContext - from isaaclab_newton.cloner import replicate as replicate_module - from isaaclab_newton.physics import NewtonManager + from isaaclab_newton.physics import NewtonBuilderCfg, NewtonManager + from isaaclab_newton.renderers import NewtonWarpRendererCfg from newton import ModelBuilder - from pxr import Usd, UsdGeom + from pxr import Usd, UsdGeom, UsdPhysics - from isaaclab.physics import PhysicsEvent, PhysicsManager - from isaaclab.scene_data import SceneDataFormat, SceneDataProvider + from isaaclab.physics import PhysicsCfg, PhysicsManager + from isaaclab.renderers import RenderContext from isaaclab.sim import SimulationContext - class ForeignPhysicsManager(PhysicsManager): + class Manager(PhysicsManager): _callbacks = {} _reset_newton_manager_state() monkeypatch.setattr(PhysicsManager, "_device", "cpu") - asset = AssetBaseCfg(prim_path="/Scene/Copy_[^/]+", spawn=SpawnerCfg(spawn_path="/Scene/Source")) + asset = AssetBaseCfg(prim_path="/Scene/Copy_[^/]+/Cube", spawn=SpawnerCfg(spawn_path="/Scene/Source/Cube")) plan = make_clone_plan((asset,), ((0,),), 2, env_template="/Scene/Copy_{}") sim = object.__new__(SimulationContext) - sim.cfg = SimpleNamespace(physics=object(), device="cpu") + sim.cfg = SimpleNamespace(physics=PhysicsCfg(), device="cpu") + sim.physics_manager = Manager sim.stage = Usd.Stage.CreateInMemory() - UsdGeom.Xform.Define(sim.stage, "/Scene/Source") - sim.physics_manager = ForeignPhysicsManager + UsdPhysics.RigidBodyAPI.Apply(UsdGeom.Cube.Define(sim.stage, "/Scene/Source/Cube").GetPrim()) sim._backend_registry = [] - body_paths = [f"/Scene/Body_{index}" for index in range(body_count)] - transforms = SceneDataFormat.Transform() - transforms.transforms = wp.zeros(body_count, dtype=wp.transformf, device="cpu") - sim._scene_data_provider = SceneDataProvider( - SimpleNamespace( - transforms=transforms, - get_transforms=lambda _format: transforms, - transforms_timestamp=0, - transform_paths=body_paths, - transform_count=body_count, - ) - ) - monkeypatch.setattr(SimulationContext, "_instance", sim) - - finalize = Mock( - side_effect=lambda device: SimpleNamespace( - body_count=body_count, - body_label=body_paths, - particle_count=particle_count, - world_count=2, - state=lambda: SimpleNamespace( - body_q=wp.empty(body_count, dtype=wp.transformf, device="cpu") if body_count else None, particle_q=None - ), - ) - ) + sim._render_context = RenderContext(sim._backend_registry) + sim.requires_usd_stage = sim.requires_newton_model = False + monkeypatch.setattr(SimulationContext, "instance", lambda: sim) + renderers = [sim.get_or_create_backend(NewtonWarpRendererCfg(enable_shadows=flag)) for flag in (False, True)] + builder_cfg = NewtonBuilderCfg(physics_cfg=sim.cfg.physics) + declared_builder = sim.get_or_create_backend(builder_cfg) + assert sim.get_or_create_backend(replace(builder_cfg)) is declared_builder + finalized = [] + original = ModelBuilder.finalize + + def finalize(builder, *, device): + finalized.append(builder) + return original(builder, device=device) + monkeypatch.setattr(ModelBuilder, "finalize", finalize) - build = Mock(wraps=replicate_module._replicate_newton) - monkeypatch.setattr(replicate_module, "_replicate_newton", build) context = NewtonReplicateContext(sim) - assert NewtonManager.get_model() is None - assert NewtonManager.get_state() is None - build.assert_not_called() - builder, _, _ = context.replicate(plan, (0,)) - assert isinstance(builder, ModelBuilder) - build.assert_called_once() - finalize.assert_not_called() - assert not sim._backend_registry - - ForeignPhysicsManager.dispatch_event(PhysicsEvent.PHYSICS_READY) - if body_count: - assert NewtonManager.get_state_0().body_q is transforms.transforms - first_model = NewtonManager.get_model() - first_state = NewtonManager.get_state() - ForeignPhysicsManager.dispatch_event(PhysicsEvent.PHYSICS_READY) - assert NewtonManager.get_model() is first_model - assert NewtonManager.get_state() is first_state - assert (first_model.body_count, first_model.particle_count) == (body_count, particle_count) - assert first_model.num_envs == NewtonManager.get_num_envs() == 2 - finalize.assert_called_once_with(device="cpu") - - ForeignPhysicsManager.dispatch_event(PhysicsEvent.STOP) - assert NewtonManager.get_model() is None and NewtonManager.get_state() is None - assert not sim._backend_registry - ForeignPhysicsManager.dispatch_event(PhysicsEvent.PHYSICS_READY) - assert NewtonManager.get_model() is not first_model - assert finalize.call_count == 2 - assert build.call_count == 1 - ForeignPhysicsManager.dispatch_event(PhysicsEvent.STOP) + assert builder is declared_builder + assert finalized == [] + assert len(sim._backend_registry) == 3 + assert not hasattr(sim, "newton_cfg") and not hasattr(sim, "fabric_cfg") + assert renderers[0].newton_cfg == renderers[1].newton_cfg + first = sim.get_or_create_backend(renderers[0].newton_cfg) + assert sim.get_or_create_backend(renderers[1].newton_cfg) is first + assert first.model.world_count == len(plan.topology.world_prototype_layout) + assert first.model.body_count == 2 + assert first.model.body_label == ["/Scene/Copy_0/Cube", "/Scene/Copy_1/Cube"] + late = sim.get_or_create_backend(NewtonWarpRendererCfg(max_distance=12)) + assert sim.get_or_create_backend(late.newton_cfg) is first + assert finalized == [builder] + assert NewtonManager.backend is None + sim.close_backend(first) + assert len(sim._backend_registry) == 4 + assert first.model is first.state_0 is None + assert sim.get_or_create_backend(builder_cfg) is builder + second = sim.get_or_create_backend(late.newton_cfg) + assert second is not first + assert all(sim.get_or_create_backend(renderer.newton_cfg) is second for renderer in (*renderers, late)) + assert finalized == [builder, builder] + sim.close_backend(second) @pytest.mark.parametrize("invalidate", ["invalidate_body_state", "invalidate_fk"]) @@ -174,7 +152,7 @@ def test_native_publication_reuses_clean_fk_and_refreshes_writes_and_swaps(monke matrices = output.matrices NewtonManager.pre_render() NewtonManager._eval_fk.assert_not_called() - NewtonManager.get_state(provider) + assert provider.get_transforms(SceneDataFormat.Transform()) assert provider.get_transforms(output) assert output.matrices is matrices assert wp.launch.call_count == 1 @@ -203,141 +181,17 @@ def test_native_publication_reuses_clean_fk_and_refreshes_writes_and_swaps(monke np.testing.assert_allclose(output.matrices.numpy()[0, :3, 3], [3, 2, 1]) -def test_resolve_scene_data_body_paths_uses_joint_body_targets(): - """PhysX visualization sync maps Newton joint labels to the actual body prim path.""" - pytest.importorskip("pxr") - from isaaclab_newton.physics import NewtonManager - - from pxr import Usd, UsdGeom, UsdPhysics - - stage = Usd.Stage.CreateInMemory() - body_prim = UsdGeom.Xform.Define(stage, "/World/envs/env_0/Robot/robot0_forearm").GetPrim() - UsdPhysics.RigidBodyAPI.Apply(body_prim) - joint = UsdPhysics.FixedJoint.Define(stage, "/World/envs/env_0/Robot/joints/robot0_forearm") - joint.GetBody1Rel().SetTargets([body_prim.GetPath()]) - - body_paths = ["/World/envs/env_0/Robot/joints/robot0_forearm"] - resolved_paths = NewtonManager._resolve_scene_data_body_paths(body_paths, stage) - - assert resolved_paths == ["/World/envs/env_0/Robot/robot0_forearm"] - - -@pytest.mark.parametrize("layout", ["identity", "reordered", "missing", "duplicate"]) -def test_update_visualization_state_shares_sdp_transforms(monkeypatch, layout): - """Native and reordered layouts bind shared output once and refresh only on publication.""" - import numpy as np - import warp as wp - from isaaclab_newton.physics import NewtonManager - - from isaaclab.scene_data import SceneDataFormat, SceneDataProvider - - _reset_newton_manager_state() - monkeypatch.setattr(NewtonManager, "_backend_is_newton", classmethod(lambda cls, provider=None: False)) - - body_paths = ["/World/envs/env_0/Object", "/World/envs/env_1/Object"] - render_paths = { - "identity": body_paths, - "reordered": body_paths[::-1], - "missing": [body_paths[0], "/World/Missing"], - "duplicate": [body_paths[0], body_paths[0]], - }[layout] - source_transforms = wp.array( - [ - [1.0, 2.0, 3.0, 0.0, 0.0, 0.0, 1.0], - [4.0, 5.0, 6.0, 0.0, 0.0, 0.0, 1.0], - ], - dtype=wp.transformf, - device="cpu", - ) - source_data = SceneDataFormat.Transform() - source_data.transforms = source_transforms - provider = SceneDataProvider( - SimpleNamespace( - transforms=source_data, - get_transforms=lambda _format: source_data, - transforms_timestamp=0, - transform_paths=body_paths, - transform_count=len(body_paths), - ) - ) - monkeypatch.setattr(SceneDataProvider, "usd_stage", property(lambda self: None)) - - destination = wp.zeros(len(body_paths), dtype=wp.transformf, device="cpu") - monkeypatch.setattr( - NewtonManager, - "backend", - SimpleNamespace( - model=SimpleNamespace(body_label=render_paths, body_count=len(body_paths)), - state_0=SimpleNamespace(body_q=destination, particle_q=None), - ), - ) - - if layout in ("missing", "duplicate"): - with pytest.raises(ValueError, match="one unique SDP transform path"): - NewtonManager.update_visualization_state(provider) - return - - remapped = layout == "reordered" - NewtonManager.update_visualization_state(provider) - shared = NewtonManager.get_state(provider).body_q - assert (shared is source_transforms) is not remapped - np.testing.assert_allclose(shared.numpy(), source_transforms.numpy()[:: -1 if remapped else 1]) - assert NewtonManager.get_state(provider).body_q is shared - - source_data.transforms = wp.array(source_transforms.numpy() + 1.0, dtype=wp.transformf, device="cpu") - provider.backend.transforms_timestamp += 1 - NewtonManager.update_visualization_state(provider) - np.testing.assert_allclose( - NewtonManager.backend.state_0.body_q.numpy(), source_data.transforms.numpy()[:: -1 if remapped else 1] - ) - - -def test_update_visualization_state_writes_final_geometry_once_per_timestamp(monkeypatch): - """SDP writes directly into native render slots, preserving gaps and the destination allocation.""" - import warp as wp - from isaaclab_newton.physics import NewtonManager - - from isaaclab.scene_data import SceneDataFormat, SceneDataProvider - - _reset_newton_manager_state() - data = SceneDataFormat.Points() - data.points = wp.array(np.arange(18, dtype=np.float32).reshape(6, 3), dtype=wp.vec3f, device="cpu") - ranges = {"/Cells/Cloth/mesh": (1, 2), "/Shared/Particles": (4, 1)} - backend = SimpleNamespace(geometry_timestamp=0, get_geometry_batches=lambda _format: [(data, ranges)]) - provider = SceneDataProvider(backend) - particle_q = wp.zeros(6, dtype=wp.vec3f, device="cpu") - monkeypatch.setattr( - NewtonManager, - "backend", - SimpleNamespace(model=SimpleNamespace(), state_0=SimpleNamespace(body_q=None, particle_q=particle_q)), - ) - monkeypatch.setattr(NewtonManager, "_geometry_offsets", {"/Shared/Particles": 1, "/Cells/Cloth/mesh": 3}) - monkeypatch.setattr(NewtonManager, "_mark_sensor_state_dirty", Mock()) - monkeypatch.setattr(wp, "launch", Mock(wraps=wp.launch)) - - for timestamp in (0, 1): - if timestamp: - data.points = wp.array(data.points.numpy() + 10.0, dtype=wp.vec3f, device="cpu") - backend.geometry_timestamp += 1 - NewtonManager.update_visualization_state(provider) - NewtonManager.update_visualization_state(provider) - expected = np.zeros((6, 3), dtype=np.float32) - expected[1] = data.points.numpy()[4] - expected[3:5] = data.points.numpy()[1:3] - assert NewtonManager.backend.state_0.particle_q is particle_q - np.testing.assert_array_equal(particle_q.numpy(), expected) - assert wp.launch.call_count == timestamp + 1 - assert NewtonManager._mark_sensor_state_dirty.call_count == timestamp + 1 - - @pytest.mark.parametrize("global_path", ["/World/Assets/Cloth", "/World/Assets"]) def test_clone_visualization_builder_imports_only_declared_global_deformables(monkeypatch, global_path): """Global ancestors do not route excluded clone sources into the shadow model.""" from isaaclab_newton.cloner import NewtonReplicateContext + from isaaclab_newton.physics import NewtonBackendCfg from newton import ModelBuilder from pxr import Sdf, UsdGeom + from isaaclab.sim import SimulationContext + stage = _make_surface_cloth_stage(path="/World/Assets/Cloth") Sdf.CopySpec(stage.GetRootLayer(), "/World/Assets/Cloth", stage.GetRootLayer(), "/World/UnplannedCloth") sources = ("/World/Assets/Selected", "/World/Assets/Excluded") @@ -350,12 +204,11 @@ def test_clone_visualization_builder_imports_only_declared_global_deformables(mo ) assets += (AssetBaseCfg(prim_path=global_path),) plan = make_clone_plan(assets, ((0, 1),), 2, shared_assets=(2,), env_template="/Copies/env_{}") - sim = SimpleNamespace( - cfg=SimpleNamespace(physics=object()), - device="cpu", - stage=stage, - physics_manager=SimpleNamespace(register_callback=Mock()), - ) + sim = object.__new__(SimulationContext) + sim.cfg = SimpleNamespace(physics=object(), device="cpu") + sim.physics_manager = SimpleNamespace(get_device=lambda: "cpu") + sim.stage, sim._backend_registry = stage, [] + monkeypatch.setattr(SimulationContext, "instance", lambda: sim) usd_imports = [] add_usd = ModelBuilder.add_usd @@ -365,11 +218,13 @@ def import_usd(builder, *args, **kwargs): monkeypatch.setattr(ModelBuilder, "add_usd", import_usd) - builder, _, _ = NewtonReplicateContext(sim).replicate(plan, (0, 2)) - callback = sim.physics_manager.register_callback.call_args.args[0] - geometry = callback.args[1] + context = NewtonReplicateContext(sim) + builder, _, _ = context.replicate(plan, (0, 2)) + backend = sim.get_or_create_backend(NewtonBackendCfg(physics_cfg=sim.cfg.physics, device=sim.device)) + geometry = backend.geometry_offsets assert sorted(kwargs["root_path"] for kwargs in usd_imports) == sorted([global_path, sources[0]]) assert set(geometry) == {"/World/Assets/Cloth", "/Copies/env_0/Selected/Cloth", "/Copies/env_1/Selected/Cloth"} assert sorted(geometry.values()) == [0, 3, 6] assert builder.particle_count == 9 + backend.close() diff --git a/source/isaaclab/test/sim/test_physics_manager_lifecycle.py b/source/isaaclab/test/sim/test_physics_manager_lifecycle.py index c692baa1bc77..160e7f24a2be 100644 --- a/source/isaaclab/test/sim/test_physics_manager_lifecycle.py +++ b/source/isaaclab/test/sim/test_physics_manager_lifecycle.py @@ -28,8 +28,12 @@ def test_backend_registry_identity_and_lifecycle(): sim_package = Path(__file__).parents[2] / "isaaclab" / "sim" assert not (sim_package / "service_locator.py").exists() assert not hasattr(SimulationContext, "services") + assert not hasattr(SimulationContext, "register_backend") + assert not hasattr(SimulationContext, "get_backend") + assert not hasattr(BackendCfg, "close") assert issubclass(RendererCfg, BackendCfg) assert not hasattr(RenderContext, "get_renderer") + assert "BACKEND_CFG_READY" not in PhysicsEvent.__members__ assert "_renderer_entries" not in RenderContext.__slots__ assert tuple(inspect.signature(SimulationContext.get_or_create_backend).parameters) == ("self", "cfg") assert all( @@ -93,6 +97,25 @@ class OtherBackend(Backend): context.close_backend(second) assert second.close.call_count == 2 + # Backend declarations require close(); plain construction cfgs only share Python-owned data. + data_cfg = Cfg(class_type=lambda cfg: list(cfg.values), values=[3]) + data = context.get_or_create_backend(data_cfg) + with pytest.raises(AttributeError, match="close"): + context.close_backend(data) + assert context.get_or_create_backend(data_cfg) is data + + @dataclass + class DataCfg: + class_type: object + values: list[int] + + data_cfg = DataCfg(class_type=data_cfg.class_type, values=data_cfg.values) + data = context.get_or_create_backend(data_cfg) + assert context.get_or_create_backend(replace(data_cfg)) is data + context.close_backend(data) + assert context.get_or_create_backend(data_cfg) == data + assert context.get_or_create_backend(data_cfg) is not data + def test_physics_manager_close_only_clears_active_manager_binding(monkeypatch): """Only the active physics manager can clear shared SimulationContext state.""" @@ -277,6 +300,7 @@ def close(self): context.get_or_create_backend( cfg_type(class_type=lambda cfg, name=name, error=error: Resource(name, error)) ) + context.get_or_create_backend(SimpleNamespace(class_type=lambda cfg: Resource("data"))) monkeypatch.setattr(SimulationContext, "_instance", context) monkeypatch.setattr(context_module.stage_utils, "close_stage", lambda: events.append("stage")) monkeypatch.setattr(context_module, "clear_resolve_matching_names_cache", lambda: events.append("cache")) diff --git a/source/isaaclab/test/sim/test_simulation_context_visualizers.py b/source/isaaclab/test/sim/test_simulation_context_visualizers.py index 8c04c6c67411..76d3673acf29 100644 --- a/source/isaaclab/test/sim/test_simulation_context_visualizers.py +++ b/source/isaaclab/test/sim/test_simulation_context_visualizers.py @@ -16,6 +16,9 @@ import isaaclab_visualizers.rerun.rerun_visualizer as rerun_visualizer import isaaclab_visualizers.viser.viser_visualizer as viser_visualizer import pytest +import warp as wp +from isaaclab_newton.physics import NewtonBackendCfg +from isaaclab_newton.renderers import NewtonWarpRendererCfg from isaaclab_visualizers.kit.kit_visualizer_cfg import KitVisualizerCfg from isaaclab_visualizers.newton.newton_visualizer_cfg import ( NewtonGLVisualizerCfg, @@ -250,10 +253,9 @@ def test_newton_visualizer_is_initialized_and_rebound_before_capture(): reset_calls = [] camera_calls = [] - class _Cfg: - cloning_contexts = () - + class _Cfg(VisualizerCfg): def __init__(self, visualizer_type, enable_picking=False): + super().__init__() self.visualizer_type = visualizer_type self.enable_picking = enable_picking self.headless = False @@ -337,10 +339,34 @@ def usd_stage(self): def get_camera_transforms(self): return {} + def create_mapping(self, paths): + return None + + def get_transforms(self, output, **kwargs): + output.transforms = wp.zeros(1, dtype=wp.transform, device="cpu") + return True + + +@pytest.fixture +def web_backend(monkeypatch): + model = SimpleNamespace(body_label=["/Object"], body_count=1, num_envs=4) + backend = SimpleNamespace(model=model, state_0=SimpleNamespace(body_q=None), geometry_offsets={}) + sim = SimpleNamespace( + cfg=SimpleNamespace(physics=object(), device="cpu"), + device="cpu", + get_or_create_backend=Mock(return_value=backend), + ) + monkeypatch.setattr(SimulationContext, "instance", lambda: sim) + return sim + class _DummyViserViewer: def __init__(self): self.calls = [] + self._server = object() + self.set_model = Mock() + self.set_visible_worlds = Mock() + self.set_world_offsets = Mock() def begin_frame(self, sim_time: float) -> None: self.calls.append(("begin_frame", sim_time)) @@ -355,7 +381,7 @@ def is_running(self) -> bool: return True -def test_viser_visualizer_initialize_and_step_uses_newton_manager_state(monkeypatch: pytest.MonkeyPatch): +def test_viser_visualizer_reads_sdp_and_rebinds_native_resource(monkeypatch, web_backend): provider = _DummyViserSceneDataProvider() viewer = _DummyViserViewer() @@ -365,39 +391,32 @@ def _fake_create_viewer(self, record_to_viser: str | None, metadata: dict | None self._viewer = viewer monkeypatch.setattr(viser_visualizer.ViserVisualizer, "_create_viewer", _fake_create_viewer) + monkeypatch.setattr(viser_visualizer.ViserVisualizer, "_setup_isaaclab_sidebar", lambda self, server: None) - state_calls: list[object] = [] - - class _FakeNewtonManager: - @staticmethod - def get_model(): - return "dummy-model" - - @staticmethod - def get_state(scene_data_provider=None): - state_calls.append(scene_data_provider) - return {"state_call": len(state_calls)} - - @staticmethod - def get_num_envs() -> int: - return 1 - - import isaaclab_newton.physics as _np_mod - - monkeypatch.setattr(_np_mod, "NewtonManager", _FakeNewtonManager) - + cfg = NewtonBackendCfg(physics_cfg=web_backend.cfg.physics, device=web_backend.device) visualizer = viser_visualizer.ViserVisualizer(ViserVisualizerCfg()) visualizer.initialize(cast(Any, provider)) visualizer.step(0.25) assert visualizer.is_initialized - assert state_calls == [provider, provider] + backend = web_backend.get_or_create_backend.return_value + assert visualizer.backend is backend + assert backend.state_0.body_q.shape == (1,) assert visualizer._sim_time == pytest.approx(0.25) assert viewer.calls[0][0] == "begin_frame" assert viewer.calls[0][1] == pytest.approx(0.25) - # log_state passes NewtonManager.get_state(provider) through as-is; no env_ids merged in. - assert viewer.calls[1] == ("log_state", {"state_call": 2}) + assert viewer.calls[1] == ("log_state", backend.state_0) assert viewer.calls[2] == ("end_frame",) + replacement = SimpleNamespace(model=backend.model, state_0=SimpleNamespace(body_q=None), geometry_offsets={}) + web_backend.get_or_create_backend.return_value = replacement + visualizer.reset(soft=True) + assert visualizer.backend is backend + visualizer.reset() + visualizer.reset() + viewer.set_model.assert_called_once_with(replacement.model) + web_backend.get_or_create_backend.assert_called_with(cfg) + visualizer.step(0.25) + assert viewer.calls[-2] == ("log_state", replacement.state_0) @pytest.mark.parametrize( @@ -464,7 +483,7 @@ def share_url(self) -> str | None: randomly_sample_visible_envs=False, ) visualizer = viser_visualizer.ViserVisualizer(cfg) - visualizer._model = "dummy-model" + visualizer.backend = SimpleNamespace(model="dummy-model") visualizer._env_ids = None # normally set by initialize() -> _compute_visualized_env_ids() visualizer._create_viewer(record_to_viser="record.viser", metadata={"num_envs": 8}) @@ -484,6 +503,7 @@ def share_url(self) -> str | None: ) def test_rerun_visualizer_initialize_applies_visible_worlds_and_world_offsets( monkeypatch: pytest.MonkeyPatch, + web_backend, cfg_max_visible_envs: int | None, expected_visible: list[int] | None, ): @@ -527,36 +547,6 @@ def set_world_offsets(self, spacing) -> None: def close(self) -> None: captured["closed"] = True - class _DummyRerunSceneDataProvider: - @property - def num_envs(self) -> int: - return 4 - - @property - def usd_stage(self): - return None - - def get_camera_transforms(self): - return {} - - class _FakeNewtonManager: - @staticmethod - def get_model(): - return "dummy-model" - - @staticmethod - def get_state(scene_data_provider=None): - captured["state_provider"] = scene_data_provider - return {"ok": True} - - @staticmethod - def get_num_envs() -> int: - return 1 - - import isaaclab_newton.physics as _np_mod - - monkeypatch.setattr(_np_mod, "NewtonManager", _FakeNewtonManager) - monkeypatch.setattr(rerun_visualizer, "NewtonViewerRerun", _FakeNewtonViewerRerun) monkeypatch.setattr( rerun_visualizer, "_ensure_rerun_server", lambda **kwargs: ("rerun+http://127.0.0.1:9876/proxy", False) @@ -575,11 +565,18 @@ def get_num_envs() -> int: randomly_sample_visible_envs=False, ) visualizer = rerun_visualizer.RerunVisualizer(cfg) - visualizer.initialize(cast(Any, _DummyRerunSceneDataProvider())) + visualizer.initialize(cast(Any, _DummyViserSceneDataProvider())) - assert captured["set_model"] == "dummy-model" + assert captured["set_model"] is web_backend.get_or_create_backend.return_value.model assert captured["visible_worlds"] == expected_visible assert captured["set_world_offsets"] == (0.0, 0.0, 0.0) + replacement = SimpleNamespace(model=SimpleNamespace(body_label=["/Replacement"])) + web_backend.get_or_create_backend.return_value = replacement + visualizer.reset() + assert visualizer.backend is replacement + web_backend.get_or_create_backend.assert_called_with(visualizer.newton_cfg) + assert captured["set_model"] is replacement.model + assert captured["visible_worlds"] == expected_visible def test_kit_visualizer_default_camera_source_does_not_require_camera_prim(monkeypatch: pytest.MonkeyPatch): @@ -738,12 +735,11 @@ def test_get_cli_visualizer_types_handles_non_string_setting_without_crashing(): # --------------------------------------------------------------------------- -class _FakeVisualizerCfg: +class _FakeVisualizerCfg(VisualizerCfg): """Minimal visualizer config for testing initialize_visualizers.""" - cloning_contexts = () - def __init__(self, visualizer_type: str, *, fail_construct: bool = False, fail_init: bool = False): + super().__init__() self.visualizer_type = visualizer_type self.class_type = ( self._raise_construction_error @@ -849,6 +845,7 @@ def _make_context_with_settings( ctx._visualizers = [] ctx._pending_visualizers = [] ctx._render_context = SimpleNamespace(clone_contexts=set()) + ctx.physics_manager = SimpleNamespace(register_callback=Mock()) ctx._scene_data_provider = _FakeProvider() ctx.requires_usd_stage = False ctx.requires_newton_model = False @@ -897,7 +894,8 @@ def test_cli_type_newton_rtx_resolves_to_newton_rtx_visualizer_cfg(): assert isinstance(cfgs[0], NewtonRTXVisualizerCfg) -def test_default_visualizer_cfg_applies_to_explicit_visualizer_cfgs(): +@pytest.mark.parametrize("renderer_cfg", [None, NewtonWarpRendererCfg(use_cuda_graph=False)]) +def test_default_visualizer_cfg_applies_to_explicit_visualizer_cfgs(renderer_cfg): """default_visualizer_cfg fills in env-level hints (eye, lookat) on explicit cfgs. When visualizer_cfgs is set directly (e.g. for video recording), fields that are @@ -915,6 +913,8 @@ def test_default_visualizer_cfg_applies_to_explicit_visualizer_cfgs(): lookat=(0.0, 0.0, 0.5), streaming_cam_target_prim_path="/World/envs/*/Object", ) + if renderer_cfg is not None: + default_cfg.streaming_cam_renderer_cfg = renderer_cfg # Explicit Newton cfg with only window_width customized; eye/lookat at class defaults. explicit_cfg = NewtonGLVisualizerCfg(window_width=320, window_height=240) ctx = _make_context_with_settings(settings, visualizer_cfgs=[explicit_cfg], default_visualizer_cfg=default_cfg) @@ -931,6 +931,9 @@ def test_default_visualizer_cfg_applies_to_explicit_visualizer_cfgs(): assert cfgs[0].window_height == 240 assert cfgs[0].class_type.__name__ == "NewtonGLVisualizer" assert cfgs[0].visualizer_type == "newton_gl" + # Inherit explicit choices, not the source visualizer's native renderer default. + assert cfgs[0].streaming_cam_renderer_cfg == (renderer_cfg or NewtonWarpRendererCfg()) + assert cfgs[0].cloning_contexts == NewtonGLVisualizerCfg().cloning_contexts def test_default_visualizer_cfg_does_not_override_explicitly_customized_fields(): diff --git a/source/isaaclab/test/visualizers/test_visualizer.py b/source/isaaclab/test/visualizers/test_visualizer.py index b88d567625d6..2af1897b1c6d 100644 --- a/source/isaaclab/test/visualizers/test_visualizer.py +++ b/source/isaaclab/test/visualizers/test_visualizer.py @@ -14,6 +14,7 @@ import torch from isaaclab.envs.utils.camera_view import apply_camera_view_from_origins, prim_world_positions +from isaaclab.renderers import RendererCfg from isaaclab.utils.string import ResolvableString from isaaclab.visualizers.base_visualizer import BaseVisualizer from isaaclab.visualizers.visualizer_cfg import VisualizerCfg @@ -37,9 +38,16 @@ ) def test_visualizer_cfg_names_its_implementation(module_name, cfg_name, implementation): cfg_type = getattr(pytest.importorskip(module_name), cfg_name) - class_type = cfg_type().class_type + cfg = cfg_type() + class_type = cfg.class_type assert isinstance(class_type, ResolvableString) assert class_type.__name__ == implementation + renderer_cfg = cfg.streaming_cam_renderer_cfg + assert renderer_cfg.renderer_type == ("isaac_rtx" if cfg_name == "KitVisualizerCfg" else "newton_warp") + assert isinstance(renderer_cfg.class_type, ResolvableString) + custom = RendererCfg(class_type="my_renderers:CustomRenderer", renderer_type="custom") + assert cfg_type(streaming_cam_renderer_cfg=custom).streaming_cam_renderer_cfg == custom + assert not hasattr(cfg, "streaming_cam_renderer") def test_visualizer_cfg_streaming_view_is_opt_in(): @@ -48,7 +56,7 @@ def test_visualizer_cfg_streaming_view_is_opt_in(): assert cfg.background_color == (0.3, 0.55, 0.82) assert cfg.streaming_view is False assert cfg.streaming_envs == 32 - assert cfg.streaming_cam_renderer is None + assert cfg.streaming_cam_renderer_cfg is None def test_visualizer_cfg_validates_background_color(): @@ -86,7 +94,7 @@ def _make_cfg(**kwargs): "randomly_sample_visible_envs": False, } cfg.update(kwargs) - return SimpleNamespace(**cfg) + return VisualizerCfg(**cfg) _HAS_ISAACLAB_VIZ = importlib.util.find_spec("isaaclab_visualizers") is not None diff --git a/source/isaaclab_contrib/changelog.d/coupler-builder-preparation.rst b/source/isaaclab_contrib/changelog.d/coupler-builder-preparation.rst new file mode 100644 index 000000000000..4c3ac28d25f2 --- /dev/null +++ b/source/isaaclab_contrib/changelog.d/coupler-builder-preparation.rst @@ -0,0 +1,4 @@ +Fixed +^^^^^ + +* Prepared coupled Newton builders once per selected solver manager, avoiding unnecessary or repeated VBD coloring. diff --git a/source/isaaclab_contrib/isaaclab_contrib/coupling/coupler.py b/source/isaaclab_contrib/isaaclab_contrib/coupling/coupler.py index 03ca32fe34d0..03aa37bcc60b 100644 --- a/source/isaaclab_contrib/isaaclab_contrib/coupling/coupler.py +++ b/source/isaaclab_contrib/isaaclab_contrib/coupling/coupler.py @@ -202,10 +202,10 @@ def _registers_builder_attributes_from_solver(cls, solver_cls: type[SolverBase]) @classmethod def _prepare_builder_for_finalize(cls, builder: ModelBuilder) -> None: - """Normalize kinematic colliders when a coupled entry uses implicit MPM.""" - super()._prepare_builder_for_finalize(builder) - for entry in PhysicsManager._cfg.solver_cfg.entries: - entry.solver_cfg.class_type._prepare_builder_for_finalize(builder) + """Prepare the shared builder once per selected solver manager.""" + entries = PhysicsManager._cfg.solver_cfg.entries + for prepare in dict.fromkeys(entry.solver_cfg.class_type._prepare_builder_for_finalize for entry in entries): + prepare(builder) @classmethod def _initialize_contacts(cls) -> None: diff --git a/source/isaaclab_contrib/test/coupling/test_coupler.py b/source/isaaclab_contrib/test/coupling/test_coupler.py index 4348ca0716f7..589e9bbd806a 100644 --- a/source/isaaclab_contrib/test/coupling/test_coupler.py +++ b/source/isaaclab_contrib/test/coupling/test_coupler.py @@ -505,13 +505,15 @@ def test_coupler_clear_releases_nested_manager_state(monkeypatch): assert events == ["vbd", "mpm"] -def test_mpm_entry_reuses_builder_lifecycle_hooks(monkeypatch): - """Coupled MPM entries normalize kinematic colliders before finalize.""" +@pytest.mark.parametrize("vbd_entries", [0, 2]) +def test_mpm_entry_reuses_builder_lifecycle_hooks(monkeypatch, vbd_entries): + """Prepare the shared builder once per selected solver, without unconditional VBD coloring.""" events: list[tuple[str, object]] = [] - builder = object() - solver_cfg = CouplerProxyCfg( - entries=[CouplerEntryCfg(name="media", solver_cfg=MPMSolverCfg())], - ) + builder = ModelBuilder() + monkeypatch.setattr(builder, "color", lambda *, balance_colors: events.append(("color", balance_colors))) + entries = [CouplerEntryCfg(name="media", solver_cfg=MPMSolverCfg())] + entries.extend(CouplerEntryCfg(name=f"cloth_{index}", solver_cfg=VBDSolverCfg()) for index in range(vbd_entries)) + solver_cfg = CouplerProxyCfg(entries=entries) monkeypatch.setattr(coupler.PhysicsManager, "_cfg", SimpleNamespace(solver_cfg=solver_cfg)) monkeypatch.setattr( coupler.NewtonMPMManager, @@ -521,7 +523,7 @@ def test_mpm_entry_reuses_builder_lifecycle_hooks(monkeypatch): NewtonCouplerManager._prepare_builder_for_finalize(builder) - assert events == [("finalize", builder)] + assert events == [("finalize", builder)] + ([("color", False)] if vbd_entries else []) def test_nested_solvers_register_their_builder_attributes(monkeypatch): diff --git a/source/isaaclab_newton/changelog.d/native-query-ownership.major.rst b/source/isaaclab_newton/changelog.d/native-query-ownership.major.rst new file mode 100644 index 000000000000..3c2ffc21bcba --- /dev/null +++ b/source/isaaclab_newton/changelog.d/native-query-ownership.major.rst @@ -0,0 +1,30 @@ +Changed +^^^^^^^ + +* Moved Newton camera and ray-cast consumers to the simulation-owned native backend. Consumers + requested transforms and geometry directly through SDP and shared native BVH refits while owning + their individual query graphs. ``NewtonWarpRendererCfg.use_cuda_graph`` controlled camera query + capture independently of physics capture. Deformable triangle-mesh rendering stayed eager because + native mesh updates required host reads and allocations. + Stateless query and capture functions lived in ``NewtonQueries`` alongside ``NewtonManager``; + the container retained no model or scheduling state. +* Applied ray-cast BVH requirements to the shared builder before finalization, including when + another consumer acquired the builder first. +* **Breaking:** Replaced the native builder input in ``NewtonBackendCfg`` with the selected physics + configuration. Acquire builders through ``NewtonBuilderCfg(physics_cfg=sim.cfg.physics)`` + and finalized models through ``NewtonBackendCfg(physics_cfg=sim.cfg.physics, device=sim.device)`` + using ``sim.get_or_create_backend(cfg)``. Non-Newton physics selected a render-only representation; + model allocation followed cloning rather than occurring inside it. Kept construction and runtime + cfgs independent: closing a backend released its native buffers but retained the builder for hard reset. + +Removed +^^^^^^^ + +* **Breaking:** Removed ``NewtonManager.set_builder()``. Acquire and populate the shared builder with + ``sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics))`` before ``sim.reset()``. +* **Breaking:** Removed ``NewtonManager.get_state()`` and ``update_visualization_state()``. + Acquire the native backend through ``sim.get_or_create_backend(backend_cfg)`` and request + ``SceneDataProvider.get_transforms()`` and ``get_geometry_points()`` directly instead. +* **Breaking:** Removed ``NewtonManager.sync_transforms_to_fabric()`` and ``sync_transforms_to_usd()``. + Kit/Isaac RTX rendering already requested published poses through SDP. Custom Fabric consumers + should call ``FabricBackend.update_transforms(provider)`` instead of asking the physics manager to render. diff --git a/source/isaaclab_newton/isaaclab_newton/assets/articulation/articulation.py b/source/isaaclab_newton/isaaclab_newton/assets/articulation/articulation.py index a341eafe481a..e905d1adb0e7 100644 --- a/source/isaaclab_newton/isaaclab_newton/assets/articulation/articulation.py +++ b/source/isaaclab_newton/isaaclab_newton/assets/articulation/articulation.py @@ -28,6 +28,7 @@ from isaaclab.assets.articulation import ordering_kernels from isaaclab.assets.articulation.base_articulation import BaseArticulation from isaaclab.physics import PhysicsEvent +from isaaclab.sim import SimulationContext from isaaclab.sim.utils.queries import resolve_matching_prims_from_source from isaaclab.utils.string import resolve_matching_names, resolve_matching_names_values from isaaclab.utils.version import get_isaac_sim_version, has_kit @@ -37,6 +38,7 @@ from isaaclab_newton.assets import kernels as shared_kernels from isaaclab_newton.assets.articulation import kernels as articulation_kernels from isaaclab_newton.assets.articulation.joint_coordinates import scatter_joint_coordinates +from isaaclab_newton.physics import NewtonBuilderCfg from isaaclab_newton.physics import NewtonManager as SimulationManager from .actuator_control import NewtonActuatorControl @@ -84,63 +86,6 @@ def has_articulation_root_api(prim) -> bool: return resolve_matching_prims_from_source(cfg.prim_path, **resolve_kwargs)[0][1] -def _configure_builder_joint_target_modes(builder, cfg: ArticulationCfg) -> None: - """Resolve configured actuator gains into Newton builder target modes before finalization.""" - root_prim_path_regex = _resolve_articulation_root_prim_path_expr(cfg) - articulation_ids, _ = resolve_matching_names( - root_prim_path_regex, builder.articulation_label, raise_when_no_match=False - ) - source_dof_ids = None - for articulation_id in articulation_ids: - joint_start = builder.articulation_start[articulation_id] - joint_end = builder.articulation_end[articulation_id] - dof_ids: list[int] = [] - dof_names: list[str] = [] - for joint_id in range(joint_start, joint_end): - if builder.joint_type[joint_id] in (JointType.FREE, JointType.FIXED): - continue - dof_start = builder.joint_qd_start[joint_id] - dof_end = ( - builder.joint_qd_start[joint_id + 1] - if joint_id + 1 < len(builder.joint_qd_start) - else len(builder.joint_target_mode) - ) - joint_name = builder.joint_label[joint_id].rsplit("/", maxsplit=1)[-1] - for axis_index, dof_id in enumerate(range(dof_start, dof_end)): - dof_ids.append(dof_id) - dof_names.append(joint_name if dof_end - dof_start == 1 else f"{joint_name}:{axis_index}") - - if source_dof_ids is not None: - for source_dof_id, dof_id in zip(source_dof_ids, dof_ids, strict=True): - builder.joint_target_mode[dof_id] = builder.joint_target_mode[source_dof_id] - continue - - for actuator_cfg in cfg.actuators.values(): - matched_indices, matched_names = resolve_matching_names( - actuator_cfg.joint_names_expr, dof_names, raise_when_no_match=False - ) - if not matched_indices: - continue - selected_dof_ids = [dof_ids[index] for index in matched_indices] - stiffness_values = _resolve_actuator_gain_values( - actuator_cfg.stiffness, - matched_names, - [builder.joint_target_ke[dof_id] for dof_id in selected_dof_ids], - ) - damping_values = _resolve_actuator_gain_values( - actuator_cfg.damping, - matched_names, - [builder.joint_target_kd[dof_id] for dof_id in selected_dof_ids], - ) - for dof_id, stiffness, damping in zip(selected_dof_ids, stiffness_values, damping_values, strict=True): - builder.joint_target_mode[dof_id] = int( - _target_mode_from_gains(stiffness, damping) - if _is_implicit_actuator_cfg(actuator_cfg) - else JointTargetMode.EFFORT - ) - source_dof_ids = dof_ids - - class Articulation(BaseArticulation): """An articulation asset class. @@ -212,8 +157,6 @@ def __init__(self, cfg: ArticulationCfg): Args: cfg: A configuration instance. """ - from isaaclab.sim import SimulationContext # noqa: PLC0415 - super().__init__(cfg) sim_ctx = SimulationContext.instance() @@ -232,10 +175,62 @@ def _register_callbacks(self) -> None: ) def _configure_joint_target_modes(self, _event) -> None: - """Apply configured actuator modes to the private Newton model builder.""" - builder = SimulationManager._builder - if builder is not None: - _configure_builder_joint_target_modes(builder, self.cfg) + """Apply configured actuator modes to the shared builder before model allocation.""" + sim = SimulationContext.instance() + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics)) + root_prim_path_regex = _resolve_articulation_root_prim_path_expr(self.cfg) + articulation_ids, _ = resolve_matching_names( + root_prim_path_regex, builder.articulation_label, raise_when_no_match=False + ) + source_dof_ids = None + for articulation_id in articulation_ids: + joint_start = builder.articulation_start[articulation_id] + joint_end = builder.articulation_end[articulation_id] + dof_ids: list[int] = [] + dof_names: list[str] = [] + for joint_id in range(joint_start, joint_end): + if builder.joint_type[joint_id] in (JointType.FREE, JointType.FIXED): + continue + dof_start = builder.joint_qd_start[joint_id] + dof_end = ( + builder.joint_qd_start[joint_id + 1] + if joint_id + 1 < len(builder.joint_qd_start) + else len(builder.joint_target_mode) + ) + joint_name = builder.joint_label[joint_id].rsplit("/", maxsplit=1)[-1] + for axis_index, dof_id in enumerate(range(dof_start, dof_end)): + dof_ids.append(dof_id) + dof_names.append(joint_name if dof_end - dof_start == 1 else f"{joint_name}:{axis_index}") + + if source_dof_ids is not None: + for source_dof_id, dof_id in zip(source_dof_ids, dof_ids, strict=True): + builder.joint_target_mode[dof_id] = builder.joint_target_mode[source_dof_id] + continue + + for actuator_cfg in self.cfg.actuators.values(): + matched_indices, matched_names = resolve_matching_names( + actuator_cfg.joint_names_expr, dof_names, raise_when_no_match=False + ) + if not matched_indices: + continue + selected_dof_ids = [dof_ids[index] for index in matched_indices] + stiffness_values = _resolve_actuator_gain_values( + actuator_cfg.stiffness, + matched_names, + [builder.joint_target_ke[dof_id] for dof_id in selected_dof_ids], + ) + damping_values = _resolve_actuator_gain_values( + actuator_cfg.damping, + matched_names, + [builder.joint_target_kd[dof_id] for dof_id in selected_dof_ids], + ) + for dof_id, stiffness, damping in zip(selected_dof_ids, stiffness_values, damping_values, strict=True): + builder.joint_target_mode[dof_id] = int( + _target_mode_from_gains(stiffness, damping) + if _is_implicit_actuator_cfg(actuator_cfg) + else JointTargetMode.EFFORT + ) + source_dof_ids = dof_ids """ Properties diff --git a/source/isaaclab_newton/isaaclab_newton/cloner/replicate.py b/source/isaaclab_newton/isaaclab_newton/cloner/replicate.py index 03b004a23df7..3e0e61c0ce48 100644 --- a/source/isaaclab_newton/isaaclab_newton/cloner/replicate.py +++ b/source/isaaclab_newton/isaaclab_newton/cloner/replicate.py @@ -13,14 +13,14 @@ import numpy as np import warp as wp -from newton import ModelBuilder +from newton import Axis, ModelBuilder from pxr import Sdf, Usd, UsdGeom from isaaclab.assets import AssetBaseCfg from isaaclab.cloner import ClonePlan, PrototypeWorldTopology from isaaclab.cloner import path as cloner_path -from isaaclab.physics import PhysicsEvent, PhysicsManager +from isaaclab.physics import PhysicsManager from isaaclab.scene_data import SceneDataFormat from isaaclab.scene_data.deformable_discovery import ( deformable_geometry_batches, @@ -36,7 +36,7 @@ build_source_builders, replicate_builder_mapping, ) -from isaaclab_newton.physics import NewtonBackendCfg, NewtonCfg, NewtonManager +from isaaclab_newton.physics import NewtonBuilderCfg, NewtonCfg, NewtonManager from isaaclab_newton.sim.spawners.mpm.mpm import _SIMULATION_POINTS_SUFFIX if TYPE_CHECKING: @@ -136,7 +136,8 @@ def _replicate_newton( load_visual_shapes = cfg.load_visual_shapes if simulation else True if load_visual_shapes is None: load_visual_shapes = sim.is_rendering or sim.can_render_rgb_array() or sim.visual_shapes_required - builder = create_builder() + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=cfg)) + builder.up_axis = Axis.from_string(up_axis) import_paths = (sim.cfg.physics_prim_path, *global_paths) if simulation else global_paths source_paths = list(dict.fromkeys(sources[index] for index in asset_prototype_ids if sources[index] is not None)) if simulation: @@ -160,7 +161,6 @@ def _replicate_newton( options = dict(ignore_paths=ignore_paths, load_visual_shapes=load_visual_shapes) options.update(skip_mesh_approximation=not simulation, import_results_out=import_results) source_builders = build_source_builders(stage, source_paths, create_builder, schema_resolvers, **options) - source_geometry = {source: {} for source in source_builders} if simulation: entries = [add_deformable_from_usd(source_builders[path], stage, root_path=path) for path in deformable_paths] else: @@ -169,9 +169,11 @@ def _replicate_newton( ancestors = reversed(Sdf.Path(entry.root_path).GetPrefixes()) source = next(str(path) for path in ancestors if str(path) in source_builders) native = source_builders[source] - source_geometry[source][entry.vis_mesh_path] = native.particle_count pose = dict(pos=wp.vec3(*entry.init_pos), rot=wp.quat(*entry.init_rot), scale=1.0, vel=wp.vec3()) - if entry.deformable_type == "surface" or entry.vis_mesh_path != entry.sim_mesh_path: + surface = entry.deformable_type == "surface" or entry.vis_mesh_path != entry.sim_mesh_path + particle_start, tri_start = native.particle_count, len(native.tri_indices) + edge_start, tet_start = len(native.edge_indices), len(native.tet_indices) + if surface: add_mesh = native.add_cloth_mesh mesh = dict(vertices=entry.vis_vertices, indices=entry.vis_indices, density=1.0) mesh.update(tri_ke=1e4, tri_ka=1e4, tri_kd=1.5e-6, edge_ke=5.0, edge_kd=1e-2, particle_radius=0.008) @@ -179,13 +181,20 @@ def _replicate_newton( add_mesh = native.add_soft_mesh mesh = dict(vertices=entry.vertices, indices=entry.indices, density=1000.0) mesh.update(k_mu=1e5, k_lambda=1e5, k_damp=0.0) - add_mesh(**mesh, **pose) + add_mesh(label=entry.vis_mesh_path, **mesh, **pose) + # Remove private recording when the pinned Newton includes #3326. + particle_range = particle_start, native.particle_count + if surface: + tri_range, edge_range = (tri_start, len(native.tri_indices)), (edge_start, len(native.edge_indices)) + native._record_cloth_group(entry.vis_mesh_path, particle_range, tri_range, edge_range) + else: + native._record_soft_group(entry.vis_mesh_path, particle_range, (tet_start, len(native.tet_indices))) # Resolve native capsule indices once per source, not by rediscovering labels after cloning. source_cables = {} for source, imported in import_results.items(): cables = source_cables[source] = {} - if not imported["path_cable_map"]: + if not simulation or not imported["path_cable_map"]: continue shapes = {label: index for index, label in enumerate(source_builders[source].shape_label)} for path, (bodies, _) in imported["path_cable_map"].items(): @@ -194,7 +203,6 @@ def _replicate_newton( if len(UsdGeom.BasisCurves(stage.GetPrimAtPath(path)).GetCurveVertexCountsAttr().Get()) != 1: continue cables[path] = [shapes[f"{path}_edge_capsule_{segment}"] for segment in range(len(bodies))] - if simulation: global_sites, source_sites, root_sites = NewtonManager._cl_inject_sites(builder, source_builders) else: @@ -204,7 +212,7 @@ def _replicate_newton( imported.shape_collision_group[:] = [0] * imported.shape_count global_sites, source_sites, root_sites = {}, {}, {} - particle_ranges, visual_ranges, cable_bindings, geometry_offsets = {}, {}, {}, {} + particle_ranges, visual_ranges, cable_bindings = {}, {}, {} # The USD importer owns simulation ranges; MPM's spawner authors a separate visible point prim. particle_visual_paths = { path: path.removesuffix(_SIMULATION_POINTS_SUFFIX) + "/Particles" @@ -222,20 +230,18 @@ def record_geometry(source: str, destination: str, shape_offset: int, particle_o particle_ranges[cloner_path.rebase(path, source, destination)] = native_range if path in particle_visual_paths: visual_ranges[cloner_path.rebase(particle_visual_paths[path], source, destination)] = native_range - for path, offset in source_geometry[source].items(): - geometry_offsets[cloner_path.rebase(path, source, destination)] = particle_offset + offset options = dict(env_ids=env_ids, source_site_indices=source_sites, env_root_sites=root_sites) options["per_world_builder_hooks"] = NewtonManager._per_world_builder_hooks if simulation else () has_geometry = any(source_cables.values()) or any(result["path_particle_map"] for result in import_results.values()) - options["source_builder_added"] = record_geometry if has_geometry or any(source_geometry.values()) else None + options["source_builder_added"] = record_geometry if has_geometry else None local_site_map, world_xforms = replicate_builder_mapping( builder, plan, positions, quaternions, source_builders, **options ) site_index_map = {label: (idx, None) for label, idx in global_sites.items()} site_index_map.update((label, (None, per_world)) for label, per_world in local_site_map.items()) - NewtonManager._cable_bindings = cable_bindings if simulation: + NewtonManager._cable_bindings = cable_bindings geometry = expand_deformable_entries(entries, plan, env_ids, positions) ranges = { label: start @@ -252,16 +258,8 @@ def record_geometry(source: str, destination: str, shape_offset: int, particle_o NewtonManager._cl_site_index_map = site_index_map NewtonManager._world_xforms = world_xforms NewtonManager._cl_protos = source_builders - NewtonManager.set_builder(builder, particle_ranges=particle_ranges) + NewtonManager._particle_ranges = particle_ranges NewtonManager._num_envs = len(env_ids) - else: - backend_cfg = NewtonBackendCfg(builder=builder, device=sim.device, num_envs=len(env_ids), simulation=False) - sim.physics_manager.register_callback( - partial(NewtonManager._initialize_visualization_model, backend_cfg, geometry_offsets), - PhysicsEvent.PHYSICS_READY, - name="newton_visualization_model", - wrap_weak_ref=False, - ) return builder, stage_info, site_index_map @@ -276,7 +274,7 @@ def __init__(self, sim_context: SimulationContext, *, up_axis: str = "Z"): self.up_axis = up_axis def replicate(self, plan: ClonePlan, asset_prototype_ids: tuple[int, ...]) -> tuple[ModelBuilder, object, dict]: - """Build and publish a Newton model from this context's source declarations.""" + """Populate the shared Newton builder from this context's source declarations.""" env_ids = np.arange(len(plan.topology.world_prototype_layout)) options = dict(plan=plan, asset_prototype_ids=asset_prototype_ids, positions=plan.positions) return _replicate_newton(self._sim.stage, env_ids, self._sim, up_axis=self.up_axis, **options) diff --git a/source/isaaclab_newton/isaaclab_newton/physics/__init__.pyi b/source/isaaclab_newton/isaaclab_newton/physics/__init__.pyi index 871591d216ec..39a2887807d7 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/__init__.pyi +++ b/source/isaaclab_newton/isaaclab_newton/physics/__init__.pyi @@ -19,11 +19,13 @@ __all__ = [ "MJWarpSolverCfg", "NewtonCfg", "NewtonBackendCfg", + "NewtonBuilderCfg", "NewtonCollisionPipelineCfg", "NewtonFeatherstoneManager", "NewtonKaminoManager", "NewtonMPMManager", "NewtonManager", + "NewtonQueries", "NewtonMJWarpManager", "NewtonShapeCfg", "NewtonSoftContactCfg", @@ -32,6 +34,7 @@ __all__ = [ "VBDSolverCfg", "NewtonXPBDManager", "XPBDSolverCfg", + "create_newton_builder", ] from .featherstone_manager import NewtonFeatherstoneManager @@ -53,9 +56,10 @@ from .mjwarp_manager_cfg import MJWarpSolverCfg from .mpm_manager import NewtonMPMManager from .mpm_manager_cfg import MPMSolverCfg from .newton_collision_cfg import HydroelasticSDFCfg, NewtonCollisionPipelineCfg -from .newton_manager import NewtonManager +from .newton_manager import NewtonManager, NewtonQueries, create_newton_builder from .newton_manager_cfg import ( NewtonBackendCfg, + NewtonBuilderCfg, NewtonCfg, NewtonShapeCfg, NewtonSoftContactCfg, diff --git a/source/isaaclab_newton/isaaclab_newton/physics/mpm_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/mpm_manager.py index a3f6906aad4f..049f237790ee 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/mpm_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/mpm_manager.py @@ -202,9 +202,7 @@ def _register_builder_attributes(cls, builder: ModelBuilder) -> None: that ``add_particles(custom_attributes=...)`` succeeds and so that ``builder.finalize()`` allocates the matching model arrays. - Idempotent: ``has_custom_attribute`` guards against re-registration - when the hook is invoked multiple times (e.g. once via - :meth:`create_builder` and again via :meth:`start_simulation`). + ``create_builder()`` registers these on each prototype and the shared builder. """ if not builder.has_custom_attribute("mpm:young_modulus"): SolverImplicitMPM.register_custom_attributes(builder) diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py index ede502de264d..0d01ef42dc7a 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py @@ -14,6 +14,7 @@ import re from abc import abstractmethod from collections.abc import Callable, Sequence +from functools import partial from typing import TYPE_CHECKING, Any, ClassVar import numpy as np @@ -54,7 +55,13 @@ from isaaclab_newton.physics.featherstone_manager_cfg import FeatherstoneSolverCfg from isaaclab_newton.physics.mjwarp_manager_cfg import MJWarpSolverCfg -from isaaclab_newton.physics.newton_manager_cfg import NewtonBackendCfg, NewtonCfg, NewtonShapeCfg, NewtonSolverCfg +from isaaclab_newton.physics.newton_manager_cfg import ( + NewtonBackendCfg, + NewtonBuilderCfg, + NewtonCfg, + NewtonShapeCfg, + NewtonSolverCfg, +) from isaaclab_newton.physics.xpbd_manager_cfg import XPBDSolverCfg from isaaclab_newton.renderers.visual_material import ( VisualMaterialWriter, @@ -79,30 +86,6 @@ """Extended state attributes that MuJoCo Warp's sensor stage fills via ``rne_postconstraint``.""" -@contextlib.contextmanager -def _paused_gc(): - """Pause Python garbage collection for the duration of a CUDA graph capture. - - A garbage-collection pass inside a capture window can drop the last - reference to an array allocated earlier in the capture. While the capture - is paused for a ``wp.capture_while``/``wp.capture_if`` conditional body, - Warp then inserts the memory free node into the body graph with dependency - nodes from the parent graph, which fails and latches a sticky CUDA error - that poisons a later, unrelated copy. Reference-count-driven frees are - deterministic solver behavior and remain allowed; only the collector is - deferred, and a collection runs immediately after the capture window, - where freeing graph-scoped allocations is handled correctly. - """ - was_enabled = gc.isenabled() - gc.disable() - try: - yield - finally: - if was_enabled: - gc.enable() - gc.collect() - - def _compile_label_pattern(expr: str | list[str] | None) -> re.Pattern[str] | None: """Compile selector expressions for Newton's full label matching.""" if not expr: @@ -167,42 +150,145 @@ def _scatter_world_reset_mask_from_ids(env_ids: wp.array(dtype=wp.int32), world_ world_mask[env_ids[wp.tid()]] = True +def create_newton_builder(cfg: NewtonBuilderCfg) -> ModelBuilder: + """Construct a native builder through the selected physics manager's factory. + + Args: + cfg: The selected physics configuration. + + Returns: + An empty builder with the selected solver schemas and shape/BVH defaults. + """ + if isinstance(cfg.physics_cfg, NewtonCfg): + return cfg.physics_cfg.class_type.create_builder(physics_cfg=cfg.physics_cfg) + return ModelBuilder() + + class NewtonBackend: """Own one finalized Newton model and its native state and control buffers.""" def __init__(self, cfg: NewtonBackendCfg): - self.model = cfg.builder.finalize(device=cfg.device) - self.particle_ranges = cfg.particle_ranges + builder = SimulationContext.instance().get_or_create_backend(NewtonBuilderCfg(physics_cfg=cfg.physics_cfg)) + self.model = builder.finalize(device=cfg.device) + self.particle_ranges: dict[str, tuple[int, int]] = {} # Newton 1.6 preserves groups through builder replication but not finalization. # Remove this snapshot when the pinned Newton includes newton-physics/newton#3326. self.deformable_ranges = { label: (start, end - start, kind) for family, kind in (("cloth", "surface"), ("soft", "volume")) for label, start, end in zip( - getattr(cfg.builder, f"_{family}_label"), - getattr(cfg.builder, f"_{family}_particle_start"), - getattr(cfg.builder, f"_{family}_particle_end"), + getattr(builder, f"_{family}_label"), + getattr(builder, f"_{family}_particle_start"), + getattr(builder, f"_{family}_particle_end"), strict=True, ) } - self.model.num_envs = self.model.world_count if cfg.num_envs is None else cfg.num_envs - if cfg.gravity is not None: - self.model.set_gravity(cfg.gravity) - if cfg.soft_contact_cfg is not None: - self.model.soft_contact_ke = float(cfg.soft_contact_cfg.soft_contact_ke) - self.model.soft_contact_kd = float(cfg.soft_contact_cfg.soft_contact_kd) - self.model.soft_contact_mu = float(cfg.soft_contact_cfg.soft_contact_mu) - if cfg.contact_attributes: - self.model.request_contact_attributes(*cfg.contact_attributes) + self.model.num_envs = self.model.world_count + simulation = isinstance(cfg.physics_cfg, NewtonCfg) + soft_contact = cfg.physics_cfg.soft_contact_cfg if simulation else None + if soft_contact is not None: + self.model.soft_contact_ke = float(soft_contact.soft_contact_ke) + self.model.soft_contact_kd = float(soft_contact.soft_contact_kd) + self.model.soft_contact_mu = float(soft_contact.soft_contact_mu) self.state_0 = self.model.state() - self.state_1 = self.model.state() if cfg.simulation else None - self.control = self.model.control() if cfg.simulation else None + self.state_1 = self.model.state() if simulation else None + self.control = self.model.control() if simulation else None + self.geometry_offsets = ( + {} if simulation else {path: bounds[0] for path, bounds in self.deformable_ranges.items()} + ) + self.bvh_refit = TimestampedBuffer() + + def create_visual_material_writer(self, batches: tuple[VisualMaterialBatch, ...]) -> VisualMaterialWriter: + """Bind material writes to this resource's native shape-color buffer.""" + return VisualMaterialWriter(self.model, batches) def close(self) -> None: """Drop native handles after consumers release their bindings.""" self.control = self.state_1 = self.state_0 = self.model = None self.deformable_ranges.clear() self.particle_ranges = {} + self.geometry_offsets = {} + self.bvh_refit = TimestampedBuffer() + + +class NewtonQueries: + """Stateless query and capture functions operating on an explicit Newton resource.""" + + @staticmethod + def run_query( + backend: NewtonBackend, + timestamp: int, + query: Callable[[], None], + graph: tuple[tuple[int, ...], wp.Graph] | None, + *, + use_cuda_graph: bool = True, + ) -> tuple[tuple[int, ...], wp.Graph] | None: + """Refresh shared BVHs once per publication and run a consumer-owned query graph. + + Args: + backend: Shared model, state, and acceleration structures. + timestamp: Sum of the monotonic SDP transform and geometry timestamps for this state. + query: Camera or ray-cast work using the supplied backend. + graph: This consumer's previous captured query and pointer layout, or None. + use_cuda_graph: Whether to capture GPU queries. CPU queries always run eagerly. + + Returns: + The consumer's captured query and pointer layout, or None for eager execution. + """ + model, state = backend.model, backend.state_0 + pointers = tuple(array.ptr if array is not None else 0 for array in (state.body_q, state.particle_q)) + cached = backend.bvh_refit + if cached.timestamp != timestamp: + if model.device.is_cuda and use_cuda_graph: + if cached.data is None or cached.data[0] != pointers: + refit = partial(NewtonQueries._refit_bvh, backend) + cached.data = pointers, NewtonQueries.capture_graph(str(model.device), refit, relaxed=has_kit()) + wp.capture_launch(cached.data[1]) + else: + NewtonQueries._refit_bvh(backend) + cached.timestamp = timestamp + if not model.device.is_cuda or not use_cuda_graph: + query() + return None + if graph is None or graph[0] != pointers: + graph = pointers, NewtonQueries.capture_graph(str(model.device), query, relaxed=has_kit()) + wp.capture_launch(graph[1]) + return graph + + @staticmethod + def capture_graph(device: str, capture_target: Callable[[], None], *, relaxed: bool = False) -> wp.Graph: + """Record work without an eager warmup or graph replay, using Warp's public capture API.""" + stream = wp.get_stream(device) + mode = wp.CaptureMode.THREAD_LOCAL + if relaxed: + # RTX uses the legacy CUDA stream. A nonblocking stream avoids implicit synchronization. + stream = wp.stream_from_torch(torch.cuda.Stream(device=device)) + mode = wp.CaptureMode.RELAXED + with NewtonQueries._paused_gc(), wp.ScopedStream(stream): + with wp.ScopedCapture(stream=stream, capture_mode=mode) as capture: + capture_target() + return capture.graph + + @staticmethod + def _refit_bvh(backend: NewtonBackend) -> None: + model, state = backend.model, backend.state_0 + if model.shape_count: + model.bvh_refit_shapes(state) + if model.particle_count: + model.bvh_refit_particles(state) + + @staticmethod + @contextlib.contextmanager + def _paused_gc(): + """Keep collection-driven frees out of CUDA graph capture.""" + was_enabled = gc.isenabled() + gc.disable() + try: + yield + finally: + if was_enabled: + gc.enable() + gc.collect() class NewtonSceneDataBackend(SceneDataBackend): @@ -354,7 +440,6 @@ class NewtonManager(PhysicsManager): """Whether the solver consumes applied rigid-body forces from :class:`State`.""" # Native ownership and physics orchestration - _builder: ModelBuilder = None backend: ClassVar[NewtonBackend | None] = None """Borrowed native resource shared by physics and scene consumers; the simulation registry owns it.""" _solver: SolverBase | None = None @@ -407,18 +492,7 @@ class NewtonManager(PhysicsManager): _graph = None _graph_capture_pending: bool = False - # Newton scene-query scheduling and graph execution. - _sensor_tasks: dict[str, Callable[[], None]] = {} - _sensor_eager_tasks: set[str] = set() - _sensor_graph: wp.Graph | None = None - _sensor_flags: wp.array | None = None - _sensor_flags_host: np.ndarray | None = None - _sensor_state: State | None = None - _sensor_state_dirty: bool = True - _sensor_graph_capture_failed: bool = False - _sensor_bvh_shape_flags: ShapeFlags = ShapeFlags.VISIBLE - - # USD/Fabric sync + # Native scene-data publication. transforms_may_change_on_graph_replay: bool = False _cable_bindings: dict[str, list[int]] = {} @@ -431,15 +505,6 @@ class NewtonManager(PhysicsManager): # Scene data backend _scene_data_backend: NewtonSceneDataBackend | None = None - # Visualization-only state used when the sim backend is PhysX. Populated - # from the clone plan in :meth:`_initialize_visualization_model` and updated each render - # frame in :meth:`update_visualization_state`. - _transform_mapping: wp.array | None = None - _visualization_transforms: ClassVar[TimestampedBuffer] = TimestampedBuffer() - _geometry_offsets: dict[str, int] = {} - _geometry_timestamp: int = -1 - _visualization_stop_callback: CallbackHandle | None = None - _builder_attribute_solvers: tuple[type[SolverBase], ...] = () _particle_ranges: dict[str, tuple[int, int]] = {} @@ -501,6 +566,8 @@ def reset(cls, soft: bool = False) -> None: soft: If True, skip full reinitialization. """ if not soft: + if NewtonManager.backend is not None: + cls.dispatch_event(PhysicsEvent.STOP) # Release the cached collision pipeline, contacts and CUDA graph; # they point at the old model's freed buffers (CUDA 700 on next step). NewtonManager._graph = None @@ -511,8 +578,6 @@ def reset(cls, soft: bool = False) -> None: NewtonManager._eval_fk = _eval_fk_unbound NewtonManager._reset_solver_internals_delegate = _reset_solver_internals_unbound NewtonManager._adapter = None - cls._invalidate_sensor_graph() - NewtonManager._sensor_state = None if NewtonManager.backend is not None: SimulationContext.instance().close_backend(NewtonManager.backend) NewtonManager.backend = None @@ -560,35 +625,12 @@ def forward(cls) -> None: if cls._world_reset_mask is not None: cls._world_reset_mask.zero_() NewtonManager.kinematics_dirty = False - cls._mark_sensor_state_dirty() @classmethod def video_capture_backend(cls) -> str: """Newton GL headless perspective video capture.""" return "newton_gl" - @classmethod - def sync_transforms_to_fabric(cls) -> None: - """Publish rigid-body poses through SDP to Fabric, leaving authored USD untouched.""" - sim = PhysicsManager._sim - if sim.fabric_cfg is None or cls.backend is None: - return - sim.get_or_create_backend(sim.fabric_cfg).update_transforms(sim.get_scene_data_provider()) - - @classmethod - def sync_transforms_to_usd(cls) -> None: - """Write Newton body_q to Fabric world matrices for Kit viewport / RTX rendering. - - .. deprecated:: v6.3.0 - Renamed to :meth:`sync_transforms_to_fabric`, which describes where the write - actually lands. This alias will be removed in a future release. - """ - logger.warning( - "The method 'NewtonManager.sync_transforms_to_usd' is deprecated because it writes Fabric, not the USD" - " stage. Please use 'NewtonManager.sync_transforms_to_fabric' instead." - ) - cls.sync_transforms_to_fabric() - @classmethod def _mark_transforms_changed(cls) -> None: """Publish authored rigid-body changes and invalidate cable geometry.""" @@ -687,7 +729,6 @@ def step(cls) -> None: PhysicsManager._sim_time += physics_dt cls._mark_transforms_changed() - cls._mark_sensor_state_dirty() cls._check_solver_status() @@ -730,12 +771,7 @@ def is_fabric_enabled(cls) -> bool: @classmethod def clear(cls): """Clear all Newton-specific state (callbacks cleared by super().close()).""" - callback = NewtonManager._visualization_stop_callback - NewtonManager._visualization_stop_callback = None - if callback is not None: - callback.deregister() NewtonManager._num_envs = None - NewtonManager._builder = None NewtonManager._solver = None NewtonManager._use_single_state = None NewtonManager._supports_rigid_body_force_input = False @@ -769,21 +805,11 @@ def clear(cls): NewtonManager.kinematics_dirty = False NewtonManager._graph = None NewtonManager._graph_capture_pending = False - NewtonManager._sensor_tasks = {} - NewtonManager._sensor_eager_tasks = set() - NewtonManager._invalidate_sensor_graph() - NewtonManager._sensor_state = None - NewtonManager._sensor_state_dirty = True - NewtonManager._sensor_bvh_shape_flags = ShapeFlags.VISIBLE NewtonManager.transforms_may_change_on_graph_replay = False NewtonManager._cable_bindings = {} NewtonManager._particle_ranges = {} NewtonManager._per_world_builder_hooks = [] NewtonManager._up_axis = "Z" - NewtonManager._transform_mapping = None - NewtonManager._visualization_transforms = TimestampedBuffer() - NewtonManager._geometry_offsets = {} - NewtonManager._geometry_timestamp = -1 NewtonManager._model_changes = set() NewtonManager._scene_data_backend = None NewtonManager._cl_pending_sites = {} @@ -801,13 +827,9 @@ def clear(cls): NewtonManager.backend = None @classmethod - def set_builder(cls, builder: ModelBuilder, *, particle_ranges: dict[str, tuple[int, int]] | None = None) -> None: - """Set the Newton model builder and its imported point-to-particle ranges.""" - NewtonManager._builder = builder - NewtonManager._particle_ranges = {} if particle_ranges is None else particle_ranges - - @classmethod - def create_builder(cls, up_axis: str | None = None, **kwargs) -> ModelBuilder: + def create_builder( + cls, up_axis: str | None = None, *, physics_cfg: NewtonCfg | None = None, **kwargs + ) -> ModelBuilder: """Create a :class:`ModelBuilder` configured with default settings. Forwards :class:`NewtonShapeCfg` defaults onto Newton's upstream @@ -818,6 +840,7 @@ def create_builder(cls, up_axis: str | None = None, **kwargs) -> ModelBuilder: Args: up_axis: Override for the up-axis. Defaults to ``None``, which uses the manager's ``_up_axis``. + physics_cfg: Explicit builder settings; None uses the active physics configuration. **kwargs: Forwarded to :class:`ModelBuilder`. Returns: @@ -826,14 +849,14 @@ def create_builder(cls, up_axis: str | None = None, **kwargs) -> ModelBuilder: # Resolve which NewtonShapeCfg to apply: user override if active config # is NewtonCfg, else the wrapper's own defaults so callers from non-Newton # contexts (tests, early construction) still get the rough-terrain margin. - cfg = PhysicsManager._cfg + cfg = PhysicsManager._cfg if physics_cfg is None else physics_cfg builder = ModelBuilder(up_axis=up_axis or cls._up_axis, **kwargs) builder.default_bvh_cfg = ModelBuilder.BvhConfig( mesh_constructor=cfg.bvh_constructor_geometry if isinstance(cfg, NewtonCfg) else None, gaussian_constructor=cfg.bvh_constructor_gaussian if isinstance(cfg, NewtonCfg) else None, shape_constructor=cfg.bvh_constructor_scene if isinstance(cfg, NewtonCfg) else None, - shape_flags=cls._sensor_bvh_shape_flags, + shape_flags=ShapeFlags.VISIBLE, ) cls._register_builder_attributes(builder) @@ -1133,45 +1156,38 @@ def start_simulation(cls) -> None: This function finalizes the model and initializes the simulation state. Note: Collision pipeline is initialized later in initialize_solver() after we determine whether the solver needs external collision detection. - - Raises: - RuntimeError: If neither clone-plan replication nor :meth:`set_builder` supplied a builder. """ - logger.debug(f"Builder: {cls._builder}") - if cls._builder is None: - raise RuntimeError( - "Newton simulation requires an explicitly supplied builder. Replicate a ClonePlan or call" - " NewtonManager.set_builder() before starting the simulation." - ) - + sim = SimulationContext.instance() + cfg = NewtonBackendCfg(physics_cfg=sim.cfg.physics, device=sim.device) + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=cfg.physics_cfg)) cls._drain_stale_cuda_error() - cls._register_builder_attributes(cls._builder) logger.info("Dispatching MODEL_INIT callbacks") cls.dispatch_event(PhysicsEvent.MODEL_INIT) # Explicit builders and MODEL_INIT callbacks use the same site import as clone prototypes. if cls._cl_pending_sites: - global_sites, body_sites, world_sites = cls._cl_inject_sites(cls._builder, {}) + global_sites, body_sites, world_sites = cls._cl_inject_sites(builder, {}) cls._cl_site_index_map.update((label, (index, None)) for label, index in global_sites.items()) cls._cl_site_index_map.update( - (label, (None, [indices])) for label, indices in body_sites.get(id(cls._builder), {}).items() + (label, (None, [indices])) for label, indices in body_sites.get(id(builder), {}).items() ) for label, xform in world_sites.items(): - index = cls._builder.add_site(body=-1, xform=xform, label=label) + index = builder.add_site(body=-1, xform=xform, label=label) cls._cl_site_index_map[label] = (None, [[index]]) device = PhysicsManager._device logger.info(f"Finalizing model on device: {device}") - cls._builder.up_axis = Axis.from_string(cls._up_axis) + builder.up_axis = Axis.from_string(cls._up_axis) # Forward pending extended attribute requests to builder and clear them. The requests are # retained because initialize_solver() runs afterwards and must know which sensors depend # on state that MuJoCo's sensor stage fills. if cls._pending_extended_state_attributes: - cls._builder.request_state_attributes(*cls._pending_extended_state_attributes) + builder.request_state_attributes(*cls._pending_extended_state_attributes) NewtonManager._active_extended_state_attributes |= cls._pending_extended_state_attributes NewtonManager._pending_extended_state_attributes = set() - cls._prepare_builder_for_finalize(cls._builder) + cls._prepare_builder_for_finalize(builder) + builder.request_contact_attributes(*sorted(cls._pending_extended_contact_attributes)) with Timer( name="newton_finalize_builder", msg="Finalize builder took:", @@ -1179,16 +1195,11 @@ def start_simulation(cls) -> None: synchronize="both", device=device, ): - cfg = NewtonBackendCfg( - builder=cls._builder, - particle_ranges=cls._particle_ranges, - device=device, - num_envs=cls._num_envs, - gravity=cls._gravity_vector, - soft_contact_cfg=PhysicsManager._cfg.soft_contact_cfg, - contact_attributes=tuple(sorted(cls._pending_extended_contact_attributes)), - ) - NewtonManager.backend = SimulationContext.instance().get_or_create_backend(cfg) + NewtonManager.backend = sim.get_or_create_backend(cfg) + cls.backend.particle_ranges = cls._particle_ranges + cls.backend.model.set_gravity(cls._gravity_vector) + if cls._num_envs is not None: + cls.backend.model.num_envs = cls._num_envs NewtonManager._num_envs = cls.backend.model.num_envs NewtonManager._pending_extended_contact_attributes = set() # The initial body-state update from joint coordinates is deferred to the tail of @@ -1612,18 +1623,9 @@ def _supports_cuda_graph_capture(cls) -> bool: @classmethod def _capture_graph(cls, capture_target: Callable[[], None]) -> wp.Graph: """Record physics or scene queries without an eager warmup or graph replay.""" - device = PhysicsManager._device - stream = wp.get_stream(device) - mode = wp.CaptureMode.THREAD_LOCAL sim = PhysicsManager._sim - if has_kit() and (sim.has_gui or sim.has_offscreen_render): - # RTX uses the legacy CUDA stream. A nonblocking stream avoids implicit synchronization. - stream = wp.stream_from_torch(torch.cuda.Stream(device=device)) - mode = wp.CaptureMode.RELAXED - with _paused_gc(), wp.ScopedStream(stream): - with wp.ScopedCapture(stream=stream, capture_mode=mode) as capture: - capture_target() - return capture.graph + relaxed = has_kit() and (sim.has_gui or sim.has_offscreen_render) + return NewtonQueries.capture_graph(PhysicsManager._device, capture_target, relaxed=relaxed) # ------------------------------------------------------------------ # Building blocks — used by _simulate_full / _simulate_physics_only @@ -1730,7 +1732,7 @@ def _simulate_physics_only(cls) -> None: @classmethod def create_visual_material_writer(cls, batches: tuple[VisualMaterialBatch, ...]) -> VisualMaterialWriter: """Compile material-to-shape addresses for the active Newton model.""" - return VisualMaterialWriter(cls.get_model(), batches) + return cls.backend.create_visual_material_writer(batches) @classmethod def create_visual_shape_color_writer( @@ -1750,7 +1752,7 @@ def create_visual_shape_color_writer( @classmethod def get_model(cls) -> Model: - """Return the native or render-only model initialized from the clone-built representation.""" + """Return the active physics model. Render consumers acquire their backend from the registry.""" return None if cls.backend is None else cls.backend.model @classmethod @@ -1758,293 +1760,19 @@ def get_state_0(cls) -> State: """Get the current state.""" return None if cls.backend is None else cls.backend.state_0 - @classmethod - def get_state(cls, scene_data_provider: SceneDataProvider | None = None) -> State: - """Get the current Newton state with derived transforms refreshed. - - Use this method from sensors, visualizers, renderers, and video recorders that need - a backend-agnostic Newton ``State``. When the sim backend is PhysX this - refreshes the shadow ``backend.state_0.body_q`` from the live PhysX scene via - :meth:`update_visualization_state` before returning, so callers never - observe stale transforms. Under the Newton sim backend, pending - forward kinematics is applied before returning the live state. - """ - if scene_data_provider is None: - scene_data_provider = cls.get_scene_data_provider() - if cls._backend_is_newton(scene_data_provider): - scene_data_provider.get_transforms(SceneDataFormat.Transform()) - else: - cls.update_visualization_state(scene_data_provider) - return cls.get_state_0() - @classmethod def get_contacts(cls) -> Contacts | None: """Get the current Newton contact buffer, if the active solver exposes one.""" return cls._contacts - @classmethod - def _register_sensor_task(cls, name: str, update_fn: Callable[[], None], *, graph_capturable: bool = True) -> None: - """Register a scene-query task. - - Args: - name: Unique task name. - update_fn: Callable run by :meth:`_update_sensor_tasks`. - graph_capturable: Whether ``update_fn`` supports conditional CUDA graph capture. - """ - if name in cls._sensor_tasks: - raise ValueError(f"Newton sensor task '{name}' is already registered.") - model = cls.get_model() - state = cls.get_state_0() - if model is None or state is None: - raise RuntimeError("Registering a Newton sensor task requires an initialized model and state.") - if model.shape_count > 0 and model.bvh_shapes is None: - model.bvh_build_shapes(state) - if model.particle_count > 0 and model.bvh_particles is None: - model.bvh_build_particles(state) - cls._sensor_tasks[name] = update_fn - if not graph_capturable: - cls._sensor_eager_tasks.add(name) - cls._sensor_state = state - cls._sensor_state_dirty = True - cls._invalidate_sensor_graph() - - @classmethod - def _unregister_sensor_task(cls, name: str) -> None: - """Remove a scene-query task, ignoring unknown names.""" - if cls._sensor_tasks.pop(name, None) is not None: - cls._sensor_eager_tasks.discard(name) - cls._invalidate_sensor_graph() - - @classmethod - def _update_sensor_tasks(cls, *names: str) -> None: - """Refresh derived state, refit the BVHs, and run the requested scene-query tasks.""" - for name in names: - if name not in cls._sensor_tasks: - raise KeyError(f"Newton sensor task '{name}' is not registered.") - - # Resolve pending FK before entering the graph-capturable sensor pipeline. - state = cls.get_state() - if state is not cls._sensor_state: - cls._sensor_state = state - cls._sensor_state_dirty = True - cls._invalidate_sensor_graph() - run_eagerly = bool(cls._sensor_eager_tasks.intersection(names)) - cfg = PhysicsManager._cfg - use_cuda_graph = ( - not run_eagerly and isinstance(cfg, NewtonCfg) and cfg.use_cuda_graph and "cuda" in PhysicsManager._device - ) - if use_cuda_graph and cls._sensor_graph is None and not cls._sensor_graph_capture_failed: - cls._capture_sensor_graph() - if run_eagerly or cls._sensor_graph is None: - if cls._sensor_state_dirty: - cls._refit_sensor_bvh() - cls._sensor_state_dirty = False - for name in names: - cls._sensor_tasks[name]() - return - - assert cls._sensor_flags_host is not None - assert cls._sensor_flags is not None - cls._sensor_flags_host.fill(0) - cls._sensor_flags_host[0] = int(cls._sensor_state_dirty) - task_names = tuple(name for name in cls._sensor_tasks if name not in cls._sensor_eager_tasks) - for name in names: - cls._sensor_flags_host[1 + task_names.index(name)] = 1 - cls._sensor_flags.assign(cls._sensor_flags_host) - wp.capture_launch(cls._sensor_graph) - cls._sensor_state_dirty = False - - @classmethod - def _mark_sensor_state_dirty(cls) -> None: - """Bind the current state and mark the shape and particle BVHs stale. - - Writes through :class:`NewtonManager` rather than ``cls`` because the - sensor-task registry and its dirty flag are singleton state owned by the - base class (sensors and renderers reach it via ``NewtonManager``). This - method is invoked from the step loop where ``cls`` is the active solver - subclass, so assigning through ``cls`` would shadow the base attribute - and the per-step refit request would never reach the sensor tasks. - """ - if NewtonManager.backend is None: - return - if NewtonManager.backend.state_0 is not NewtonManager._sensor_state: - NewtonManager._sensor_state = NewtonManager.backend.state_0 - NewtonManager._invalidate_sensor_graph() - NewtonManager._sensor_state_dirty = True - - @classmethod - def _refit_sensor_bvh(cls) -> None: - """Refit the model shape and particle BVHs against the current state.""" - if cls.backend is None: - return - - model = cls.backend.model - refit_shapes = model.shape_count > 0 and model.bvh_shapes is not None - refit_particles = model.particle_count > 0 and model.bvh_particles is not None - if not refit_shapes and not refit_particles: - return - - if cls._sensor_state is None: - raise RuntimeError("Refitting Newton sensor BVHs requires an initialized sensor state.") - - if refit_shapes: - model.bvh_refit_shapes(cls._sensor_state) - - if refit_particles: - model.bvh_refit_particles(cls._sensor_state) - - @classmethod - def _invalidate_sensor_graph(cls) -> None: - """Discard captured scene-query graph resources.""" - cls._sensor_graph = None - cls._sensor_flags = None - cls._sensor_flags_host = None - cls._sensor_graph_capture_failed = False - - @classmethod - def _capture_sensor_graph(cls) -> None: - """Capture BVH refit and scene-query tasks into a conditional graph.""" - graph_tasks = tuple( - update_fn for name, update_fn in cls._sensor_tasks.items() if name not in cls._sensor_eager_tasks - ) - cls._sensor_flags = wp.zeros(1 + len(graph_tasks), dtype=wp.int32, device=PhysicsManager._device) - cls._sensor_flags_host = np.zeros(1 + len(graph_tasks), dtype=np.int32) - - def pipeline() -> None: - assert cls._sensor_flags is not None - wp.capture_if(cls._sensor_flags[0:1], cls._refit_sensor_bvh) - for index, update_fn in enumerate(graph_tasks): - wp.capture_if(cls._sensor_flags[index + 1 : index + 2], update_fn) - - try: - cls._sensor_graph = cls._capture_graph(pipeline) - except Exception: - logger.exception("[NewtonManager] sensor CUDA graph capture failed") - cls._sensor_graph = None - if cls._sensor_graph is None: - cls._sensor_flags = None - cls._sensor_flags_host = None - cls._sensor_graph_capture_failed = True - logger.warning("Newton sensor graph capture failed; falling back to eager execution.") - else: - logger.info("Captured Newton sensor graph with %d task(s).", len(graph_tasks)) - @classmethod def get_num_envs(cls) -> int: return cls._num_envs - @classmethod - def _backend_is_newton(cls, scene_data_provider: SceneDataProvider | None = None) -> bool: - """Return ``True`` when the active sim backend is Newton.""" - if scene_data_provider is not None: - return isinstance(scene_data_provider.backend, NewtonSceneDataBackend) - return isinstance(cls.get_scene_data_provider().backend, NewtonSceneDataBackend) - - @classmethod - def _initialize_visualization_model( - cls, cfg: NewtonBackendCfg, geometry_offsets: dict[str, int], _event: Any - ) -> None: - """Acquire the completed clone representation when foreign physics becomes ready.""" - if cls.backend is not None: - return - sim = SimulationContext.instance() - NewtonManager.backend = sim.get_or_create_backend(cfg) - NewtonManager._num_envs = cls.backend.model.num_envs - NewtonManager._transform_mapping = None - NewtonManager._visualization_transforms = TimestampedBuffer() - NewtonManager._geometry_offsets = geometry_offsets - NewtonManager._geometry_timestamp = -1 - cls.update_visualization_state() - NewtonManager._visualization_stop_callback = sim.physics_manager.register_callback( - lambda _payload: NewtonManager.clear(), - PhysicsEvent.STOP, - name="newton_visualization_state", - wrap_weak_ref=False, - ) - @classmethod def get_scene_data_provider(cls) -> SceneDataProvider: """Return the active scene data provider.""" - sim = SimulationContext.instance() - assert sim is not None - return sim.get_scene_data_provider() - - @classmethod - def update_visualization_state(cls, scene_data_provider: SceneDataProvider | None = None) -> None: - """Refresh visualization state for the active sim backend. - - Newton sim backend: no-op — ``_state_0`` is the live, authoritative state - already advanced by :meth:`step` / forward kinematics. - - PhysX / OVPhysX sim backend: bind shared SDP rigid-body transforms to - ``state_0.body_q`` without copying. Consumers must treat this array as - read-only. Deformable points still copy into the shadow ``particle_q`` - buffer, including simulation-to-visual mesh remapping. - - Invoked lazily from :meth:`get_state` so consumers do not need to - coordinate the sync explicitly. - """ - - provider = cls.get_scene_data_provider() if scene_data_provider is None else scene_data_provider - assert provider is not None - - if cls._backend_is_newton(provider): - return - - if cls.backend is None: - return - - if cls.backend.state_0.body_q is not None: - model = cls.backend.model - cached = cls._visualization_transforms - if cached.data is None: - body_paths = cls._resolve_scene_data_body_paths(list(model.body_label), provider.usd_stage) - unique_paths = set(body_paths) - if len(unique_paths) != model.body_count or not unique_paths.issubset(provider.backend.transform_paths): - raise ValueError("Every Newton render body must have one unique SDP transform path.") - cls._transform_mapping = provider.create_mapping(body_paths) - cached.data = SceneDataFormat.Transform() - - if provider.get_transforms(cached.data, mapping=cls._transform_mapping, count=model.body_count): - if cls.backend.state_0.body_q is not cached.data.transforms: - cls.backend.state_0.body_q = cached.data.transforms - cls._invalidate_sensor_graph() - if cached.timestamp != provider.backend.transforms_timestamp: - cls._mark_sensor_state_dirty() - cached.timestamp = provider.backend.transforms_timestamp - - if cls._geometry_offsets: - provider.get_geometry_points(output=cls.backend.state_0.particle_q, offsets=cls._geometry_offsets) - timestamp = provider.backend.geometry_timestamp - if cls._geometry_timestamp != timestamp: - cls._mark_sensor_state_dirty() - cls._geometry_timestamp = timestamp - - @staticmethod - def _resolve_scene_data_body_paths(body_paths: list[str | None], stage) -> list[str | None]: - """Map Newton joint labels to their target rigid-body prim paths.""" - if stage is None: - return body_paths - - from pxr import UsdPhysics - - def _joint_body_path(prim): - joint = UsdPhysics.Joint(prim) - for rel in (joint.GetBody1Rel(), joint.GetBody0Rel()): - for target_path in rel.GetTargets(): - target_prim = stage.GetPrimAtPath(target_path) - if target_prim.IsValid() and target_prim.HasAPI(UsdPhysics.RigidBodyAPI): - return target_path.pathString - return None - - resolved_paths = body_paths.copy() - for index, body_path in enumerate(body_paths): - if body_path is None: - continue - prim = stage.GetPrimAtPath(body_path) - if prim.IsValid() and prim.IsA(UsdPhysics.Joint): - resolved_paths[index] = _joint_body_path(prim) or body_path - return resolved_paths + return SimulationContext.instance().get_scene_data_provider() @classmethod def get_state_1(cls) -> State: diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager_cfg.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager_cfg.py index 081cbb636ffd..3fe3e283206b 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager_cfg.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager_cfg.py @@ -8,6 +8,7 @@ from __future__ import annotations import logging +from collections.abc import Callable from dataclasses import MISSING, field from typing import TYPE_CHECKING, Literal @@ -27,27 +28,24 @@ logger = logging.getLogger(__name__) +@configclass +class NewtonBuilderCfg: + """Share mutable construction data, populated before model allocation and retained across hard resets.""" + + class_type: Callable[[NewtonBuilderCfg], ModelBuilder] | str = "{DIR}.newton_manager:create_newton_builder" + physics_cfg: PhysicsCfg = field(kw_only=True, metadata={"copy": False}) + """Selected physics settings; non-Newton physics requires a render-only Newton representation.""" + + @configclass class NewtonBackendCfg(BackendCfg): - """Native allocation inputs; the populated builder is borrowed without copying.""" + """Allocate model and state from the matching builder; closing them leaves the builder intact.""" class_type: type[NewtonBackend] | str = "{DIR}.newton_manager:NewtonBackend" - builder: ModelBuilder = field(kw_only=True, metadata={"copy": False}) - """Populated clone builder. Reusing this builder and matching settings shares one resource.""" - particle_ranges: dict[str, tuple[int, int]] = field(default_factory=dict) - """Imported point paths to native particle offsets and counts, retained until Newton exposes composed ranges.""" + physics_cfg: PhysicsCfg = field(kw_only=True, metadata={"copy": False}) + """Selected physics settings, also identifying the shared construction builder.""" device: str = MISSING - """Allocation device, such as ``cpu`` or ``cuda:0``.""" - num_envs: int | None = None - """Environment count; None uses the finalized model's world count.""" - gravity: tuple[float, float, float] | None = None - """Gravity [m/s^2]; None preserves the builder's authored gravity.""" - soft_contact_cfg: NewtonSoftContactCfg | None = None - """Optional soft-contact settings applied before state allocation.""" - contact_attributes: tuple[str, ...] = () - """Additional contact attributes requested before state allocation.""" - simulation: bool = True - """Allocate two states and control for physics, or one state for rendering only.""" + """Device on which to allocate the model and native buffers.""" @configclass diff --git a/source/isaaclab_newton/isaaclab_newton/physics/vbd_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/vbd_manager.py index f0923ebe9d35..b6857de8384c 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/vbd_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/vbd_manager.py @@ -7,7 +7,7 @@ from __future__ import annotations -from newton import Model +from newton import Model, ModelBuilder from newton.solvers import SolverVBD from .newton_manager import NewtonManager @@ -18,11 +18,9 @@ class NewtonVBDManager(NewtonManager): """Newton manager specialization for the VBD solver.""" @classmethod - def start_simulation(cls) -> None: - """Color the prebuilt model before starting simulation.""" - if cls._builder is not None: - cls._builder.color(balance_colors=False) - super().start_simulation() + def _prepare_builder_for_finalize(cls, builder: ModelBuilder) -> None: + """Color the completed builder before allocating the model.""" + builder.color(balance_colors=False) @classmethod def _create_solver(cls, model: Model, solver_cfg: VBDSolverCfg) -> SolverVBD: diff --git a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py index 028c9c280d34..5265417d3fca 100644 --- a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py +++ b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py @@ -9,6 +9,7 @@ import logging from dataclasses import dataclass +from functools import partial from typing import TYPE_CHECKING, Any, NoReturn import newton @@ -16,11 +17,11 @@ from isaaclab.renderers import BaseRenderer, RenderBufferKind, RenderBufferSpec from isaaclab.renderers.camera_render_spec import CameraRenderSpec -from isaaclab.scene_data import REQUIRES_STAGE_AND_MODEL +from isaaclab.scene_data import REQUIRES_STAGE_AND_MODEL, SceneDataFormat from isaaclab.sim import SimulationContext from isaaclab.utils.warp.warp_math import replace_background_depth_wp -from ..physics.newton_manager import NewtonManager +from ..physics import NewtonBackendCfg, NewtonQueries from .newton_warp_renderer_cfg import NewtonWarpRendererCfg from .segmentation import NewtonSegmentationMapper, NewtonSegmentationMapping @@ -134,9 +135,7 @@ def __init__( self.camera_rays: wp.array(dtype=wp.vec3f, ndim=4) = None self.camera_transforms: wp.array(dtype=wp.transformf, ndim=2) = None self._intrinsic_status = wp.zeros(1, dtype=wp.int32, device=newton_sensor.model.device) - # Name under which this camera's render launch is registered with the - # Newton sensor manager (set on first render). - self.sensor_task_name: str | None = None + self.graph = None self.outputs = RenderData.CameraOutputs() # Requested depth-family destination views keyed by data-type name. Each view aliases the # caller's output buffer as ``(world_count, 1, H, W)`` float32. @@ -440,18 +439,20 @@ def __init__(self, cfg: NewtonWarpRendererCfg): self._seg_mapper: NewtonSegmentationMapper | None = None sim = SimulationContext.instance() + self.newton_cfg = NewtonBackendCfg(physics_cfg=sim.cfg.physics, device=sim.device) requires_stage, requires_model = REQUIRES_STAGE_AND_MODEL["newton_warp"] sim.requires_usd_stage |= requires_stage sim.requires_newton_model |= requires_model def initialize(self) -> None: - """Post-physics setup: read the built Newton model and construct the sensor.""" - model = NewtonManager.get_model() - if model is None: - raise RuntimeError("NewtonWarpRenderer requires a clone-built model before initialization.") + """Acquire the clone-built native resource and bind its SDP layout.""" + sim = SimulationContext.instance() + self.backend = sim.get_or_create_backend(self.newton_cfg) + self._scene_data_provider = sim.get_scene_data_provider() + self._transform_mapping = self._scene_data_provider.create_mapping(list(self.backend.model.body_label)) self.newton_sensor = newton.sensors.SensorTiledCamera( - model, + self.backend.model, default_render_config=newton.sensors.SensorTiledCamera.RenderConfig( enable_textures=self.cfg.enable_textures, enable_shadows=self.cfg.enable_shadows, @@ -479,7 +480,7 @@ def initialize(self) -> None: @property def visual_material_writer(self): """Return the shared Newton model color-writer factory.""" - return NewtonManager.create_visual_material_writer + return self.backend.create_visual_material_writer def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: """Publish the per-output layout this Newton Warp backend writes. @@ -586,17 +587,20 @@ def update_camera_intrinsics(self, render_data: RenderData, intrinsics: wp.array def render(self, render_data: RenderData): """Render and write to output buffers. See :meth:`~isaaclab.renderers.base_renderer.BaseRenderer.render`.""" - if render_data.sensor_task_name is None: - render_data.sensor_task_name = f"newton_warp_render:{id(render_data)}" - tri_indices = self.newton_sensor.model.tri_indices - # Warp mesh refits allocate graph nodes and are not supported inside a conditional graph body. - graph_capturable = tri_indices is None or tri_indices.shape[0] == 0 - NewtonManager._register_sensor_task( - render_data.sensor_task_name, - lambda: self._launch_render(render_data), - graph_capturable=graph_capturable, - ) - NewtonManager._update_sensor_tasks(render_data.sensor_task_name) + backend, provider = self.backend, self._scene_data_provider + poses = SceneDataFormat.Transform() + if provider.get_transforms(poses, mapping=self._transform_mapping, count=backend.model.body_count): + backend.state_0.body_q = poses.transforms + if backend.geometry_offsets: + provider.get_geometry_points(output=backend.state_0.particle_q, offsets=backend.geometry_offsets) + render_data.graph = NewtonQueries.run_query( + self.backend, + provider.backend.transforms_timestamp + provider.backend.geometry_timestamp, + partial(self._launch_render, render_data), + render_data.graph, + # Native triangle-mesh updates read back indices and may allocate after pointer swaps. + use_cuda_graph=self.cfg.use_cuda_graph and backend.model.tri_count == 0, + ) # Post-render PPISP: HDR scene-linear → LDR RGBA. Source/destination # tensors were bound once in ``set_outputs``. @@ -626,7 +630,7 @@ def _launch_render(self, render_data: RenderData) -> None: ) self.newton_sensor.update( - NewtonManager.get_state_0(), + self.backend.state_0, render_data.camera_transforms, render_data.camera_rays, color_image=render_data.outputs.color_image, @@ -669,10 +673,12 @@ def read_output(self, render_data: RenderData, camera_data: CameraData) -> None: camera_data.info[output_name] = info def cleanup(self, render_data: RenderData | None): - """Release resources and drop the camera's sensor task. + """Release the camera's buffers and captured query. See :meth:`~isaaclab.renderers.base_renderer.BaseRenderer.cleanup`.""" if render_data: - if render_data.sensor_task_name is not None: - NewtonManager._unregister_sensor_task(render_data.sensor_task_name) - render_data.sensor_task_name = None - render_data.sensor = None + render_data.graph = None + render_data.newton_sensor = None + + def close(self) -> None: + """Release borrowed native handles and SDP bindings after camera cleanup.""" + self.newton_sensor = self.backend = self._scene_data_provider = self._transform_mapping = None diff --git a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py index 01c2cc32ae5f..cb828f3cee92 100644 --- a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py +++ b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer_cfg.py @@ -31,6 +31,9 @@ class NewtonWarpRendererCfg(RendererCfg): cloning_contexts: tuple[type | str, ...] = ("isaaclab_newton.cloner:NewtonReplicateContext",) + use_cuda_graph: bool = True + """Capture camera queries independently of the physics solver's graph setting.""" + enable_textures: bool = True """Enable texture-mapped rendering for meshes.""" diff --git a/source/isaaclab_newton/isaaclab_newton/sensors/frame_transformer/frame_transformer.py b/source/isaaclab_newton/isaaclab_newton/sensors/frame_transformer/frame_transformer.py index d8a88bbe0102..2ba3cb5f345b 100644 --- a/source/isaaclab_newton/isaaclab_newton/sensors/frame_transformer/frame_transformer.py +++ b/source/isaaclab_newton/isaaclab_newton/sensors/frame_transformer/frame_transformer.py @@ -143,7 +143,7 @@ def _initialize_impl(self): source_indices, target_per_world, self._target_frame_body_names, - NewtonManager._builder.shape_label, + NewtonManager.backend.model.shape_label, world_origin_idx, num_envs, ) diff --git a/source/isaaclab_newton/isaaclab_newton/sensors/ray_caster/newton_raycast_sensor.py b/source/isaaclab_newton/isaaclab_newton/sensors/ray_caster/newton_raycast_sensor.py index a30eebbe4c87..30b3f49803af 100644 --- a/source/isaaclab_newton/isaaclab_newton/sensors/ray_caster/newton_raycast_sensor.py +++ b/source/isaaclab_newton/isaaclab_newton/sensors/ray_caster/newton_raycast_sensor.py @@ -15,11 +15,12 @@ import isaaclab.sim as sim_utils from isaaclab import cloner from isaaclab.cloner.cloner_cfg import DEFAULT_ENV_TEMPLATE +from isaaclab.scene_data import SceneDataFormat from isaaclab.sensors.ray_caster.base_ray_caster import BaseRayCaster from isaaclab.sensors.ray_caster.kernels import ALIGNMENT_BASE, update_ray_caster_kernel from isaaclab.utils.warp import ProxyArray -from isaaclab_newton.physics import NewtonManager +from isaaclab_newton.physics import NewtonBackendCfg, NewtonBuilderCfg, NewtonManager, NewtonQueries from .newton_raycast_sensor_cfg import NewtonRaycastSensorCfg from .newton_raycast_sensor_data import NewtonRaycastSensorData @@ -85,7 +86,7 @@ def _identity_offsets(count: int, device: str) -> tuple[wp.array, wp.array]: class _NewtonRayCasterPoseMixin: - """Register Newton sensor sites and update ray poses from the Newton manager state.""" + """Register Newton sensor sites and update ray poses from a shared backend.""" @property def count(self: Any) -> int: @@ -96,6 +97,8 @@ def __init__(self: Any, cfg): """Register the sensor site before Newton model finalization.""" super().__init__(cfg) # pyright: ignore[reportCallIssue] self._sensor_site_labels = self._register_sites_for_expr(self.cfg.prim_path) + sim = sim_utils.SimulationContext.instance() + self.newton_cfg = NewtonBackendCfg(physics_cfg=sim.cfg.physics, device=sim.device) def _register_sites_for_expr(self, prim_expr: str) -> list[str]: """Register Newton sites for a prim expression.""" @@ -125,6 +128,10 @@ def _register_sites_for_expr(self, prim_expr: str) -> list[str]: def _initialize_pose_tracking(self: Any) -> None: """Resolve registered site labels and allocate pose buffers.""" + sim = sim_utils.SimulationContext.instance() + self.backend = sim.get_or_create_backend(self.newton_cfg) + self._scene_data_provider = sim.get_scene_data_provider() + self._transform_mapping = self._scene_data_provider.create_mapping(list(self.backend.model.body_label)) site_indices = self._resolve_site_indices(self._sensor_site_labels, self.cfg.prim_path, self._num_envs) self._view = self self._view_count = len(site_indices) @@ -166,7 +173,12 @@ def _update_ray_infos(self: Any, env_mask: wp.array) -> None: def get_world_poses(self: Any, indices=None) -> tuple[ProxyArray, ProxyArray]: """Return current world poses after resolving pending FK.""" - NewtonManager.get_state() + backend = self.backend + poses = SceneDataFormat.Transform() + if self._scene_data_provider.get_transforms( + poses, mapping=self._transform_mapping, count=backend.model.body_count + ): + backend.state_0.body_q = poses.transforms self._update_newton_site_transforms( self._sensor_site_indices, self._newton_pose_w, self._newton_pos_w.warp, self._newton_quat_w.warp ) @@ -192,9 +204,8 @@ def _update_newton_site_transforms( pos_buf: wp.array, quat_buf: wp.array, ) -> None: - """Update site transforms from manager state already refreshed by the caller.""" - model = NewtonManager.get_model() - state = NewtonManager.get_state_0() + """Update site transforms from the shared state already refreshed through SDP.""" + model, state = self.backend.model, self.backend.state_0 wp.launch( _newton_site_world_poses_kernel, dim=site_indices.shape[0], @@ -262,9 +273,8 @@ class NewtonRaycastSensor(_NewtonRayCasterPoseMixin, BaseRayCaster): Rays are cast with :func:`newton.intersect_ray` against every collision shape in the sensor's own world plus the global world (e.g. terrain), so dynamic bodies are hit without configuring target meshes. The full update - (sensor pose, ray transform, BVH query, hit resolve) is registered as a - task with :class:`~isaaclab_newton.physics.NewtonManager`, which owns - the shared BVH refit and sensor execution graph. + (sensor pose, ray transform, BVH query, hit resolve) uses a sensor-owned + CUDA graph and the registry-owned model's shared acceleration structures. """ cfg: NewtonRaycastSensorCfg @@ -273,10 +283,12 @@ class NewtonRaycastSensor(_NewtonRayCasterPoseMixin, BaseRayCaster): def __init__(self, cfg: NewtonRaycastSensorCfg): if cfg.max_distance <= 0.0: raise ValueError(f"max_distance must be positive, received {cfg.max_distance}.") - NewtonManager._sensor_bvh_shape_flags |= newton.ShapeFlags.COLLIDE_SHAPES super().__init__(cfg) + sim = sim_utils.SimulationContext.instance() + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics)) + builder.default_bvh_cfg.shape_flags |= newton.ShapeFlags.COLLIDE_SHAPES self._data = NewtonRaycastSensorData() - self._sensor_task_name: str | None = None + self._graph = None @property def data(self) -> NewtonRaycastSensorData: @@ -326,14 +338,13 @@ def _initialize_impl(self) -> None: self._hit_dist = wp.empty(ray_count, dtype=wp.float32, device=self._device) self._hit_normal = wp.empty(ray_count, dtype=wp.vec3f, device=self._device) - self._sensor_task_name = f"newton_raycast:{self.cfg.prim_path}:{id(self)}" - NewtonManager._register_sensor_task(self._sensor_task_name, self._launch_raycast) + self._graph = None def _launch_raycast(self) -> None: """Sensor pose + ray transform + BVH query + hit resolve (graph-capturable).""" self._update_ray_infos(self._is_outdated) newton.intersect_ray( - NewtonManager.get_model(), + self.backend.model, ray_origins=self._ray_starts_w_flat, ray_directions=self._ray_directions_w_flat, ray_worlds=self._ray_worlds, @@ -365,13 +376,22 @@ def _update_buffers_impl(self, env_mask: wp.array) -> None: # The captured graph is bound to ``_is_outdated``; mirror any other mask into it. if env_mask.ptr != self._is_outdated.ptr: wp.copy(self._is_outdated, env_mask) - assert self._sensor_task_name is not None - NewtonManager._update_sensor_tasks(self._sensor_task_name) + backend, provider = self.backend, self._scene_data_provider + poses = SceneDataFormat.Transform() + if provider.get_transforms(poses, mapping=self._transform_mapping, count=backend.model.body_count): + backend.state_0.body_q = poses.transforms + if backend.geometry_offsets: + provider.get_geometry_points(output=backend.state_0.particle_q, offsets=backend.geometry_offsets) + self._graph = NewtonQueries.run_query( + self.backend, + provider.backend.transforms_timestamp + provider.backend.geometry_timestamp, + self._launch_raycast, + self._graph, + use_cuda_graph=self.cfg.use_cuda_graph, + ) def _invalidate_initialize_callback(self, event) -> None: - if self._sensor_task_name is not None: - NewtonManager._unregister_sensor_task(self._sensor_task_name) - self._sensor_task_name = None + self._graph = None super()._invalidate_initialize_callback(event) diff --git a/source/isaaclab_newton/isaaclab_newton/sim/views/newton_site_frame_view.py b/source/isaaclab_newton/isaaclab_newton/sim/views/newton_site_frame_view.py index b6d824eba7a1..3f38d2151ae6 100644 --- a/source/isaaclab_newton/isaaclab_newton/sim/views/newton_site_frame_view.py +++ b/source/isaaclab_newton/isaaclab_newton/sim/views/newton_site_frame_view.py @@ -562,7 +562,10 @@ def _mirror_to_fabric(self) -> None: # Bodies sync at render cadence, so after a ``render=False`` step the local derivation below # would read a stale parent. No-op when clean. sim = sim_utils.SimulationContext.instance() - sim.get_or_create_backend(sim.fabric_cfg).update_transforms(sim.get_scene_data_provider()) + from isaaclab_physx.renderers.fabric import FabricBackendCfg # noqa: PLC0415 - requires Kit + + fabric = sim.get_or_create_backend(FabricBackendCfg(stage=sim.stage, device=sim.device)) + fabric.update_transforms(sim.get_scene_data_provider()) count = self._fabric_sel.count pos_ta, quat_ta = self._get_world_poses_impl(None) diff --git a/source/isaaclab_newton/test/assets/test_articulation.py b/source/isaaclab_newton/test/assets/test_articulation.py index ec141b7b3e3e..95bbfc051039 100644 --- a/source/isaaclab_newton/test/assets/test_articulation.py +++ b/source/isaaclab_newton/test/assets/test_articulation.py @@ -40,9 +40,8 @@ import warp as wp from isaaclab_newton.assets import Articulation from isaaclab_newton.assets.articulation.actuator_control import NewtonActuatorControl -from isaaclab_newton.assets.articulation.articulation import _configure_builder_joint_target_modes from isaaclab_newton.assets.articulation.articulation_data import ArticulationData -from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg +from isaaclab_newton.physics import MJWarpSolverCfg, NewtonBuilderCfg, NewtonCfg from isaaclab_newton.physics import NewtonManager as SimulationManager from isaaclab_physx.sim.schemas import PhysxJointCfg from newton import JointTargetMode, JointType, ModelBuilder, ModelFlags, ShapeFlags @@ -569,10 +568,14 @@ def sim(request): def _make_target_mode_builder( - joint_names: list[str], target_modes: list[JointTargetMode], stiffness: list[float], damping: list[float] + monkeypatch, joint_names: list[str], modes: list[JointTargetMode], stiffness: list[float], damping: list[float] ) -> ModelBuilder: """Build a zero-gain articulated model builder for target-mode tests.""" - builder = ModelBuilder() + sim = object.__new__(sim_utils.SimulationContext) + sim.cfg = SimpleNamespace(physics=NewtonCfg()) + sim._backend_registry = [] + monkeypatch.setattr(sim_utils.SimulationContext, "instance", lambda: sim) + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics)) inertia = wp.mat33(1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0) parent = -1 joint_ids = [] @@ -590,7 +593,7 @@ def _make_target_mode_builder( parent = link builder.add_articulation(joint_ids, label="/World/Env_0/Robot") builder.articulation_label = ["/World/Env_0/Robot"] - builder.joint_target_mode = [int(mode) for mode in target_modes] + builder.joint_target_mode = [int(mode) for mode in modes] builder.joint_target_ke = stiffness builder.joint_target_kd = damping return builder @@ -673,7 +676,7 @@ def test_prepare_native_actuators_activates_only_explicit_groups(monkeypatch, ac ], ) def test_actuator_cfg_sets_newton_target_mode_before_solver_init( - actuator_cfg, expected_mode, expected_actuator_indices + monkeypatch, actuator_cfg, expected_mode, expected_actuator_indices ): """Resolve configured modes before finalization constructs MuJoCo actuators.""" articulation_cfg = ArticulationCfg( @@ -682,9 +685,9 @@ def test_actuator_cfg_sets_newton_target_mode_before_solver_init( actuators={"joint": actuator_cfg}, ) builder = _make_target_mode_builder( - ["left_joint", "right_joint"], [JointTargetMode.NONE, JointTargetMode.NONE], [0.0, 0.0], [0.0, 0.0] + monkeypatch, ["left_joint", "right_joint"], [JointTargetMode.NONE, JointTargetMode.NONE], [0.0, 0.0], [0.0, 0.0] ) - _configure_builder_joint_target_modes(builder, articulation_cfg) + Articulation._configure_joint_target_modes(SimpleNamespace(cfg=articulation_cfg), None) model = builder.finalize(device="cpu") solver = SolverMuJoCo(model, use_mujoco_cpu=True) assert model.joint_target_mode.numpy().tolist() == [int(expected_mode), int(expected_mode)] @@ -693,16 +696,16 @@ def test_actuator_cfg_sets_newton_target_mode_before_solver_init( ) == expected_actuator_indices -def test_actuator_cfg_matches_explicit_descendant_articulation_root(): +def test_actuator_cfg_matches_explicit_descendant_articulation_root(monkeypatch): """Match target modes against an explicitly configured descendant articulation root.""" articulation_cfg = ArticulationCfg( prim_path="/World/Env_[^/]*/Robot", articulation_root_prim_path="/base", actuators={"joint": ImplicitActuatorCfg(joint_names_expr=[".*"], stiffness=10.0, damping=0.0)}, ) - builder = _make_target_mode_builder(["joint"], [JointTargetMode.NONE], [0.0], [0.0]) + builder = _make_target_mode_builder(monkeypatch, ["joint"], [JointTargetMode.NONE], [0.0], [0.0]) builder.articulation_label = ["/World/Env_0/Robot/base"] - _configure_builder_joint_target_modes(builder, articulation_cfg) + Articulation._configure_joint_target_modes(SimpleNamespace(cfg=articulation_cfg), None) assert builder.joint_target_mode == [int(JointTargetMode.POSITION)] @@ -716,39 +719,39 @@ def test_actuator_cfg_matches_clone_plan_root_expr(monkeypatch): "isaaclab_newton.assets.articulation.articulation.resolve_matching_prims_from_source", lambda *_args, **_kwargs: [(None, "/World/envs/env_[^/]+/Robot/base")], ) - builder = _make_target_mode_builder(["joint"], [JointTargetMode.NONE], [0.0], [0.0]) + builder = _make_target_mode_builder(monkeypatch, ["joint"], [JointTargetMode.NONE], [0.0], [0.0]) builder.articulation_label = ["/World/envs/env_0/Robot/base"] - _configure_builder_joint_target_modes(builder, articulation_cfg) + Articulation._configure_joint_target_modes(SimpleNamespace(cfg=articulation_cfg), None) assert builder.joint_target_mode == [int(JointTargetMode.POSITION)] @pytest.mark.parametrize("joint_type", [JointType.FREE, JointType.FIXED]) -def test_actuator_cfg_leaves_excluded_joint_types_imported(joint_type): +def test_actuator_cfg_leaves_excluded_joint_types_imported(monkeypatch, joint_type): """Leave target modes for free and fixed joints unchanged.""" articulation_cfg = ArticulationCfg( prim_path="/World/Env_[^/]*/Robot", articulation_root_prim_path="", actuators={"joint": ImplicitActuatorCfg(joint_names_expr=[".*"], stiffness=10.0, damping=0.0)}, ) - builder = _make_target_mode_builder(["joint"], [JointTargetMode.NONE], [0.0], [0.0]) + builder = _make_target_mode_builder(monkeypatch, ["joint"], [JointTargetMode.NONE], [0.0], [0.0]) builder.joint_type[0] = joint_type - _configure_builder_joint_target_modes(builder, articulation_cfg) + Articulation._configure_joint_target_modes(SimpleNamespace(cfg=articulation_cfg), None) assert builder.joint_target_mode == [int(JointTargetMode.NONE)] -def test_actuator_cfg_uses_imported_gain_for_none_stiffness(): +def test_actuator_cfg_uses_imported_gain_for_none_stiffness(monkeypatch): """Retain the imported stiffness when an implicit actuator config leaves it unset.""" articulation_cfg = ArticulationCfg( prim_path="/World/Env_[^/]*/Robot", articulation_root_prim_path="", actuators={"joint": ImplicitActuatorCfg(joint_names_expr=[".*"], stiffness=None, damping=0.0)}, ) - builder = _make_target_mode_builder(["joint"], [JointTargetMode.EFFORT], [10.0], [0.0]) - _configure_builder_joint_target_modes(builder, articulation_cfg) + builder = _make_target_mode_builder(monkeypatch, ["joint"], [JointTargetMode.EFFORT], [10.0], [0.0]) + Articulation._configure_joint_target_modes(SimpleNamespace(cfg=articulation_cfg), None) assert builder.joint_target_mode == [int(JointTargetMode.POSITION)] -def test_actuator_cfg_leaves_unconfigured_newton_target_modes_imported(): +def test_actuator_cfg_leaves_unconfigured_newton_target_modes_imported(monkeypatch): """Leave target modes for DOFs outside an actuator group unchanged.""" subset_cfg = ArticulationCfg( prim_path="/World/Env_[^/]*/Robot", @@ -758,12 +761,13 @@ def test_actuator_cfg_leaves_unconfigured_newton_target_modes_imported(): }, ) builder = _make_target_mode_builder( + monkeypatch, ["left_shoulder", "right_shoulder"], [JointTargetMode.NONE, JointTargetMode.VELOCITY], [0.0, 0.0], [0.0, 2.0], ) - _configure_builder_joint_target_modes(builder, subset_cfg) + Articulation._configure_joint_target_modes(SimpleNamespace(cfg=subset_cfg), None) assert builder.joint_target_mode == [int(JointTargetMode.POSITION), int(JointTargetMode.VELOCITY)] @@ -774,7 +778,7 @@ def test_actuator_cfg_leaves_unconfigured_newton_target_modes_imported(): ({"left_joint": 10.0}, {"right_joint": 2.0}, [JointTargetMode.POSITION, JointTargetMode.VELOCITY]), ], ) -def test_actuator_cfg_aligns_partial_dictionary_gains_by_joint_name(stiffness, damping, expected_modes): +def test_actuator_cfg_aligns_partial_dictionary_gains_by_joint_name(monkeypatch, stiffness, damping, expected_modes): """Resolve sparse stiffness and damping dictionaries independently by joint name.""" articulation_cfg = ArticulationCfg( prim_path="/World/Env_[^/]*/Robot", @@ -782,9 +786,9 @@ def test_actuator_cfg_aligns_partial_dictionary_gains_by_joint_name(stiffness, d actuators={"joint": ImplicitActuatorCfg(joint_names_expr=[".*"], stiffness=stiffness, damping=damping)}, ) builder = _make_target_mode_builder( - ["left_joint", "right_joint"], [JointTargetMode.NONE, JointTargetMode.NONE], [0.0, 0.0], [0.0, 0.0] + monkeypatch, ["left_joint", "right_joint"], [JointTargetMode.NONE, JointTargetMode.NONE], [0.0, 0.0], [0.0, 0.0] ) - _configure_builder_joint_target_modes(builder, articulation_cfg) + Articulation._configure_joint_target_modes(SimpleNamespace(cfg=articulation_cfg), None) assert builder.joint_target_mode == [int(mode) for mode in expected_modes] diff --git a/source/isaaclab_newton/test/assets/test_deformable_object.py b/source/isaaclab_newton/test/assets/test_deformable_object.py index f8495a7b63fc..f109014a7831 100644 --- a/source/isaaclab_newton/test/assets/test_deformable_object.py +++ b/source/isaaclab_newton/test/assets/test_deformable_object.py @@ -21,7 +21,7 @@ import torch import warp as wp from flaky import flaky -from isaaclab_newton.physics import NewtonCfg, NewtonManager, VBDSolverCfg +from isaaclab_newton.physics import NewtonBuilderCfg, NewtonCfg, NewtonManager, VBDSolverCfg from isaaclab_newton.sim.schemas import NewtonDeformableBodyPropertiesCfg from isaaclab_newton.sim.spawners.materials import ( NewtonDeformableBodyMaterialCfg, @@ -235,7 +235,8 @@ def test_surface_initialization_and_freefall(sim): """Test initialization and stepping for surface deformable objects.""" num_cloths = 2 cloth_object = generate_cloth_scene(num_cloths=num_cloths, height=5.0) - rest_angles = np.asarray(NewtonManager._builder.edge_rest_angle, dtype=np.float32) + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics)) + rest_angles = np.asarray(builder.edge_rest_angle, dtype=np.float32) assert np.any(np.abs(rest_angles) > 0.1) sim.reset() diff --git a/source/isaaclab_newton/test/cloner/test_newton_builder_world_hook.py b/source/isaaclab_newton/test/cloner/test_newton_builder_world_hook.py index 8a2c36745448..bdaa4c7904b6 100644 --- a/source/isaaclab_newton/test/cloner/test_newton_builder_world_hook.py +++ b/source/isaaclab_newton/test/cloner/test_newton_builder_world_hook.py @@ -21,7 +21,7 @@ from isaaclab.assets import AssetBaseCfg from isaaclab.cloner import make_clone_plan -from isaaclab.sim import SimulationCfg, SpawnerCfg, build_simulation_context +from isaaclab.sim import SimulationCfg, SimulationContext, SpawnerCfg, build_simulation_context replicate_module = importlib.import_module("isaaclab_newton.cloner.replicate") @@ -117,24 +117,22 @@ def import_usd(builder, *args, **kwargs): _get_usd_import_schema_resolvers=NewtonManager._get_usd_import_schema_resolvers, _inject_terrain_heightfields=mock.Mock(return_value=[]), ) - monkeypatch.setattr( - replicate_module.PhysicsManager, - "_sim", - SimpleNamespace( - physics_manager=manager, - device="cpu", - cfg=SimpleNamespace( - physics=NewtonCfg(load_visual_shapes=load_visual_shapes), physics_prim_path="/physicsScene" - ), - is_rendering=is_rendering, - can_render_rgb_array=lambda: rgb_array, - visual_shapes_required=visual_shapes_required, + sim = SimpleNamespace( + physics_manager=manager, + device="cpu", + cfg=SimpleNamespace( + physics=NewtonCfg(load_visual_shapes=load_visual_shapes), physics_prim_path="/physicsScene", device="cpu" ), + is_rendering=is_rendering, + can_render_rgb_array=lambda: rgb_array, + visual_shapes_required=visual_shapes_required, + _backend_registry=[], ) + sim.get_or_create_backend = lambda cfg: SimulationContext.get_or_create_backend(sim, cfg) + monkeypatch.setattr(replicate_module.PhysicsManager, "_sim", sim) monkeypatch.setattr(NewtonManager, "_scene_data_backend", NewtonSceneDataBackend()) monkeypatch.setattr(replicate_module.NewtonManager, "_cl_inject_sites", mock.Mock(return_value=({}, {}, {}))) monkeypatch.setattr(NewtonManager, "_per_world_builder_hooks", ()) - monkeypatch.setattr(NewtonManager, "_builder", None) monkeypatch.setattr(NewtonManager, "_cl_site_index_map", {}) monkeypatch.setattr(NewtonManager, "_world_xforms", None) monkeypatch.setattr(NewtonManager, "_cl_protos", {}) diff --git a/source/isaaclab_newton/test/cloner/test_rename_builder_labels.py b/source/isaaclab_newton/test/cloner/test_rename_builder_labels.py index 00f9ff9fbc3f..387f9482a9e6 100644 --- a/source/isaaclab_newton/test/cloner/test_rename_builder_labels.py +++ b/source/isaaclab_newton/test/cloner/test_rename_builder_labels.py @@ -16,6 +16,7 @@ from isaaclab_newton.cloner import newton_clone_utils as newton_clone_utils_module from isaaclab_newton.cloner import replicate as replicate_module from isaaclab_newton.cloner.newton_clone_utils import replicate_builder_mapping +from isaaclab_newton.physics import NewtonBackendCfg from pxr import Sdf, Usd, UsdGeom, UsdPhysics, UsdShade @@ -23,7 +24,7 @@ from isaaclab.cloner import ClonePlan, PrototypeWorldTopology, make_clone_plan from isaaclab.cloner import path as cloner_path from isaaclab.sensors import SensorBaseCfg -from isaaclab.sim import SpawnerCfg +from isaaclab.sim import SimulationContext, SpawnerCfg from isaaclab.sim.schemas import define_deformable_curve_properties @@ -102,12 +103,17 @@ def test_local_and_env_root_sites_keep_indices_labels_and_world_positions(self): class TestVisualizationClonePlan(unittest.TestCase): def setUp(self): - self.sim = SimpleNamespace( - cfg=SimpleNamespace(physics=object()), - device="cpu", - stage=None, - physics_manager=SimpleNamespace(register_callback=mock.Mock()), - ) + self.sim = object.__new__(SimulationContext) + self.sim.cfg = SimpleNamespace(physics=object(), device="cpu") + self.sim.physics_manager = SimpleNamespace(get_device=lambda: "cpu") + self.sim.stage, self.sim._backend_registry = None, [] + patch = mock.patch.object(SimulationContext, "instance", return_value=self.sim) + patch.start() + self.addCleanup(patch.stop) + + def tearDown(self): + for _, resource in tuple(self.sim._backend_registry): + self.sim.close_backend(resource) @staticmethod def _define_xform(stage, path, translation=None): @@ -131,12 +137,16 @@ def test_visualization_builder_imports_only_declared_global_roots(self): self.assertEqual(builder.body_label, ["/World/Declared"]) self.assertIsNone(stage_info) self.assertEqual(site_index_map, {}) + for _, resource in tuple(self.sim._backend_registry): + self.sim.close_backend(resource) # A non-cloning sensor must not exclude the body selected from its owner's subtree. cfgs = AssetBaseCfg(prim_path="/World"), SensorBaseCfg(prim_path="/World/Declared") plan = make_clone_plan(cfgs, ((),), 1, shared_assets=(0, 1)) builder, _, _ = NewtonReplicateContext(self.sim).replicate(plan, (0,)) self.assertCountEqual(builder.body_label, ["/World/Declared", "/World/Undeclared", "/World/Excluded"]) + for _, resource in tuple(self.sim._backend_registry): + self.sim.close_backend(resource) assets = ( AssetBaseCfg(prim_path="/Copies/env_[^/]+/Body", spawn=SpawnerCfg(spawn_path="/World/Declared")), @@ -158,6 +168,8 @@ def test_visualization_builder_imports_only_declared_global_roots(self): np.testing.assert_allclose(source_position, np.zeros(3) if positions is None else positions[0]) offset = np.zeros(3) if positions is None else positions[1] - positions[0] np.testing.assert_allclose(target_position - source_position, offset) + for _, resource in tuple(self.sim._backend_registry): + self.sim.close_backend(resource) # The same definition can also be shared and appear twice in each replicated world. plan = make_clone_plan((AssetBaseCfg(prim_path="/World/Declared"),), ((0, 0),), 2, shared_assets=(0,)) @@ -169,7 +181,7 @@ def test_visualization_builder_imports_only_declared_global_roots(self): self.assertEqual(builder.body_world, [-1, 0, 0, 1, 1]) self.assertEqual(len(set(builder.body_label)), 5) - def test_cable_import_binds_only_supported_native_instances_without_destination_prims(self): + def test_cable_import_keeps_physics_bindings_without_destination_prims(self): stage = self.sim.stage = Usd.Stage.CreateInMemory() source = "/Scene/copy_7/Rope" shared = ("/Scene/SharedRope", "/Scene/PeriodicRope", "/Scene/MultiRope", "/Scene/CubicRope", "/Scene/OnePoint") @@ -195,14 +207,13 @@ def test_cable_import_binds_only_supported_native_instances_without_destination_ ) assets += tuple(AssetBaseCfg(prim_path=path) for path in shared) plan = make_clone_plan(assets, ((0, 1),), 2, shared_assets=range(2, len(assets)), env_template="/Scene/copy_{}") - builder, _, _ = NewtonReplicateContext(self.sim).replicate(plan, (0, *range(2, len(shared) + 2))) - model = builder.finalize(device="cpu") - bindings = replicate_module.NewtonManager._cable_bindings - self.assertEqual(set(bindings), {"/Scene/SharedRope", "/Scene/copy_0/Rope", "/Scene/copy_1/Rope"}) - for path, shape_ids in bindings.items(): - self.assertEqual( - [model.shape_label[index] for index in shape_ids], [f"{path}_edge_capsule_{i}" for i in range(2)] - ) + bindings = {"/PhysicsOwned": [0]} + with mock.patch.object(replicate_module.NewtonManager, "_cable_bindings", bindings): + builder, _, _ = NewtonReplicateContext(self.sim).replicate(plan, (0, *range(2, len(shared) + 2))) + self.assertIs(replicate_module.NewtonManager._cable_bindings, bindings) + for path in ("/Scene/SharedRope", "/Scene/copy_0/Rope", "/Scene/copy_1/Rope"): + self.assertTrue(all(f"{path}_edge_capsule_{index}" in builder.shape_label for index in range(2))) + self.assertFalse(any("OtherRope" in path for path in builder.shape_label)) self.assertFalse(stage.GetPrimAtPath("/Scene/copy_1/Rope")) def test_visualization_builder_disables_collision_pairs(self): @@ -235,6 +246,9 @@ def test_visualization_builder_disables_collision_pairs(self): self.assertEqual(model.shape_count, 4) self.assertEqual(len(model.shape_collision_filter_pairs), 0) + self.assertEqual( + model.body_label, [f"/World/envs/env_{env}/Robot/{body}" for env in range(2) for body in ("A", "B")] + ) self.assertEqual(model.shape_contact_pair_count, 0) def test_visualization_builder_uses_clone_plan_sources_and_rewrites_labels(self): @@ -315,6 +329,8 @@ def test_render_deformables_import_once_then_follow_native_replication(self): quaternions = np.asarray([[0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 2**-0.5, 2**-0.5]], dtype=np.float32) for positions in (np.array([[10, 0, 0], [20, 0, 0], [30, 0, 0]], dtype=np.float32), None): with self.subTest(positions=positions): + for _, resource in tuple(self.sim._backend_registry): + self.sim.close_backend(resource) options = dict(shared_assets=(3,), env_template="/Scene/copy_{}") options.update(clone_strategy=lambda _weights, _count: np.array([0, 1, 0]), positions=positions) plan = make_clone_plan(assets, ((0,), (1, 1, 2)), 3, **options) @@ -324,9 +340,11 @@ def test_render_deformables_import_once_then_follow_native_replication(self): options = dict(plan=plan, asset_prototype_ids=range(4)) options.update(positions=positions, quaternions=quaternions) builder, _, _ = replicate_module._replicate_newton(stage, env_ids, self.sim, **options) + cfg = NewtonBackendCfg(physics_cfg=self.sim.cfg.physics, device=self.sim.device) + backend = self.sim.get_or_create_backend(cfg) + offsets = backend.geometry_offsets self.assertEqual(add_cloth.call_count, 3) # Three prototypes, not five destination meshes. np.testing.assert_array_equal(np.bincount(np.asarray(builder.particle_world) + 1), [3, 3, 16, 3]) - offsets = self.sim.physics_manager.register_callback.call_args.args[0].args[1] expected = {"/Shared/sim": vertices[:3]} origins = np.zeros((3, 3)) if positions is None else positions for world, env_id in enumerate(env_ids): @@ -341,7 +359,8 @@ def test_render_deformables_import_once_then_follow_native_replication(self): self.assertEqual(set(offsets), set(expected)) for path, points in expected.items(): start = offsets[path] - np.testing.assert_allclose(builder.particle_q[start : start + len(points)], points, atol=1e-5) + native = backend.state_0.particle_q.numpy()[start : start + len(points)] + np.testing.assert_allclose(native, points, atol=1e-5) self.assertFalse(stage.GetPrimAtPath("/Scene/copy_12")) diff --git a/source/isaaclab_newton/test/physics/test_newton_fabric_body_sync.py b/source/isaaclab_newton/test/physics/test_newton_fabric_body_sync.py index a62d6e366457..943c81481bbb 100644 --- a/source/isaaclab_newton/test/physics/test_newton_fabric_body_sync.py +++ b/source/isaaclab_newton/test/physics/test_newton_fabric_body_sync.py @@ -301,7 +301,8 @@ def test_root_pose_sync_preserves_authored_scale(device, renderer_cfg): sim.reset() scene.reset() - sim.get_or_create_backend(sim.fabric_cfg).bind_transforms(sim.get_scene_data_provider()) + fabric = sim.get_or_create_backend(FabricBackendCfg(stage=sim.stage, device=sim.device)) + fabric.bind_transforms(sim.get_scene_data_provider()) _render(sim, scene) torch.testing.assert_close(_fabric_scale(body_path), authored_scale, rtol=0.0, atol=1.0e-5) @@ -314,7 +315,7 @@ def test_root_pose_sync_preserves_authored_scale(device, renderer_cfg): scene["cube"].write_root_link_pose_to_sim_index(root_pose=target_pose) if isinstance(renderer_cfg, NewtonWarpRendererCfg): assert not sim.visualizers - sim.get_or_create_backend(sim.fabric_cfg).update_transforms(sim.get_scene_data_provider()) + fabric.update_transforms(sim.get_scene_data_provider()) _render(sim, scene) torch.testing.assert_close(_fabric_position(body_path), target_pose[0, :3].cpu(), rtol=0.0, atol=1.0e-4) @@ -393,7 +394,7 @@ def test_fabric_geometry_sink_uses_sdp_world_points_and_frame_cadence(monkeypatc get_geometry_batches=lambda _format=SceneDataFormat.Points: batches, ) ) - fabric = sim.get_or_create_backend(sim.fabric_cfg) + fabric = sim.get_or_create_backend(FabricBackendCfg(stage=sim.stage, device=sim.device)) fabric.update_geometries(provider, 0) simulation_app.update() wp.synchronize_device(sim.device) diff --git a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py index 69254e4843c3..a123629772d3 100644 --- a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py +++ b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py @@ -23,7 +23,6 @@ import contextlib import logging -from inspect import signature from types import SimpleNamespace import isaaclab_newton.physics.newton_manager as newton_manager_module @@ -42,6 +41,7 @@ KaminoPADMMSolverCfg, MJWarpSolverCfg, MPMSolverCfg, + NewtonBuilderCfg, NewtonCfg, NewtonCollisionPipelineCfg, NewtonFeatherstoneManager, @@ -49,6 +49,7 @@ NewtonManager, NewtonMJWarpManager, NewtonMPMManager, + NewtonQueries, NewtonShapeCfg, NewtonVBDManager, NewtonXPBDManager, @@ -56,16 +57,18 @@ XPBDSolverCfg, ) from isaaclab_newton.physics.mpm_manager import _make_solver_config +from isaaclab_newton.physics.newton_manager_cfg import NewtonBackendCfg from isaaclab_newton.renderers.newton_warp_renderer import NewtonWarpRenderer -from newton import JointTargetMode, JointType, ModelBuilder, ModelFlags, ShapeFlags +from newton import JointTargetMode, JointType, ModelBuilder, ModelFlags from newton.actuators import DrivePID from newton.selection import ArticulationView from newton.solvers import SolverFeatherstone, SolverImplicitMPM, SolverKamino, SolverMuJoCo, SolverVBD, SolverXPBD from isaaclab.actuators import ImplicitActuatorCfg from isaaclab.physics import PhysicsEvent, PhysicsManager -from isaaclab.scene_data import SceneDataFormat -from isaaclab.sim import SimulationCfg, build_simulation_context +from isaaclab.scene_data import SceneDataFormat, SceneDataProvider +from isaaclab.sim import SimulationCfg, SimulationContext, build_simulation_context +from isaaclab.test.utils import test_devices # --------------------------------------------------------------------------- # Lightweight (no sim) parametrisation @@ -264,144 +267,76 @@ def contacts(self): assert NewtonManager._contacts.rigid_contact_max == 2 -def test_refit_sensor_bvh_rejects_missing_sensor_state(monkeypatch): - """BVH refitting raises when a particle BVH exists without an initialized sensor state.""" - model = SimpleNamespace(shape_count=0, particle_count=1, bvh_particles=object()) - monkeypatch.setattr(NewtonManager, "backend", SimpleNamespace(model=model)) - monkeypatch.setattr(NewtonManager, "_sensor_state", None, raising=False) - - with pytest.raises(RuntimeError, match="requires an initialized sensor state"): - NewtonManager._refit_sensor_bvh() - - -def test_sensor_task_builds_and_refits_bvhs_before_rendering(monkeypatch): - """One state refresh precedes BVH refits and rendering, including explicit transform updates.""" - - state = object() - status = {"state_refreshes": 0, "shape_refit": 0, "particle_refit": 0, "rendered": False} - - class FakeModel: - shape_count = 1 - particle_count = 1 - bvh_shapes = None - bvh_particles = None - tri_indices = None - - def bvh_build_shapes(self, current_state): - assert current_state is state - self.bvh_shapes = object() - - def bvh_build_particles(self, current_state): - assert current_state is state - self.bvh_particles = object() - - def bvh_refit_shapes(self, current_state): - assert current_state is state - status["shape_refit"] += 1 - - def bvh_refit_particles(self, current_state): - assert current_state is state - status["particle_refit"] += 1 - - model = FakeModel() - - def render(): - assert status["state_refreshes"] > 0 - assert model.bvh_shapes is not None - assert model.bvh_particles is not None - assert status["shape_refit"] - assert status["particle_refit"] - status["rendered"] = True - - def get_state(cls): - status["state_refreshes"] += 1 - return state - - monkeypatch.setattr(NewtonManager, "get_model", classmethod(lambda cls: model)) - monkeypatch.setattr(NewtonManager, "get_state_0", classmethod(lambda cls: state)) - monkeypatch.setattr(NewtonManager, "get_state", classmethod(get_state)) - monkeypatch.setattr(NewtonManager, "backend", SimpleNamespace(model=model, state_0=state)) - monkeypatch.setattr(NewtonManager, "_sensor_tasks", {}, raising=False) - monkeypatch.setattr(NewtonManager, "_sensor_state", None, raising=False) - monkeypatch.setattr(NewtonManager, "_sensor_state_dirty", True, raising=False) - monkeypatch.setattr(NewtonManager, "_sensor_graph", None, raising=False) - monkeypatch.setattr(NewtonManager, "_sensor_flags", None, raising=False) - monkeypatch.setattr(NewtonManager, "_sensor_flags_host", None, raising=False) - monkeypatch.setattr(NewtonManager, "_sensor_graph_capture_failed", False, raising=False) - monkeypatch.setattr(PhysicsManager, "_cfg", SimpleNamespace(use_cuda_graph=False), raising=False) - - renderer = object.__new__(NewtonWarpRenderer) - renderer.newton_sensor = SimpleNamespace(model=model) - monkeypatch.setattr(renderer, "_launch_render", lambda _data: render()) - renderer.update_transforms() - render_data = SimpleNamespace(sensor_task_name=None, ppisp_pipeline=None) - renderer.render(render_data) - renderer.render(render_data) - assert status["shape_refit"] == status["particle_refit"] == 1 - NewtonManager._mark_sensor_state_dirty() - renderer.render(render_data) - assert status["shape_refit"] == status["particle_refit"] == 2 - assert status["rendered"] - - -def test_newton_warp_renderer_runs_triangle_mesh_refit_eagerly(monkeypatch): - """Allocation-backed triangle-mesh rendering runs without attempting CUDA graph capture.""" - state = object() - model = SimpleNamespace( - shape_count=0, particle_count=0, bvh_shapes=None, bvh_particles=None, tri_indices=SimpleNamespace(shape=(1, 3)) +@pytest.mark.parametrize("cloth", [False, True]) +@pytest.mark.parametrize("device", test_devices()) +def test_queries_share_native_bvhs_and_read_only_through_sdp(monkeypatch, cloth, device): + """Two consumers reuse one refit and see fresh rigid and deformable publications.""" + sim = object.__new__(SimulationContext) + sim._backend_registry = [] + monkeypatch.setattr(SimulationContext, "instance", lambda: sim) + cfg = NewtonBackendCfg(physics_cfg=object(), device=device) + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=cfg.physics_cfg)) + builder.add_shape_sphere(builder.add_body(label="/Object")) + for position in ((0.0, 0.0, 0.0), (1.0, 0.0, 0.0), (0.0, 1.0, 0.0)): + builder.add_particle(pos=wp.vec3(position), vel=wp.vec3(), mass=1.0) + if cloth: + builder.add_triangle(0, 1, 2) + backend = sim.get_or_create_backend(cfg) + backend.geometry_offsets = {"/Cloth": 0} + transforms, points = SceneDataFormat.Transform(), SceneDataFormat.Points() + transforms.transforms = wp.array([wp.transform_identity()], dtype=wp.transform, device=device) + points.points = wp.array(np.tile([1.0, 2.0, 3.0], (3, 1)), dtype=wp.vec3, device=device) + publication = SimpleNamespace( + transforms=transforms, + transform_paths=["/Object"], + transforms_timestamp=0, + geometry_timestamp=0, + get_transforms=lambda output_format: transforms, + get_geometry_batches=lambda output_format: [(points, {"/Cloth": (0, 3)})], ) - calls: list[str] = [] - - monkeypatch.setattr(NewtonManager, "get_model", classmethod(lambda cls: model)) - monkeypatch.setattr(NewtonManager, "get_state_0", classmethod(lambda cls: state)) - monkeypatch.setattr(NewtonManager, "get_state", classmethod(lambda cls: state)) - monkeypatch.setattr(NewtonManager, "backend", SimpleNamespace(model=model, state_0=state)) - monkeypatch.setattr(NewtonManager, "_sensor_tasks", {}, raising=False) - monkeypatch.setattr(NewtonManager, "_sensor_eager_tasks", set(), raising=False) - monkeypatch.setattr(NewtonManager, "_sensor_state", None, raising=False) - monkeypatch.setattr(NewtonManager, "_sensor_state_dirty", True, raising=False) - monkeypatch.setattr(NewtonManager, "_sensor_graph", None, raising=False) - monkeypatch.setattr(NewtonManager, "_sensor_flags", None, raising=False) - monkeypatch.setattr(NewtonManager, "_sensor_flags_host", None, raising=False) - monkeypatch.setattr(NewtonManager, "_sensor_graph_capture_failed", False, raising=False) - monkeypatch.setattr(PhysicsManager, "_cfg", SimpleNamespace(use_cuda_graph=True), raising=False) - monkeypatch.setattr(PhysicsManager, "_device", "cuda:0", raising=False) - monkeypatch.setattr( - NewtonManager, - "_capture_sensor_graph", - classmethod(lambda cls: pytest.fail("Non-graph-capturable task attempted CUDA graph capture.")), - ) - + provider = SceneDataProvider(publication) renderer = object.__new__(NewtonWarpRenderer) - renderer.newton_sensor = SimpleNamespace(model=model) - monkeypatch.setattr(renderer, "_launch_render", lambda _data: calls.append("render")) - renderer.render(SimpleNamespace(sensor_task_name=None, ppisp_pipeline=None)) - - assert calls == ["render"] - - -def test_sensor_bvh_shape_flags_are_fixed_before_builder_creation(monkeypatch): - """Builder finalization includes collision-only shapes without a later BVH rebuild.""" - import newton - - flags = ShapeFlags.VISIBLE | ShapeFlags.COLLIDE_SHAPES - monkeypatch.setattr(NewtonManager, "_sensor_bvh_shape_flags", flags) - monkeypatch.setattr(PhysicsManager, "_cfg", NewtonCfg()) - builder = NewtonManager.create_builder() - body = builder.add_body() - builder.add_shape_sphere(body, cfg=newton.ModelBuilder.ShapeConfig(is_visible=False)) - - model = builder.finalize(device="cpu") - - assert builder.default_bvh_cfg.shape_flags == flags - assert model.bvh_shape_count_enabled == 1 - assert model.bvh_shapes is not None - - -def test_sensor_task_registration_has_no_raycast_bvh_fallback(): - """Raycast BVH requirements belong to builder creation, not task registration.""" - assert "include_collision_shapes" not in signature(NewtonManager._register_sensor_task).parameters - assert not hasattr(NewtonManager, "_sensor_bvh_has_collision_shapes") + renderer.backend = backend + renderer._scene_data_provider = provider + renderer._transform_mapping = provider.create_mapping(list(backend.model.body_label)) + # Triangle rendering performs host work and must stay eager even when capture is enabled. + renderer.cfg = SimpleNamespace(use_cuda_graph=cloth) + refits, observed = [], [] + refit = backend.model.bvh_refit_shapes + monkeypatch.setattr(backend.model, "bvh_refit_shapes", lambda state: (refits.append(state), refit(state))) + + def read(_data): + assert backend.state_0.body_q is transforms.transforms + np.testing.assert_array_equal(backend.state_0.particle_q.numpy(), points.points.numpy()) + observed.append(backend.state_0.body_q.numpy().copy()) + + monkeypatch.setattr(renderer, "_launch_render", read) + for _ in range(2): + render_data = SimpleNamespace(graph=None, ppisp_pipeline=None) + renderer.render(render_data) + assert render_data.graph is None + assert len(refits) == 1 + transforms.transforms = wp.array( + [wp.transform(wp.vec3(4.0, 5.0, 6.0), wp.quat_identity())], dtype=wp.transform, device=device + ) + points.points.assign(np.tile(np.asarray([7.0, 8.0, 9.0], dtype=np.float32), (3, 1))) + publication.transforms_timestamp += 1 + publication.geometry_timestamp += 1 + renderer.render(render_data) + assert len(refits) == 2 + np.testing.assert_array_equal(observed[-1][0, :3], [4.0, 5.0, 6.0]) + for expected, attribute in enumerate(("geometry_timestamp", "transforms_timestamp"), start=3): + setattr(publication, attribute, getattr(publication, attribute) + 1) + renderer.render(render_data) + renderer.render(render_data) + assert len(refits) == expected + # Render/query consumers may not restore a manager gateway or another state-update wrapper. + for name in ("_register_sensor_task", "get_state", "update_visualization_state"): + assert not hasattr(NewtonManager, name) + assert not hasattr(NewtonQueries, "get_state") + assert not hasattr(NewtonQueries, "update_scene_data") + assert not hasattr(backend, "transforms") + backend.close() def test_newton_shape_cfg_defaults_match_newton_shape_config(): @@ -762,18 +697,12 @@ def test_production_imports_scope_mujoco_joint_properties( joint.GetPrim().CreateAttribute("physxJoint:armature", Sdf.ValueTypeNames.Float, True).Set(0.21) physics_cfg = NewtonCfg(solver_cfg=solver_cfg, load_visual_shapes=False) - monkeypatch.setattr( - PhysicsManager, - "_sim", - SimpleNamespace( - physics_manager=manager_cls, - device="cpu", - cfg=SimpleNamespace(physics=physics_cfg, physics_prim_path=physics_prim_path), - ), - ) + sim = object.__new__(SimulationContext) + sim.cfg = SimpleNamespace(physics=physics_cfg, physics_prim_path=physics_prim_path, device="cpu") + sim.physics_manager, sim._backend_registry = manager_cls, [] + monkeypatch.setattr(PhysicsManager, "_sim", sim) monkeypatch.setattr(PhysicsManager, "_cfg", physics_cfg) monkeypatch.setattr(PhysicsManager, "_device", "cpu") - monkeypatch.setattr(NewtonManager, "_builder", None) monkeypatch.setattr(NewtonManager, "_scene_data_backend", newton_manager_module.NewtonSceneDataBackend()) monkeypatch.setattr(NewtonManager, "_cl_pending_sites", {}) monkeypatch.setattr(NewtonManager, "_per_world_builder_hooks", []) @@ -859,7 +788,7 @@ def test_mpm_project_outside_colliders_gates_projection(project_outside): ) with build_simulation_context(sim_cfg=sim_cfg) as sim: - builder = sim.physics_manager.create_builder() + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics)) # MPM custom attrs must exist on the builder before particles use them (the production path). assert builder.has_custom_attribute("mpm:young_modulus") builder.add_particles( @@ -877,7 +806,6 @@ def test_mpm_project_outside_colliders_gates_projection(project_outside): "mpm:damping": 0.0, }, ) - NewtonManager.set_builder(builder) sim.reset() assert isinstance(NewtonManager._solver, SolverImplicitMPM) @@ -1172,13 +1100,11 @@ def reset(self, state, world_mask=None, flags=0): def test_clear_resets_rigid_body_force_capability(monkeypatch): """Teardown clears the canonical solver capability without subclass shadowing.""" monkeypatch.setattr(NewtonManager, "_supports_rigid_body_force_input", True) - monkeypatch.setattr(NewtonManager, "_sensor_bvh_shape_flags", ShapeFlags.COLLIDE_SHAPES) monkeypatch.setattr(NewtonManager, "_num_envs", 4) NewtonManager.clear() assert NewtonManager._supports_rigid_body_force_input is False - assert NewtonManager._sensor_bvh_shape_flags == ShapeFlags.VISIBLE assert NewtonManager._num_envs is None for manager in ( NewtonMJWarpManager, @@ -1219,7 +1145,9 @@ def count_actuator_resolutions(name_keys, names, *args, **kwargs): return original(name_keys, names, *args, **kwargs) monkeypatch.setattr(articulation_module, "resolve_matching_names", count_actuator_resolutions) - articulation_module._configure_builder_joint_target_modes(builder, cfg) + sim = SimpleNamespace(cfg=SimpleNamespace(physics=object()), get_or_create_backend=lambda cfg: builder) + monkeypatch.setattr(SimulationContext, "instance", lambda: sim) + articulation_module.Articulation._configure_joint_target_modes(SimpleNamespace(cfg=cfg), None) assert builder.joint_target_mode == [int(JointTargetMode.POSITION)] * 2 assert actuator_resolutions == 1 @@ -1228,6 +1156,7 @@ def count_actuator_resolutions(name_keys, names, *args, **kwargs): def test_initialize_solver_prepares_picking_after_scene_data(monkeypatch): """Initial and hard resets realize native layouts before consumers and picking.""" events: list[str] = [] + allocations = [] sim_cfg = SimulationCfg( dt=1.0 / 120.0, device="cpu", @@ -1238,12 +1167,19 @@ def test_initialize_solver_prepares_picking_after_scene_data(monkeypatch): def on_physics_ready(_): events.append("ready") + backend = sim.get_or_create_backend(NewtonBackendCfg(physics_cfg=sim.cfg.physics, device=sim.device)) + assert backend is NewtonManager.backend + allocations.append(backend) + assert not hasattr(sim, "newton_cfg") and not hasattr(sim, "fabric_cfg") + different = NewtonCfg( + solver_cfg=MJWarpSolverCfg(), load_visual_shapes=not sim.cfg.physics.load_visual_shapes + ) + assert sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=different)) is not builder sim.get_scene_data_provider().get_transforms(SceneDataFormat.Transform()) - builder = sim.physics_manager.create_builder() + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics)) body = builder.add_body(mass=1.0) builder.add_joint_revolute(parent=-1, child=body, axis=(0, 0, 1)) - NewtonManager.set_builder(builder) monkeypatch.setattr(sim, "_prepare_newton_visualizer_for_capture", lambda: events.append("prepare")) sim.physics_manager.register_callback( on_physics_ready, @@ -1255,6 +1191,7 @@ def on_physics_ready(_): sim.reset() assert events == ["ready", "prepare"] * 2 + assert len(allocations) == 2 and allocations[0] is not allocations[1] # --------------------------------------------------------------------------- @@ -1313,7 +1250,7 @@ def test_initialize_solver_populates_canonical_state( assert resolved_manager.__name__ == expected_manager.__name__ assert resolved_manager.__name__.lower().startswith("newton") - builder = resolved_manager.create_builder() + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics)) if expected_solver_cls is SolverImplicitMPM: assert builder.has_custom_attribute("mpm:young_modulus") builder.add_particle_grid( @@ -1349,7 +1286,6 @@ def test_initialize_solver_populates_canonical_state( if isinstance(solver_cfg, (KaminoPADMMSolverCfg, KaminoDVISolverCfg)) and solver_cfg.use_collision_detector: builder.add_shape_sphere(body=body, radius=0.05) builder.add_ground_plane() - NewtonManager.set_builder(builder) # Force resolution and bring up the solver. expected_supports_force_input = RIGID_BODY_FORCE_INPUT_SUPPORT[expected_manager] @@ -1402,10 +1338,9 @@ def test_mjwarp_internal_contacts_with_collision_cfg_raises(): ) with build_simulation_context(sim_cfg=sim_cfg) as sim: - builder = sim.physics_manager.create_builder() + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics)) body = builder.add_body(mass=1.0) builder.add_joint_revolute(parent=-1, child=body, axis=(0, 0, 1)) - NewtonManager.set_builder(builder) with pytest.raises(ValueError, match="collision_cfg cannot be set"): sim.reset() @@ -1443,7 +1378,7 @@ def test_collision_decimation_invokes_mid_loop_collide(num_substeps, collision_d ) with build_simulation_context(sim_cfg=sim_cfg) as sim: - builder = sim.physics_manager.create_builder() + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics)) body = builder.add_body(mass=1.0) builder.add_joint_free(child=body) builder.add_shape_sphere(body=body, radius=0.05) @@ -1451,7 +1386,6 @@ def test_collision_decimation_invokes_mid_loop_collide(num_substeps, collision_d # Lift the sphere to 0.5 m above the plane so the scene is non-degenerate. # joint_q for a free joint is [tx, ty, tz, qx, qy, qz, qw]. builder.joint_q[-7:] = [0.0, 0.0, 0.5, 0.0, 0.0, 0.0, 1.0] - NewtonManager.set_builder(builder) sim.reset() # Wrap collide() with a counter — must run after sim.reset() so the @@ -1567,10 +1501,9 @@ def test_reset_lands_in_state_0_after_odd_kamino_steps_without_cuda_graph(): ) with build_simulation_context(sim_cfg=sim_cfg) as sim: - builder = NewtonManager.create_builder() + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics)) body = builder.add_body(mass=1.0) builder.add_joint_revolute(parent=-1, child=body, axis=(0, 0, 1)) - NewtonManager.set_builder(builder) sim.reset() # Kamino keeps separate input/output states; the bug only exists there. @@ -1612,10 +1545,9 @@ def test_graph_capture_preserves_first_step_and_recapture(monkeypatch, solver_cf physics=NewtonCfg(solver_cfg=solver_cfg, num_substeps=1, use_cuda_graph=True), ) with build_simulation_context(sim_cfg=sim_cfg) as sim: - builder = NewtonManager.create_builder() + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics)) builder.add_body(mass=1.0) builder.joint_q[-7:] = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0] - NewtonManager.set_builder(builder) sim.reset() if rtx_capture: monkeypatch.setattr(newton_manager_module, "has_kit", lambda: True) @@ -1647,14 +1579,13 @@ def test_stateful_actuator_graph_matches_eager_across_decimation_changes(monkeyp ), ) with build_simulation_context(sim_cfg=sim_cfg) as sim: - builder = NewtonManager.create_builder() + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics)) builder.begin_world() body = builder.add_link(mass=1.0) joint = builder.add_joint_prismatic(parent=-1, child=body, axis=(1.0, 0.0, 0.0)) builder.add_articulation([joint]) builder.add_actuator(DrivePID, index=0, kp=1.0, kd=0.0, ki=0.5, delay_steps=2) builder.end_world() - NewtonManager.set_builder(builder) sim.reset() monkeypatch.setattr(newton_manager_module, "has_kit", lambda: True) monkeypatch.setattr(sim, "_has_offscreen_render", True) @@ -1684,13 +1615,12 @@ def _build_collision_scene(sim, num_boxes=8): Uses ``MJWarpSolverCfg(use_mujoco_contacts=False)`` so the Newton collision pipeline / contacts are allocated on ``sim.reset()``. """ - builder = sim.physics_manager.create_builder() + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics)) for _ in range(num_boxes): body = builder.add_body(mass=1.0) builder.add_joint_free(child=body) builder.add_shape_box(body=body, hx=0.1, hy=0.1, hz=0.1) builder.add_ground_plane() - NewtonManager.set_builder(builder) # Model device arrays ``CollisionPipeline.collide()`` reads off its cached model. diff --git a/source/isaaclab_newton/test/physics/test_vbd_core.py b/source/isaaclab_newton/test/physics/test_vbd_core.py index 3fd73bd439e8..54641e10b664 100644 --- a/source/isaaclab_newton/test/physics/test_vbd_core.py +++ b/source/isaaclab_newton/test/physics/test_vbd_core.py @@ -11,28 +11,29 @@ from types import SimpleNamespace import pytest -from isaaclab_newton.physics import NewtonBackendCfg, NewtonManager, NewtonSoftContactCfg +from isaaclab_newton.physics import NewtonBackendCfg, NewtonBuilderCfg, NewtonCfg, NewtonManager, NewtonSoftContactCfg from newton import ModelBuilder from newton.solvers import SolverVBD -from isaaclab.sim import SimulationContext +from isaaclab.sim import BackendCfg, SimulationContext +from isaaclab.utils import replace # The soft-contact and simulation axes are independent, so each value is covered once. @pytest.mark.parametrize( ("soft_contact_cfg", "expected", "simulation"), [ - pytest.param(None, (7.0, 8.0, 9.0), True, id="preserve-physics"), + pytest.param(None, (7.0, 8.0, 9.0), False, id="preserve-render"), pytest.param( NewtonSoftContactCfg(soft_contact_ke=11.0, soft_contact_kd=12.0, soft_contact_mu=13.0), (11.0, 12.0, 13.0), - False, - id="override-render", + True, + id="override-physics", ), ], ) -def test_soft_contact_cfg_updates_finalized_model(soft_contact_cfg, expected, simulation): - """Registry construction shares the builder and applies model options before native allocation.""" +def test_soft_contact_cfg_updates_finalized_model(monkeypatch, soft_contact_cfg, expected, simulation): + """Cfg-keyed construction shares the builder and applies physics settings before state allocation.""" state_values = [] class Model: @@ -42,9 +43,6 @@ class Model: world_count = 0 articulation_count = 0 - def set_gravity(self, gravity): - pass - def state(self): state_values.append((self.soft_contact_ke, self.soft_contact_kd, self.soft_contact_mu)) return object() @@ -52,32 +50,36 @@ def state(self): def control(self): return object() - class Builder(ModelBuilder): - def finalize(self, *, device): - return model - - def __deepcopy__(self, memo): - pytest.fail("A native builder must be borrowed without copying.") - model = Model() - builder = Builder() - cfg = NewtonBackendCfg(builder=builder, device="cpu", soft_contact_cfg=soft_contact_cfg, simulation=simulation) + monkeypatch.setattr(ModelBuilder, "finalize", lambda self, device: model) + physics_cfg = NewtonCfg(soft_contact_cfg=soft_contact_cfg) if simulation else object() + builder_cfg = NewtonBuilderCfg(physics_cfg=physics_cfg) + assert not isinstance(builder_cfg, BackendCfg) + assert not hasattr(builder_cfg, "close") + cfg = NewtonBackendCfg(physics_cfg=physics_cfg, device="cpu") sim = object.__new__(SimulationContext) sim._backend_registry = [] + monkeypatch.setattr(SimulationContext, "instance", lambda: sim) + builder = sim.get_or_create_backend(builder_cfg) backend = sim.get_or_create_backend(cfg) - assert all(name not in vars(NewtonManager) for name in ("_backend", "_model", "_state_0", "_state_1", "_control")) - assert cfg.builder is builder - assert backend is sim.get_or_create_backend(cfg) + assert not {"_builder", "set_builder", "_model", "_state_0", "_state_1"}.intersection(vars(NewtonManager)) + assert not {"sync_transforms_to_fabric", "sync_transforms_to_usd"}.intersection(vars(NewtonManager)) + assert cfg.physics_cfg is physics_cfg + assert backend is sim.get_or_create_backend(replace(cfg)) + assert builder is sim.get_or_create_backend(builder_cfg) assert (model.soft_contact_ke, model.soft_contact_kd, model.soft_contact_mu) == expected assert state_values == [expected] * (2 if simulation else 1) assert (backend.state_1 is not None) == simulation assert (backend.control is not None) == simulation sim.close_backend(backend) assert backend.model is backend.state_0 is backend.state_1 is backend.control is None + assert sim.get_or_create_backend(builder_cfg) is builder + sim.close_backend(builder) + assert sim._backend_registry == [] -def test_vbd_colors_prebuilt_builder_before_start(monkeypatch): - """VBD colors a prebuilt builder before starting simulation.""" +def test_vbd_colors_builder_before_finalization(): + """VBD colors the completed builder in the existing pre-finalization hook.""" physics = importlib.import_module("isaaclab_newton.physics") events = [] @@ -85,12 +87,8 @@ class Builder: def color(self, *, balance_colors): events.append(("color", balance_colors)) - monkeypatch.setattr(physics.NewtonVBDManager, "_builder", Builder()) - monkeypatch.setattr(NewtonManager, "start_simulation", classmethod(lambda cls: events.append("start"))) - - physics.NewtonVBDManager.start_simulation() - - assert events == [("color", False), "start"] + physics.NewtonVBDManager._prepare_builder_for_finalize(Builder()) + assert events == [("color", False)] def test_vbd_solver_force_input_capability(monkeypatch): diff --git a/source/isaaclab_newton/test/renderers/test_visual_material.py b/source/isaaclab_newton/test/renderers/test_visual_material.py index e553571b405f..c873f109bb87 100644 --- a/source/isaaclab_newton/test/renderers/test_visual_material.py +++ b/source/isaaclab_newton/test/renderers/test_visual_material.py @@ -7,10 +7,9 @@ import torch import warp as wp -from isaaclab_newton.physics import NewtonManager +from isaaclab_newton.physics import NewtonBackendCfg, NewtonBuilderCfg from isaaclab_newton.renderers.newton_warp_renderer import NewtonWarpRenderer from isaaclab_newton.renderers.visual_material import ( - VisualMaterialWriter, VisualShapeColorWriter, import_builder_visual_material_paths, ) @@ -20,15 +19,20 @@ from pxr import Sdf, Usd, UsdGeom, UsdShade from isaaclab.renderers.base_renderer import VisualMaterialBatch +from isaaclab.sim import SimulationContext def _srgb(colors: torch.Tensor) -> torch.Tensor: return torch.where(colors <= 0.0031308, 12.92 * colors, 1.055 * colors.pow(1.0 / 2.4) - 0.055) -def _material_model(material_paths: list[str]): +def _material_backend(material_paths: list[str], monkeypatch): stage = Usd.Stage.CreateInMemory() - builder = ModelBuilder() + sim = object.__new__(SimulationContext) + sim._backend_registry = [] + monkeypatch.setattr(SimulationContext, "instance", lambda: sim) + cfg = NewtonBackendCfg(physics_cfg=object(), device="cpu") + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=cfg.physics_cfg)) for index, material_path in enumerate(material_paths): shape = UsdGeom.Cube.Define(stage, f"/World/shape_{index}") builder.add_shape_box(-1, label=str(shape.GetPath())) @@ -36,7 +40,7 @@ def _material_model(material_paths: list[str]): material = UsdShade.Material.Define(stage, material_path) UsdShade.MaterialBindingAPI.Apply(shape.GetPrim()).Bind(material) import_builder_visual_material_paths(builder, stage) - return builder.finalize(device="cpu") + return sim.get_or_create_backend(cfg) def test_import_captures_effective_material_binding() -> None: @@ -70,12 +74,15 @@ def test_import_preserves_binding_to_a_logical_clone_not_yet_on_stage() -> None: def test_material_writer_scatters_only_dirty_material_rows(monkeypatch) -> None: - model = _material_model(["/Looks/a", "/Looks/b", "/Looks/a", ""]) + backend = _material_backend(["/Looks/a", "/Looks/b", "/Looks/a", ""], monkeypatch) + model = backend.model values = torch.tensor([[0.1, 0.2, 0.3], [0.8, 0.6, 0.4]]) batch = VisualMaterialBatch( "color", ("/Looks/a", "/Looks/b"), ("/Looks/a/Shader", "/Looks/b/Shader"), ("color", "color"), values ) - writer = VisualMaterialWriter(model, (batch,)) + renderer = object.__new__(NewtonWarpRenderer) + renderer.backend = backend + writer = renderer.visual_material_writer((batch,)) before = wp.to_torch(model.shape_color).clone() launch = wp.launch launches = 0 @@ -139,8 +146,3 @@ def test_shape_writer_samples_each_body_and_selected_environment_independently() torch.testing.assert_close(actual[1], original.reshape(3, 3, 3)[1]) torch.testing.assert_close(actual[2, :2], expected[1, 0].expand(2, 3)) torch.testing.assert_close(actual[2, 2], expected[1, 1]) - - -def test_newton_renderer_exposes_shared_writer_factory() -> None: - renderer = object.__new__(NewtonWarpRenderer) - assert renderer.visual_material_writer == NewtonManager.create_visual_material_writer diff --git a/source/isaaclab_newton/test/sensors/test_newton_raycast_sensor.py b/source/isaaclab_newton/test/sensors/test_newton_raycast_sensor.py index 670b698f8a80..6de5c0e0edc4 100644 --- a/source/isaaclab_newton/test/sensors/test_newton_raycast_sensor.py +++ b/source/isaaclab_newton/test/sensors/test_newton_raycast_sensor.py @@ -12,7 +12,7 @@ import pytest import torch -from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg, NewtonManager +from isaaclab_newton.physics import MJWarpSolverCfg, NewtonBuilderCfg, NewtonCfg, NewtonManager from isaaclab_newton.renderers import NewtonWarpRendererCfg from isaaclab_newton.sensors import ( LegacyMultiMeshRayCaster, @@ -136,11 +136,12 @@ def test_rays_hit_ground_plane(sim, generic_cfg): The generic row uses the backend-dispatching :class:`RayCasterCfg` with ``global_world_only=True``, which must select the Newton BVH implementation. """ + # Another consumer may acquire the shared builder before this sensor is constructed. + builder = sim.get_or_create_backend(NewtonBuilderCfg(physics_cfg=sim.cfg.physics)) scene_cfg = GenericRaycastTestSceneCfg(num_envs=2) if generic_cfg else RaycastTestSceneCfg(num_envs=2) scene = InteractiveScene(scene_cfg) expected_bvh_flags = ShapeFlags.VISIBLE | ShapeFlags.COLLIDE_SHAPES - assert NewtonManager._sensor_bvh_shape_flags == expected_bvh_flags - assert NewtonManager._builder.default_bvh_cfg.shape_flags == expected_bvh_flags + assert builder.default_bvh_cfg.shape_flags == expected_bvh_flags sim.reset() sensor = _step_and_read(sim, scene) @@ -261,50 +262,55 @@ class RaycastCameraSceneCfg(RaycastTestSceneCfg): ) -def test_renderer_and_raycast_share_newton_manager_graph(sim): - """Shared graph replay preserves camera depth, clipping, and ray hits after a pose write.""" +def test_renderer_and_raycast_share_backend_with_independent_query_graphs(sim): + """Independent queries preserve depth, clipping, and ray hits across pose writes and hard resets.""" cfg = RaycastCameraSceneCfg(num_envs=1) + cfg.raycast.use_cuda_graph = sim.cfg.physics.use_cuda_graph + cfg.camera.renderer_cfg.use_cuda_graph = sim.cfg.physics.use_cuda_graph cfg.camera.update_latest_camera_pose = True cfg.camera.renderer_cfg.depth_clipping_behavior = "max" if sim.cfg.physics.use_cuda_graph else "none" scene = InteractiveScene(cfg) - sim.reset() - sim.step() - scene.update(sim.get_physics_dt()) - - camera = scene["camera"] - matrix = camera.data.intrinsic_matrices.torch[0] y, x = torch.meshgrid( torch.arange(cfg.camera.height, device=sim.device), torch.arange(cfg.camera.width, device=sim.device), indexing="ij", ) - ray_length = torch.sqrt( - 1 + ((x + 0.5 - matrix[0, 2]) / matrix[0, 0]) ** 2 + ((y + 0.5 - matrix[1, 2]) / matrix[1, 1]) ** 2 - ) far_clip = cfg.camera.spawn.clipping_range[1] background = far_clip if cfg.camera.renderer_cfg.depth_clipping_behavior == "max" else 0.0 - for height in (RAY_START_HEIGHT, RAY_START_HEIGHT + 0.1): - carrier = scene["sensor_body"] - pose = carrier.data.root_link_pose_w.torch.clone() - pose[:, 2] = height + RAY_OFFSET - carrier.write_root_pose_to_sim_index(root_pose=pose) + for _ in range(2): + sim.reset() sim.step() scene.update(sim.get_physics_dt()) - # A downward camera sees constant planar depth; oblique rays travel farther to the ground. - expected_ray = height * ray_length - hit = expected_ray < far_clip - assert hit.any() and (~hit).any() - expected_plane = torch.where(hit, height, background) - expected_ray = torch.where(hit, expected_ray, background) - outputs = camera.data.output - for name in ("depth", "distance_to_image_plane"): - torch.testing.assert_close(outputs[name].torch[0, ..., 0], expected_plane, atol=1e-5, rtol=1e-5) - torch.testing.assert_close(outputs["distance_to_camera"].torch[0, ..., 0], expected_ray, atol=1e-5, rtol=1e-5) - distances = scene["raycast"].data.ray_distances.torch - torch.testing.assert_close(distances, torch.full_like(distances, height), atol=1e-3, rtol=0) - - task_names = sorted(NewtonManager._sensor_tasks) - assert any(name.startswith("newton_raycast:") for name in task_names) - assert any(name.startswith("newton_warp_render:") for name in task_names) - assert (NewtonManager._sensor_graph is not None) == sim.cfg.physics.use_cuda_graph + camera = scene["camera"] + matrix = camera.data.intrinsic_matrices.torch[0] + ray_length = torch.sqrt( + 1 + ((x + 0.5 - matrix[0, 2]) / matrix[0, 0]) ** 2 + ((y + 0.5 - matrix[1, 2]) / matrix[1, 1]) ** 2 + ) + for height in (RAY_START_HEIGHT, RAY_START_HEIGHT + 0.1): + carrier = scene["sensor_body"] + pose = carrier.data.root_link_pose_w.torch.clone() + pose[:, 2] = height + RAY_OFFSET + carrier.write_root_pose_to_sim_index(root_pose=pose) + sim.step() + scene.update(sim.get_physics_dt()) + + # A downward camera sees constant planar depth; oblique rays travel farther to the ground. + expected_ray = height * ray_length + hit = expected_ray < far_clip + assert hit.any() and (~hit).any() + expected_plane = torch.where(hit, height, background) + expected_ray = torch.where(hit, expected_ray, background) + outputs = camera.data.output + for name in ("depth", "distance_to_image_plane"): + torch.testing.assert_close(outputs[name].torch[0, ..., 0], expected_plane, atol=1e-5, rtol=1e-5) + torch.testing.assert_close( + outputs["distance_to_camera"].torch[0, ..., 0], expected_ray, atol=1e-5, rtol=1e-5 + ) + distances = scene["raycast"].data.ray_distances.torch + torch.testing.assert_close(distances, torch.full_like(distances, height), atol=1e-3, rtol=0) + + camera, raycast = scene["camera"], scene["raycast"] + assert camera._renderer.backend is raycast.backend is NewtonManager.backend + assert (raycast._graph is not None) == cfg.raycast.use_cuda_graph + assert (camera._render_data.graph is not None) == cfg.camera.renderer_cfg.use_cuda_graph diff --git a/source/isaaclab_ov/changelog.d/consumer-owned-ovrtx-dependencies.rst b/source/isaaclab_ov/changelog.d/consumer-owned-ovrtx-dependencies.rst new file mode 100644 index 000000000000..bcbe9bf136d0 --- /dev/null +++ b/source/isaaclab_ov/changelog.d/consumer-owned-ovrtx-dependencies.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Loaded OVRTX's bundled native dependency when constructing its backend, so renderer creation + no longer depended on viewer-specific setup. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index dbf891bd92ae..1cbb8e319741 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -20,6 +20,7 @@ from __future__ import annotations import contextlib +import ctypes import logging import math import os @@ -269,6 +270,11 @@ class OVRTXBackend: """Own one native renderer and its optional detached stage, without camera or transport policy.""" def __init__(self, cfg: OVRTXBackendCfg): + # Resolve the wheel's native dependency here, including callers without a viewer. + dependency = Path(ovstage.__file__).parent / "bin/plugins/libosdCPU.so.3.6.0" + if dependency.exists(): + with contextlib.suppress(OSError): + ctypes.CDLL(str(dependency)) native_cfg = RendererConfig( log_file_path=cfg.renderer_cfg.log_file_path, log_level=cfg.renderer_cfg.log_level, diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index 6ee815f78cc8..e6a231f3fc11 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -6,6 +6,7 @@ """Tests for the OVRTX renderer output contract.""" import contextlib +import ctypes import importlib.util import sys import types @@ -99,13 +100,23 @@ def _simulation_registry(monkeypatch): @pytest.mark.parametrize("use_ovstage", [False, True]) -def test_ovrtx_renderer_config_enables_supported_runtime_options(monkeypatch: pytest.MonkeyPatch, use_ovstage): +def test_ovrtx_renderer_config_enables_supported_runtime_options(monkeypatch, tmp_path, use_ovstage): """Equal cfgs share one native resource; closing borrowers leaves it owned by the registry.""" config_kwargs: dict[str, object] = {} - destroyed = [] + destroyed, redirected = [], [] + dependency = tmp_path / "bin/plugins/libosdCPU.so.3.6.0" + dependency.parent.mkdir(parents=True) + dependency.touch() + loaded = [] + monkeypatch.setattr(ovrtx_renderer_module.ovstage, "__file__", str(tmp_path / "__init__.py")) + monkeypatch.setattr(ctypes, "CDLL", lambda path: loaded.append(path)) + # Cache redirection can load the SDK too; isolate it with the other native entry points. + monkeypatch.setenv("OVRTX_SHADER_CACHE_PATH", str(tmp_path / "shader-cache")) + monkeypatch.setattr(ovrtx_renderer_module, "redirect_shader_cache", redirected.append) class RecordingRendererConfig: def __init__(self, **kwargs): + assert loaded, "The renderer must load its native dependencies without viewer setup." config_kwargs.update(kwargs) monkeypatch.setattr(ovrtx_renderer_module, "RendererConfig", RecordingRendererConfig) @@ -120,6 +131,8 @@ def __init__(self, **kwargs): shared = OVRTXRenderer(renderer.cfg) assert shared.backend is renderer.backend + assert loaded == [str(dependency)] + assert len(redirected) == 1 assert renderer.backend.renderer is not None assert config_kwargs["suppress_deprecation_warnings"] is True assert config_kwargs["texture_streaming_mode"] is ovrtx_renderer_module.TextureStreamingMode.SYNCHRONOUS @@ -136,6 +149,7 @@ def __init__(self, **kwargs): SimulationContext.instance().close_backend(renderer.backend) SimulationContext.instance().close_backend(other.backend) assert len(destroyed) == 2 + assert redirected == destroyed assert not SimulationContext.instance()._backend_registry diff --git a/source/isaaclab_physx/changelog.d/consumer-owned-fabric-cfg.rst b/source/isaaclab_physx/changelog.d/consumer-owned-fabric-cfg.rst new file mode 100644 index 000000000000..c5bd118a4154 --- /dev/null +++ b/source/isaaclab_physx/changelog.d/consumer-owned-fabric-cfg.rst @@ -0,0 +1,11 @@ +Changed +^^^^^^^ + +* Acquired the shared Fabric resource from an explicit stage/device configuration in Kit rendering + consumers, without relying on a backend-specific field on ``SimulationContext``. + +Fixed +^^^^^ + +* Waited for GPU-to-host copies before PhysX consumed rigid-body mass, center-of-mass, and inertia + updates, preventing stale pinned-memory reads. diff --git a/source/isaaclab_physx/isaaclab_physx/assets/rigid_object/rigid_object.py b/source/isaaclab_physx/isaaclab_physx/assets/rigid_object/rigid_object.py index 6019efbec07a..cb95f52bdc62 100644 --- a/source/isaaclab_physx/isaaclab_physx/assets/rigid_object/rigid_object.py +++ b/source/isaaclab_physx/isaaclab_physx/assets/rigid_object/rigid_object.py @@ -768,6 +768,8 @@ def set_masses_index( # Set into simulation, note that when updating "model" properties with PhysX we need to do it on CPU. cpu_env_ids = self._get_cpu_env_ids(env_ids, sim_env_ids) wp.copy(self._cpu_body_mass, self.data._body_mass) + # PhysX reads pinned host data immediately; the asynchronous copy must finish first. + wp.synchronize_stream(self.device) self.root_view.set_masses(self._cpu_body_mass, indices=cpu_env_ids) def set_masses_mask( @@ -859,6 +861,7 @@ def set_coms_index( # Set into simulation, note that when updating "model" properties with PhysX we need to do it on CPU. cpu_env_ids = self._get_cpu_env_ids(env_ids, sim_env_ids) wp.copy(self._cpu_body_coms, self.data._body_com_pose_b.data.view(wp.float32)) + wp.synchronize_stream(self.device) self.root_view.set_coms(self._cpu_body_coms, indices=cpu_env_ids) def set_coms_mask( @@ -953,6 +956,7 @@ def set_inertias_index( # Set into simulation, note that when updating "model" properties with PhysX we need to do it on CPU. cpu_env_ids = self._get_cpu_env_ids(env_ids, sim_env_ids) wp.copy(self._cpu_body_inertia, self.data._body_inertia) + wp.synchronize_stream(self.device) self.root_view.set_inertias(self._cpu_body_inertia.flatten(), indices=cpu_env_ids) def set_inertias_mask( diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index 16a851a56dbe..bab4a9213b08 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -30,6 +30,7 @@ from isaaclab.utils.warp.kernels import reshape_tiled_image from isaaclab.utils.warp.warp_math import clamp_depth_to_inf_wp, replace_inf_depth_wp +from .fabric import FabricBackendCfg from .isaac_rtx_renderer_utils import ( apply_isaac_rtx_determinism_settings, apply_isaac_rtx_global_settings, @@ -200,7 +201,7 @@ def __init__(self, cfg: IsaacRtxRendererCfg): def initialize(self) -> None: """Bind shared Fabric destinations after scene creation.""" sim = SimulationContext.instance() - self._fabric = sim.get_or_create_backend(sim.fabric_cfg) + self._fabric = sim.get_or_create_backend(FabricBackendCfg(stage=sim.stage, device=sim.device)) self._fabric.bind_transforms(sim.get_scene_data_provider()) @property diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_utils.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_utils.py index 67d487e4ae56..41207312ebd6 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_utils.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_utils.py @@ -233,7 +233,10 @@ def ensure_isaac_rtx_render_update(force: bool = False) -> None: if not force and not sim.is_rendering: return - sim.get_or_create_backend(sim.fabric_cfg).update_transforms(sim.get_scene_data_provider()) + from .fabric import FabricBackendCfg # noqa: PLC0415 - requires Kit + + fabric = sim.get_or_create_backend(FabricBackendCfg(stage=sim.stage, device=sim.device)) + fabric.update_transforms(sim.get_scene_data_provider()) import omni.kit.app diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_utils.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_utils.py index 4c7878231184..2761242d1f83 100644 --- a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_utils.py +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_utils.py @@ -40,6 +40,8 @@ def _reset_globals(monkeypatch): """Restore module-level state so tests are isolated.""" monkeypatch.setattr(rtx_utils, "_last_render_update_key", (0, -1, -1)) + fabric = types.SimpleNamespace(FabricBackendCfg=types.SimpleNamespace) + monkeypatch.setitem(sys.modules, "isaaclab_physx.renderers.fabric", fabric) @pytest.fixture() @@ -190,7 +192,9 @@ def test_visualizer_pumps_only_after_initial_render_update( rtx_utils.ensure_isaac_rtx_render_update() mock_app.update.assert_not_called() - mock_sim.get_or_create_backend.assert_called_once_with(mock_sim.fabric_cfg) + cfg = mock_sim.get_or_create_backend.call_args.args[0] + assert cfg.stage is mock_sim.stage and cfg.device is mock_sim.device + mock_sim.get_or_create_backend.assert_called_once() update_transforms.assert_called_once_with(provider) mock_sim.physics_manager.forward.assert_not_called() diff --git a/source/isaaclab_visualizers/changelog.d/native-query-ownership.major.rst b/source/isaaclab_visualizers/changelog.d/native-query-ownership.major.rst new file mode 100644 index 000000000000..831569041f5b --- /dev/null +++ b/source/isaaclab_visualizers/changelog.d/native-query-ownership.major.rst @@ -0,0 +1,18 @@ +Changed +^^^^^^^ + +* Acquired Newton viewer models from the simulation's shared backend registry instead of the + physics manager. Viewers requested transforms and geometry directly through SDP; headless GL + and RTX captures requested current arrays only when a frame was requested. +* Rebound GL, RTX, Rerun, and Viser resources after hard resets, including when picking was disabled. +* Shared the selected Newton representation with streaming-camera renderers; initialization acquired + the clone-built model through ``get_or_create_backend(cfg)`` without cfg notifications. +* **Breaking:** Replaced streaming renderer nicknames with ``streaming_cam_renderer_cfg``. + Kit defaulted to ``IsaacRtxRendererCfg()``; Newton, Rerun, and Viser defaulted to + ``NewtonWarpRendererCfg()``. Explicit renderer failures propagated instead of switching renderer + or disabling the stream. Visualizers registered their configured auto-camera renderer before cloning. + Pass a renderer configuration to customize construction. +* Honored explicit auto-camera targets in Rerun and Viser instead of substituting a scene camera. + Omitting the target continued to adopt an existing camera, as documented. +* Consolidated GL/RTX headless and paused frame handling without changing pause behavior or + frame readback types. diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py index 2d5bdbd3504c..ae1916120a10 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py @@ -60,30 +60,6 @@ } -def _preload_ovrtx_native_deps() -> None: - """Pre-load ``libosdCPU.so`` from ``ovstage`` so ``ovrtx.Renderer`` can resolve it. - - ``libovrtx.dylib.so`` depends on ``libosdCPU.so.3.6.0`` which ships in the - ``ovstage`` wheel but is not on the system ``LD_LIBRARY_PATH``. Loading it - explicitly via :func:`ctypes.CDLL` places it in the process-wide ``dlopen`` cache - so the subsequent ``Renderer()`` instantiation in ``OVRTXRenderer.__init__`` can - find it. - """ - import ctypes - import importlib.util - import pathlib - - spec = importlib.util.find_spec("ovstage") - if spec is None: - return - lib = pathlib.Path(spec.origin).parent / "bin" / "plugins" / "libosdCPU.so.3.6.0" - if lib.exists(): - import contextlib - - with contextlib.suppress(OSError): - ctypes.CDLL(str(lib)) - - class KitVisualizer(BaseVisualizer): """Kit visualizer using Isaac Sim viewport.""" @@ -134,13 +110,6 @@ def __init__(self, cfg: KitVisualizerCfg): # Camera tracking state (replaces ViewportCameraController) self._interactive_scene = None # set from SimulationContext._interactive_scene in initialize() self._viewer_origin: torch.Tensor | None = None # world-space origin offset for eye/lookat - if ( - cfg.streaming_view - and get_settings_manager().get("/isaaclab/cameras_enabled", False) - and cfg.streaming_sensor_prim_path is None - and cfg.streaming_cam_target_prim_path is not None - ): - SimulationContext.instance().get_or_create_backend(self._resolve_streaming_renderer_cfg()) # ---- Lifecycle ------------------------------------------------------------------------ @@ -201,7 +170,9 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: self._setup_streaming_view(num_envs) sim = SimulationContext.instance() - self._fabric = sim.get_or_create_backend(sim.fabric_cfg) + from isaaclab_physx.renderers.fabric import FabricBackendCfg # noqa: PLC0415 - requires Kit + + self._fabric = sim.get_or_create_backend(FabricBackendCfg(stage=sim.stage, device=sim.device)) self._fabric.bind_transforms(scene_data_provider) self._is_initialized = True self._setup_initial_camera_view() @@ -689,27 +660,6 @@ def _uses_streaming_view(self) -> bool: """Return whether Kit should display a streaming camera image panel.""" return bool(self.cfg.streaming_view) - def _resolve_streaming_renderer_cfg(self): - """Return a renderer cfg for the auto-created streaming camera. - - Respects :attr:`~VisualizerCfg.streaming_cam_renderer`. When ``None``, - defaults to ``IsaacRtxRendererCfg`` (the Kit-native renderer). - """ - renderer_name = self.cfg.streaming_cam_renderer - if renderer_name == "ovrtx": - _preload_ovrtx_native_deps() - from isaaclab_ov.renderers import OVRTXRendererCfg - - return OVRTXRendererCfg() - if renderer_name == "newton_warp": - from isaaclab_newton.renderers import NewtonWarpRendererCfg - - return NewtonWarpRendererCfg() - # Default (None or "isaac_rtx"): use the Kit-native IsaacRtx renderer - from isaaclab_physx.renderers import IsaacRtxRendererCfg - - return IsaacRtxRendererCfg() - def _setup_streaming_view(self, num_envs: int) -> None: """Resolve or create the Camera sensor backing the streaming image panel.""" if not self._uses_streaming_view(): @@ -780,7 +730,6 @@ def _setup_streaming_view(self, num_envs: int) -> None: tile_w, tile_h, ) - renderer_cfg = self._resolve_streaming_renderer_cfg() ( self._camera_sensor, self._generated_camera_prim_paths, @@ -790,7 +739,7 @@ def _setup_streaming_view(self, num_envs: int) -> None: num_envs=num_envs, width=tile_w, height=tile_h, - renderer_cfg=renderer_cfg, + renderer_cfg=self.cfg.streaming_cam_renderer_cfg, data_types=sensor_keys_for_gt_types(gt_types), streaming_envs=tuple(int(i) for i in env_ids), ) diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer_cfg.py b/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer_cfg.py index f04a0c23d7a4..f8dab64276df 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer_cfg.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer_cfg.py @@ -9,6 +9,9 @@ from typing import TYPE_CHECKING +from isaaclab_physx.renderers import IsaacRtxRendererCfg + +from isaaclab.renderers import RendererCfg from isaaclab.utils import configclass from isaaclab.visualizers.visualizer_cfg import VisualizerCfg @@ -33,6 +36,9 @@ class KitVisualizerCfg(VisualizerCfg): visualizer_type: str = "kit" """Type identifier for Kit visualizer.""" + streaming_cam_renderer_cfg: RendererCfg = IsaacRtxRendererCfg() + """Renderer configuration for the auto-created streaming camera.""" + viewport_name: str | None = None """Name for a new viewport window when :attr:`create_viewport` is ``True``. diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py index 0cca84fb263a..acb4b123070a 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py @@ -32,7 +32,7 @@ del _pyglet_headless_init import newton -from isaaclab_newton.physics import NewtonManager +from isaaclab_newton.physics import NewtonBackendCfg, NewtonManager from newton.viewer import ViewerGL, ViewerRTX from pyglet.math import Vec3 as PygletVec3 @@ -55,6 +55,8 @@ remove_generated_prims, resolve_streaming_envs, ) +from isaaclab.scene_data import SceneDataFormat +from isaaclab.sim import SimulationContext from isaaclab.visualizers.base_visualizer import BaseVisualizer from isaaclab_visualizers.desktop_entry import write_desktop_entry @@ -992,7 +994,7 @@ class NewtonVisualizer(BaseVisualizer): @property def visual_material_writer(self): """Return the shared Newton model color-writer factory.""" - return NewtonManager.create_visual_material_writer + return self.backend.create_visual_material_writer class _ViewerPickingBinding: """Stable Newton-manager callback for viewer picking. @@ -1045,9 +1047,7 @@ def __init__(self, cfg: NewtonVisualizerCfg): self._sim_time = 0.0 self._step_counter = 0 self._runtime_headless: bool = False - self._model = None - self._state = None - self._update_frequency = cfg.update_frequency + self.backend = None self._last_camera_pose: tuple[tuple[float, float, float], tuple[float, float, float]] | None = None self._headless_no_viewer = False self._resolved_visible_env_ids: list[int] | None = None @@ -1078,8 +1078,6 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: scene_data_provider: Scene data provider used to fetch model/state data. """ - from isaaclab.sim import SimulationContext - if self._is_initialized: logger.debug("[%s] initialize() called while already initialized.", type(self).__name__) return @@ -1102,7 +1100,8 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: " `--viz kit` with a Kit-compatible physics backend." ) newton_backend_active = self.physics_backend == "newton" - physics_manager = SimulationContext.instance().physics_manager + sim = SimulationContext.instance() + physics_manager = sim.physics_manager picking_supported = newton_backend_active and bool( getattr(physics_manager, "_supports_rigid_body_force_input", False) ) @@ -1110,10 +1109,9 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: metadata = {"num_envs": num_envs} self._env_ids = self._compute_visualized_env_ids() self._resolved_visible_env_ids = resolve_visible_env_indices(self._env_ids, self.cfg.max_visible_envs, num_envs) - self._model = NewtonManager.get_model() - self._state = ( - NewtonManager.get_state_0() if newton_backend_active else NewtonManager.get_state(self._scene_data_provider) - ) + self.newton_cfg = NewtonBackendCfg(physics_cfg=sim.cfg.physics, device=sim.device) + self.backend = sim.get_or_create_backend(self.newton_cfg) + self._transform_mapping = scene_data_provider.create_mapping(list(self.backend.model.body_label)) runtime_headless = self.cfg.headless or ( sys.platform not in ("win32", "darwin") and not os.environ.get("DISPLAY") @@ -1144,7 +1142,7 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: self._viewer = self._create_viewer(runtime_headless, metadata) if self._viewer is not None: - self._viewer.set_model(self._model) + self._viewer.set_model(self.backend.model) if self._picking_enabled: # Keep Newton's public force path scoped to picking for this integration. self._viewer.wind = None @@ -1225,47 +1223,43 @@ def step(self, dt: float) -> None: self._sim_time += dt self._step_counter += 1 - # Headless mode renders on demand via render_rgb_array(). Keep the latest - # physics state available without paying the per-step render cost. - if self._runtime_headless: - self._state = NewtonManager.get_state(self._scene_data_provider) + # Headless capture requests current SDP arrays in render_rgb_array(). + if self._runtime_headless or self._viewer is None: return - if self._viewer is None: - self._state = NewtonManager.get_state(self._scene_data_provider) - return - - update_frequency = self._viewer._update_frequency if self._viewer else self._update_frequency - if self._step_counter % update_frequency != 0: + if self._step_counter % self._viewer._update_frequency != 0: return self._pre_step() - num_envs = NewtonManager.get_num_envs() + num_envs = self.backend.model.num_envs try: if not self._viewer.is_paused(): - self._state = NewtonManager.get_state(self._scene_data_provider) + backend, provider = self.backend, self._scene_data_provider + poses = SceneDataFormat.Transform() + if provider.get_transforms(poses, mapping=self._transform_mapping, count=backend.model.body_count): + backend.state_0.body_q = poses.transforms + if backend.geometry_offsets: + provider.get_geometry_points(output=backend.state_0.particle_q, offsets=backend.geometry_offsets) self._viewer.begin_frame(self._sim_time) try: - if self._state is not None: - body_q = getattr(self._state, "body_q", None) - if hasattr(body_q, "shape") and body_q.shape[0] == 0: - self._log_pending_meshes() - return - self._viewer.log_state(self._state) - contacts = NewtonManager.get_contacts() - if contacts is not None: - self._viewer.log_contacts(contacts, self._state) - else: - self._log_scene_contact_sensor_arrows(num_envs) - if self.cfg.enable_markers and not isinstance(self._viewer, NewtonViewerRTX): - # ViewerRTX uses a USD stage whose prim paths are not set up - # for the debug mesh overlays that markers require; skip for RTX. - render_newton_visualization_markers( - self._viewer, self._resolved_visible_env_ids, num_envs=num_envs - ) - self._log_streaming_image() - self._render_live_plots() + state = backend.state_0 + if state.body_q is not None and state.body_q.shape[0] == 0: + self._log_pending_meshes() + return + self._viewer.log_state(state) + contacts = NewtonManager.get_contacts() + if contacts is not None: + self._viewer.log_contacts(contacts, state) + else: + self._log_scene_contact_sensor_arrows(num_envs) + if self.cfg.enable_markers and not isinstance(self._viewer, NewtonViewerRTX): + # RTX's USD scene does not support the GL marker overlays. + render_newton_visualization_markers( + self._viewer, self._resolved_visible_env_ids, num_envs=num_envs + ) + self._log_streaming_image() + self._render_live_plots() self._log_pending_meshes() finally: self._viewer.end_frame() @@ -1307,16 +1301,17 @@ def consume_reset_request(self) -> bool: def reset(self, soft: bool = False) -> None: """Rebind viewer resources after a hard Newton model reset.""" - if soft or not self._picking_enabled or not self._is_initialized or self._is_closed: + if soft or not self._is_initialized or self._is_closed: return - model = NewtonManager.get_model() - if model is self._model: + sim = SimulationContext.instance() + backend = sim.get_or_create_backend(self.newton_cfg) + if backend is self.backend: return - self._model = model - self._state = NewtonManager.get_state_0() + self.backend = backend + self._transform_mapping = self._scene_data_provider.create_mapping(list(backend.model.body_label)) if self._viewer is not None: - self._viewer.set_model(self._model) + self._viewer.set_model(self.backend.model) if self._picking_enabled: self._viewer.wind = None self._viewer._register_isaaclab_ui_callbacks() @@ -1374,6 +1369,7 @@ def close(self) -> None: evict_visualizer_camera(self._streaming_camera_key) remove_generated_prims(self._generated_camera_prim_paths) self._camera_sensor = None + self.backend = self._scene_data_provider = self._transform_mapping = None self._is_closed = True def is_running(self) -> bool: @@ -1549,7 +1545,11 @@ def _apply_camera_focal_length(self) -> None: def _pump_paused(self) -> None: """Keep the event loop alive while simulation is paused without advancing state.""" - raise NotImplementedError + self._viewer.begin_frame(self._sim_time) + try: + self._log_pending_meshes() + finally: + self._viewer.end_frame() def _pre_step(self) -> None: """Per-frame hook called before the render block. No-op by default.""" @@ -1558,6 +1558,29 @@ def render_rgb_array(self) -> np.ndarray | None: """Return the latest RGB frame as a uint8 array with shape ``(H, W, 3)``.""" raise NotImplementedError + def _render_headless_frame(self) -> None: + """Render on demand, borrowing current SDP arrays and preserving paused frames.""" + if not self._runtime_headless or self._viewer.is_paused(): + return + backend, provider = self.backend, self._scene_data_provider + poses = SceneDataFormat.Transform() + if provider.get_transforms(poses, mapping=self._transform_mapping, count=backend.model.body_count): + backend.state_0.body_q = poses.transforms + if backend.geometry_offsets: + provider.get_geometry_points(output=backend.state_0.particle_q, offsets=backend.geometry_offsets) + self._pre_step() + self._viewer.begin_frame(self._sim_time) + try: + self._viewer.log_state(backend.state_0) + # RTX's USD scene does not support the GL marker overlays. + if self.cfg.enable_markers and not isinstance(self._viewer, NewtonViewerRTX): + render_newton_visualization_markers( + self._viewer, self._resolved_visible_env_ids, num_envs=backend.model.num_envs + ) + self._log_pending_meshes() + finally: + self._viewer.end_frame() + def _log_streaming_image(self) -> None: """Push the composited streaming frame into the viewer image panel.""" @@ -1658,26 +1681,6 @@ def _resolve_initial_camera_pose(self) -> tuple[tuple[float, float, float], tupl """Resolve initial camera pose from config or USD camera path.""" return self._resolve_cfg_camera_pose(type(self).__name__) - def _resolve_streaming_renderer_cfg(self): - """Return the renderer cfg for the auto-created streaming camera. - - Uses :attr:`~isaaclab.visualizers.VisualizerCfg.streaming_cam_renderer` - when set, otherwise falls back to :class:`~isaaclab_newton.renderers.NewtonWarpRendererCfg`. - """ - from isaaclab_newton.renderers import NewtonWarpRendererCfg - - renderer_name = self.cfg.streaming_cam_renderer - if renderer_name is None or renderer_name == "newton_warp": - return NewtonWarpRendererCfg() - if renderer_name == "ovrtx": - from isaaclab_ov.renderers import OVRTXRendererCfg - - return OVRTXRendererCfg() - raise ValueError( - f"[{type(self).__name__}] streaming_cam_renderer={renderer_name!r} is not supported. " - "Valid values for Newton visualizers: 'newton_warp', 'ovrtx', None." - ) - def _setup_streaming_view(self, num_envs: int) -> None: """Resolve or create the camera sensor for the streaming view.""" if not self._uses_streaming_view(): @@ -1731,30 +1734,20 @@ def _setup_streaming_view(self, num_envs: int) -> None: ) return - renderer_cfg = self._resolve_streaming_renderer_cfg() count = max(1, len(env_ids)) tile_w, tile_h = compute_tile_resolution( self.cfg.window_width, self.cfg.window_height, count, n_gt=len(gt_types) ) - try: - result = create_visualizer_camera( - num_envs=num_envs, - width=tile_w, - height=tile_h, - renderer_cfg=renderer_cfg, - data_types=sensor_keys_for_gt_types(gt_types), - target_prim_path=self.cfg.streaming_cam_target_prim_path, - eye=self.cfg.streaming_cam_eye, - streaming_envs=tuple(int(i) for i in env_ids), - ) - except Exception: - logger.warning( - "[%s] Streaming view disabled: could not auto-create a camera sensor. " - "Add a TiledCamera to the scene config or set streaming_sensor_prim_path " - "to point to an existing camera.", - type(self).__name__, - ) - return + result = create_visualizer_camera( + num_envs=num_envs, + width=tile_w, + height=tile_h, + renderer_cfg=self.cfg.streaming_cam_renderer_cfg, + data_types=sensor_keys_for_gt_types(gt_types), + target_prim_path=self.cfg.streaming_cam_target_prim_path, + eye=self.cfg.streaming_cam_eye, + streaming_envs=tuple(int(i) for i in env_ids), + ) self._camera_sensor, self._generated_camera_prim_paths, self._camera_is_owned, self._streaming_camera_key = ( result ) @@ -2263,18 +2256,11 @@ def _apply_camera_focal_length(self) -> None: return self._viewer.camera.fov = self._focal_length_to_vertical_fov_degrees() - def _pump_paused(self) -> None: - self._viewer.begin_frame(self._sim_time) - try: - self._log_pending_meshes() - finally: - self._viewer.end_frame() - def render_rgb_array(self) -> np.ndarray: """Return the latest RGB frame rendered by the Newton GL viewer. - In headless mode, a full render cycle is executed on demand using the - state captured during the most recent :meth:`step` call. + In headless mode, current transforms and geometry are requested from SDP + only when a frame is captured. Returns: The latest viewer framebuffer as a uint8 array with shape ``(H, W, 3)``. @@ -2284,22 +2270,7 @@ def render_rgb_array(self) -> np.ndarray: """ if self._viewer is None: raise RuntimeError("NewtonGLVisualizer must be initialized before capturing an RGB frame.") - if self._runtime_headless and self._state is not None and not self._viewer.is_paused(): - self._pre_step() - self._viewer.begin_frame(self._sim_time) - try: - self._viewer.log_state(self._state) - # The interactive render path logs markers every frame; a capture that - # skips them records the scene without its goal poses and command arrows. - if self.cfg.enable_markers: - render_newton_visualization_markers( - self._viewer, - self._resolved_visible_env_ids, - num_envs=NewtonManager.get_num_envs(), - ) - self._log_pending_meshes() - finally: - self._viewer.end_frame() + self._render_headless_frame() return self._viewer.get_frame().numpy() def _log_streaming_image(self) -> None: @@ -2445,22 +2416,11 @@ def _apply_rtx_fov_if_pending(self) -> None: def _pre_step(self) -> None: self._apply_rtx_fov_if_pending() - def _pump_paused(self) -> None: - # Both begin_frame/end_frame are required to close the imgui frame each tick. - # log_state() is skipped so the scene is frozen; the path-tracer accumulates - # samples on it, producing a progressively cleaner image while paused. - # Note: full RTX render cost is incurred every tick even while paused. - self._viewer.begin_frame(self._sim_time) - try: - self._log_pending_meshes() - finally: - self._viewer.end_frame() - def render_rgb_array(self) -> np.ndarray | None: """Return the latest RGB frame rendered by the Newton RTX viewer. - In headless mode, render the state captured during the latest simulation step - before reading back the path-traced LDR framebuffer. + In headless mode, request current transforms and geometry from SDP before + rendering and reading back the path-traced LDR framebuffer. Returns: The latest viewer framebuffer as a uint8 array with shape ``(H, W, 3)``, @@ -2468,12 +2428,5 @@ def render_rgb_array(self) -> np.ndarray | None: """ if self._viewer is None: return None - if self._runtime_headless and self._state is not None and not self._viewer.is_paused(): - self._pre_step() - self._viewer.begin_frame(self._sim_time) - try: - self._viewer.log_state(self._state) - self._log_pending_meshes() - finally: - self._viewer.end_frame() + self._render_headless_frame() return self._viewer.get_frame() diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer_cfg.py b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer_cfg.py index 9eb8fb5fa314..dc5c4052abf7 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer_cfg.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer_cfg.py @@ -10,6 +10,9 @@ import warnings from typing import TYPE_CHECKING, Any +from isaaclab_newton.renderers import NewtonWarpRendererCfg + +from isaaclab.renderers import RendererCfg from isaaclab.utils import configclass from isaaclab.visualizers.visualizer_cfg import VisualizerCfg @@ -34,6 +37,9 @@ class NewtonVisualizerCfg(VisualizerCfg): cloning_contexts: tuple[type | str, ...] = ("isaaclab_newton.cloner:NewtonReplicateContext",) + streaming_cam_renderer_cfg: RendererCfg = NewtonWarpRendererCfg() + """Renderer configuration for the auto-created streaming camera.""" + def __post_init__(self) -> None: super().__post_init__() if type(self) is NewtonVisualizerCfg: diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/newton_adapter.py b/source/isaaclab_visualizers/isaaclab_visualizers/newton_adapter.py index 6535f460849c..9508b9f38fda 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/newton_adapter.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/newton_adapter.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Shared helpers for viewer env selection (Newton viewers and Kit partial USD visibility).""" +"""Shared Newton viewer configuration, geometry, and environment selection.""" from __future__ import annotations diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer.py index 59c43b2af69b..01452bff2963 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer.py @@ -20,8 +20,11 @@ import numpy as np import rerun as rr import rerun.blueprint as rrb +from isaaclab_newton.physics import NewtonBackendCfg from newton.viewer import ViewerRerun +from isaaclab.scene_data import SceneDataFormat +from isaaclab.sim import SimulationContext from isaaclab.visualizers.base_visualizer import BaseVisualizer from isaaclab_visualizers.newton.newton_visualization_markers import render_newton_visualization_markers @@ -39,54 +42,6 @@ logger = logging.getLogger(__name__) -def _preload_ovrtx_native_deps() -> None: - """Pre-load ``libosdCPU.so`` from ``ovstage`` so ``ovrtx.Renderer`` can resolve it. - - ``libovrtx.dylib.so`` depends on ``libosdCPU.so.3.6.0`` which ships inside the - ``ovstage`` wheel but is not on the system ``LD_LIBRARY_PATH``. Loading it - explicitly places it in the process-wide ``dlopen`` cache. - """ - import ctypes - import importlib.util - import pathlib - - spec = importlib.util.find_spec("ovstage") - if spec is None: - return - lib = pathlib.Path(spec.origin).parent / "bin" / "plugins" / "libosdCPU.so.3.6.0" - if lib.exists(): - with contextlib.suppress(OSError): - ctypes.CDLL(str(lib)) - - -def _resolve_streaming_renderer_cfg(renderer_name: str | None): - """Return a renderer cfg for the auto-created streaming camera.""" - from isaaclab_newton.renderers import NewtonWarpRendererCfg - - if renderer_name is None or renderer_name == "newton_warp": - return NewtonWarpRendererCfg() - if renderer_name == "ovrtx": - _preload_ovrtx_native_deps() - from isaaclab_ov.renderers import OVRTXRendererCfg - - return OVRTXRendererCfg() - if renderer_name == "isaac_rtx": - try: - from isaaclab_physx.renderers import IsaacRtxRendererCfg - - import omni.replicator.core # noqa: F401 - - return IsaacRtxRendererCfg() - except ModuleNotFoundError: - logger.info( - "[RerunVisualizer] streaming_cam_renderer='isaac_rtx' unavailable (kitless); using newton_warp." - ) - return NewtonWarpRendererCfg() - raise ValueError( - f"streaming_cam_renderer={renderer_name!r} unsupported. Use 'newton_warp', 'ovrtx', 'isaac_rtx', or None." - ) - - _BACKEND_DISPLAY_NAMES = { "physx": "PhysX", "ovphysx": "OVPhysX", @@ -322,8 +277,7 @@ def __init__(self, cfg: RerunVisualizerCfg): self._backend_display: str | None = None self._sim_time = 0.0 self._step_counter = 0 - self._model = None - self._state = None + self.backend = None self._last_camera_pose: tuple[tuple[float, float, float], tuple[float, float, float]] | None = None self._resolved_visible_env_ids: list[int] | None = None self._camera_sensor = None @@ -341,16 +295,16 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: Args: scene_data_provider: Scene data provider used to fetch model/state data. """ - from isaaclab_newton.physics import NewtonManager - if self._is_initialized: return scene_data_provider = self._set_scene_data_provider(scene_data_provider) num_envs = scene_data_provider.num_envs self._env_ids = self._compute_visualized_env_ids() - self._model = NewtonManager.get_model() - self._state = NewtonManager.get_state(self._scene_data_provider) + sim = SimulationContext.instance() + self.newton_cfg = NewtonBackendCfg(physics_cfg=sim.cfg.physics, device=sim.device) + self.backend = sim.get_or_create_backend(self.newton_cfg) + self._transform_mapping = scene_data_provider.create_mapping(list(self.backend.model.body_label)) grpc_port = int(self.cfg.grpc_port) web_port = int(self.cfg.web_port) @@ -384,7 +338,7 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: self._log_viewer_url("RerunVisualizer", viewer_url) if self.cfg.open_browser and not start_server_in_viewer: _open_rerun_web_viewer(viewer_host, web_port, rerun_address) - self._viewer.set_model(self._model) + self._viewer.set_model(self.backend.model) self._viewer.show_particles = self.cfg.show_particles apply_viewer_visible_worlds( self._viewer, @@ -436,30 +390,31 @@ def step(self, dt: float) -> None: Args: dt: Simulation time-step in seconds. """ - from isaaclab_newton.physics import NewtonManager - if not self._is_initialized or self._is_closed or self._viewer is None: return self._sim_time += dt self._step_counter += 1 - self._state = NewtonManager.get_state(self._scene_data_provider) - num_envs = NewtonManager.get_num_envs() + num_envs = self.backend.model.num_envs if not self._viewer.is_paused(): + backend, provider = self.backend, self._scene_data_provider + poses = SceneDataFormat.Transform() + if provider.get_transforms(poses, mapping=self._transform_mapping, count=backend.model.body_count): + backend.state_0.body_q = poses.transforms + if backend.geometry_offsets: + provider.get_geometry_points(output=backend.state_0.particle_q, offsets=backend.geometry_offsets) self._viewer.begin_frame(self._sim_time) try: - if self._state is not None: - body_q = getattr(self._state, "body_q", None) - # Skip log_state for empty body arrays but do not return: _push_streaming_frame - # must still run after end_frame() so the streaming panel stays live. - if not (hasattr(body_q, "shape") and body_q.shape[0] == 0): - self._viewer.log_state(self._state) - if self.cfg.enable_markers: - render_newton_visualization_markers( - self._viewer, self._resolved_visible_env_ids, num_envs=num_envs - ) + # Empty body arrays skip log_state, but streaming still runs after end_frame. + body_q = backend.state_0.body_q + if body_q is None or body_q.shape[0]: + self._viewer.log_state(backend.state_0) + if self.cfg.enable_markers: + render_newton_visualization_markers( + self._viewer, self._resolved_visible_env_ids, num_envs=num_envs + ) self._render_live_plots() finally: self._viewer.end_frame() @@ -475,6 +430,20 @@ def step(self, dt: float) -> None: else: self._push_streaming_frame() + def reset(self, soft: bool = False) -> None: + """Rebind the viewer when a hard reset replaces the shared native model.""" + if soft or not self._is_initialized or self._is_closed: + return + sim = SimulationContext.instance() + backend = sim.get_or_create_backend(self.newton_cfg) + if backend is self.backend: + return + self.backend = backend + self._transform_mapping = self._scene_data_provider.create_mapping(list(backend.model.body_label)) + self._viewer.set_model(backend.model) + self._viewer.set_visible_worlds(self._resolved_visible_env_ids) + self._viewer.set_world_offsets((0.0, 0.0, 0.0)) + def close(self) -> None: """Close viewer/session resources.""" if self._is_closed: @@ -499,6 +468,7 @@ def close(self) -> None: rr.disconnect() except Exception as exc: logger.warning("[RerunVisualizer] Failed while disconnecting rerun: %s", exc) + self.backend = self._scene_data_provider = self._transform_mapping = None self._is_closed = True def is_running(self) -> bool: @@ -554,41 +524,24 @@ def _setup_streaming_view(self, num_envs: int) -> None: self._viewer._streaming_view_active = True return - # Auto-detect fallback: with Newton MJWarp replicate_physics=True, post-init prim - # spawning only survives at env_0. Reuse the first scene camera with matching - # renderer_type (or any scene camera with the right count as secondary fallback). - renderer_cfg = _resolve_streaming_renderer_cfg(self.cfg.streaming_cam_renderer) - renderer_type = getattr(renderer_cfg, "renderer_type", None) - scene_cameras = self._scene_data_provider.get_camera_sensors() - _fallback_cam = None - for cam in scene_cameras.values(): - if cam._view.count != num_envs: - continue - if getattr(getattr(cam.cfg, "renderer_cfg", None), "renderer_type", None) == renderer_type: - _fallback_cam = cam - break - if _fallback_cam is None: - _fallback_cam = cam - if _fallback_cam is not None: - self._camera_sensor = _fallback_cam - self._camera_sensor_indices = env_ids - self._streaming_view_active = True - self._viewer._streaming_view_active = True + if self.cfg.streaming_cam_target_prim_path is None: + cameras = self._scene_data_provider.get_camera_sensors() + if cameras: + self._camera_sensor = next(iter(cameras.values())) + self._camera_sensor_indices = env_ids + self._streaming_view_active = self._viewer._streaming_view_active = True return - tile_w, tile_h = 320, 240 # default resolution for Rerun stream (no window size) - try: - result = create_visualizer_camera( - num_envs=num_envs, - width=tile_w, - height=tile_h, - renderer_cfg=renderer_cfg, - data_types=sensor_keys_for_gt_types(gt_types), - streaming_envs=tuple(int(i) for i in env_ids), - ) - except Exception as e: - logger.warning("[RerunVisualizer] Streaming view disabled: could not auto-create a camera sensor (%s).", e) - return + result = create_visualizer_camera( + num_envs=num_envs, + width=320, + height=240, + renderer_cfg=self.cfg.streaming_cam_renderer_cfg, + data_types=sensor_keys_for_gt_types(gt_types), + target_prim_path=self.cfg.streaming_cam_target_prim_path, + eye=self.cfg.streaming_cam_eye, + streaming_envs=tuple(int(i) for i in env_ids), + ) self._camera_sensor, self._generated_camera_prim_paths, self._camera_is_owned, self._streaming_camera_key = ( result ) diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer_cfg.py b/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer_cfg.py index 97ff43fe739f..a0cbdf6b7763 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer_cfg.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer_cfg.py @@ -9,6 +9,9 @@ from typing import TYPE_CHECKING +from isaaclab_newton.renderers import NewtonWarpRendererCfg + +from isaaclab.renderers import RendererCfg from isaaclab.utils import configclass from isaaclab.visualizers.visualizer_cfg import VisualizerCfg @@ -28,6 +31,9 @@ class RerunVisualizerCfg(VisualizerCfg): cloning_contexts: tuple[type | str, ...] = ("isaaclab_newton.cloner:NewtonReplicateContext",) + streaming_cam_renderer_cfg: RendererCfg = NewtonWarpRendererCfg() + """Renderer configuration for the auto-created streaming camera.""" + app_id: str = "isaaclab-simulation" """Application identifier shown in viewer title.""" diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer.py index e2c6f411418b..ea402472f892 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer.py @@ -18,8 +18,11 @@ import newton import numpy as np +from isaaclab_newton.physics import NewtonBackendCfg from newton.viewer import ViewerViser +from isaaclab.scene_data import SceneDataFormat +from isaaclab.sim import SimulationContext from isaaclab.visualizers.base_visualizer import BaseVisualizer from isaaclab_visualizers.newton.newton_visualization_markers import render_newton_visualization_markers @@ -34,49 +37,6 @@ logger = logging.getLogger(__name__) -def _preload_ovrtx_native_deps() -> None: - """Pre-load ``libosdCPU.so`` from ``ovstage`` so ``ovrtx.Renderer`` can resolve it.""" - import ctypes - import importlib.util - import pathlib - - spec = importlib.util.find_spec("ovstage") - if spec is None: - return - lib = pathlib.Path(spec.origin).parent / "bin" / "plugins" / "libosdCPU.so.3.6.0" - if lib.exists(): - with contextlib.suppress(OSError): - ctypes.CDLL(str(lib)) - - -def _resolve_streaming_renderer_cfg(renderer_name: str | None): - """Return a renderer cfg for the auto-created streaming camera.""" - from isaaclab_newton.renderers import NewtonWarpRendererCfg - - if renderer_name is None or renderer_name == "newton_warp": - return NewtonWarpRendererCfg() - if renderer_name == "ovrtx": - _preload_ovrtx_native_deps() - from isaaclab_ov.renderers import OVRTXRendererCfg - - return OVRTXRendererCfg() - if renderer_name == "isaac_rtx": - try: - from isaaclab_physx.renderers import IsaacRtxRendererCfg - - import omni.replicator.core # noqa: F401 - - return IsaacRtxRendererCfg() - except ModuleNotFoundError: - logger.info( - "[ViserVisualizer] streaming_cam_renderer='isaac_rtx' unavailable (kitless); using newton_warp." - ) - return NewtonWarpRendererCfg() - raise ValueError( - f"streaming_cam_renderer={renderer_name!r} unsupported. Use 'newton_warp', 'ovrtx', 'isaac_rtx', or None." - ) - - if TYPE_CHECKING: from isaaclab.scene_data import SceneDataProvider @@ -380,8 +340,7 @@ def __init__(self, cfg: ViserVisualizerCfg): super().__init__(cfg) self.cfg: ViserVisualizerCfg = cfg self._viewer: NewtonViewerViser | None = None - self._model: Any | None = None - self._state = None + self.backend = None self._sim_time = 0.0 self._active_record_path: str | None = None self._last_camera_pose: tuple[tuple[float, float, float], tuple[float, float, float]] | None = None @@ -405,8 +364,6 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: Args: scene_data_provider: Scene data provider used to fetch model/state data. """ - from isaaclab_newton.physics import NewtonManager - if self._is_initialized: logger.debug("[ViserVisualizer] initialize() called while already initialized.") return @@ -415,8 +372,10 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: num_envs = scene_data_provider.num_envs metadata = {"num_envs": num_envs} self._env_ids = self._compute_visualized_env_ids() - self._model = NewtonManager.get_model() - self._state = NewtonManager.get_state(self._scene_data_provider) + sim = SimulationContext.instance() + self.newton_cfg = NewtonBackendCfg(physics_cfg=sim.cfg.physics, device=sim.device) + self.backend = sim.get_or_create_backend(self.newton_cfg) + self._transform_mapping = scene_data_provider.create_mapping(list(self.backend.model.body_label)) self._active_record_path = self.cfg.record_to_viser self._create_viewer(record_to_viser=self.cfg.record_to_viser, metadata=metadata) @@ -447,15 +406,12 @@ def step(self, dt: float) -> None: Args: dt: Simulation time-step in seconds. """ - from isaaclab_newton.physics import NewtonManager - if not self._is_initialized or self._viewer is None or self._scene_data_provider is None: return self._apply_pending_camera_pose() - self._state = NewtonManager.get_state(self._scene_data_provider) - num_envs = NewtonManager.get_num_envs() + num_envs = self.backend.model.num_envs self._sim_time += dt @@ -484,7 +440,13 @@ def step(self, dt: float) -> None: # When streaming_view is active, skip the 3D Newton scene so the # background streaming composite is the only content visible. if not self.cfg.streaming_view: - self._viewer.log_state(self._state) + backend, provider = self.backend, self._scene_data_provider + poses = SceneDataFormat.Transform() + if provider.get_transforms(poses, mapping=self._transform_mapping, count=backend.model.body_count): + backend.state_0.body_q = poses.transforms + if backend.geometry_offsets: + provider.get_geometry_points(output=backend.state_0.particle_q, offsets=backend.geometry_offsets) + self._viewer.log_state(self.backend.state_0) if self.cfg.enable_markers: self._render_markers(num_envs) self._render_live_plots() @@ -531,39 +493,23 @@ def _setup_streaming_view(self, num_envs: int) -> None: self._camera_sensor_indices = env_ids return - # Auto-detect fallback: with Newton MJWarp replicate_physics=True, post-init prim - # spawning only survives at env_0. Reuse the first scene camera with matching - # renderer_type (or any scene camera with the right count as secondary fallback). - renderer_cfg = _resolve_streaming_renderer_cfg(self.cfg.streaming_cam_renderer) - renderer_type = getattr(renderer_cfg, "renderer_type", None) - scene_cameras = self._scene_data_provider.get_camera_sensors() - _fallback_cam = None - for cam in scene_cameras.values(): - if cam._view.count != num_envs: - continue - if getattr(getattr(cam.cfg, "renderer_cfg", None), "renderer_type", None) == renderer_type: - _fallback_cam = cam - break - if _fallback_cam is None: - _fallback_cam = cam - if _fallback_cam is not None: - self._camera_sensor = _fallback_cam - self._camera_sensor_indices = env_ids + if self.cfg.streaming_cam_target_prim_path is None: + cameras = self._scene_data_provider.get_camera_sensors() + if cameras: + self._camera_sensor = next(iter(cameras.values())) + self._camera_sensor_indices = env_ids return - tile_w, tile_h = 320, 240 # default resolution for Viser stream - try: - result = create_visualizer_camera( - num_envs=num_envs, - width=tile_w, - height=tile_h, - renderer_cfg=renderer_cfg, - data_types=sensor_keys_for_gt_types(gt_types), - streaming_envs=tuple(int(i) for i in env_ids), - ) - except Exception as e: - logger.warning("[ViserVisualizer] Streaming view disabled: could not auto-create a camera sensor (%s).", e) - return + result = create_visualizer_camera( + num_envs=num_envs, + width=320, + height=240, + renderer_cfg=self.cfg.streaming_cam_renderer_cfg, + data_types=sensor_keys_for_gt_types(gt_types), + target_prim_path=self.cfg.streaming_cam_target_prim_path, + eye=self.cfg.streaming_cam_eye, + streaming_envs=tuple(int(i) for i in env_ids), + ) self._camera_sensor, self._generated_camera_prim_paths, self._camera_is_owned, self._streaming_camera_key = ( result ) @@ -663,6 +609,21 @@ def _render_markers(self, num_envs: int) -> None: else: logger.debug("[ViserVisualizer] Marker rendering failed: %s", exc) + def reset(self, soft: bool = False) -> None: + """Rebind the viewer when a hard reset replaces the shared native model.""" + if soft or not self._is_initialized or self._is_closed: + return + sim = SimulationContext.instance() + backend = sim.get_or_create_backend(self.newton_cfg) + if backend is self.backend: + return + self.backend = backend + self._transform_mapping = self._scene_data_provider.create_mapping(list(backend.model.body_label)) + self._viewer.set_model(backend.model) + self._setup_isaaclab_sidebar(self._viewer._server) + self._viewer.set_visible_worlds(self._resolved_visible_env_ids) + self._viewer.set_world_offsets((0.0, 0.0, 0.0)) + def close(self) -> None: """Close viewer resources and finalize optional recording.""" if not self._is_initialized: @@ -681,6 +642,7 @@ def close(self) -> None: self._viewer = None self._is_initialized = False + self.backend = self._scene_data_provider = self._transform_mapping = None self._is_closed = True self._active_record_path = None self._pending_camera_pose = None @@ -763,9 +725,6 @@ def _create_viewer(self, record_to_viser: str | None, metadata: dict | None = No record_to_viser: Optional output path for viser recording. metadata: Optional metadata passed to viewer. """ - if self._model is None: - raise RuntimeError("Viser visualizer requires a Newton model.") - self._viewer = NewtonViewerViser( port=self.cfg.port, bind_address=self.cfg.bind_address, @@ -788,7 +747,7 @@ def _create_viewer(self, record_to_viser: str | None, metadata: dict | None = No viewer_url, ) num_envs = int((metadata or {}).get("num_envs", 0)) - self._viewer.set_model(self._model) + self._viewer.set_model(self.backend.model) self._viewer.show_particles = self.cfg.show_particles # Set up sidebar AFTER set_model() — set_model calls clear_model() internally, # which would destroy any GUI elements created before it. diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer_cfg.py b/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer_cfg.py index 0657745c77bc..cefc8a4a6298 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer_cfg.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer_cfg.py @@ -9,6 +9,9 @@ from typing import TYPE_CHECKING +from isaaclab_newton.renderers import NewtonWarpRendererCfg + +from isaaclab.renderers import RendererCfg from isaaclab.utils import configclass from isaaclab.visualizers.visualizer_cfg import VisualizerCfg @@ -28,6 +31,9 @@ class ViserVisualizerCfg(VisualizerCfg): cloning_contexts: tuple[type | str, ...] = ("isaaclab_newton.cloner:NewtonReplicateContext",) + streaming_cam_renderer_cfg: RendererCfg = NewtonWarpRendererCfg() + """Renderer configuration for the auto-created streaming camera.""" + port: int = 8080 """Port of the local viser web server.""" diff --git a/source/isaaclab_visualizers/test/test_kit_visualizer_scene_partitioning.py b/source/isaaclab_visualizers/test/test_kit_visualizer_scene_partitioning.py index 158e027f24e5..5b952dc550c6 100644 --- a/source/isaaclab_visualizers/test/test_kit_visualizer_scene_partitioning.py +++ b/source/isaaclab_visualizers/test/test_kit_visualizer_scene_partitioning.py @@ -16,6 +16,7 @@ from pxr import Sdf, Usd, UsdGeom +from isaaclab.renderers import RendererCfg from isaaclab.utils.renderers import ISAAC_RTX_SHOW_ALL_PARTITIONS_BY_DEFAULT_SETTING @@ -47,14 +48,14 @@ def test_viewport_pose_publication_is_deferred_for_headless_capture(monkeypatch, @pytest.mark.parametrize("generated", [False, True]) def test_streaming_renderer_registers_before_visualizer_initialization(monkeypatch, generated): sim = MagicMock() - settings = MagicMock() - settings.get.return_value = True - renderer_cfg = object() + renderer_cfg = RendererCfg(class_type="my_renderers:CustomRenderer", renderer_type="custom") monkeypatch.setattr(kit_visualizer_module.SimulationContext, "instance", lambda: sim) - monkeypatch.setattr(kit_visualizer_module, "get_settings_manager", lambda: settings) - monkeypatch.setattr(KitVisualizer, "_resolve_streaming_renderer_cfg", lambda self: renderer_cfg) - - KitVisualizer(KitVisualizerCfg(streaming_view=True, streaming_cam_target_prim_path="/Robot" if generated else None)) + cfg = KitVisualizerCfg( + streaming_view=True, + streaming_cam_target_prim_path="/Robot" if generated else None, + streaming_cam_renderer_cfg=renderer_cfg, + ) + KitVisualizer(cfg) if generated: sim.get_or_create_backend.assert_called_once_with(renderer_cfg) diff --git a/source/isaaclab_visualizers/test/test_newton_adapter.py b/source/isaaclab_visualizers/test/test_newton_adapter.py index de8d4947d680..4306acc33554 100644 --- a/source/isaaclab_visualizers/test/test_newton_adapter.py +++ b/source/isaaclab_visualizers/test/test_newton_adapter.py @@ -30,6 +30,13 @@ log_geo_with_expanded_plane_scale, resolve_visible_env_indices, ) +from isaaclab_visualizers.rerun import RerunVisualizerCfg +from isaaclab_visualizers.viser import ViserVisualizerCfg + +from isaaclab.envs.utils import camera_view +from isaaclab.renderers import RendererCfg +from isaaclab.sim import SimulationContext +from isaaclab.utils import instantiate @pytest.mark.parametrize( @@ -210,8 +217,13 @@ def __init__(self): assert visualizer.cfg.lookat == (0.0, 0.0, 1.0) -def test_newton_visualizer_auto_creates_streaming_camera_when_scene_camera_exists(monkeypatch): +@pytest.mark.parametrize( + "cfg_type", [NewtonGLVisualizerCfg, NewtonRTXVisualizerCfg, RerunVisualizerCfg, ViserVisualizerCfg] +) +def test_visualizer_uses_declared_streaming_renderer(monkeypatch, cfg_type): """Auto-create mode should not silently replace its configured view with a scene camera.""" + sim = Mock() + monkeypatch.setattr(SimulationContext, "instance", lambda: sim) existing_camera = SimpleNamespace( _view=SimpleNamespace(count=4), cfg=SimpleNamespace( @@ -227,30 +239,40 @@ def _create_visualizer_camera(**kwargs): return generated_camera, ["/World/envs/env_0/VisualizerCamera"], True, ("camera-key",) monkeypatch.setattr(newton_visualizer_module, "create_visualizer_camera", _create_visualizer_camera) - - visualizer = NewtonGLVisualizer( - NewtonGLVisualizerCfg( - streaming_view=True, - streaming_envs=4, - streaming_cam_eye=(2.25, 0.0, 1.25), - streaming_cam_target_prim_path="/World/envs/*/Robot", - window_width=400, - window_height=400, - ) + monkeypatch.setattr(camera_view, "create_visualizer_camera", _create_visualizer_camera) + cfg = cfg_type( + streaming_view=True, + streaming_envs=4, + streaming_cam_eye=(2.25, 0.0, 1.25), + streaming_cam_target_prim_path="/World/envs/*/Robot", + streaming_cam_renderer_cfg=RendererCfg(class_type="my_renderers:CustomRenderer", renderer_type="custom"), ) + visualizer = instantiate(cfg) + sim.get_or_create_backend.assert_called_once_with(cfg.streaming_cam_renderer_cfg) + visualizer._viewer = SimpleNamespace() visualizer._scene_data_provider = SimpleNamespace( get_camera_sensors=lambda: {"task_camera": existing_camera}, ) visualizer._update_owned_camera_poses = lambda: None + visualizer._apply_streaming_camera_pose = lambda env_ids: None visualizer._setup_streaming_view(num_envs=4) assert visualizer._camera_sensor is generated_camera assert len(create_calls) == 1 - assert create_calls[0]["width"] == 200 - assert create_calls[0]["height"] == 200 assert create_calls[0]["target_prim_path"] == "/World/envs/*/Robot" assert create_calls[0]["eye"] == (2.25, 0.0, 1.25) + assert create_calls[0]["renderer_cfg"] is visualizer.cfg.streaming_cam_renderer_cfg + + unavailable = Mock(side_effect=ModuleNotFoundError("custom_renderer")) + monkeypatch.setattr(newton_visualizer_module, "create_visualizer_camera", unavailable) + monkeypatch.setattr(camera_view, "create_visualizer_camera", unavailable) + with pytest.raises(ModuleNotFoundError, match="custom_renderer"): + visualizer._setup_streaming_view(num_envs=4) + cfg.streaming_cam_target_prim_path = None + visualizer._setup_streaming_view(num_envs=4) + assert visualizer._camera_sensor is existing_camera + assert unavailable.call_count == 1 def test_newton_visualizer_render_rgb_array_returns_viewer_frame(): @@ -503,10 +525,6 @@ def test_newton_viewer_inactive_mpm_particles_use_newton_filter(monkeypatch): assert fallback_calls == [state] -class _BodyQ: - shape = (1,) - - class _Viewer: _update_frequency = 1 @@ -581,16 +599,30 @@ def __init__(self, contact_sensors=None): def get_contact_sensors(self): return self._contact_sensors + def create_mapping(self, paths): + return None -def _make_newton_visualizer(viewer, scene_data_provider=None): - visualizer = NewtonGLVisualizer(NewtonGLVisualizerCfg(enable_markers=False)) + def get_transforms(self, output, **kwargs): + output.transforms = self.poses + return True + + +def _make_newton_visualizer(viewer, scene_data_provider=None, state=None, *, cfg=None): + cfg = cfg or NewtonGLVisualizerCfg(enable_markers=False) + visualizer = instantiate(cfg) visualizer._is_initialized = True visualizer._is_closed = False visualizer._sim_time = 0.0 visualizer._step_counter = 0 visualizer._runtime_headless = False visualizer._viewer = viewer - visualizer._scene_data_provider = scene_data_provider + state = state or SimpleNamespace(body_q=wp.empty(1, dtype=wp.transform, device="cpu")) + visualizer.backend = SimpleNamespace( + model=SimpleNamespace(num_envs=1, body_count=len(state.body_q)), state_0=state, geometry_offsets={} + ) + visualizer._scene_data_provider = scene_data_provider or _SceneDataProvider() + visualizer._scene_data_provider.poses = state.body_q + visualizer._transform_mapping = None visualizer._resolved_visible_env_ids = None visualizer._live_plot_sources = [] if viewer is not None: @@ -624,13 +656,15 @@ def test_newton_visualizer_forwards_and_neutralizes_picking(): assert visualizer._viewer_picking_binding._retained_picking is None -def test_newton_visualizer_hard_reset_rebinds_viewer_model(monkeypatch): - from isaaclab_newton.physics import NewtonManager +@pytest.mark.parametrize("picking", [False, True]) +def test_newton_visualizer_hard_reset_rebinds_viewer_model(monkeypatch, picking): + from isaaclab_newton.physics import NewtonBackendCfg - new_model = object() + new_model = SimpleNamespace(body_label=["/Object"]) new_state = object() - monkeypatch.setattr(NewtonManager, "get_model", lambda: new_model) - monkeypatch.setattr(NewtonManager, "get_state_0", lambda: new_state) + backend = SimpleNamespace(model=new_model, state_0=new_state) + sim = SimpleNamespace(get_or_create_backend=Mock(return_value=backend)) + monkeypatch.setattr(SimulationContext, "instance", lambda: sim) viewer = _Viewer() viewer.picking_enabled = False @@ -639,38 +673,38 @@ def test_newton_visualizer_hard_reset_rebinds_viewer_model(monkeypatch): viewer.set_visible_worlds = Mock() viewer.set_world_offsets = Mock() visualizer = _make_newton_visualizer(viewer) + cfg = visualizer.newton_cfg = NewtonBackendCfg(physics_cfg=object(), device="cpu") visualizer._resolved_visible_env_ids = [1, 3] - visualizer._picking_enabled = True + visualizer._picking_enabled = picking visualizer.cfg.world_spacing = (2.0, 0.0, 0.0) visualizer.cfg.show_contacts = True visualizer.reset(soft=False) visualizer.reset(soft=False) - assert visualizer._model is new_model - assert visualizer._state is new_state + assert visualizer.backend is backend + sim.get_or_create_backend.assert_called_with(cfg) viewer.set_model.assert_called_once_with(new_model) viewer._register_isaaclab_ui_callbacks.assert_called_once_with() viewer.set_visible_worlds.assert_called_once_with([1, 3]) viewer.set_world_offsets.assert_called_once_with((2.0, 0.0, 0.0)) assert viewer.show_contacts is True - assert viewer.picking_enabled is True - assert viewer.wind is None + assert viewer.picking_enabled is picking + if picking: + assert viewer.wind is None assert visualizer._viewer_picking_binding._viewer is viewer def test_newton_visualizer_logs_native_contacts_when_available(monkeypatch): from isaaclab_newton.physics import NewtonManager - state = SimpleNamespace(body_q=_BodyQ()) + state = SimpleNamespace(body_q=wp.empty(1, dtype=wp.transform, device="cpu")) contacts = object() viewer = _Viewer() - monkeypatch.setattr(NewtonManager, "get_state", lambda _scene_data_provider=None: state) monkeypatch.setattr(NewtonManager, "get_contacts", lambda: contacts) - monkeypatch.setattr(NewtonManager, "get_num_envs", lambda: 1) - _make_newton_visualizer(viewer).step(0.1) + _make_newton_visualizer(viewer, state=state).step(0.1) assert viewer.logged_state is state assert viewer.logged_contacts == (contacts, state) @@ -679,15 +713,13 @@ def test_newton_visualizer_logs_native_contacts_when_available(monkeypatch): def test_newton_visualizer_logs_staged_mesh_inside_frame(monkeypatch): from isaaclab_newton.physics import NewtonManager - state = SimpleNamespace(body_q=_BodyQ()) + state = SimpleNamespace(body_q=wp.empty(1, dtype=wp.transform, device="cpu")) viewer = _Viewer() - visualizer = _make_newton_visualizer(viewer) + visualizer = _make_newton_visualizer(viewer, state=state) points = wp.zeros(3, dtype=wp.vec3) indices = wp.zeros(3, dtype=wp.int32) - monkeypatch.setattr(NewtonManager, "get_state", lambda _scene_data_provider=None: state) monkeypatch.setattr(NewtonManager, "get_contacts", lambda: None) - monkeypatch.setattr(NewtonManager, "get_num_envs", lambda: 1) normals = wp.zeros(3, dtype=wp.vec3) @@ -726,17 +758,12 @@ def test_newton_visualizer_logs_staged_mesh_inside_frame(monkeypatch): def test_newton_visualizer_logs_staged_mesh_for_bodyless_state(monkeypatch): - from isaaclab_newton.physics import NewtonManager - - state = SimpleNamespace(body_q=SimpleNamespace(shape=(0,))) + state = SimpleNamespace(body_q=wp.empty(0, dtype=wp.transform, device="cpu")) viewer = _Viewer() - visualizer = _make_newton_visualizer(viewer) + visualizer = _make_newton_visualizer(viewer, state=state) points = wp.zeros(3, dtype=wp.vec3) indices = wp.zeros(3, dtype=wp.int32) - monkeypatch.setattr(NewtonManager, "get_state", lambda _scene_data_provider=None: state) - monkeypatch.setattr(NewtonManager, "get_num_envs", lambda: 1) - visualizer.log_mesh("/surface", points, indices, dynamic=True) visualizer.step(0.1) @@ -745,16 +772,12 @@ def test_newton_visualizer_logs_staged_mesh_for_bodyless_state(monkeypatch): def test_newton_gl_visualizer_logs_staged_mesh_while_paused(monkeypatch): - from isaaclab_newton.physics import NewtonManager - viewer = _Viewer() viewer.paused = True visualizer = _make_newton_visualizer(viewer) points = wp.zeros(3, dtype=wp.vec3) indices = wp.zeros(3, dtype=wp.int32) - monkeypatch.setattr(NewtonManager, "get_num_envs", lambda: 1) - visualizer.log_mesh("/surface", points, indices, dynamic=True) visualizer.step(0.1) @@ -762,32 +785,55 @@ def test_newton_gl_visualizer_logs_staged_mesh_while_paused(monkeypatch): assert viewer.logged_state is None -def test_newton_visualizer_headless_renders_frame_on_demand(monkeypatch): - """Headless EGL should defer rendering until a frame is requested.""" - from isaaclab_newton.physics import NewtonManager - - state = SimpleNamespace(body_q=_BodyQ()) - viewer = _Viewer() +@pytest.mark.parametrize("cfg_type", [NewtonGLVisualizerCfg, NewtonRTXVisualizerCfg]) +def test_newton_visualizer_headless_renders_frame_on_demand(monkeypatch, cfg_type): + """Headless viewers share on-demand binding, preserve pause, and close frames even on errors.""" - monkeypatch.setattr(NewtonManager, "get_state", lambda _scene_data_provider=None: state) - monkeypatch.setattr(NewtonManager, "get_contacts", lambda: None) - monkeypatch.setattr(NewtonManager, "get_num_envs", lambda: 1) + class RTXViewer(_Viewer, NewtonViewerRTX): + def get_frame(self): + return super().get_frame().numpy() - visualizer = _make_newton_visualizer(viewer) + is_rtx = cfg_type is NewtonRTXVisualizerCfg + state = SimpleNamespace( + body_q=wp.empty(1, dtype=wp.transform, device="cpu"), particle_q=wp.empty(3, dtype=wp.vec3, device="cpu") + ) + viewer = RTXViewer() if is_rtx else _Viewer() + markers = Mock() + monkeypatch.setattr(newton_visualizer_module, "render_newton_visualization_markers", markers) + visualizer = _make_newton_visualizer(viewer, state=state, cfg=cfg_type(enable_markers=True)) + provider = visualizer._scene_data_provider + provider.get_transforms = Mock(wraps=provider.get_transforms) + provider.get_geometry_points = Mock() + visualizer.backend.geometry_offsets = {"/Cloth": 0} visualizer._runtime_headless = True visualizer.step(0.1) - assert viewer.logged_state is None + provider.get_transforms.assert_not_called() + + assert visualizer.render_rgb_array().shape == (4, 6, 3) + assert viewer.logged_state is state + assert viewer.events == ["begin_frame", "log_state", "end_frame"] + assert markers.call_count == (0 if is_rtx else 1) + provider.get_geometry_points.assert_called_once_with(output=state.particle_q, offsets={"/Cloth": 0}) + viewer.paused = True visualizer.render_rgb_array() + assert len(viewer.events) == 3 + provider.get_transforms.assert_called_once() - assert viewer.logged_state is state + viewer.paused = False + provider.poses = wp.empty(1, dtype=wp.transform, device="cpu") + viewer.log_state = Mock(side_effect=RuntimeError("render failed")) + with pytest.raises(RuntimeError, match="render failed"): + visualizer.render_rgb_array() + assert state.body_q is provider.poses + assert viewer.events[-2:] == ["begin_frame", "end_frame"] def test_newton_visualizer_contact_sensor_fallback_obeys_show_contacts(monkeypatch): from isaaclab_newton.physics import NewtonManager - state = SimpleNamespace(body_q=_BodyQ()) + state = SimpleNamespace(body_q=wp.empty(1, dtype=wp.transform, device="cpu")) viewer = _Viewer() sensor = _ContactSensor( net_normal_forces_w=torch.tensor([[[0.0, 0.0, 2.0], [0.0, 0.0, 0.5]]], dtype=torch.float32), @@ -796,11 +842,9 @@ def test_newton_visualizer_contact_sensor_fallback_obeys_show_contacts(monkeypat ) scene_data_provider = _SceneDataProvider({"contact_forces": sensor}) - monkeypatch.setattr(NewtonManager, "get_state", lambda _scene_data_provider=None: state) monkeypatch.setattr(NewtonManager, "get_contacts", lambda: None) - monkeypatch.setattr(NewtonManager, "get_num_envs", lambda: 1) - visualizer = _make_newton_visualizer(viewer, scene_data_provider) + visualizer = _make_newton_visualizer(viewer, scene_data_provider, state=state) visualizer.step(0.1) assert viewer.logged_arrows == ("/contacts", None, None, None) @@ -1086,6 +1130,7 @@ def test_newton_rtx_visualizer_allows_ovphysx_backend(monkeypatch): def test_newton_rtx_visualizer_setup_streaming_view_creates_owned_camera(monkeypatch): """_setup_streaming_view must create the owned camera sensor on RTX, not return early.""" + monkeypatch.setattr(SimulationContext, "instance", Mock()) generated_camera = object() create_calls = [] diff --git a/source/isaaclab_visualizers/test/test_newton_visual_material.py b/source/isaaclab_visualizers/test/test_newton_visual_material.py index 6fa0c7525dab..079518d36b07 100644 --- a/source/isaaclab_visualizers/test/test_newton_visual_material.py +++ b/source/isaaclab_visualizers/test/test_newton_visual_material.py @@ -5,15 +5,8 @@ """Visualizer material-writer contracts.""" -from isaaclab_newton.physics import NewtonManager from isaaclab_physx.renderers.visual_material import FabricVisualMaterialWriter from isaaclab_visualizers.kit.kit_visualizer import KitVisualizer -from isaaclab_visualizers.newton.newton_visualizer import NewtonVisualizer - - -def test_newton_visualizer_exposes_shared_writer_factory() -> None: - visualizer = object.__new__(NewtonVisualizer) - assert visualizer.visual_material_writer == NewtonManager.create_visual_material_writer def test_kit_visualizer_publishes_fabric_visual_material_writer() -> None: diff --git a/source/isaaclab_visualizers/test/test_newton_visualizer_viewer_release.py b/source/isaaclab_visualizers/test/test_newton_visualizer_viewer_release.py index e9618dec3269..41bdf87604a8 100644 --- a/source/isaaclab_visualizers/test/test_newton_visualizer_viewer_release.py +++ b/source/isaaclab_visualizers/test/test_newton_visualizer_viewer_release.py @@ -28,6 +28,8 @@ from __future__ import annotations +from types import SimpleNamespace + import isaaclab_visualizers.newton.newton_visualizer as newton_visualizer import pytest from isaaclab_visualizers.newton.newton_visualizer import NewtonVisualizer @@ -182,7 +184,7 @@ def _arm_for_step_failure(visualizer: NewtonVisualizer, viewer: _SpyRTXViewer) - visualizer._disable_viewer_on_step_exception = True visualizer._sim_time = 0.0 visualizer._step_counter = 0 - visualizer._state = None + visualizer.backend = SimpleNamespace(model=SimpleNamespace(num_envs=1)) visualizer._scene_data_provider = None visualizer._update_frequency = 1 viewer._update_frequency = 1 @@ -206,7 +208,6 @@ def test_step_failure_releases_the_viewer(monkeypatch: pytest.MonkeyPatch) -> No visualizer._picking_enabled = True visualizer._viewer_picking_binding.bind(viewer) # type: ignore[arg-type] _arm_for_step_failure(visualizer, viewer) - monkeypatch.setattr(newton_visualizer.NewtonManager, "get_num_envs", staticmethod(lambda: 1), raising=False) NewtonVisualizer.step(visualizer, dt=0.01) # must not raise @@ -236,7 +237,6 @@ def test_step_contains_a_failing_viewer_teardown(monkeypatch: pytest.MonkeyPatch viewer = _SpyRTXViewer(raises=True) visualizer = _make_visualizer(viewer) _arm_for_step_failure(visualizer, viewer) - monkeypatch.setattr(newton_visualizer.NewtonManager, "get_num_envs", staticmethod(lambda: 1), raising=False) NewtonVisualizer.step(visualizer, dt=0.01) # must not raise