From 5eef1d70f3c7f3af1e1c99eae85192b61cda52ac Mon Sep 17 00:00:00 2001 From: Antoine RICHARD Date: Tue, 29 Sep 2026 16:59:15 +0200 Subject: [PATCH 1/3] Fix DR Legs Kamino P-ADMM divergence (#7638) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Stabilize the DR Legs Kamino P-ADMM preset under bounded random joint targets by disabling the driven-joint solver effort limit only for Newton/Kamino. The PhysX presets retain the authored 3.1 N m limit, and the Kamino penalty parameter is restored to its original ``rho_0=0.02``. Re-enable ``IsaacContrib-DrLegs-Walk`` in the contributed-environment smoke suite and add coverage proving that the nested effort-limit preset resolves to ``inf`` for Newton/default and ``3.1`` for both PhysX aliases. The finite effort limit adds solver-side constraint rows; the failure is in their interaction with the closed-loop Kamino P-ADMM solve, rather than ordinary controller-side torque clipping. This is a temporary Isaac Lab workaround for https://github.com/newton-physics/newton/issues/4173. Its explicit tradeoff is that Newton no longer enforces the robot's 3.1 N m driven-joint torque cap until the upstream constraint instability is fixed. PhysX behavior is unchanged. Fixes #7625 ## Type of change - Bug fix (non-breaking change which fixes an issue) ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Screenshots Not applicable. ## Testing - ``uv run isaaclab -f`` - ``uv run python tools/changelog/cli.py check develop`` - Effort-limit preset regression: failed for both Newton/default variants before the fix; all four Newton and PhysX variants pass after the fix - ``test_contrib_environments[IsaacContrib-DrLegs-Walk]`` (CUDA): passed in 12/12 fresh 20-step runs - Final focused CUDA run: 5/5 tests passed together ## Checklist Docker and GPU tests run on demand. Push the commits you want tested, then comment `run-ci` on the pull request. - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `./isaaclab.sh --format` - [x] I have made corresponding changes to the documentation (not applicable; no public API or workflow changed) - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] I have added a changelog fragment under `source//changelog.d/` for every touched package (do **not** edit `CHANGELOG.rst` or bump `extension.toml` — CI handles that) - [x] I have added my name to the `CONTRIBUTORS.md` or my name already exists there --- .../antoiner-fix-drlegs-padmm-divergence.rst | 5 +++ .../contrib/dr_legs/hold_pose_env_cfg.py | 21 +++++++++++-- .../test/contrib/contrib_env_test_utils.py | 3 -- .../test/contrib/test_dr_legs_env_cfg.py | 31 +++++++++++++++++++ 4 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 source/isaaclab_tasks/changelog.d/antoiner-fix-drlegs-padmm-divergence.rst create mode 100644 source/isaaclab_tasks/test/contrib/test_dr_legs_env_cfg.py diff --git a/source/isaaclab_tasks/changelog.d/antoiner-fix-drlegs-padmm-divergence.rst b/source/isaaclab_tasks/changelog.d/antoiner-fix-drlegs-padmm-divergence.rst new file mode 100644 index 000000000000..0ec40801a7be --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/antoiner-fix-drlegs-padmm-divergence.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Stabilized the DR Legs Kamino P-ADMM preset by disabling its driven-joint effort limit; PhysX retained + the 3.1 N m limit. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/dr_legs/hold_pose_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/dr_legs/hold_pose_env_cfg.py index e81734599621..fba248e818be 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/dr_legs/hold_pose_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/dr_legs/hold_pose_env_cfg.py @@ -8,11 +8,13 @@ The robot must keep its pelvis upright at a target height. """ +import math + from isaaclab_newton.physics import KaminoPADMMCfg, KaminoPADMMSolverCfg, NewtonCfg, NewtonShapeCfg from isaaclab_physx.physics import PhysxCfg import isaaclab.sim as sim_utils -from isaaclab.assets import AssetBaseCfg +from isaaclab.assets import ArticulationCfg, AssetBaseCfg from isaaclab.envs import ManagerBasedRLEnvCfg from isaaclab.managers import EventTermCfg as EventTerm from isaaclab.managers import ObservationGroupCfg as ObsGroup @@ -27,7 +29,7 @@ from isaaclab.visualizers import VisualizerCfg import isaaclab_tasks.contrib.dr_legs.mdp as mdp -from isaaclab_tasks.utils import PresetCfg +from isaaclab_tasks.utils import PresetCfg, preset from isaaclab_assets.robots.dr_legs import DR_LEGS_ACTUATED_JOINTS, DR_LEGS_IMPLICIT_PD_CFG @@ -77,6 +79,19 @@ class DrLegsPhysicsCfg(PresetCfg): physx: PhysxAutoCfg = PhysxAutoCfg(isaacsim_physx=isaacsim_physx) +def _dr_legs_robot_cfg() -> ArticulationCfg: + """Return the task-local DR Legs articulation configuration.""" + cfg = replace(DR_LEGS_IMPLICIT_PD_CFG, prim_path="{ENV_REGEX_NS}/Robot") + # Work around Newton issue #4173: finite implicit-PD effort rows can destabilize closed-loop P-ADMM. + cfg.actuators["driven_joints"].joint_effort_limit = preset( + default=math.inf, + newton_kamino=math.inf, + isaacsim_physx=3.1, + physx=3.1, + ) + return cfg + + ## # Scene definition ## @@ -89,7 +104,7 @@ class HoldPoseSceneCfg(InteractiveSceneCfg): spawn=sim_utils.GroundPlaneCfg(size=(100.0, 100.0), physics_material=_PHYSICS_MATERIAL), ) - robot = replace(DR_LEGS_IMPLICIT_PD_CFG, prim_path="{ENV_REGEX_NS}/Robot") + robot = _dr_legs_robot_cfg() dome_light = AssetBaseCfg( prim_path="/World/DomeLight", diff --git a/source/isaaclab_tasks/test/contrib/contrib_env_test_utils.py b/source/isaaclab_tasks/test/contrib/contrib_env_test_utils.py index 4ffc72f255ea..97602390279a 100644 --- a/source/isaaclab_tasks/test/contrib/contrib_env_test_utils.py +++ b/source/isaaclab_tasks/test/contrib/contrib_env_test_utils.py @@ -36,9 +36,6 @@ "IsaacContrib-AutoMate-Disassembly-Direct": "Requires CUDA support outside the standard environment test runner.", } _SKIPPED_TASK_SUBSTRINGS = { - # The DR Legs solver intermittently produces NaN robot state under random actions. - # Re-enable the family once the tracked solver instability is resolved. - "DrLegs": "Kamino solver intermittently produces NaN robot state under random actions.", "RmpFlow": "Uses SingleArticulation, which requires an update.", "Skillgen": "Requires cuRobo-specific coverage.", "Suction": "Requires CPU simulation.", diff --git a/source/isaaclab_tasks/test/contrib/test_dr_legs_env_cfg.py b/source/isaaclab_tasks/test/contrib/test_dr_legs_env_cfg.py new file mode 100644 index 000000000000..5197fadd45ea --- /dev/null +++ b/source/isaaclab_tasks/test/contrib/test_dr_legs_env_cfg.py @@ -0,0 +1,31 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the DR Legs task configuration.""" + +import math + +import pytest + +import isaaclab_tasks # noqa: F401 +from isaaclab_tasks.utils.parse_cfg import parse_env_cfg + + +@pytest.mark.parametrize( + ("overrides", "expected_effort_limit"), + [ + ((), math.inf), + (("physics=newton_kamino",), math.inf), + (("physics=isaacsim_physx",), 3.1), + (("physics=physx",), 3.1), + ], +) +def test_dr_legs_driven_joint_effort_limit_matches_physics_backend( + overrides: tuple[str, ...], expected_effort_limit: float +): + """Disable the unstable effort constraint only for the Kamino backend.""" + env_cfg = parse_env_cfg("IsaacContrib-DrLegs-Walk", overrides=overrides) + actual_effort_limit = env_cfg.scene.robot.actuators["driven_joints"].joint_effort_limit + assert actual_effort_limit == expected_effort_limit From d176bf025ea5af1a2da9456cd3de46af41038957 Mon Sep 17 00:00:00 2001 From: Maximilian Krause <99733341+maxkra15@users.noreply.github.com> Date: Tue, 29 Sep 2026 18:25:01 +0200 Subject: [PATCH 2/3] Support heterogeneous OvPhysX cloning (#7890) ## Summary Enable heterogeneous rigid-body and articulation cloning with the **released OVPhysX 0.6.3 wheel**. No native patch, dependency update, or monkey-patch is required. - Build native copies from the clone plan. Worlds containing source assets or USD-only physics import their complete composition; other worlds copy from a complete original with one environment ID. This preserves within-world contacts without destination placeholders. - Keep the CPU full-USD path because 0.6.3 does not support CPU environment-ID filtering. - Preserve numeric environment ordering and contact-filter pairing. Reuse resolved paths inside `OvPhysxView`; assets do not orchestrate binding reuse. - Check active D6 rotation axes and tendon layouts once per variant prototype; native metadata validates the remaining body/joint layout. Defer unused rendering bindings during headless startup. - Match the scene's declared step rate to `SimulationCfg.dt`, fixing oversized automatic contact offsets. Enable Kuka Allegro's existing heterogeneous shapes preset for OvPhysX. ## Validation - 26 focused clone/export tests passed, including selective USD materialization that keeps unaffected worlds on native cloning. - 16 heterogeneous integration cases passed on CPU and 16 on GPU, using the unmodified 0.6.3 wheel. Coverage includes indexed state, contact isolation, USD-only supports with and without their native ancestor, incompatible D6/tendon layouts, and compatible disabled-joint and translation-limit differences. Regression cases were verified to fail before their fixes. - Formatting and changelog checks passed. Docker/GPU CI requested with `run-ci`; documentation validation is left to CI. ### Released-wheel benchmark `Isaac-Lift-KukaAllegro`, 4096 environments, 16 object variants, RTX 5090 GPU 1, no renderer/viewer, seed 42. One warm sample; 20 warmup steps followed by 100 measured steps, synchronized around the measurement window. | Environment creation | Environment step | Environment steps/s | | ---: | ---: | ---: | | 97.97 s | 86.05 ms | 47,600 | Environment creation is `gym.make`, not whole-process startup. This measurement uses stock OVPhysX. Further native startup optimizations are submitted separately in [NVIDIA-Omniverse/PhysX#532](https://github.com/NVIDIA-Omniverse/PhysX/pull/532); they are not required by this PR or included in its numbers. ```bash CUDA_VISIBLE_DEVICES=1 PXR_WORK_THREAD_LIMIT=1 OVPHYSX_NVTX=0 \ uv run --extra ovphysx isaaclab benchmark runtime \ --task Isaac-Lift-KukaAllegro --num_envs 4096 --seed 42 \ --num_steps 100 --warmup_steps 20 --device cuda:0 --visualizer none physics=ovphysx ``` Deformable native cloning remains outside this PR. CPU kinematic-velocity reset limitations and heterogeneous rendering-golden coverage are unchanged. ## Type of change - Bug fix - Documentation update ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Checklist - [x] Contribution guidelines and formatting checks reviewed. - [x] Documentation and focused regression tests updated. - [x] Changelog fragments cover the changed source packages. - [x] Contributor name is present in `CONTRIBUTORS.md`. --------- Co-authored-by: Octi Zhang --- docs/source/concepts/ovphysx.rst | 38 +- ...nk-heterogeneous-cloning-ovphysx.minor.rst | 16 + source/isaaclab_ov/isaaclab_ov/_clone.py | 16 + .../assets/articulation/articulation.py | 5 +- .../assets/rigid_object/rigid_object.py | 5 +- .../isaaclab_ov/cloner/replicate.py | 185 ++++++++-- .../isaaclab_ov/physics/ovphysx_manager.py | 300 +++++----------- .../sensors/contact_sensor/contact_sensor.py | 72 ++-- .../isaaclab_ov/sim/views/ovphysx_view.py | 44 ++- .../test/assets/test_articulation.py | 102 +++++- .../test/assets/test_rigid_object.py | 107 +++++- .../physics/test_ovphysx_manager_lifecycle.py | 23 +- .../test_ovphysx_scene_data_backend.py | 324 ++---------------- .../test/sensors/test_contact_sensor.py | 82 ++++- .../isaaclab_ov/test/sim/test_ovphysx_view.py | 30 ++ source/isaaclab_ov/test/test_cloner.py | 259 ++++++++++++-- ...nk-heterogeneous-cloning-ovphysx.minor.rst | 4 + .../isaaclab_tasks/core/lift/lift_env_cfg.py | 3 +- .../core/test_lift_kuka_ovphysx_presets.py | 18 +- .../test/core/test_ovphysx_camera_presets.py | 4 +- ...test_rendering_lift_kuka_hetero_kitless.py | 3 +- 21 files changed, 1021 insertions(+), 619 deletions(-) create mode 100644 source/isaaclab_ov/changelog.d/maximiliank-heterogeneous-cloning-ovphysx.minor.rst create mode 100644 source/isaaclab_tasks/changelog.d/maximiliank-heterogeneous-cloning-ovphysx.minor.rst diff --git a/docs/source/concepts/ovphysx.rst b/docs/source/concepts/ovphysx.rst index c955be7704aa..4c0fd13449e5 100644 --- a/docs/source/concepts/ovphysx.rst +++ b/docs/source/concepts/ovphysx.rst @@ -63,12 +63,43 @@ validated at the time of writing. The following pieces are available on `PR #5678 `_. * :class:`~isaaclab.assets.DeformableObject` — experimental volume- and surface-deformable support on CUDA simulation devices. +* Fast-path cloning of heterogeneous rigid-body and articulation geometry + variants with matching native tensor layouts. Additional OvPhysX work remains in flight. IMU, Frame Transformer, Joint Wrench, PVA, Ray Caster, and rendering support are not documented as supported here until their implementations land on ``develop`` and pass the backend smoke tests. +Heterogeneous cloning +--------------------- + +OvPhysX 0.6.3 is required for heterogeneous runtime cloning. Tensor bindings use +numeric environment order so indexed resets, actions and observations address +the correct variant. Variants may differ in geometry but must preserve body, +joint and tendon layout. Isaac Lab checks prototype rotation axes and tendon layouts; +OVPhysX checks body and joint metadata when binding articulations. + +The clone context compiles the plan's prototypes and world assignments once. +The physics manager attaches one exported stage, replays the native copies, and +warms the runtime before assets and sensors bind. Hard reset reuses the same +declarations; there is no separate consumable queue. Binding paths come from the +plan's world layout and environment template, not from native clone order. + +Authored sources all receive native environment ID zero. Every world containing +a source or USD-only physics therefore imports all its declared assets as originals. Other worlds +clone their assets from a complete original world of the same composition, +using one native environment ID per destination world. This keeps a robot, +table and object able to contact one another without destination placeholders. +Only declared asset subtrees are copied; unrelated authored assets are retained. + +Keep :attr:`~isaaclab.scene.InteractiveSceneCfg.filter_collisions` enabled to +isolate original worlds with USD collision groups. GPU copies additionally use +native environment-ID filtering. CPU has no native environment-ID filtering +in OvPhysX 0.6.3, so it imports all declared USD copies and uses collision groups. +Large overlapping original layouts can still exhaust broadphase pair capacity; +use spatially separated environments. + Deformable limitations ---------------------- @@ -80,7 +111,8 @@ padded state that would produce incorrect reductions. Deformable scenes also require full-stage materialization. Startup cost therefore grows with the number of authored environments. Use this path for small validation scenes; training-scale workloads with thousands of environments are not currently -supported. +supported. Deformable bodies are not supported by the heterogeneous fast-path +cloner. Installation ------------ @@ -92,7 +124,9 @@ The ``ovphysx`` extra requires OvPhysX 0.6.3. Install it from the repository roo uv sync --inexact --extra ovphysx The ``--inexact`` flag preserves packages installed through other extras. -Use ``--extra ov`` to install both public OvPhysX and OVRTX runtimes. The legacy +Use ``--extra ov`` to install both public OvPhysX and OVRTX runtimes. The combined +extra pairs OVRTX 0.5.0.377615 with OVStage 0.2; OVRTX 0.4.1 is not compatible +with this runtime combination. The legacy Isaac Lab installer also supports ``./isaaclab.sh -i 'ov[ovphysx]'`` and ``./isaaclab.sh -i 'ov[all]'``. diff --git a/source/isaaclab_ov/changelog.d/maximiliank-heterogeneous-cloning-ovphysx.minor.rst b/source/isaaclab_ov/changelog.d/maximiliank-heterogeneous-cloning-ovphysx.minor.rst new file mode 100644 index 000000000000..43ebcc61e693 --- /dev/null +++ b/source/isaaclab_ov/changelog.d/maximiliank-heterogeneous-cloning-ovphysx.minor.rst @@ -0,0 +1,16 @@ +Added +^^^^^ + +* Added plan-driven OvPhysX cloning for heterogeneous rigid-body and articulation geometry variants. GPU cloning imported complete original worlds and assigned one native environment ID per copied world; CPU imported full USD copies with collision grouping. + +Fixed +^^^^^ + +* Preserved numeric environment order in tensor bindings, including non-cloned multi-instance assets, so indexed state reads and writes address the intended instances. +* Paired each cloned contact sensor with its own environment's filters and shared filters, and rejected missing filters that would silently remove sensor rows. +* Preserved all active source variants and independently authored assets during physics export, including custom environment templates and scaled asset roots. +* Preserved contacts with USD-only physics in each world composition and rejected incompatible active rotation-axis and tendon layouts before combining articulation variants. +* Removed destination placeholders and empty exported environment roots that caused quadratic native binding lookup costs. +* Registered Newton USD schemas before OVStage parsing. This affected all authored physics assets, including mimic-joint constraints, not only cloned assets. +* Deferred scene-data tensor bindings until visualization requests them, avoiding unnecessary headless-training startup cost. +* Matched the native scene's declared step rate to ``SimulationCfg.dt`` so automatic contact offsets used the actual simulation timestep instead of the 60 Hz default. diff --git a/source/isaaclab_ov/isaaclab_ov/_clone.py b/source/isaaclab_ov/isaaclab_ov/_clone.py index ed518245fe40..30fe4163c8cb 100644 --- a/source/isaaclab_ov/isaaclab_ov/_clone.py +++ b/source/isaaclab_ov/isaaclab_ov/_clone.py @@ -5,9 +5,25 @@ """Internal clone payload definitions shared by the OvPhysX cloner and manager.""" +import re from collections.abc import Sequence +from fnmatch import fnmatchcase CloneTransform = tuple[float, float, float, float, float, float, float] +# Source, destinations, destination poses, destination world IDs, source world ID. +CloneRecipe = tuple[str, list[str], list[CloneTransform], list[int] | None, int | None] + + +def ordered_clone_paths(paths: list[str], patterns: list[str]) -> list[str]: + """Order binding rows numerically within each requested body pattern.""" + if paths == patterns: + return paths + + def order_key(path: str) -> tuple: + group = next((i for i, pattern in enumerate(patterns) if fnmatchcase(path, pattern)), len(patterns)) + return group, tuple(int(part) if part.isdigit() else part for part in re.split(r"(\d+)", path)) + + return sorted(paths, key=order_key) def clone_transforms_from_positions(positions: Sequence[Sequence[float]]) -> list[CloneTransform]: diff --git a/source/isaaclab_ov/isaaclab_ov/assets/articulation/articulation.py b/source/isaaclab_ov/isaaclab_ov/assets/articulation/articulation.py index 8fc7852c6875..b1137fe2ff93 100644 --- a/source/isaaclab_ov/isaaclab_ov/assets/articulation/articulation.py +++ b/source/isaaclab_ov/isaaclab_ov/assets/articulation/articulation.py @@ -40,7 +40,7 @@ from isaaclab_ov.assets import kernels as shared_kernels from isaaclab_ov.physics import OvPhysxManager from isaaclab_ov.physics.ovphysx_compat import OVPHYSX_VERSION, requires_legacy_joint_sign_correction -from isaaclab_ov.sim.views.ovphysx_view import OvPhysxView +from isaaclab_ov.sim.views.ovphysx_view import OvPhysxView, _expand_env_pattern from .actuator_control import OvPhysxActuatorControl from .articulation_data import ArticulationData @@ -3952,7 +3952,8 @@ def has_articulation_root_api(prim) -> bool: TT.BODY_COM_POSE, TT.BODY_INERTIA, ] - self._root_view = OvPhysxView(self._ovphysx, pattern=pattern, device=self._device) + paths = _expand_env_pattern(pattern, PhysicsManager._sim.get_clone_plan()) + self._root_view = OvPhysxView(self._ovphysx, prim_paths=paths, device=self._device) # ``try_binding_for`` creates and caches each binding, returning ``None`` for tensor # types that do not apply to these prims (so a minimal articulation that lacks some # of these types is skipped rather than failing the whole init). diff --git a/source/isaaclab_ov/isaaclab_ov/assets/rigid_object/rigid_object.py b/source/isaaclab_ov/isaaclab_ov/assets/rigid_object/rigid_object.py index 05d6056c80c0..7ddac7f6ff84 100644 --- a/source/isaaclab_ov/isaaclab_ov/assets/rigid_object/rigid_object.py +++ b/source/isaaclab_ov/isaaclab_ov/assets/rigid_object/rigid_object.py @@ -28,7 +28,7 @@ from isaaclab_ov.assets import kernels as shared_kernels from isaaclab_ov.assets.kernels import _body_wrench_to_world from isaaclab_ov.physics import OvPhysxManager -from isaaclab_ov.sim.views.ovphysx_view import OvPhysxView +from isaaclab_ov.sim.views.ovphysx_view import OvPhysxView, _expand_env_pattern from .rigid_object_data import RigidObjectData @@ -957,7 +957,8 @@ def has_rigid_body_api(prim) -> bool: # Eagerly create every binding the data container reads at init, so failures # surface here with a helpful message rather than as a raw wheel exception # (or a KeyError) at first writer call. - self._root_view = OvPhysxView(self._ovphysx, pattern=pattern, device=self._device) + paths = _expand_env_pattern(pattern, OvPhysxManager._sim.get_clone_plan()) + self._root_view = OvPhysxView(self._ovphysx, prim_paths=paths, device=self._device) for tt in ( TT.RIGID_BODY_POSE, TT.RIGID_BODY_VELOCITY, diff --git a/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py b/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py index 6552b1614771..6d6893840e13 100644 --- a/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py +++ b/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py @@ -12,12 +12,12 @@ import numpy as np -from pxr import Gf, Usd, UsdGeom +from pxr import Gf, Sdf, Usd, UsdGeom, UsdPhysics from isaaclab import cloner from isaaclab.physics import PhysicsManager -from isaaclab_ov._clone import CloneTransform +from isaaclab_ov._clone import CloneRecipe if TYPE_CHECKING: from isaaclab.cloner import ClonePlan @@ -30,18 +30,53 @@ def _clone_recipes( env_ids: np.ndarray, positions: np.ndarray | None, quaternions: np.ndarray | None, -) -> list[tuple[str, list[str], list[CloneTransform], list[int]]]: +) -> list[CloneRecipe]: """Build OvPhysX clone recipes from the selected native instance groups.""" if positions is not None and positions.shape != (len(env_ids), 3): raise ValueError(f"positions must have shape [num_envs, 3], got {list(positions.shape)}.") if quaternions is not None and quaternions.shape != (len(env_ids), 4): raise ValueError(f"quaternions must have shape [num_envs, 4], got {list(quaternions.shape)}.") + grouped = {} + for pair, columns in copies: + if len(columns) and columns[0] != -1: + grouped.setdefault(pair, []).append(columns) + copies = [(pair, np.unique(np.concatenate(columns))) for pair, columns in grouped.items()] + + # OVPhysX's native metatype omits effective D6 axes and tendon layout. Check only variant sources. + variants, layouts = {}, {} + for source, template in grouped: + variants.setdefault(template, []).append(source) + for sources in variants.values(): + if len(sources) < 2: + continue + for source in sources: + if source not in layouts: + root = stage.GetPrimAtPath(source) + if not root: + raise ValueError(f"OvPhysX clone source prim is not valid on the stage: {source}") + layout = layouts[source] = {} + for prim in Usd.PrimRange(root, Usd.TraverseInstanceProxies()): + relative = str(prim.GetPath().MakeRelativePath(root.GetPath())) + if prim.GetTypeName() == "PhysicsJoint" and UsdPhysics.Joint(prim).GetJointEnabledAttr().Get(): + # Native articulations always lock translation, regardless of authored limits. + axes = [] + for axis in ("rotX", "rotY", "rotZ"): + limit = UsdPhysics.LimitAPI(prim, axis) + if not limit or limit.GetLowAttr().Get() <= limit.GetHighAttr().Get(): + axes.append(axis) + layout[relative, "rotation axes"] = tuple(axes) + for schema in prim.GetPrimTypeInfo().GetAppliedAPISchemas(): + if schema.startswith(("PhysxTendonAxisRootAPI:", "PhysxTendonAttachmentRootAPI:")): + layout[relative, schema] = None + if layouts[source] != layouts[sources[0]]: + raise ValueError( + f"OvPhysX variants {sources[0]!r} and {source!r} have incompatible rotation axes or tendon layouts." + ) + xform_cache = UsdGeom.XformCache(Usd.TimeCode.Default()) recipes = [] for (source, template), columns in copies: - if not len(columns) or columns[0] == -1: - continue prefix, _, suffix = template.partition("{}") env_template = prefix + "{}" + suffix.split("/", 1)[0] matched = cloner.path.match(source, env_template) @@ -65,8 +100,6 @@ def _clone_recipes( for env_id, column in zip(env_ids[columns], columns, strict=True): env_id = int(env_id) destination = template.format(env_id) - if destination == source: - continue targets.append(destination) target_env_ids.append(env_id) target_env_world = Gf.Matrix4d(1.0) @@ -76,10 +109,9 @@ def _clone_recipes( q = quaternions[column] target_env_world.SetRotateOnly(Gf.Quatd(float(q[3]), Gf.Vec3d(*map(float, q[:3])))) pose = (source_relative * target_env_world).RemoveScaleShear() - quat = pose.ExtractRotationQuat() - target_transforms.append((*pose.ExtractTranslation(), *quat.GetImaginary(), quat.GetReal())) - if targets: - recipes.append((source, targets, target_transforms, target_env_ids)) + rotation = pose.ExtractRotationQuat() + target_transforms.append((*pose.ExtractTranslation(), *rotation.GetImaginary(), rotation.GetReal())) + recipes.append((source, targets, target_transforms, target_env_ids, self_env_id)) return recipes @@ -105,22 +137,22 @@ def replicate(self, plan: ClonePlan, asset_prototype_ids: tuple[int, ...]) -> No asset_prototype_ids: Asset definitions routed to OVPhysX. Raises: - ValueError: If positions are malformed or an active source or source anchor prim is invalid. + ValueError: If positions are malformed, a source or source anchor is invalid, or variants + for one destination differ in effective rotation axes or tendon layout. """ sources = cloner.path.get_asset_prototype_paths(plan) templates, starts, world_ids, world_starts = cloner.path.get_world_prototype_asset_templates( plan, include_world_indices=True ) - copies = {} + copies = [] for group in np.flatnonzero(np.diff(world_starts[1:])) + 1: targets = world_ids[world_starts[group] : world_starts[group + 1]] for index in range(*starts[group : group + 2]): if (asset := plan.topology.world_prototypes[index]) in asset_prototype_ids: - copies.setdefault((sources[asset], templates[index]), []).append(targets) - copies = ((key, np.concatenate(groups)) for key, groups in copies.items()) + copies.append(((sources[asset], templates[index]), targets)) env_ids = np.arange(len(plan.topology.world_prototype_layout)) - for recipe in _clone_recipes(self.stage, copies, env_ids, plan.positions, None): - self._sim.physics_manager._register_clone_transforms(*recipe) + recipes = _clone_recipes(self.stage, copies, env_ids, plan.positions, None) + self._sim.physics_manager._clone_recipes.extend(recipes) def ovphysx_replicate( @@ -145,13 +177,126 @@ def ovphysx_replicate( Raises: RuntimeError: If no simulation context is active. - ValueError: If transforms are malformed or a source or source anchor is invalid. + ValueError: If source/destination lengths or mapping/transform shapes are inconsistent, + or an active source or source anchor is invalid. """ + if len(sources) != len(destinations): + raise ValueError(f"Expected one destination per source, got {len(sources)} and {len(destinations)}.") + if mapping.shape != (len(sources), len(env_ids)): + raise ValueError( + f"mapping must have shape [num_sources, num_envs], got {list(mapping.shape)} for " + f"{len(sources)} sources and {len(env_ids)} environments." + ) pairs = zip(sources, destinations, strict=True) copies = ((pair, np.flatnonzero(mapping[index])) for index, pair in enumerate(pairs)) recipes = _clone_recipes(stage, copies, env_ids, positions, quaternions) sim = PhysicsManager._sim if sim is None: raise RuntimeError("OvPhysX replication requires an active SimulationContext.") - for recipe in recipes: - sim.physics_manager._register_clone_transforms(*recipe) + sim.physics_manager._clone_recipes.extend(recipes) + + +def _serialize_stage( + stage: Usd.Stage, recipes: Sequence[CloneRecipe], full_stage: bool, plan: ClonePlan | None = None +) -> tuple[str, list[CloneRecipe]]: + """Export complete original worlds once and compile copies from those originals. + + Parsed bodies have native environment ID zero. An original-bearing world must therefore + import all its assets together; its copies inherit one collision group and environment ID. + CPU and features without native cloning import every declared world instead. + """ + # Group worlds by their declared asset memberships, not by completed-stage discovery. + sources = tuple(Sdf.Path(source) for source, _, _, _, _ in recipes) + memberships = {} + originals = {world for _, _, _, _, world in recipes if world is not None} + if plan is not None and not full_stage: + # USD-only physics keeps native ID zero. Coverage requires the same copy in each destination world. + asset_sources = cloner.path.get_asset_prototype_paths(plan) + templates, starts, world_ids, world_starts = cloner.path.get_world_prototype_asset_templates( + plan, include_world_indices=True + ) + for group in np.flatnonzero(np.diff(world_starts[1:])) + 1: + worlds = world_ids[world_starts[group] : world_starts[group + 1]] + for index in range(*starts[group : group + 2]): + uncovered = set(worlds) - originals + if not uncovered: + break + source, template = asset_sources[plan.topology.world_prototypes[index]], templates[index] + for root, targets, *_ in recipes: + suffix = cloner.path.relative_to(source, root) + if suffix is not None: + copied_paths = {target + suffix for target in targets} + uncovered = {world for world in uncovered if template.format(world) not in copied_paths} + if not uncovered: + continue + prim = stage.GetPrimAtPath(source) + if prim and any( + child.HasAPI(UsdPhysics.RigidBodyAPI) or child.HasAPI(UsdPhysics.CollisionAPI) + for child in Usd.PrimRange(prim, Usd.TraverseInstanceProxies()) + ): + originals.update(uncovered) + for index, (_, _, _, world_ids, _) in enumerate(recipes): + for world in world_ids or (): + memberships.setdefault(world, []).append(index) + prototypes = {tuple(memberships[world]): world for world in sorted(originals) if world in memberships} + representatives = { + world: prototypes.setdefault(tuple(members), world) for world, members in sorted(memberships.items()) + } + originals.update(prototypes.values()) + + # Materialize only originals on GPU. Each native destination is absent from the export. + layer = stage.Flatten() + exported = Usd.Stage.Open(layer) + xforms = UsdGeom.XformCache() + native = {} + for recipe in sorted(recipes, key=lambda recipe: recipe[0].count("/")): + source, targets, transforms, world_ids, source_world = recipe + target_by_world = dict(zip(world_ids, targets, strict=True)) if world_ids is not None else {} + source_pose = xforms.GetLocalToWorldTransform(stage.GetPrimAtPath(source)) + for index, target in enumerate(targets): + if target == source: + continue + target_path = Sdf.Path(target) + if any(path.HasPrefix(target_path) for path in sources): + raise ValueError(f"OvPhysX clone target {target!r} overlaps a clone source.") + world = world_ids[index] if world_ids is not None else None + if full_stage or world in originals: + # References preserve joint relationships and any authored target opinions. + prim = exported.GetPrimAtPath(target) + authored = bool(prim) + prim = prim or exported.DefinePrim(target, "Xform") + prim.GetReferences().AddInternalReference(source) + if transforms and not authored: + pose = transforms[index] + anchor = Gf.Matrix4d().SetRotate(Gf.Quatd(pose[6], Gf.Vec3d(*pose[3:6]))) + anchor.SetTranslateOnly(Gf.Vec3d(*pose[:3])) + world_pose = source_pose * source_pose.RemoveScaleShear().GetInverse() * anchor + xform = UsdGeom.Xformable(prim) + xform.MakeMatrixXform().Set(world_pose) + xform.SetResetXformStack(True) + continue + + # Clone all members of a world from the same complete original, with one native ID. + original = representatives[world] if world is not None else source_world + native_source = target_by_world[original] if world is not None else source + operation = native.setdefault( + native_source, (native_source, [], [], [] if world_ids is not None else None, original) + ) + operation[1].append(target) + if transforms: + operation[2].append(transforms[index]) + if world_ids is not None: + operation[3].append(world) + with Sdf.ChangeBlock(): + if (spec := layer.GetPrimAtPath(target_path)) is not None: + del spec.nameParent.nameChildren[spec.name] + # Empty clone ancestors make native literal-path lookup enumerate every world. + spec = layer.GetPrimAtPath(target_path.GetParentPath()) + while spec and not spec.nameChildren and spec.typeName in ("", "Xform"): + if spec.HasInfo("apiSchemas") or any(source.HasPrefix(spec.path) for source in sources): + break + parent = spec.nameParent + del (parent.nameChildren if parent else layer.rootPrims)[spec.name] + spec = parent + + return layer.ExportToString(), list(native.values()) diff --git a/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager.py b/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager.py index 3dd1d9f26269..ae5ce203e9e4 100644 --- a/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager.py +++ b/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager.py @@ -14,10 +14,10 @@ import atexit import contextlib +import importlib.util import logging import math import os -import re import stat from collections.abc import Sequence from typing import TYPE_CHECKING, Any, ClassVar @@ -37,9 +37,10 @@ from isaaclab.sim.simulation_context import SimulationContext from isaaclab.utils.buffers import TimestampedBuffer -from isaaclab_ov._clone import CloneTransform, clone_transforms_from_positions +from isaaclab_ov._clone import CloneRecipe, clone_transforms_from_positions from isaaclab_ov._runtime import import_ovphysx from isaaclab_ov.cloner import OvPhysxReplicateContext +from isaaclab_ov.cloner.replicate import _serialize_stage from isaaclab_ov.sim.views.ovphysx_view import OvPhysxView from isaaclab_ov.stage import create_ovstage @@ -88,6 +89,17 @@ def _prepare_default_cache_dir(cache_dir: str) -> str: logger = logging.getLogger(__name__) +def _newton_schema_root() -> str | None: + """Return the installed Newton USD schema plugin root, if available.""" + spec = importlib.util.find_spec("newton_usd_schemas") + if spec is None or spec.origin is None: + return None + schema_root = os.path.dirname(spec.origin) + if not os.path.isfile(os.path.join(schema_root, "plugInfo.json")): + return None + return schema_root + + class OvPhysxSceneDataBackend(SceneDataBackend): """Scene-data backend for the OVPhysX physics manager. @@ -102,16 +114,50 @@ def __init__(self): self.geometry_timestamp = 0 self._geometry = TimestampedBuffer() self._deformable_bindings: list[tuple[OvPhysxView, Any, wp.array]] = [] + self._pending_setup: tuple[OvPhysxBackend, str, Sequence[DeformableStageEntry] | None] | None = None + + def _defer_setup( + self, backend: OvPhysxBackend, device: str, entries: Sequence[DeformableStageEntry] | None = None + ) -> None: + """Defer renderer-only bindings until scene data is requested.""" + self.backend = None + self._transforms.data.transforms = None + self.transforms_timestamp += 1 + self._deformable_bindings = [] + self._geometry = TimestampedBuffer() + self.geometry_timestamp += 1 + self._pending_setup = (backend, device, entries) + + def _ensure_setup(self) -> None: + if self._pending_setup is not None: + backend, device, entries = self._pending_setup + # Native output metadata names the actual bodies, not articulation-root aliases. + # Keep these renderer-only bindings out of headless simulation startup. + from ovphysx.types import SimObjectType + from ovstage import PathDictionary + + body_paths = [] + with PathDictionary() as paths: + for kind in (SimObjectType.RIGID_BODY, SimObjectType.ARTICULATION_LINK): + with backend.physx.read(kind, ["mass"]) as bodies: + for group in bodies.groups: + body_paths.extend(paths.get_path_strings(group.prim_list)) + if body_paths: + backend.rigid_body_view = backend.physx.create_tensor_binding(prim_paths=body_paths) + self.setup(backend, device, entries) + self._pending_setup = None @property def transform_count(self) -> int: """Number of poses in the native publication.""" + self._ensure_setup() poses = self._transforms.data.transforms return 0 if poses is None else len(poses) @property def transform_paths(self) -> list[str]: """Native body paths in the same order as the published poses.""" + self._ensure_setup() view = None if self.backend is None else self.backend.rigid_body_view return [] if view is None else view.prim_paths @@ -197,6 +243,7 @@ def get_geometry_batches(self, output_format: Any = SceneDataFormat.Points) -> l Raises: RuntimeError: If scene geometry was not initialized from a clone plan. """ + self._ensure_setup() if self._geometry.data is None: raise RuntimeError("Declare and replicate a ClonePlan before requesting scene geometry.") if self._geometry.timestamp != self.geometry_timestamp: @@ -212,6 +259,7 @@ def native_geometry_formats(self) -> tuple[Any, ...]: Raises: RuntimeError: If scene geometry was not initialized from a clone plan. """ + self._ensure_setup() if self._geometry.data is None: raise RuntimeError("Declare and replicate a ClonePlan before requesting scene geometry.") return tuple(dict.fromkeys(publication._cls for publication, _ in self._geometry.data)) @@ -219,6 +267,7 @@ def native_geometry_formats(self) -> tuple[Any, ...]: @property def transforms(self) -> SceneDataFormat.Transform: """Publish native rigid-body poses [m, xyzw].""" + self._ensure_setup() if self._transforms.timestamp != self.transforms_timestamp: OvPhysxManager.pre_render() if self._transforms.data.transforms is not None: @@ -242,6 +291,7 @@ def __init__(self, cfg: OvPhysxBackendCfg): "/physics/updateToUsd": False, "/physics/updateVelocitiesToUsd": False, "/physics/updateParticlesToUsd": False, + "/ovphysx/clone/useEnvIds": is_gpu, } if is_gpu: carbonite_overrides.update({"/physics/suppressReadback": True, "/physics/suppressFabricUpdate": True}) @@ -325,12 +375,8 @@ class OvPhysxManager(PhysicsManager): _requires_full_stage: ClassVar[bool] = False # Device mode is process-wide; later contexts must reuse the first selected device. _locked_device: ClassVar[str | None] = None - # Active clone recipes survive the consumable pending queue so a forced - # re-warmup can rebuild serialized-stage or runtime-only clones. - _active_clone_recipes: ClassVar[list[tuple[str, list[str], list[CloneTransform], list[int] | None]]] = [] - # Consumable snapshot of the active recipes. Full-stage warmup materializes - # these into serialized USDA; env-0-only warmup replays them with physx.clone(). - _pending_clones: ClassVar[list[tuple[str, list[str], list[CloneTransform], list[int] | None]]] = [] + # Retain construction inputs for hard reset; serialization and replay never consume them. + _clone_recipes: ClassVar[list[CloneRecipe]] = [] _atexit_registered: ClassVar[bool] = False _scene_data_backend: ClassVar[OvPhysxSceneDataBackend | None] = None kinematics_dirty: ClassVar[bool] = False @@ -374,21 +420,7 @@ def register_clone( target roots. Each position uses an identity rotation. """ target_transforms = clone_transforms_from_positions(parent_positions or []) - cls._register_clone_transforms(source, targets, target_transforms) - - @classmethod - def _register_clone_transforms( - cls, source: str, targets: list[str], target_transforms: list[CloneTransform], env_ids: list[int] | None = None - ) -> None: - """Register final target-root world poses for the current simulation context.""" - recipe = (source, list(targets), list(target_transforms), None if env_ids is None else list(env_ids)) - cls._active_clone_recipes.append(recipe) - cls._pending_clones.append(recipe) - - @classmethod - def _rearm_pending_clones(cls) -> None: - """Refresh the consumable clone queue from active context recipes.""" - cls._pending_clones = cls._active_clone_recipes.copy() + cls._clone_recipes.append((source, targets, target_transforms, None, 0)) _physx_schemas_registered: ClassVar[bool] = False @@ -399,12 +431,13 @@ def _prepare_stage_creation(cls) -> None: @classmethod def _ensure_physx_schemas_registered(cls) -> None: - """Register the codeless USD schemas published by the OVPhysX wheel. + """Register the USD schemas consumed by the OVPhysX runtime. OVStage maintains its own USD schema registry, so register the wheel's - schema root there even when the host USD runtime already provides the - same plugins. For the host USD registry, only register providers that - are not already available from a compiled plugin. + schema root and the separately packaged Newton schemas there even when + the host USD runtime already provides the same plugins. For the host USD + registry, only register providers that are not already available from a + compiled plugin. """ if cls._physx_schemas_registered: return @@ -421,8 +454,11 @@ def _ensure_physx_schemas_registered(cls) -> None: else: schema_root = getattr(ovphysx, "codeless_schema_root", None) register_ovstage_schemas = getattr(getattr(ovstage, "population", None), "register_usd_schemas", None) - if callable(schema_root) and callable(register_ovstage_schemas): - register_ovstage_schemas(str(schema_root())) + if callable(register_ovstage_schemas): + if callable(schema_root): + register_ovstage_schemas(str(schema_root())) + if (newton_schema_root := _newton_schema_root()) is not None: + register_ovstage_schemas(newton_schema_root) registry = Plug.Registry() registered_names = {plugin.name.casefold() for plugin in registry.GetAllPlugins()} # The wheel documents ``/resources`` as its stable layout and its @@ -451,8 +487,7 @@ def initialize(cls, sim_context: SimulationContext) -> None: cls._warmup_done = False cls._requires_full_stage = False cls._stage_usda = None - cls._pending_clones = [] - cls._active_clone_recipes = [] + cls._clone_recipes = [] # Construct the SceneDataBackend eagerly so :class:`SimulationContext` # captures a real instance (not ``None``) when it builds the central # :class:`~isaaclab.scene.scene_data_provider.SceneDataProvider` in @@ -548,8 +583,7 @@ def close(cls) -> None: cls._stage_usda = None cls._warmup_done = False cls._requires_full_stage = False - cls._active_clone_recipes = [] - cls._pending_clones = [] + cls._clone_recipes = [] # Drop the SceneDataBackend singleton: its cached bindings and buffers # belong to the runtime instance just released. The next # SimulationContext re-creates it in initialize(). @@ -690,143 +724,6 @@ def get_scene_data_backend(cls) -> SceneDataBackend: # Internal helpers # ------------------------------------------------------------------ - @classmethod - def _materialize_pending_clones_in_layer(cls, layer: Any) -> int: - """Materialize queued clone targets into a flattened stage layer. - - OVPhysX runtime cloning is unsafe after a heterogeneous full-stage load: - cloning one leaf can disturb tensor discovery for already loaded sibling - assets. Missing targets are copied into the flattened layer. When another - clone has already created a target ancestor, an internal reference overlays - the source physics without replacing authored descendants. The live USD - stage remains unchanged. - - Args: - layer: Flattened stage layer to augment. - - Returns: - Number of clone targets materialized in the layer. - """ - from pxr import Sdf, Usd # noqa: PLC0415 - - pending_clones = list(cls._pending_clones) - cls._pending_clones.clear() - if not pending_clones: - return 0 - - exported_stage = Usd.Stage.Open(layer) - if exported_stage is None: - raise RuntimeError("OvPhysxManager: failed to open the flattened full-stage layer.") - - envs_path = Sdf.Path("/World/envs") - operations: list[tuple[Sdf.Path, Sdf.Path, bool]] = [] - processed_targets: set[Sdf.Path] = set() - for source, targets, _, _ in pending_clones: - source_path = Sdf.Path(source) - if layer.GetPrimAtPath(source_path) is None: - raise RuntimeError(f"OvPhysxManager: clone source {source!r} is absent from the full stage.") - for target in targets: - target_path = Sdf.Path(target) - if target_path in processed_targets: - continue - boundary_path = target_path.GetParentPath() - for prefix in target_path.GetPrefixes(): - if prefix.GetParentPath() == envs_path: - boundary_path = prefix - break - if layer.GetPrimAtPath(boundary_path) is None: - raise RuntimeError(f"OvPhysxManager: clone target parent is absent for {target!r}.") - operations.append((source_path, target_path, layer.GetPrimAtPath(target_path) is not None)) - processed_targets.add(target_path) - - operations.sort(key=lambda operation: len(operation[1].GetPrefixes())) - for source_path, target_path, target_exists in operations: - parent_path = target_path.GetParentPath() - parent_spec = layer.GetPrimAtPath(parent_path) - if parent_spec is None: - generated_paths: list[Sdf.Path] = [] - # ``CreatePrimInLayer`` authors missing ancestors as ``over`` specs. Track only - # paths absent before creation so generated ancestors become defined without - # changing existing authored specs. - ancestor_path = parent_path - while layer.GetPrimAtPath(ancestor_path) is None: - generated_paths.append(ancestor_path) - ancestor_path = ancestor_path.GetParentPath() - if Sdf.CreatePrimInLayer(layer, parent_path) is None: - raise RuntimeError( - f"OvPhysxManager: failed to materialize clone target parent {str(parent_path)!r}." - ) - for generated_path in generated_paths: - generated_spec = layer.GetPrimAtPath(generated_path) - if generated_spec is not None and generated_spec.specifier == Sdf.SpecifierOver: - generated_spec.specifier = Sdf.SpecifierDef - if target_exists: - target_prim = exported_stage.GetPrimAtPath(target_path) - if not target_prim.GetReferences().AddInternalReference(source_path): - raise RuntimeError(f"OvPhysxManager: failed to overlay clone target {str(target_path)!r}.") - elif not Sdf.CopySpec(layer, source_path, layer, target_path): - raise RuntimeError(f"OvPhysxManager: failed to materialize clone target {str(target_path)!r}.") - - if operations: - logger.info("OvPhysxManager: materialized %d clone targets in the full-stage layer", len(operations)) - return len(operations) - - @staticmethod - def _strip_nonzero_environments(layer: Any) -> int: - """Strip authored ``env_`` prims other than ``env_0`` from a stage layer.""" - envs_spec = layer.GetPrimAtPath("/World/envs") - if envs_spec is None or not envs_spec: - return 0 - - env_name_re = re.compile(r"^env_(\d+)$") - names_to_remove = [ - child_name - for child_name in list(envs_spec.nameChildren.keys()) - if (match := env_name_re.match(child_name)) and match.group(1) != "0" - ] - for child_name in names_to_remove: - del envs_spec.nameChildren[child_name] - return len(names_to_remove) - - @classmethod - def _serialize_selected_stage(cls, sim_stage: Any) -> str: - """Serialize the selected stage representation for OVStage population.""" - layer = sim_stage.Flatten() - if cls._requires_full_stage: - cls._materialize_pending_clones_in_layer(layer) - logger.info("OvPhysxManager: serialized the full USD stage in memory") - else: - removed_count = cls._strip_nonzero_environments(layer) - if removed_count: - logger.info( - "OvPhysxManager: stripped %d env_ subtrees from in-memory USD (kept env_0 + globals)", - removed_count, - ) - else: - logger.debug("OvPhysxManager: no cloned environments to strip — serialized stage as-is.") - return layer.ExportToString() - - @classmethod - def _replay_pending_clones(cls, physx: Any, requires_full_stage: bool) -> None: - pending_clones = list(cls._pending_clones) - cls._pending_clones.clear() - - if requires_full_stage: - return - - for source, targets, target_transforms, env_ids in pending_clones: - if not targets: - continue - logger.info( - "OvPhysxManager: cloning %s -> %d targets (%s ... %s)", - source, - len(targets), - targets[0], - targets[-1], - ) - # Assets cloned separately into the same world must still collide with each other. - physx.wait_op(physx.clone(source, targets, target_transforms or None, env_ids=env_ids)) - @classmethod def _warmup_and_load(cls) -> None: """Serialize the USD stage and attach it to the ovphysx runtime. @@ -849,9 +746,9 @@ def _warmup_and_load(cls) -> None: sim = PhysicsManager._sim if sim is None: raise RuntimeError("OvPhysxManager: SimulationContext is not set.") - + plan = sim.get_clone_plan() entries = None - if (plan := sim.get_clone_plan()) is not None: + if plan is not None: env_ids = np.arange(len(plan.topology.world_prototype_layout)) entries = expand_deformable_entries(deformable_prototypes(sim.stage, plan), plan, env_ids, plan.positions) @@ -866,32 +763,12 @@ def _warmup_and_load(cls) -> None: scene_prim = sim.stage.GetPrimAtPath(sim.cfg.physics_prim_path) if scene_prim.IsValid(): - if cls._active_clone_recipes: + if cls._clone_recipes: scene_prim.CreateAttribute("physxScene:envIdInBoundsBitCount", Sdf.ValueTypeNames.Int).Set(4) cls._configure_physx_scene_prim(scene_prim, PhysicsManager._cfg, ovphysx_device) - # Flatten the current USD stage to USDA text so OVStage can populate it - # without an intermediate file. - # - # When ``InteractiveScene`` runs with ``clone_usd=True``, the live USD - # stage carries env_0..N's full asset subtrees as authored copies. - # Handing that whole stage to OVStage would make the wheel ingest - # all 4096 envs as independent USD-defined bodies, defeating the - # ``physx.clone()`` fast path and turning every subsequent - # ``create_tensor_binding`` call into an O(N) USD enumeration -- the - # hang you'd see at large env counts. - # - # By default, strip ``/World/envs/env_`` for i != 0 from the - # flattened layer before handing it to the wheel. Sensors that read - # USD directly (RayCaster, Camera, ContactSensor discovery) still see - # the full N-env stage; only the wheel-side physics ingestion is - # scoped to env_0, and ``physx.clone()`` re-populates env_1..N in - # the physics runtime with proper clone lineage (which is what the - # binding fast path expects). Features that need distinct authored - # physics in every environment request the full stage; missing - # heterogeneous clone targets are materialized in its flattened layer. - cls._rearm_pending_clones() - stage_usda = cls._serialize_selected_stage(sim.stage) + full_stage = cls._requires_full_stage or ovphysx_device == "cpu" + stage_usda, native_clones = _serialize_stage(sim.stage, cls._clone_recipes, full_stage, plan) cls._stage_usda = stage_usda previous_backend = cls.backend @@ -914,29 +791,17 @@ def _warmup_and_load(cls) -> None: cls._attach_ovstage(stage_usda) logger.info("OvPhysxManager: attached OVStage to ovphysx (device=%s)", ovphysx_device) - cls._replay_pending_clones(cls.backend.physx, requires_full_stage=cls._requires_full_stage) + for source, targets, transforms, env_ids, _ in native_clones: + cls.backend.physx.wait_op(cls.backend.physx.clone(source, targets, transforms or None, env_ids=env_ids)) # Native metadata and bindings must see the newly attached bodies, including on CPU. cls._warmup_physx(cls.backend.physx) - # Bind SDP after the native runtime has realized the stage and clones. + # The central SceneDataProvider can request these bindings later. Headless + # training never consumes them, so avoid binding every rigid link here. if cls._scene_data_backend is None: cls._scene_data_backend = OvPhysxSceneDataBackend() - # Native output metadata names the actual bodies, not articulation-root aliases. - # SDK imports follow runtime initialization so inactive backends remain optional. - from ovphysx.types import SimObjectType - from ovstage import PathDictionary - - body_paths = [] - with PathDictionary() as paths: - for kind in (SimObjectType.RIGID_BODY, SimObjectType.ARTICULATION_LINK): - # Mass is host-resident: obtaining paths needs no GPU pose read or another step. - with cls.backend.physx.read(kind, ["mass"]) as bodies: - for group in bodies.groups: - body_paths.extend(paths.get_path_strings(group.prim_list)) - if body_paths: - cls.backend.rigid_body_view = cls.backend.physx.create_tensor_binding(prim_paths=body_paths) - cls._scene_data_backend.setup(cls.backend, PhysicsManager._device, entries) + cls._scene_data_backend._defer_setup(cls.backend, PhysicsManager._device, entries) cls.dispatch_event(PhysicsEvent.MODEL_INIT, payload={}) cls._warmup_done = True @@ -988,11 +853,12 @@ def _configure_physx_scene_prim(scene_prim, cfg, device: str) -> None: schemas.prependedItems = items scene_prim.SetMetadata("apiSchemas", schemas) - # Propagate scene query support from SimulationCfg so omni.physx creates - # the scene with the correct query mode. OvPhysxCfg does not carry this field. - sim_cfg = PhysicsManager._sim.cfg if PhysicsManager._sim is not None else None - enable_sq = getattr(sim_cfg, "enable_scene_query_support", False) - scene_prim.CreateAttribute("physxScene:enableSceneQuerySupport", Sdf.ValueTypeNames.Bool).Set(enable_sq) + # PhysX uses the declared timestep to derive automatic collision contact offsets. + sim_cfg = PhysicsManager._sim.cfg + scene_prim.CreateAttribute("physxScene:timeStepsPerSecond", Sdf.ValueTypeNames.Int).Set(int(1.0 / sim_cfg.dt)) + scene_prim.CreateAttribute("physxScene:enableSceneQuerySupport", Sdf.ValueTypeNames.Bool).Set( + sim_cfg.enable_scene_query_support + ) if cfg is not None: # OvPhysX answers the backend-agnostic determinism request with enhanced determinism. diff --git a/source/isaaclab_ov/isaaclab_ov/sensors/contact_sensor/contact_sensor.py b/source/isaaclab_ov/isaaclab_ov/sensors/contact_sensor/contact_sensor.py index 1e79c252814d..ee16fbc14f90 100644 --- a/source/isaaclab_ov/isaaclab_ov/sensors/contact_sensor/contact_sensor.py +++ b/source/isaaclab_ov/isaaclab_ov/sensors/contact_sensor/contact_sensor.py @@ -12,17 +12,20 @@ import logging import re from collections.abc import Sequence +from fnmatch import fnmatchcase from typing import TYPE_CHECKING, Any import warp as wp +from isaaclab import cloner from isaaclab.sensors.contact_sensor import BaseContactSensor from isaaclab.sim.utils.queries import path_expr_to_glob, resolve_matching_prims_from_source, split_path_expr from isaaclab.utils.warp import ProxyArray import isaaclab_ov.tensor_types as TT +from isaaclab_ov._clone import ordered_clone_paths from isaaclab_ov.physics import OvPhysxManager -from isaaclab_ov.sim.views.ovphysx_view import OvPhysxView +from isaaclab_ov.sim.views.ovphysx_view import OvPhysxView, _expand_env_pattern from .contact_sensor_data import ContactSensorData from .kernels import ( @@ -223,32 +226,50 @@ def has_contact_report(prim) -> bool: # not share a parent. IsaacLab path forms map to ovphysx fnmatch globs the same way # Articulation does. sensor_patterns = [path_expr_to_glob(re.sub(r"\{ENV_REGEX_NS\}", "*", expr)) for _, expr in body_matches] - - # Build filter patterns (flat: len = n_sensors * filters_per_sensor). - filter_globs = [ - path_expr_to_glob(re.sub(r"\{ENV_REGEX_NS\}", "*", expr)) for expr in self.cfg.filter_prim_paths_expr - ] + plan = OvPhysxManager._sim.get_clone_plan() + sensor_patterns = [path for pattern in sensor_patterns for path in _expand_env_pattern(pattern, plan)] + filter_globs = [path_expr_to_glob(expr) for expr in self.cfg.filter_prim_paths_expr] filters_per_sensor = len(filter_globs) - if filters_per_sensor > 0: - filter_patterns: list[str] | None = filter_globs * self._num_sensors - else: - filter_patterns = None - - # Create the contact binding (must happen BEFORE the next step()). - # OVPhysX's ``InteractiveScene`` runs in ``clone_usd=False`` mode: - # env_1..N have no USD prim — they're physics-layer clones via - # ``physx.clone()``. The parent class's ``find_matching_prims`` walk - # therefore sees only env_0 and sets ``self._num_envs = 1`` even when - # the scene is configured for many envs. We size the - # ``max_contact_data_count`` for env_0 only here; the binding's - # ``sensor_count`` after creation gives us the real env count. + filter_worlds = [None] * filters_per_sensor + if plan is not None: + filter_worlds = [cloner.path.match(pattern, plan.env_template) for pattern in filter_globs] + filter_patterns = [] + # Each sensor sees the filters in its own world; shared filters keep their native pattern. + for sensor_path in sensor_patterns: + sensor_world = None if plan is None else cloner.path.match(sensor_path, plan.env_template) + for pattern, filter_world in zip(filter_globs, filter_worlds, strict=True): + if ( + sensor_world is not None + and filter_world is not None + and fnmatchcase(sensor_world.instance, filter_world.instance) + ): + pattern = plan.env_template.format(sensor_world.instance) + filter_world.suffix + filter_patterns.append(pattern) + + # Contact reporting must bind before the first step. max_count = self.cfg.max_contact_data_count_per_prim * self._num_sensors * self._num_envs self._contact_binding = physx_instance.create_contact_binding( sensor_patterns=sensor_patterns, - filter_patterns=filter_patterns, + filter_patterns=filter_patterns or None, filters_per_sensor=filters_per_sensor, max_contact_data_count=max_count, ) + # Contact discovery is lexical; keep the same body-major, numeric env order as asset views. + paths = list(self._contact_binding.sensor_paths) + expected_paths = {path for path in sensor_patterns if not any(char in path for char in "*?[")} + if missing := expected_paths.difference(paths): + raise RuntimeError(f"Contact binding omitted sensor bodies {sorted(missing)}; check their filter paths.") + ordered_paths = ordered_clone_paths(paths, sensor_patterns) + if ordered_paths != paths: + filters_by_path = dict(zip(paths, self._contact_binding.filter_paths)) + filters_per_sensor = self._contact_binding.filter_count + self._contact_binding.destroy() + self._contact_binding = physx_instance.create_contact_binding( + sensor_patterns=ordered_paths, + filter_patterns=[f for path in ordered_paths for f in filters_by_path[path]] or None, + filters_per_sensor=filters_per_sensor, + max_contact_data_count=max_count, + ) # Validate: sensor_count must be a non-zero multiple of num_sensors. if self._contact_binding.sensor_count == 0 or self._contact_binding.sensor_count % self._num_sensors != 0: @@ -260,10 +281,7 @@ def has_contact_report(prim) -> bool: f"\n\tBound sensors : {self._contact_binding.sensor_count}" ) - # Override ``_num_envs`` with the binding's view if it differs (it does - # for any OVPhysX scene with ``num_envs > 1`` due to ``clone_usd=False``). - # Re-allocate the env-sized buffers from the parent class so they match - # the real env count. + # A sensor may select fewer worlds than the complete plan. binding_num_envs = self._contact_binding.sensor_count // self._num_sensors if binding_num_envs != self._num_envs: self._num_envs = binding_num_envs @@ -288,13 +306,13 @@ def has_contact_report(prim) -> bool: f"under '{self.cfg.prim_path}'). Workaround: create one ContactSensor " "per body." ) - single_pose_pattern = sensor_patterns[0] - self._root_view = OvPhysxView(physx_instance, pattern=single_pose_pattern, device=self._device) + pose_paths = self._contact_binding.sensor_paths + self._root_view = OvPhysxView(physx_instance, prim_paths=pose_paths, device=self._device) self._pose_binding = self._root_view.binding_for(TT.RIGID_BODY_POSE) if self._pose_binding.count != self._contact_binding.sensor_count: raise RuntimeError( "RIGID_BODY_POSE binding count mismatch." - f"\n\tPattern: {single_pose_pattern}" + f"\n\tPattern: {self.cfg.prim_path}" f"\n\tBound : {self._pose_binding.count}" f"\n\tExpect : {self._contact_binding.sensor_count}" ) diff --git a/source/isaaclab_ov/isaaclab_ov/sim/views/ovphysx_view.py b/source/isaaclab_ov/isaaclab_ov/sim/views/ovphysx_view.py index a97dd9f297c6..486e887717b2 100644 --- a/source/isaaclab_ov/isaaclab_ov/sim/views/ovphysx_view.py +++ b/source/isaaclab_ov/isaaclab_ov/sim/views/ovphysx_view.py @@ -46,10 +46,14 @@ import logging import math +from fnmatch import fnmatchcase from typing import Any, ClassVar, Protocol import warp as wp +from isaaclab import cloner + +from isaaclab_ov._clone import ordered_clone_paths from isaaclab_ov._runtime import import_ovphysx from isaaclab_ov.tensor_types import _CPU_ONLY_TYPES @@ -58,6 +62,28 @@ # Pure-Python enum (no native dependency); safe to import regardless of USD state. TensorType = import_ovphysx("ovphysx.types").TensorType + +def _expand_env_pattern(pattern: str, plan: cloner.ClonePlan | None) -> list[str]: + """Expand planned world instances in world order; leave other native globs unchanged.""" + if plan is None: + return [pattern] + templates, starts, worlds, world_starts = cloner.path.get_world_prototype_asset_templates( + plan, include_world_indices=True + ) + paths = [] + for group in range(1, len(starts) - 1): + env_ids = worlds[world_starts[group] : world_starts[group + 1]] + for template in templates[starts[group] : starts[group + 1]]: + matched = cloner.path.match(pattern, template) + if matched is not None: + paths.extend( + (int(env_id), template.format(env_id) + matched.suffix) + for env_id in env_ids + if fnmatchcase(str(env_id), matched.instance) + ) + return list(dict.fromkeys(path for _, path in sorted(paths))) if paths else [pattern] + + # Tensor types that cannot be written. The first group is read-only by PhysX # convention (accelerations, inverse mass/inertia, projected joint force); the # computed-dynamics group (jacobian, mass matrix, coriolis, gravity) is read-only @@ -631,7 +657,14 @@ def _binding(self, tensor_type: Any) -> Any: return binding create_type = self._key_aliases.get(tensor_type, tensor_type) kwargs: dict[str, Any] = {"tensor_type": create_type} - if self._prim_paths is not None: + # Reuse resolved paths within each native family; rigid bodies and articulations may differ. + prefix = create_type.name.partition("_")[0] + "_" + previous = None + if prefix in ("RIGID_", "ARTICULATION_"): + previous = next((b for b in self._bindings.values() if b.tensor_type.name.startswith(prefix)), None) + if previous is not None: + kwargs["prim_paths"] = previous.prim_paths + elif self._prim_paths is not None: kwargs["prim_paths"] = self._prim_paths else: kwargs["pattern"] = self._pattern @@ -658,6 +691,15 @@ def _binding(self, tensor_type: Any) -> Any: f"Attribute {tensor_type_name(tensor_type)!r} is not available for {self._target_repr()} " "(no matching prims)." ) + # Variant batches are discovered in clone order, not environment order. + # Rebind only when needed; explicit path lists also preserve collection body order. + patterns = kwargs.get("prim_paths", [self._pattern]) + paths = list(binding.prim_paths) if create_type.name.startswith(("RIGID_BODY_", "ARTICULATION_")) else [] + + ordered_paths = ordered_clone_paths(paths, patterns) + if ordered_paths != paths: + binding.destroy() + binding = self._physx.create_tensor_binding(prim_paths=ordered_paths, tensor_type=create_type) self._bindings[tensor_type] = binding self._live_views.add(self) return binding diff --git a/source/isaaclab_ov/test/assets/test_articulation.py b/source/isaaclab_ov/test/assets/test_articulation.py index f075d4462872..40cea80c376a 100644 --- a/source/isaaclab_ov/test/assets/test_articulation.py +++ b/source/isaaclab_ov/test/assets/test_articulation.py @@ -60,7 +60,7 @@ import torch import warp as wp -from pxr import Gf, Usd, UsdGeom, UsdPhysics +from pxr import Gf, Sdf, Usd, UsdGeom, UsdPhysics from isaaclab.test.utils import test_devices from isaaclab.test.utils.articulation_ordering import ( @@ -89,6 +89,8 @@ import isaaclab.utils.string as string_utils # noqa: E402 from isaaclab.actuators import DelayedPDActuatorCfg, IdealPDActuatorCfg, ImplicitActuatorCfg # noqa: E402 from isaaclab.assets import ArticulationCfg, get_articulation_name_ordering # noqa: E402 +from isaaclab.managers import SceneEntityCfg # noqa: E402 +from isaaclab.scene import InteractiveScene, InteractiveSceneCfg # noqa: E402 from isaaclab.sim import SimulationCfg, build_simulation_context # noqa: E402 from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR # noqa: E402 from isaaclab.utils.warp.launch_cache import _WarpLaunchCache # noqa: E402 @@ -298,6 +300,102 @@ def _ovphysx_sim_context(device: str, **kwargs): return build_simulation_context(device=device, sim_cfg=sim_cfg, **kwargs) +@pytest.mark.parametrize("device", test_devices()) +@pytest.mark.parametrize( + "variant", ["geometry", "joint_type", "d6_rotation", "d6_translation", "disabled_joint", "fixed_tendon"] +) +def test_heterogeneous_articulation_clone_indexed_state(device, variant, tmp_path): + """Compatible variants preserve indexed state; incompatible action/tendon layouts are rejected.""" + variants = [] + for shape in (UsdGeom.Cube, UsdGeom.Sphere): + stage = Usd.Stage.CreateInMemory() + robot = UsdGeom.Xform.Define(stage, "/Robot").GetPrim() + stage.SetDefaultPrim(robot) + UsdPhysics.ArticulationRootAPI.Apply(robot) + for name in ("Base", "Tip"): + body = UsdGeom.Xform.Define(stage, f"/Robot/{name}") + body.AddTranslateOp().Set((0.0, 0.0, 0.4 if name == "Tip" else 0.0)) + UsdPhysics.RigidBodyAPI.Apply(body.GetPrim()) + UsdPhysics.MassAPI.Apply(body.GetPrim()).CreateMassAttr(1.0) + geometry = shape.Define(stage, f"/Robot/{name}/Geometry") + geometry.AddScaleOp().Set((0.1, 0.1, 0.1)) + UsdPhysics.CollisionAPI.Apply(geometry.GetPrim()) + fixed = UsdPhysics.FixedJoint.Define(stage, "/Robot/Fixed") + fixed.CreateBody1Rel().SetTargets(["/Robot/Base"]) + joint_schema = UsdPhysics.RevoluteJoint + if variant.startswith("d6_"): + joint_schema = UsdPhysics.Joint + elif variant == "joint_type" and shape is UsdGeom.Sphere: + joint_schema = UsdPhysics.PrismaticJoint + joint = joint_schema.Define(stage, "/Robot/Joint") + joint.CreateBody0Rel().SetTargets(["/Robot/Base"]) + joint.CreateBody1Rel().SetTargets(["/Robot/Tip"]) + joint.CreateLocalPos0Attr((0.0, 0.0, 0.4)) + if variant == "disabled_joint": + joint = UsdPhysics.Joint.Define(stage, "/Robot/Disabled") + joint.CreateBody0Rel().SetTargets(["/Robot/Base"]) + joint.CreateBody1Rel().SetTargets(["/Robot/Tip"]) + joint.CreateLocalPos0Attr((0.0, 0.0, 0.4)) + joint.CreateJointEnabledAttr(False) + if variant.startswith("d6_") or variant == "disabled_joint": + free_axis = "rotY" if variant != "d6_translation" and shape is UsdGeom.Sphere else "rotX" + for axis in ("transX", "transY", "transZ", "rotX", "rotY", "rotZ"): + if axis == free_axis or (axis == "transX" and variant == "d6_translation" and shape is UsdGeom.Sphere): + continue + limit = UsdPhysics.LimitAPI.Apply(joint.GetPrim(), axis) + limit.CreateLowAttr(1.0) + limit.CreateHighAttr(-1.0) + elif variant == "fixed_tendon" and shape is UsdGeom.Sphere: + prim = joint.GetPrim() + prim.AddAppliedSchema("PhysxTendonAxisRootAPI:tendon") + prim.CreateAttribute("physxTendon:tendon:stiffness", Sdf.ValueTypeNames.Float).Set(123.0) + prim.CreateAttribute("physxTendon:tendon:gearing", Sdf.ValueTypeNames.FloatArray).Set([1.0]) + path = str(tmp_path / f"{shape.__name__}.usda") + stage.Export(path) + variants.append(sim_utils.UsdFileCfg(usd_path=path)) + + with _ovphysx_sim_context(device=device, gravity_enabled=False) as sim: + cfg = InteractiveSceneCfg(num_envs=6, env_spacing=2.0) + cfg.robot = ArticulationCfg( + prim_path="{ENV_REGEX_NS}/Robot", + spawn=sim_utils.MultiAssetSpawnerCfg(assets_cfg=variants), + actuators={"joint": ImplicitActuatorCfg(joint_names_expr=["Joint.*"], stiffness=0.0, damping=0.0)}, + ) + if variant in ("d6_rotation", "fixed_tendon"): + with pytest.raises(ValueError, match="incompatible.*(rotation axes|tendon)"): + InteractiveScene(cfg) + return + scene = InteractiveScene(cfg) + if variant == "joint_type": + with pytest.raises(RuntimeError, match="heterogeneous or empty view"): + sim.reset() + return + sim.reset() + robot = scene["robot"] + assert robot.num_joints == 1 + assert robot.num_bodies == 2 + assert robot.root_view.prim_paths == [f"/World/envs/env_{i}/Robot" for i in range(scene.num_envs)] + torch.testing.assert_close(robot.data.root_pos_w.torch, scene.env_origins) + selected = torch.tensor([5, 3], device=device) + positions = torch.tensor([[0.25], [-0.4]], device=device) + robot.write_joint_state_to_sim(positions, torch.zeros_like(positions), env_ids=selected) + sim.step() + scene.update(sim.get_physics_dt()) + expected = torch.zeros((scene.num_envs, 1), device=device) + expected[selected] = positions + torch.testing.assert_close(robot.data.joint_pos.torch, expected, atol=1e-4, rtol=0.0) + for env_id in range(scene.num_envs): + binding = sim.physics_manager.get_physx_instance().create_tensor_binding( + pattern=f"/World/envs/env_{env_id}/Robot", tensor_type=TT.DOF_POSITION + ) + try: + actual = torch.empty(binding.shape, device=device) + binding.read(actual) + torch.testing.assert_close(actual, expected[env_id : env_id + 1], atol=1e-4, rtol=0.0) + finally: + binding.destroy() + + def generate_articulation_cfg( articulation_type: str, stiffness: float | None = 10.0, @@ -566,7 +664,7 @@ def test_ovphysx_effort_binding_excludes_implicit_pd(device, use_newton_actuator def test_newton_native_actuator_reset_and_gain_event_are_environment_selective(device): """Reset and randomize only the selected OVPhysX native-controller environment.""" from isaaclab.envs.mdp.events import randomize_actuator_gains # noqa: PLC0415 - from isaaclab.managers import EventTermCfg, SceneEntityCfg # noqa: PLC0415 + from isaaclab.managers import EventTermCfg # noqa: PLC0415 class Env: def __init__(self, asset): diff --git a/source/isaaclab_ov/test/assets/test_rigid_object.py b/source/isaaclab_ov/test/assets/test_rigid_object.py index 4253adb3807f..63f6dc712fd4 100644 --- a/source/isaaclab_ov/test/assets/test_rigid_object.py +++ b/source/isaaclab_ov/test/assets/test_rigid_object.py @@ -41,12 +41,15 @@ from isaaclab_ov import tensor_types as TT # noqa: E402 from isaaclab_ov.assets import RigidObject # noqa: E402 -from isaaclab_ov.physics import OvPhysxCfg # noqa: E402 +from isaaclab_ov.physics import OvPhysxCfg, OvPhysxManager # noqa: E402 import isaaclab.sim as sim_utils # noqa: E402 -from isaaclab.assets import RigidObjectCfg # noqa: E402 +from isaaclab import cloner # noqa: E402 +from isaaclab.assets import AssetBaseCfg, RigidObjectCfg # noqa: E402 +from isaaclab.scene import InteractiveScene, InteractiveSceneCfg # noqa: E402 from isaaclab.sim import SimulationCfg, build_simulation_context # noqa: E402 from isaaclab.sim.spawners import materials # noqa: E402 +from isaaclab.utils import configclass # noqa: E402 from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR # noqa: E402 from isaaclab.utils.math import ( # noqa: E402 combine_frame_transforms, @@ -109,6 +112,106 @@ def _ovphysx_sim_context(device: str, **kwargs): return build_simulation_context(device=device, sim_cfg=sim_cfg, **kwargs) +@configclass +class HeterogeneousRigidSceneCfg(InteractiveSceneCfg): + """Two object variants dropping onto an independently cloned support.""" + + support = RigidObjectCfg( + prim_path="{ENV_REGEX_NS}/Support", + spawn=sim_utils.CuboidCfg( + size=(1.0, 1.0, 0.2), + rigid_props=sim_utils.RigidBodyBaseCfg(kinematic_enabled=True), + collision_props=sim_utils.CollisionBaseCfg(), + ), + ) + object = RigidObjectCfg( + prim_path="{ENV_REGEX_NS}/Object", + spawn=sim_utils.MultiAssetSpawnerCfg( + assets_cfg=[sim_utils.CuboidCfg(size=(0.2, 0.2, 0.2)), sim_utils.SphereCfg(radius=0.1)], + rigid_props=sim_utils.RigidBodyBaseCfg(), + collision_props=sim_utils.CollisionBaseCfg(), + mass_props=sim_utils.MassPropertiesCfg(mass=1.0), + ), + init_state=RigidObjectCfg.InitialStateCfg(pos=(0.0, 0.0, 1.0)), + ) + + +@pytest.mark.parametrize("device", test_devices()) +@pytest.mark.parametrize("filter_collisions", [False, True]) +@pytest.mark.parametrize("support_cloning", ["native", "usd_and_native", "usd", "usd_nested"]) +def test_heterogeneous_clone_contacts(device, filter_collisions, support_cloning): + """Sources and clones from different variants contact their own support.""" + with _ovphysx_sim_context(device=device, dt=1.0 / 120.0) as sim: + cfg = HeterogeneousRigidSceneCfg(num_envs=6, env_spacing=2.0, filter_collisions=filter_collisions) + if support_cloning != "native": + cfg.clone_cfg.clone_template = "/Scenes/World_{}" + cfg.support.cloning_contexts = ("isaaclab.cloner:UsdReplicateContext",) + if support_cloning == "usd_and_native": + cfg.support.cloning_contexts += ("isaaclab_ov.cloner:OvPhysxReplicateContext",) + if support_cloning == "usd_nested": + cfg.support.prim_path = "{ENV_REGEX_NS}/Assembly/Support" + cfg.object.spawn.assets_cfg = cfg.object.spawn.assets_cfg[:1] + spawn = sim_utils.SpawnerCfg(func=lambda path, cfg, **kwargs: sim.stage.DefinePrim(path, "Xform")) + cfg.assembly = AssetBaseCfg(prim_path="{ENV_REGEX_NS}/Assembly", spawn=spawn, cloning_contexts=None) + cfg.clone_cfg.clone_combinations = [cloner.InclusionSet(assets=names) for names in (["assembly"], [])] + scene = InteractiveScene(cfg) + sim.reset() + for _ in range(240): + sim.step() + scene.update(sim.get_physics_dt()) + positions = scene["object"].data.root_pos_w.torch + torch.testing.assert_close(positions[:, 2], torch.full_like(positions[:, 2], 0.2), atol=0.02, rtol=0.0) + torch.testing.assert_close(positions[:, :2], scene.env_origins[:, :2], atol=0.02, rtol=0.0) + + +@pytest.mark.parametrize("device", test_devices()) +def test_heterogeneous_clone_collision_isolation(device): + """Collision groups isolate overlapping environments, including both retained sources.""" + with _ovphysx_sim_context(device=device, dt=1.0 / 120.0) as sim: + # Without GPU environment-ID filtering, 2048 overlapping worlds exhaust broadphase pairs. + num_envs = 2048 if device.startswith("cuda") else 6 + scene = InteractiveScene(HeterogeneousRigidSceneCfg(num_envs=num_envs, env_spacing=0.0)) + sim.reset() + for _ in range(240): + sim.step() + scene.update(sim.get_physics_dt()) + positions = scene["object"].data.root_pos_w.torch + expected = scene.env_origins.clone() + expected[:, 2] = 0.2 + torch.testing.assert_close(positions, expected, atol=0.02, rtol=0.0) + + +@pytest.mark.parametrize("device", test_devices()) +def test_heterogeneous_clone_indexed_state(device): + """Indexed reads and writes use environment order across six interleaved variants.""" + with _ovphysx_sim_context(device=device, gravity_enabled=False) as sim: + scene = InteractiveScene(HeterogeneousRigidSceneCfg(num_envs=6, env_spacing=2.0)) + sim.reset() + obj = scene["object"] + expected_paths = [f"/World/envs/env_{i}/Object" for i in range(scene.num_envs)] + assert obj.root_view.prim_paths == expected_paths + torch.testing.assert_close(obj.data.root_pos_w.torch[:, :2], scene.env_origins[:, :2]) + selected = torch.tensor([3], device=device) + pose = obj.data.root_pose_w.torch[selected].clone() + pose[:, 2] = 3.0 + obj.write_root_pose_to_sim(pose, env_ids=selected) + sim.step() + scene.update(sim.get_physics_dt()) + # An independent exact-path binding detects a write to the wrong physical object. + binding = OvPhysxManager.get_physx_instance().create_tensor_binding( + prim_paths=expected_paths, tensor_type=TT.RIGID_BODY_POSE + ) + try: + actual = torch.empty(binding.shape, device=device) + binding.read(actual) + expected_height = torch.ones(scene.num_envs, device=device) + expected_height[selected] = 3.0 + torch.testing.assert_close(actual[:, 2], expected_height) + torch.testing.assert_close(obj.data.root_pos_w.torch, actual[:, :3]) + finally: + binding.destroy() + + def generate_cubes_scene( num_cubes: int = 1, height=1.0, diff --git a/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py b/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py index 110f864f94ff..9b2094b9d665 100644 --- a/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py +++ b/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py @@ -64,8 +64,7 @@ def manager_module(monkeypatch): "_warmup_done": False, "_requires_full_stage": False, "_locked_device": None, - "_active_clone_recipes": [], - "_pending_clones": [], + "_clone_recipes": [], "_atexit_registered": False, "_scene_data_backend": None, "_physx_schemas_registered": False, @@ -156,11 +155,18 @@ def RegisterPlugins(self, paths): # A None entry makes importing OVStage raise ModuleNotFoundError. monkeypatch.setitem(sys.modules, "ovstage", fake_ovstage if has_registration_api is not None else None) monkeypatch.setitem(sys.modules, "pxr", fake_pxr) + newton_schema_root = "/schemas/newton" + monkeypatch.setattr(manager_module, "_newton_schema_root", lambda: newton_schema_root, raising=False) manager._ensure_physx_schemas_registered() manager._ensure_physx_schemas_registered() - assert ovstage_registrations == ([schema_root] if schema_root is not None and has_registration_api else []) + expected_ovstage_registrations = [] + if has_registration_api: + if schema_root is not None: + expected_ovstage_registrations.append(schema_root) + expected_ovstage_registrations.append(newton_schema_root) + assert ovstage_registrations == expected_ovstage_registrations assert host_registrations == ([expected_paths] if expected_paths else []) @@ -482,10 +488,10 @@ def test_retained_binding_preserves_uncaught_failure_exit_status(): _assert_no_atexit_errors(output) -@pytest.mark.parametrize("device", ["cpu", "gpu"]) +@pytest.mark.parametrize(("device", "rate"), [("cpu", 60), ("gpu", 120)]) @pytest.mark.parametrize(("override", "expected"), [(None, True), (False, False), (True, True)]) -def test_scene_external_forces_every_iteration(monkeypatch, manager_module, device, override, expected): - """Default TGS force integration and explicit overrides reach the USD physics scene.""" +def test_scene_settings(monkeypatch, manager_module, device, rate, override, expected): + """The simulation timestep, scene queries, and TGS force settings reach the native scene.""" from isaaclab_ov.physics.ovphysx_manager_cfg import OvPhysxCfg from pxr import Usd @@ -494,11 +500,14 @@ def test_scene_external_forces_every_iteration(monkeypatch, manager_module, devi cfg = OvPhysxCfg() if override is None else OvPhysxCfg(enable_external_forces_every_iteration=override) assert cfg.enable_external_forces_every_iteration is expected - monkeypatch.setattr(PhysicsManager, "_sim", None) + sim_cfg = SimpleNamespace(dt=1.0 / rate, enable_scene_query_support=device == "gpu") + monkeypatch.setattr(PhysicsManager, "_sim", SimpleNamespace(cfg=sim_cfg)) stage = Usd.Stage.CreateInMemory() prim = stage.DefinePrim("/World/PhysicsScene", "PhysicsScene") manager_module.OvPhysxManager._configure_physx_scene_prim(prim, cfg, device) assert prim.GetAttribute("physxScene:enableExternalForcesEveryIteration").Get() is expected + assert prim.GetAttribute("physxScene:timeStepsPerSecond").Get() == rate + assert prim.GetAttribute("physxScene:enableSceneQuerySupport").Get() is sim_cfg.enable_scene_query_support def test_construct_physx_forwards_cooked_collider_cache_dir(monkeypatch, manager_module, tmp_path): diff --git a/source/isaaclab_ov/test/physics/test_ovphysx_scene_data_backend.py b/source/isaaclab_ov/test/physics/test_ovphysx_scene_data_backend.py index 21b5137f14d0..9130b51116cc 100644 --- a/source/isaaclab_ov/test/physics/test_ovphysx_scene_data_backend.py +++ b/source/isaaclab_ov/test/physics/test_ovphysx_scene_data_backend.py @@ -7,7 +7,6 @@ from __future__ import annotations -import logging import sys from types import ModuleType, SimpleNamespace @@ -62,257 +61,6 @@ def _make_two_environment_stage(): return stage -def _serialize_full_stage_with_pending_clones(stage) -> str: - """Serialize ``stage`` with pending clones materialized via the production path.""" - from isaaclab_ov.physics import OvPhysxManager - - previous = OvPhysxManager._requires_full_stage - try: - OvPhysxManager._requires_full_stage = True - return OvPhysxManager._serialize_selected_stage(stage) - finally: - OvPhysxManager._requires_full_stage = previous - - -def test_manager_full_stage_materializes_only_missing_heterogeneous_targets(): - """A full-stage export copies missing heterogeneous targets without replacing authored ones.""" - from isaaclab_ov.physics import OvPhysxManager - - from pxr import Sdf, Usd - - stage = Usd.Stage.CreateInMemory() - stage.DefinePrim("/World/envs/env_0", "Xform") - stage.DefinePrim("/World/envs/env_1", "Xform") - stage.DefinePrim("/World/envs/env_2", "Xform") - source = stage.DefinePrim("/World/envs/env_0/Object", "Xform") - source.CreateAttribute("test:variant", Sdf.ValueTypeNames.String).Set("source") - existing = stage.DefinePrim("/World/envs/env_1/Object", "Xform") - existing.CreateAttribute("test:variant", Sdf.ValueTypeNames.String).Set("authored") - previous = OvPhysxManager._pending_clones - try: - OvPhysxManager._pending_clones = [ - ( - "/World/envs/env_0/Object", - ["/World/envs/env_1/Object", "/World/envs/env_2/Object"], - [(1.0, 2.0, 3.0, 0.0, 0.0, 0.0, 1.0), (4.0, 5.0, 6.0, 0.0, 0.0, 0.0, 1.0)], - [1, 2], - ) - ] - materialized_usda = _serialize_full_stage_with_pending_clones(stage) - layer = Sdf.Layer.CreateAnonymous("materialized.usda") - assert layer.ImportFromString(materialized_usda) - exported = Usd.Stage.Open(layer) - assert exported.GetPrimAtPath("/World/envs/env_1/Object").GetAttribute("test:variant").Get() == "authored" - assert exported.GetPrimAtPath("/World/envs/env_2/Object").GetAttribute("test:variant").Get() == "source" - assert OvPhysxManager._pending_clones == [] - finally: - OvPhysxManager._pending_clones = previous - - -def test_manager_full_stage_materializes_nested_targets_parent_before_child(): - """Nested clone targets materialize shallow-to-deep regardless of queue order.""" - from isaaclab_ov.physics import OvPhysxManager - - from pxr import Sdf, Usd - - stage = Usd.Stage.CreateInMemory() - stage.DefinePrim("/World/envs/env_0", "Xform") - stage.DefinePrim("/World/envs/env_1", "Xform") - groceries = stage.DefinePrim("/World/envs/env_0/Groceries", "Xform") - groceries.CreateAttribute("test:collection", Sdf.ValueTypeNames.String).Set("source") - object_prim = stage.DefinePrim("/World/envs/env_0/Groceries/Object", "Xform") - object_prim.CreateAttribute("physics:enabled", Sdf.ValueTypeNames.Bool).Set(True) - - previous = OvPhysxManager._pending_clones - try: - OvPhysxManager._pending_clones = [ - ( - "/World/envs/env_0/Groceries/Object", - ["/World/envs/env_1/Groceries/Object"], - [(1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0)], - [1], - ), - ( - "/World/envs/env_0/Groceries", - ["/World/envs/env_1/Groceries"], - [(1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0)], - [1], - ), - ] - materialized_usda = _serialize_full_stage_with_pending_clones(stage) - layer = Sdf.Layer.CreateAnonymous("materialized.usda") - assert layer.ImportFromString(materialized_usda) - exported = Usd.Stage.Open(layer) - target_parent = exported.GetPrimAtPath("/World/envs/env_1/Groceries") - target_child = exported.GetPrimAtPath("/World/envs/env_1/Groceries/Object") - assert target_parent.GetAttribute("test:collection").Get() == "source" - assert target_child.GetAttribute("physics:enabled").Get() is True - assert OvPhysxManager._pending_clones == [] - finally: - OvPhysxManager._pending_clones = previous - - -def test_manager_full_stage_promotes_generated_nested_ancestors_to_def(): - """A child-only nested target composes beneath a generated defined parent.""" - from isaaclab_ov.physics import OvPhysxManager - - from pxr import Sdf, Usd - - stage = Usd.Stage.CreateInMemory() - stage.DefinePrim("/World/envs/env_0", "Xform") - stage.DefinePrim("/World/envs/env_1", "Xform") - source = stage.DefinePrim("/World/envs/env_0/Groceries/Object", "Xform") - source.CreateAttribute("physics:enabled", Sdf.ValueTypeNames.Bool).Set(True) - - previous = OvPhysxManager._pending_clones - try: - OvPhysxManager._pending_clones = [ - ( - "/World/envs/env_0/Groceries/Object", - ["/World/envs/env_1/Groceries/Object"], - [(1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0)], - [1], - ) - ] - materialized_usda = _serialize_full_stage_with_pending_clones(stage) - layer = Sdf.Layer.CreateAnonymous("materialized.usda") - assert layer.ImportFromString(materialized_usda) - exported = Usd.Stage.Open(layer) - target_parent = exported.GetPrimAtPath("/World/envs/env_1/Groceries") - target_child = exported.GetPrimAtPath("/World/envs/env_1/Groceries/Object") - assert target_parent.IsDefined() - assert target_child.IsDefined() - assert target_child.GetAttribute("physics:enabled").Get() is True - assert OvPhysxManager._pending_clones == [] - finally: - OvPhysxManager._pending_clones = previous - - -def test_manager_full_stage_overlays_existing_ancestor_without_removing_descendants(): - """An ancestor created for another asset gains source physics while retaining descendants.""" - from isaaclab_ov.physics import OvPhysxManager - - from pxr import Sdf, Usd - - stage = Usd.Stage.CreateInMemory() - stage.DefinePrim("/World/envs/env_0", "Xform") - stage.DefinePrim("/World/envs/env_1", "Xform") - source = stage.DefinePrim("/World/envs/env_0/Robot", "Xform") - source.CreateAttribute("physics:enabled", Sdf.ValueTypeNames.Bool).Set(True) - physics = stage.DefinePrim("/World/envs/env_0/Robot/Physics", "Xform") - physics.CreateAttribute("physics:mass", Sdf.ValueTypeNames.Float).Set(3.0) - camera = stage.DefinePrim("/World/envs/env_1/Robot/Camera", "Xform") - camera.CreateAttribute("test:keep", Sdf.ValueTypeNames.Bool).Set(True) - - previous = OvPhysxManager._pending_clones - try: - OvPhysxManager._pending_clones = [ - ("/World/envs/env_0/Robot", ["/World/envs/env_1/Robot"], [(1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0)], [1]) - ] - materialized_usda = _serialize_full_stage_with_pending_clones(stage) - layer = Sdf.Layer.CreateAnonymous("materialized.usda") - assert layer.ImportFromString(materialized_usda) - exported = Usd.Stage.Open(layer) - robot = exported.GetPrimAtPath("/World/envs/env_1/Robot") - assert robot.GetAttribute("physics:enabled").Get() is True - assert exported.GetPrimAtPath("/World/envs/env_1/Robot/Physics").GetAttribute("physics:mass").Get() == 3.0 - assert exported.GetPrimAtPath("/World/envs/env_1/Robot/Camera").GetAttribute("test:keep").Get() is True - finally: - OvPhysxManager._pending_clones = previous - - -def test_manager_retains_clone_recipes_across_full_stage_serializations(): - """A second full-stage serialization rematerializes targets from active recipes.""" - from isaaclab_ov.physics import OvPhysxManager - - from pxr import Sdf, Usd - - stage = Usd.Stage.CreateInMemory() - stage.DefinePrim("/World/envs/env_0", "Xform") - stage.DefinePrim("/World/envs/env_1", "Xform") - source = stage.DefinePrim("/World/envs/env_0/Object", "Xform") - source.CreateAttribute("physics:enabled", Sdf.ValueTypeNames.Bool).Set(True) - previous_pending = OvPhysxManager._pending_clones - previous_active = OvPhysxManager._active_clone_recipes - try: - OvPhysxManager._pending_clones = [] - OvPhysxManager._active_clone_recipes = [] - OvPhysxManager.register_clone("/World/envs/env_0/Object", ["/World/envs/env_1/Object"], [(1.0, 0.0, 0.0)]) - for _ in range(2): - OvPhysxManager._rearm_pending_clones() - materialized_usda = _serialize_full_stage_with_pending_clones(stage) - layer = Sdf.Layer.CreateAnonymous("materialized.usda") - assert layer.ImportFromString(materialized_usda) - exported = Usd.Stage.Open(layer) - assert exported.GetPrimAtPath("/World/envs/env_1/Object").GetAttribute("physics:enabled").Get() is True - assert OvPhysxManager._pending_clones == [] - assert len(OvPhysxManager._active_clone_recipes) == 1 - finally: - OvPhysxManager._pending_clones = previous_pending - OvPhysxManager._active_clone_recipes = previous_active - - -def test_manager_full_stage_materialization_is_atomic_on_invalid_target(): - """A validation failure clears the queue without partially modifying the export.""" - from isaaclab_ov.physics import OvPhysxManager - - from pxr import Usd - - stage = Usd.Stage.CreateInMemory() - stage.DefinePrim("/World/envs/env_0", "Xform") - stage.DefinePrim("/World/envs/env_1", "Xform") - stage.DefinePrim("/World/envs/env_0/Object", "Xform") - previous = OvPhysxManager._pending_clones - try: - OvPhysxManager._pending_clones = [ - ( - "/World/envs/env_0/Object", - ["/World/envs/env_1/Object", "/World/envs/env_2/Object"], - [(1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0), (2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0)], - [1, 2], - ) - ] - with pytest.raises(RuntimeError, match="clone target parent is absent"): - _serialize_full_stage_with_pending_clones(stage) - assert OvPhysxManager._pending_clones == [] - assert not stage.GetPrimAtPath("/World/envs/env_1/Object").IsValid() - finally: - OvPhysxManager._pending_clones = previous - - -@pytest.mark.parametrize("requires_full_stage", [False, True]) -def test_manager_replays_pending_runtime_clones_without_full_stage_requirement(requires_full_stage): - """Only the default replay path forwards final world transforms; a full-stage load never clones.""" - from isaaclab_ov.physics import OvPhysxManager - - class FakePhysX: - def __init__(self): - self.calls = [] - - def clone(self, source, targets, transforms, env_ids): - self.calls.append(("clone", source, targets, transforms, env_ids)) - return 19 - - def wait_op(self, operation): - self.calls.append(("wait_op", operation)) - - fake = FakePhysX() - previous = OvPhysxManager._pending_clones - try: - OvPhysxManager._pending_clones = [("/env_0", ["/env_1"], [(1.0, 2.0, 3.0, 0.0, 0.0, 0.0, 1.0)], [1])] - OvPhysxManager._replay_pending_clones(fake, requires_full_stage=requires_full_stage) - if requires_full_stage: - assert fake.calls == [] - else: - assert fake.calls == [ - ("clone", "/env_0", ["/env_1"], [(1.0, 2.0, 3.0, 0.0, 0.0, 0.0, 1.0)], [1]), - ("wait_op", 19), - ] - assert OvPhysxManager._pending_clones == [] - finally: - OvPhysxManager._pending_clones = previous - - def test_manager_forced_rewarm_invalidates_bindings_before_loading(monkeypatch): """A forced re-warm invalidates views before replacing their attached stage.""" from isaaclab_ov.physics import OvPhysxManager @@ -397,6 +145,7 @@ def pinned_config(*, num_threads=None, cooked_collider_cache_dir=None, carbonite assert physx.constructor["active_cuda_gpus"] == expected_active_cuda_gpus assert physx.constructor["config"].num_threads == 8 assert physx.constructor["config"].cooked_collider_cache_dir == cache_dir + assert physx.constructor["config"].carbonite_overrides["/ovphysx/clone/useEnvIds"] is not expected_cpu_mode updates = [("step_sync", 0.02), ("update_articulations_kinematic",)] assert physx.calls == updates + [("destroy_view",), ("reset_stage",), ("wait_op", 23)] assert OvPhysxManager.backend.rigid_body_view is None @@ -436,54 +185,6 @@ def test_transforms_finish_dirty_kinematics_before_native_reads(monkeypatch): assert calls == ["fk", "read", "fk", "read"] -def test_manager_serializes_env0_only_stage_in_memory(caplog): - """The OVPhysX input keeps globals and env 0 without writing cloned envs.""" - from isaaclab_ov.physics import OvPhysxManager - - from pxr import Sdf, Usd, UsdGeom - - stage = Usd.Stage.CreateInMemory() - for path in ("/World/Ground", "/World/envs/env_0/Cube", "/World/envs/env_1/Cube"): - UsdGeom.Xform.Define(stage, path) - - previous = OvPhysxManager._requires_full_stage - try: - OvPhysxManager._requires_full_stage = False - with caplog.at_level(logging.INFO, logger=OvPhysxManager.__module__): - usda = OvPhysxManager._serialize_selected_stage(stage) - finally: - OvPhysxManager._requires_full_stage = previous - layer = Sdf.Layer.CreateAnonymous("filtered.usda") - assert layer.ImportFromString(usda) - filtered = Usd.Stage.Open(layer) - - assert filtered.GetPrimAtPath("/World/Ground").IsValid() - assert filtered.GetPrimAtPath("/World/envs/env_0/Cube").IsValid() - assert not filtered.GetPrimAtPath("/World/envs/env_1").IsValid() - assert "stripped 1 env_ subtrees from in-memory USD" in caplog.text - - -def test_manager_serializes_stage_without_envs_as_is(): - """The in-memory serializer keeps stages without the standard env namespace intact.""" - from isaaclab_ov.physics import OvPhysxManager - - from pxr import Sdf, Usd, UsdGeom - - stage = Usd.Stage.CreateInMemory() - UsdGeom.Xform.Define(stage, "/World/Ground") - - previous = OvPhysxManager._requires_full_stage - try: - OvPhysxManager._requires_full_stage = False - usda = OvPhysxManager._serialize_selected_stage(stage) - finally: - OvPhysxManager._requires_full_stage = previous - - layer = Sdf.Layer.CreateAnonymous("no_envs.usda") - assert layer.ImportFromString(usda) - assert Usd.Stage.Open(layer).GetPrimAtPath("/World/Ground").IsValid() - - def test_manager_attaches_and_releases_owned_ovstage(monkeypatch): """The registered resource releases the attached OVStage once, after PhysX.""" import isaaclab_ov.physics.ovphysx_manager as om_mod @@ -785,6 +486,29 @@ def _stop_at_stage_creation(): assert SimulationContext.instance() is None +def test_scene_data_binding_is_deferred_until_requested(): + """Headless reset defers renderer-only native reads and tensor binding.""" + from contextlib import nullcontext + + from isaaclab_ov.physics.ovphysx_manager import OvPhysxSceneDataBackend + + calls = [] + + class FakePhysX: + def read(self, kind, fields): + calls.append((kind, fields)) + return nullcontext(SimpleNamespace(groups=[])) + + native = SimpleNamespace(physx=FakePhysX(), rigid_body_view=None) + backend = OvPhysxSceneDataBackend() + backend._defer_setup(native, "cpu", ()) + assert calls == [] + assert backend.transform_count == 0 + assert len(calls) == 2 + assert backend.transform_count == 0 + assert len(calls) == 2 + + def test_transforms_read_native_buffer_only_when_dirty(): """One fused native binding fills the pose buffer directly and skips clean publications.""" import isaaclab_ov.physics.ovphysx_manager as module diff --git a/source/isaaclab_ov/test/sensors/test_contact_sensor.py b/source/isaaclab_ov/test/sensors/test_contact_sensor.py index 7d61cc909aa1..5dabbbbee0f9 100644 --- a/source/isaaclab_ov/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_ov/test/sensors/test_contact_sensor.py @@ -431,9 +431,23 @@ def _hold(pose: torch.Tensor, num_steps: int) -> None: @pytest.mark.parametrize("device", ["cuda:0", "cpu"]) -def test_cube_stack_contact_filtering(device): +@pytest.mark.parametrize("missing_filter", [False, True]) +def test_cube_stack_contact_filtering(device, monkeypatch, missing_filter): """Checks contact sensor reporting for filtering stacked cube prims.""" + from ovphysx.api import PhysX + num_envs = 6 + binding_patterns = [] + create_contact_binding = PhysX.create_contact_binding + + def record_binding_patterns(physx, sensor_patterns, filter_patterns=None, **kwargs): + if missing_filter and not binding_patterns: + filter_patterns = list(filter_patterns) + filter_patterns[3] += "/Missing" + binding_patterns.append((sensor_patterns, filter_patterns)) + return create_contact_binding(physx, sensor_patterns, filter_patterns, **kwargs) + + monkeypatch.setattr(PhysX, "create_contact_binding", record_binding_patterns) with _ovphysx_sim_context(device=device, dt=_SIM_DT, add_lighting=True) as sim: # Instance new scene for the current terrain and contact prim. # OVPhysX uses fnmatch globs (not regex), so ``Env_*`` rather than ``Env_.*``. @@ -448,7 +462,7 @@ def test_cube_stack_contact_filtering(device): # -- contact sensor 1 scene_cfg.contact_sensor = ContactSensorCfg( prim_path="{ENV_REGEX_NS}/Cube_1", - track_pose=True, + track_pose=not missing_filter, debug_vis=False, update_period=0.0, history_length=2, @@ -465,11 +479,24 @@ def test_cube_stack_contact_filtering(device): scene = InteractiveScene(scene_cfg) # Play the simulation + if missing_filter: + with pytest.raises(RuntimeError, match="omitted sensor bodies"): + sim.reset() + return sim.reset() contact_sensor: ContactSensor = scene["contact_sensor"] contact_sensor_2: ContactSensor = scene["contact_sensor_2"] + assert len(binding_patterns) == 2 + if num_envs > 1: + for sensor_patterns, filter_patterns in binding_patterns: + assert len(sensor_patterns) == num_envs + assert len(filter_patterns) == num_envs + for env_id, (sensor_path, filter_path) in enumerate(zip(sensor_patterns, filter_patterns, strict=True)): + assert f"/env_{env_id}/" in sensor_path + assert f"/env_{env_id}/" in filter_path + # Check that the filter binding was created for each sensor assert contact_sensor.contact_view.filter_count == 1 assert contact_sensor_2.contact_view.filter_count == 1 @@ -505,6 +532,57 @@ def test_cube_stack_contact_filtering(device): torch.testing.assert_close(data.normal_force_matrix_w_history.torch[1:], history[1:]) +@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("filter_collisions", [False, True]) +def test_heterogeneous_clone_contact_report_order(device, filter_collisions): + """Clone contact reporters retain numeric environment order and per-variant forces.""" + with _ovphysx_sim_context(device=device, dt=0.01) as sim: + cfg = ContactSensorSceneCfg( + num_envs=12, env_spacing=2.0, lazy_sensor_update=False, filter_collisions=filter_collisions + ) + cfg.terrain = FLAT_TERRAIN_CFG.replace(prim_path="/World/ground") + cfg.shape = CUBE_CFG.replace(prim_path="{ENV_REGEX_NS}/Object") + cfg.shape.spawn = sim_utils.MultiAssetSpawnerCfg( + assets_cfg=[ + sim_utils.CuboidCfg(size=(0.2, 0.2, 0.2), mass_props=sim_utils.MassPropertiesCfg(mass=1.0)), + sim_utils.SphereCfg(radius=0.1, mass_props=sim_utils.MassPropertiesCfg(mass=2.0)), + ], + rigid_props=sim_utils.RigidBodyBaseCfg(), + collision_props=sim_utils.CollisionBaseCfg(), + activate_contact_sensors=True, + ) + cfg.shape.init_state.pos = (0.0, 0.0, 0.5) + cfg.shape_2 = RigidObjectCfg( + prim_path="{ENV_REGEX_NS}/Support", + spawn=sim_utils.CuboidCfg( + size=(1.0, 1.0, 0.2), + rigid_props=sim_utils.RigidBodyBaseCfg(kinematic_enabled=True), + collision_props=sim_utils.CollisionBaseCfg(), + ), + init_state=RigidObjectCfg.InitialStateCfg(pos=(0.0, 0.0, 0.2)), + ) + cfg.contact_sensor = ContactSensorCfg( + prim_path="{ENV_REGEX_NS}/Object", + track_pose=True, + filter_prim_paths_expr=["/World/envs/env_*/Support", "/World/ground", "/World/envs/env_0/Object"], + ) + scene = InteractiveScene(cfg) + sim.reset() + sensor = scene["contact_sensor"] + assert sensor.contact_view.sensor_paths == [f"/World/envs/env_{i}/Object" for i in range(scene.num_envs)] + assert sensor.contact_view.filter_paths == [ + [f"/World/envs/env_{i}/Support", "/World/ground", "/World/envs/env_0/Object"] for i in range(scene.num_envs) + ] + for _ in range(240): + _perform_sim_step(sim, scene, sim.get_physics_dt()) + layout = torch.as_tensor(sim.get_clone_plan().topology.world_prototype_layout, device=device) + expected = 9.81 * (layout + 1) + torch.testing.assert_close(sensor.data.net_normal_forces_w.torch[:, 0, 2], expected, atol=0.1, rtol=0.0) + torch.testing.assert_close(sensor.data.normal_force_matrix_w.torch[:, 0, 0, 2], expected, atol=0.1, rtol=0.0) + assert torch.count_nonzero(sensor.data.normal_force_matrix_w.torch[:, :, 1:]) == 0 + torch.testing.assert_close(sensor.data.pos_w.torch[:, 0, :2], scene.env_origins[:, :2], atol=0.01, rtol=0.0) + + @pytest.mark.parametrize("device", ["cuda:0", "cpu"]) @pytest.mark.parametrize("detail_mode", [None, "points", "friction", "both"]) def test_multi_body_per_sensor_indexing(device, detail_mode): diff --git a/source/isaaclab_ov/test/sim/test_ovphysx_view.py b/source/isaaclab_ov/test/sim/test_ovphysx_view.py index 422db9180513..8bca85fad3cd 100644 --- a/source/isaaclab_ov/test/sim/test_ovphysx_view.py +++ b/source/isaaclab_ov/test/sim/test_ovphysx_view.py @@ -199,6 +199,36 @@ def _make_view(n: int = 3, unavailable: set | None = None, device: str = "cpu") return OvPhysxView(_FakePhysX(n=n, unavailable=unavailable), pattern="/World/env_*/body", device=device) +@pytest.mark.parametrize("explicit", [False, True]) +def test_variant_bindings_preserve_environment_and_collection_order(explicit): + """Numeric environment order is restored within each requested collection body.""" + paths = [f"/World/env_{i}/{body}" for body in ("sphere", "cube") for i in (0, 1, 2, 4, 3, 10, 5)] + + class VariantPhysX(_FakePhysX): + def create_tensor_binding(self, *, tensor_type, pattern=None, prim_paths=None): + binding = super().create_tensor_binding(tensor_type=tensor_type, pattern=pattern, prim_paths=prim_paths) + binding.prim_paths = list(prim_paths) if prim_paths and "*" not in prim_paths[0] else paths + return binding + + physx = VariantPhysX(n=len(paths)) + patterns = ["/World/env_*/sphere", "/World/env_*/cube"] + selection = {"prim_paths": patterns} if explicit else {"pattern": "/World/env_*/*"} + view = OvPhysxView(physx, **selection) + try: + view.binding_for(TensorType.RIGID_BODY_POSE) + expected = [f"/World/env_{i}/{body}" for body in ("sphere", "cube") for i in (0, 1, 2, 3, 4, 5, 10)] + if not explicit: + expected = [f"/World/env_{i}/{body}" for i in (0, 1, 2, 3, 4, 5, 10) for body in ("cube", "sphere")] + assert view.prim_paths == expected + view.binding_for(TensorType.RIGID_BODY_VELOCITY) + assert physx.created[-1][1:] == (None, expected) + # A different family must still resolve its own prims, not reuse rigid-body paths. + view.binding_for(TensorType.ARTICULATION_ROOT_POSE) + assert physx.created[-2][1:] == (selection.get("pattern"), selection.get("prim_paths")) + finally: + view.close() + + @pytest.fixture(autouse=True) def _close_live_views(): existing_views = set(OvPhysxView._live_views) diff --git a/source/isaaclab_ov/test/test_cloner.py b/source/isaaclab_ov/test/test_cloner.py index 0d5af4565da8..4503b1752acd 100644 --- a/source/isaaclab_ov/test/test_cloner.py +++ b/source/isaaclab_ov/test/test_cloner.py @@ -11,9 +11,10 @@ import numpy as np import pytest from isaaclab_ov.cloner import OvPhysxReplicateContext, ovphysx_replicate +from isaaclab_ov.cloner.replicate import _serialize_stage from isaaclab_ov.physics.ovphysx_manager import OvPhysxManager -from pxr import Gf, Usd, UsdGeom +from pxr import Gf, Sdf, Usd, UsdGeom, UsdPhysics from isaaclab.assets import AssetBaseCfg from isaaclab.cloner import make_clone_plan @@ -30,8 +31,7 @@ def _pose_matrix(position: tuple[float, float, float], quaternion: tuple[float, def test_nested_clone_uses_final_target_pose(monkeypatch): """Nested clone rows keep their source-local pose under the target environment.""" - monkeypatch.setattr(OvPhysxManager, "_active_clone_recipes", []) - monkeypatch.setattr(OvPhysxManager, "_pending_clones", []) + monkeypatch.setattr(OvPhysxManager, "_clone_recipes", []) half_sqrt_two = math.sqrt(0.5) source_half_angle_sin = 0.5 source_half_angle_cos = math.sqrt(0.75) @@ -45,6 +45,10 @@ def test_nested_clone_uses_final_target_pose(monkeypatch): source_row.AddTransformOp().Set( _pose_matrix((0.0, 1.0, 2.0), (source_half_angle_sin, 0.0, 0.0, source_half_angle_cos)) ) + source_row.AddScaleOp().Set((2, 2, 2)) + body = UsdGeom.Xform.Define(stage, "/World/envs/env_0/Robot/Body") + body.AddTranslateOp().Set((1, 0, 0)) + UsdPhysics.RigidBodyAPI.Apply(body.GetPrim()) monkeypatch.setattr(PhysicsManager, "_sim", SimpleNamespace(physics_manager=OvPhysxManager)) ovphysx_replicate( @@ -71,17 +75,27 @@ def test_nested_clone_uses_final_target_pose(monkeypatch): ) expected_transform = (10.0 - half_sqrt_two, 20.0 + half_sqrt_two, 32.0, *expected_orientation.tolist()) - assert len(OvPhysxManager._pending_clones) == 1 - pending_source, pending_targets, pending_transforms, pending_env_ids = OvPhysxManager._pending_clones[0] - assert pending_source == "/World/envs/env_0/Robot" - assert pending_targets == ["/World/envs/env_1/Robot"] - assert pending_env_ids == [1] - assert len(pending_transforms) == 1 - assert pending_transforms[0][:3] == pytest.approx(expected_transform[:3]) - orientation = np.asarray(pending_transforms[0][3:], dtype=np.float32) + assert len(OvPhysxManager._clone_recipes) == 1 + source, targets, poses, env_ids, source_env_id = OvPhysxManager._clone_recipes[0] + assert source == "/World/envs/env_0/Robot" + assert targets == ["/World/envs/env_0/Robot", "/World/envs/env_1/Robot"] + assert env_ids == [0, 1] + assert source_env_id == 0 + assert len(poses) == 2 + assert poses[1][:3] == pytest.approx(expected_transform[:3]) + orientation = np.asarray(poses[1][3:], dtype=np.float32) if np.dot(orientation, expected_orientation) < 0.0: orientation = -orientation assert orientation.tolist() == pytest.approx(expected_orientation.tolist()) + layer = Sdf.Layer.CreateAnonymous("export.usda") + serialized, native = _serialize_stage(stage, OvPhysxManager._clone_recipes, full_stage=True) + assert not native + layer.ImportFromString(serialized) + exported = Usd.Stage.Open(layer) + target = exported.GetPrimAtPath(targets[1] + "/Body") + actual = UsdGeom.XformCache().GetLocalToWorldTransform(target).ExtractTranslation() + expected = _pose_matrix(expected_transform[:3], expected_transform[3:]).Transform(Gf.Vec3d(2, 0, 0)) + np.testing.assert_allclose(actual, expected, atol=1e-6) def test_ovphysx_context_consumes_plan(): @@ -90,31 +104,133 @@ def test_ovphysx_context_consumes_plan(): UsdGeom.Xform.Define(stage, "/World/envs/env_0").AddTranslateOp().Set((2, 0, 0)) UsdGeom.Xform.Define(stage, "/World/envs/env_0/Robot").AddTranslateOp().Set((0.25, 0, 0)) recipes = [] - manager = SimpleNamespace(_register_clone_transforms=lambda *recipe: recipes.append(recipe)) + manager = SimpleNamespace(_clone_recipes=recipes) assets = (AssetBaseCfg(prim_path="/World/envs/env_[^/]+/Robot"),) positions = np.array([[2, 0, 0], [5, 2, 3], [8, 0, 0]], dtype=np.float32) plan = make_clone_plan(assets, ((0, 0), (0,)), 3, weights=(2, 1), positions=positions) OvPhysxReplicateContext(SimpleNamespace(stage=stage, physics_manager=manager)).replicate(plan, (0,)) assert len(recipes) == 2 - assert recipes[0][0:2] == ("/World/envs/env_0/Robot", ["/World/envs/env_1/Robot", "/World/envs/env_2/Robot"]) - assert recipes[0][2][0] == pytest.approx((5.25, 2.0, 3.0, 0.0, 0.0, 0.0, 1.0)) + assert recipes[0][0:2] == ("/World/envs/env_0/Robot", [f"/World/envs/env_{i}/Robot" for i in range(3)]) + assert recipes[0][2][1] == pytest.approx((5.25, 2.0, 3.0, 0.0, 0.0, 0.0, 1.0)) assert recipes[1][1] == ["/World/envs/env_0/Robot_1", "/World/envs/env_1/Robot_1"] np.testing.assert_allclose(recipes[1][2], [(2.25, 0, 0, 0, 0, 0, 1), (5.25, 2, 3, 0, 0, 0, 1)]) - assert recipes[0][3] == [1, 2] + assert recipes[0][3] == [0, 1, 2] assert recipes[1][3] == [0, 1] +def test_ovphysx_context_preserves_heterogeneous_world_prototypes(): + """Each world prototype keeps its authored geometry and disjoint destination IDs.""" + stage = Usd.Stage.CreateInMemory() + for env_id, geometry_type in ((0, "Cube"), (1, "Sphere")): + UsdGeom.Xform.Define(stage, f"/World/envs/env_{env_id}") + source = UsdGeom.Xform.Define(stage, f"/World/envs/env_{env_id}/Object").GetPrim() + UsdPhysics.RigidBodyAPI.Apply(source) + stage.DefinePrim(f"/World/envs/env_{env_id}/Object/Geometry", geometry_type) + + recipes = [] + manager = SimpleNamespace(_clone_recipes=recipes) + assets = (AssetBaseCfg(prim_path="/World/envs/env_[^/]+/Object"),) * 2 + plan = make_clone_plan(assets, ((0,), (1,)), 4, clone_strategy=lambda weights, count: np.arange(count) % 2) + OvPhysxReplicateContext(SimpleNamespace(stage=stage, physics_manager=manager)).replicate(plan, (0, 1)) + + assert [(source, targets, env_ids) for source, targets, _, env_ids, _ in recipes] == [ + ("/World/envs/env_0/Object", ["/World/envs/env_0/Object", "/World/envs/env_2/Object"], [0, 2]), + ("/World/envs/env_1/Object", ["/World/envs/env_1/Object", "/World/envs/env_3/Object"], [1, 3]), + ] + + +def test_raw_replicate_preserves_rigid_body_variants_and_env_ids(monkeypatch): + """Each rigid-body geometry variant becomes one clone call with its selected env ids.""" + stage = Usd.Stage.CreateInMemory() + for env_id, geometry_type in ((0, "Cube"), (1, "Sphere")): + UsdGeom.Xform.Define(stage, f"/World/envs/env_{env_id}") + source = UsdGeom.Xform.Define(stage, f"/World/envs/env_{env_id}/Object").GetPrim() + UsdPhysics.RigidBodyAPI.Apply(source) + stage.DefinePrim(f"/World/envs/env_{env_id}/Object/Geometry", geometry_type) + + recipes = [] + manager = SimpleNamespace(_clone_recipes=recipes) + monkeypatch.setattr(PhysicsManager, "_sim", SimpleNamespace(physics_manager=manager)) + ovphysx_replicate( + stage, + sources=["/World/envs/env_0/Object", "/World/envs/env_1/Object"], + destinations=["/World/envs/env_{}/Object", "/World/envs/env_{}/Object"], + env_ids=np.arange(6, dtype=np.int64), + mapping=np.array( + [[True, False, True, False, True, False], [False, True, False, True, False, True]], dtype=np.bool_ + ), + ) + + assert [(source, targets, env_ids) for source, targets, _, env_ids, _ in recipes] == [ + ("/World/envs/env_0/Object", [f"/World/envs/env_{i}/Object" for i in (0, 2, 4)], [0, 2, 4]), + ("/World/envs/env_1/Object", [f"/World/envs/env_{i}/Object" for i in (1, 3, 5)], [1, 3, 5]), + ] + assert set(recipes[0][3]).isdisjoint(recipes[1][3]) + + +def test_raw_replicate_preserves_source_only_geometry_variants(monkeypatch): + """A nonzero source environment is retained even when its variant has no clone targets.""" + stage = Usd.Stage.CreateInMemory() + for env_id in range(2): + UsdGeom.Xform.Define(stage, f"/World/envs/env_{env_id}") + source = UsdGeom.Xform.Define(stage, f"/World/envs/env_{env_id}/Object").GetPrim() + UsdPhysics.RigidBodyAPI.Apply(source) + + recipes = [] + manager = SimpleNamespace(_clone_recipes=recipes) + monkeypatch.setattr(PhysicsManager, "_sim", SimpleNamespace(physics_manager=manager)) + ovphysx_replicate( + stage, + sources=["/World/envs/env_0/Object", "/World/envs/env_1/Object"], + destinations=["/World/envs/env_{}/Object", "/World/envs/env_{}/Object"], + env_ids=np.arange(2, dtype=np.int64), + mapping=np.eye(2, dtype=np.bool_), + ) + + assert [(source, targets, env_ids) for source, targets, _, env_ids, _ in recipes] == [ + (f"/World/envs/env_{i}/Object", [f"/World/envs/env_{i}/Object"], [i]) for i in range(2) + ] + assert not _serialize_stage(stage, recipes, full_stage=False)[1] + + +def test_raw_replicate_preserves_articulation_geometry_variants_and_env_ids(monkeypatch): + """Equivalent articulations with distinct link geometry retain separate clone batches.""" + stage = Usd.Stage.CreateInMemory() + for env_id, geometry_type in ((0, "Cube"), (1, "Sphere")): + UsdGeom.Xform.Define(stage, f"/World/envs/env_{env_id}") + robot = UsdGeom.Xform.Define(stage, f"/World/envs/env_{env_id}/Robot").GetPrim() + UsdPhysics.ArticulationRootAPI.Apply(robot) + link = UsdGeom.Xform.Define(stage, f"/World/envs/env_{env_id}/Robot/Link").GetPrim() + UsdPhysics.RigidBodyAPI.Apply(link) + UsdPhysics.RevoluteJoint.Define(stage, f"/World/envs/env_{env_id}/Robot/Joint") + stage.DefinePrim(f"/World/envs/env_{env_id}/Robot/Link/Geometry", geometry_type) + + recipes = [] + manager = SimpleNamespace(_clone_recipes=recipes) + monkeypatch.setattr(PhysicsManager, "_sim", SimpleNamespace(physics_manager=manager)) + ovphysx_replicate( + stage, + sources=["/World/envs/env_0/Robot", "/World/envs/env_1/Robot"], + destinations=["/World/envs/env_{}/Robot", "/World/envs/env_{}/Robot"], + env_ids=np.arange(4, dtype=np.int64), + mapping=np.array([[True, False, True, False], [False, True, False, True]], dtype=np.bool_), + ) + + assert [(source, targets, env_ids) for source, targets, _, env_ids, _ in recipes] == [ + ("/World/envs/env_0/Robot", ["/World/envs/env_0/Robot", "/World/envs/env_2/Robot"], [0, 2]), + ("/World/envs/env_1/Robot", ["/World/envs/env_1/Robot", "/World/envs/env_3/Robot"], [1, 3]), + ] + + def test_register_clone_preserves_translation_only_compatibility(monkeypatch): """World positions become target-root poses with identity rotations.""" - monkeypatch.setattr(OvPhysxManager, "_active_clone_recipes", []) - monkeypatch.setattr(OvPhysxManager, "_pending_clones", []) + monkeypatch.setattr(OvPhysxManager, "_clone_recipes", []) OvPhysxManager.register_clone("/World/env_0", ["/World/env_1"], [(1.0, 2.0, 3.0)]) - expected_recipes = [("/World/env_0", ["/World/env_1"], [(1.0, 2.0, 3.0, 0.0, 0.0, 0.0, 1.0)], None)] - assert OvPhysxManager._active_clone_recipes == expected_recipes - assert OvPhysxManager._pending_clones == expected_recipes + expected_recipes = [("/World/env_0", ["/World/env_1"], [(1.0, 2.0, 3.0, 0.0, 0.0, 0.0, 1.0)], None, 0)] + assert OvPhysxManager._clone_recipes == expected_recipes def test_raw_replicate_validates_sources_and_pose_arrays(): @@ -143,3 +259,106 @@ def GetPrimAtPath(self, path): with pytest.raises(ValueError, match="/World/envs/env_0"): ovphysx_replicate(StageWithoutAnchor(), sources, destinations, **options) + + +@pytest.mark.parametrize("full_stage", [False, True]) +@pytest.mark.parametrize("env_template", ["/World/envs/env_{}", "/Scenes/World_{}"]) +@pytest.mark.parametrize("independent_physics", [False, True]) +@pytest.mark.parametrize("nested", [False, True], ids=["sibling", "nested"]) +def test_clone_export_preserves_sources_and_independent_assets(full_stage, env_template, independent_physics, nested): + """Export exact targets, preserve unrelated assets, and replay without consuming the recipes.""" + stage = Usd.Stage.CreateInMemory() + for env_id, variant in enumerate(("cube", "sphere", "target", "target")): + root = env_template.format(env_id) + prim = stage.DefinePrim(root + "/Object", "Xform") + prim.CreateAttribute("test:variant", Sdf.ValueTypeNames.String).Set(variant) + stage.DefinePrim("/Shared/Ground", "Xform") + independent_path = "/Support/Independent" if nested else "/Independent" + for world in range(4) if nested else (1, 3): + independent = stage.DefinePrim(env_template.format(world) + independent_path, "Xform") + if independent_physics: + UsdPhysics.RigidBodyAPI.Apply(independent) + assets = tuple( + AssetBaseCfg(prim_path=env_template.format("[^/]+") + name) + for name in ("/Object", "/Object", "/Support", independent_path) + ) + worlds = ((0, 2, 3), (1, 3)) if nested else ((0, 2), (1, 2, 3)) + plan = make_clone_plan(assets, worlds, 4, env_template=env_template, clone_strategy=lambda _, n: np.arange(n) % 2) + object_paths = [env_template.format(i) + "/Object" for i in range(4)] + recipes = [(object_paths[i], [object_paths[i], object_paths[i + 2]], [], [i, i + 2], i) for i in range(2)] + # A target can also be inside a world which retains a different prototype. + support = stage.DefinePrim(env_template.format(0) + "/Support", "Xform") + support.CreateAttribute("test:physics", Sdf.ValueTypeNames.Bool).Set(True) + support_worlds = [0, 2] if nested else list(range(4)) + targets = [env_template.format(i) + "/Support" for i in support_worlds] + recipes.append((str(support.GetPath()), targets, [], support_worlds, 0)) + if not nested: + stage.DefinePrim(targets[1], "Xform") + before = stage.GetRootLayer().ExportToString() + + for _ in range(2): + layer = Sdf.Layer.CreateAnonymous("export.usda") + serialized, native = _serialize_stage(stage, recipes, full_stage, plan) + assert layer.ImportFromString(serialized) + exported = Usd.Stage.Open(layer) + for env_id, variant in enumerate(("cube", "sphere")): + prim = exported.GetPrimAtPath(object_paths[env_id]) + assert prim.GetAttribute("test:variant").Get() == variant + assert exported.GetPrimAtPath("/Shared/Ground") + assert exported.GetPrimAtPath(env_template.format(3) + independent_path) + assert bool(exported.GetPrimAtPath(env_template.format(2))) is full_stage + for world, target in zip(support_worlds, targets, strict=True): + prim = exported.GetPrimAtPath(target) + assert bool(prim) is (full_stage or world < 2 or (independent_physics and world == 3)) + if prim: + assert prim.GetAttribute("test:physics").Get() + if full_stage: + assert not native + else: + assert { + (source, target, world) for source, paths, _, ids, _ in native for target, world in zip(paths, ids) + } == { + (env_template.format(i) + name, env_template.format(i + 2) + name, i + 2) + for i in range(2) + for name in ("/Object", "/Support") + if (not independent_physics or i == 0) and (name == "/Object" or i == 0 or not nested) + } + assert stage.GetRootLayer().ExportToString() == before + assert len(recipes) == 3 + + +def test_full_stage_export_preserves_nested_authored_opinions(): + """Copy parents before children and keep independently authored descendants.""" + stage = Usd.Stage.CreateInMemory() + source = stage.DefinePrim("/Source/Robot", "Xform") + source.CreateAttribute("test:physics", Sdf.ValueTypeNames.Bool).Set(True) + child = stage.DefinePrim("/Source/Robot/Link", "Xform") + child.CreateAttribute("test:mass", Sdf.ValueTypeNames.Float).Set(3.0) + UsdGeom.Mesh.Define(stage, "/Target/Robot") + stage.DefinePrim("/Target/Robot/Camera", "Camera") + recipes = [ + ("/Source/Robot/Link", ["/Target/Robot/Link", "/Other/Nested/Robot/Link"], [], [1, 2], 0), + ("/Source/Robot", ["/Target/Robot"], [], [1], 0), + ] + layer = Sdf.Layer.CreateAnonymous("export.usda") + serialized, native = _serialize_stage(stage, recipes, full_stage=True) + assert not native + assert layer.ImportFromString(serialized) + exported = Usd.Stage.Open(layer) + assert exported.GetPrimAtPath("/Target/Robot").GetAttribute("test:physics").Get() + assert exported.GetPrimAtPath("/Target/Robot").IsA(UsdGeom.Mesh) + assert exported.GetPrimAtPath("/Target/Robot/Camera").IsA(UsdGeom.Camera) + for root in ("/Target", "/Other/Nested"): + prim = exported.GetPrimAtPath(root + "/Robot/Link") + assert prim.IsDefined() + assert prim.GetAttribute("test:mass").Get() == 3.0 + assert not stage.GetPrimAtPath("/Target/Robot/Link") + + +def test_native_clone_export_rejects_source_overlap(): + stage = Usd.Stage.CreateInMemory() + stage.DefinePrim("/World/Source", "Xform") + before = stage.GetRootLayer().ExportToString() + with pytest.raises(ValueError, match="overlaps a clone source"): + _serialize_stage(stage, [("/World/Source", ["/World"], [], [0], 0)], full_stage=False) + assert stage.GetRootLayer().ExportToString() == before diff --git a/source/isaaclab_tasks/changelog.d/maximiliank-heterogeneous-cloning-ovphysx.minor.rst b/source/isaaclab_tasks/changelog.d/maximiliank-heterogeneous-cloning-ovphysx.minor.rst new file mode 100644 index 000000000000..095cc149d70c --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/maximiliank-heterogeneous-cloning-ovphysx.minor.rst @@ -0,0 +1,4 @@ +Changed +^^^^^^^ + +* Enabled the heterogeneous shapes preset by default for KUKA–Allegro tasks with OvPhysX. Select the ``cube`` preset explicitly to retain the previous homogeneous object setup. diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/lift/lift_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/core/lift/lift_env_cfg.py index 5a31b3f878c4..39c1d66da3c0 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/lift/lift_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/lift/lift_env_cfg.py @@ -57,7 +57,7 @@ @configclass class ObjectCfg(PresetCfg): - """Graspable object presets: a set of primitive shapes, or a single cube for OvPhysX.""" + """Graspable objects: heterogeneous shapes by default, or an explicit single-cube preset.""" shapes = sim_utils.MultiAssetSpawnerCfg( assets_cfg=[ @@ -97,7 +97,6 @@ class ObjectCfg(PresetCfg): mass_props=sim_utils.MassCfg(mass=0.2), ) default = shapes - ovphysx = cube ## diff --git a/source/isaaclab_tasks/test/core/test_lift_kuka_ovphysx_presets.py b/source/isaaclab_tasks/test/core/test_lift_kuka_ovphysx_presets.py index a57172ea8b51..d7f73c80c96b 100644 --- a/source/isaaclab_tasks/test/core/test_lift_kuka_ovphysx_presets.py +++ b/source/isaaclab_tasks/test/core/test_lift_kuka_ovphysx_presets.py @@ -9,6 +9,8 @@ import pytest +from isaaclab.sim import MultiAssetSpawnerCfg + import isaaclab_tasks # noqa: F401 from isaaclab_tasks.utils import resolve_task_config @@ -16,21 +18,19 @@ @pytest.mark.parametrize( "task_name,preset_args", [ + ("Isaac-Lift-KukaAllegro", ("physics=ovphysx",)), ("Isaac-Lift-KukaAllegro", ("physics=ovphysx", "presets=shapes")), ("Isaac-Reorient-KukaAllegro", ("presets=shapes", "physics=ovphysx")), ], ) -def test_ovphysx_rejects_heterogeneous_shapes_independent_of_argument_order( - task_name: str, preset_args: tuple[str, str] -): - """OVPhysX should reject the unsupported heterogeneous shapes preset in either CLI order.""" +def test_ovphysx_uses_heterogeneous_shapes(task_name: str, preset_args: tuple[str, ...]): + """OVPhysX defaults to heterogeneous shapes and accepts explicit selection in either CLI order.""" old_argv = sys.argv.copy() try: sys.argv = [sys.argv[0], *preset_args] - with pytest.raises( - ValueError, - match=r"Conflicting global presets: .* both define preset for 'env\.scene\.object\.spawn'", - ): - resolve_task_config(task_name, "rsl_rl_cfg_entry_point") + env_cfg, _ = resolve_task_config(task_name, "rsl_rl_cfg_entry_point") finally: sys.argv = old_argv + + assert isinstance(env_cfg.scene.object.spawn, MultiAssetSpawnerCfg) + assert len(env_cfg.scene.object.spawn.assets_cfg) == 16 diff --git a/source/isaaclab_tasks/test/core/test_ovphysx_camera_presets.py b/source/isaaclab_tasks/test/core/test_ovphysx_camera_presets.py index 74914f272864..8a1824b65952 100644 --- a/source/isaaclab_tasks/test/core/test_ovphysx_camera_presets.py +++ b/source/isaaclab_tasks/test/core/test_ovphysx_camera_presets.py @@ -12,7 +12,7 @@ from isaaclab_ov.renderers import OVRTXRendererCfg from isaaclab.app import scan -from isaaclab.sim import CuboidCfg +from isaaclab.sim import MultiAssetSpawnerCfg import isaaclab_tasks # noqa: F401 from isaaclab_tasks.utils import resolve_task_config @@ -52,7 +52,7 @@ def test_ovphysx_camera_preset_resolves_kitless(task_name: str, presets: str): robot_asset_cfg = env_cfg.events.robot_physics_material.params["asset_cfg"] object_asset_cfg = env_cfg.events.object_physics_material.params["asset_cfg"] - assert isinstance(env_cfg.scene.object.spawn, CuboidCfg) + assert isinstance(env_cfg.scene.object.spawn, MultiAssetSpawnerCfg) assert robot_asset_cfg.body_names == ".*" assert object_asset_cfg.body_names == ".*" assert env_cfg.events.variable_gravity is not None diff --git a/source/isaaclab_tasks/test/core/test_rendering_lift_kuka_hetero_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_lift_kuka_hetero_kitless.py index d9ac63963c4e..92aa1df826be 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_lift_kuka_hetero_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_lift_kuka_hetero_kitless.py @@ -18,8 +18,7 @@ rendering_test_lift_kuka, ) -# The OVPhysX preset selects the homogeneous cube because heterogeneous multi-asset scenes are unsupported; -# OVPhysX Kuka goldens live in test_rendering_lift_kuka_homo_kitless.py. +# OVPhysX heterogeneous rendering goldens are not available yet. _RENDERING_PARAMS = group_rendering_params( [param for param in make_kitless_rendering_params_lift() if param.values[1] != "ovphysx"] ) From eb6617cdad62aec29b6dd6d1a43e9b4d1e4ca4a2 Mon Sep 17 00:00:00 2001 From: Peter Verswyvelen Date: Tue, 29 Sep 2026 18:50:33 +0200 Subject: [PATCH 3/3] feat: Adds an opt-in asynchronous render path (ovrtx only) (#7075) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Enable pipelined rendering with `OVRTXRendererCfg(async_rendering=True)`; the default stays synchronous. Rendering overlaps the next simulation step, at the cost of returning each camera's previous capture. - Each camera owns its pending images. Matching capture pose, calibration, and frame indices are in `camera.data.info[output]["capture"]`; live camera fields remain current. - Reset discards pending observations and primes a fresh image. Partial resets re-prime that camera's entire tiled product. - SDP fills retained GPU buffers for transforms and geometry. OVRTX writes are asynchronous, and buffers are reused only after consumption. The same path handles deformable meshes, MPM particles, and cables. - Preserve #8116: step all registered native render products, but publish only requested cameras. Other renderers retain their existing behavior. Latency is one **capture**, not necessarily one physics step. Training must account for this changed observation timing; learning performance has not been evaluated here. The ovstage path remains synchronous. Its async implementation and the existing fixed render timestep are separate follow-ups. ## Validation - 100 focused tests passed across renderer contracts, SDP bindings, material updates, and camera lifecycle, plus four Cartpole golden-image cases covering Newton and OVPhysX in sync and async modes. Native GPU coverage checks mesh/particle/curve depth, source-buffer reuse, stream changes, independent cameras, and resets. - Actual Newton cloth, volume-deformable, cable, and MPM tasks passed simulation, camera reads, and reset in both modes. - `uv run isaaclab -f` and skill validation passed. Documentation build is left to CI. ### Small-workload sync/async measurements GPU 0 (RTX 5090), OVRTX legacy path, 128×128 RGB and depth, seed 42. One warm sample per mode, 10 warmup / 50 measured environment steps, including camera reads and a final drain. Cloth/soft/cable use Newton MJWarp + VBD; MPM uses the UR10 particle-push coupled solver. These are throughput checks, not training or scaling results. | Workload | Envs | Env creation, sync / async | Env-steps/s, sync / async | Throughput change | |---|---:|---:|---:|---:| | Franka cloth | 16 | 7.49 / 7.53 s | 628 / 744 | +18.4% | | Franka soft body | 16 | 7.84 / 7.71 s | 612 / 755 | +23.5% | | Franka cable | 16 | 6.59 / 6.59 s | 570 / 690 | +21.0% | | UR10 MPM particles | 4 | 8.90 / 9.36 s | 173 / 203 | +17.3% | No startup improvement is claimed. ## Type of change - New feature (opt-in, non-breaking) ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Checklist - [x] Followed the contribution guidelines - [x] Ran formatting and lint checks - [x] Updated documentation and package changelog fragments - [x] Extended existing focused tests rather than adding a parallel suite - [x] No new implementation warnings --------- Co-authored-by: pv-nvidia <197907000+pv-nvidia@users.noreply.github.com> Co-authored-by: Octi Zhang --- .github/test-subsets/postmerge-rendering.toml | 2 + docs/source/concepts/renderers.rst | 53 ++- .../benchmarking/run_benchmarks.rst | 18 + skills/user/select-backends/SKILL.md | 1 + skills/user/use-sensors-actuators/SKILL.md | 2 + .../ovrtx-async-capture-metadata.minor.rst | 5 + ...-ovrtx-async-frame-announcements.minor.rst | 7 + .../isaaclab/renderers/base_renderer.py | 33 +- .../isaaclab/sensors/camera/camera.py | 6 + .../isaaclab/sensors/camera/camera_data.py | 6 +- .../test_simulation_render_context.py | 4 + .../ovrtx-async-operation-ownership.rst | 5 + .../pv-ovrtx-async-slot-buffers.minor.rst | 9 + .../isaaclab_ov/renderers/ovrtx_renderer.py | 322 +++++++++++++----- .../renderers/ovrtx_renderer_cfg.py | 10 + .../isaaclab_ov/renderers/visual_materials.py | 31 +- .../isaaclab_ov/test/test_ovrtx_clone_plan.py | 1 + .../test/test_ovrtx_deformable_bindings.py | 88 +++-- .../test/test_ovrtx_renderer_contract.py | 252 ++++++++++++-- .../test/test_ovrtx_visual_material_sync.py | 3 + .../pv-ovrtx-async-slot-buffers.skip | 0 .../core/test_rendering_cartpole_kitless.py | 20 +- .../test/rendering_test_utils.py | 3 + 23 files changed, 722 insertions(+), 159 deletions(-) create mode 100644 source/isaaclab/changelog.d/ovrtx-async-capture-metadata.minor.rst create mode 100644 source/isaaclab/changelog.d/pv-ovrtx-async-frame-announcements.minor.rst create mode 100644 source/isaaclab_ov/changelog.d/ovrtx-async-operation-ownership.rst create mode 100644 source/isaaclab_ov/changelog.d/pv-ovrtx-async-slot-buffers.minor.rst create mode 100644 source/isaaclab_tasks/changelog.d/pv-ovrtx-async-slot-buffers.skip diff --git a/.github/test-subsets/postmerge-rendering.toml b/.github/test-subsets/postmerge-rendering.toml index 99926c74cea7..4206e2ef1a8f 100644 --- a/.github/test-subsets/postmerge-rendering.toml +++ b/.github/test-subsets/postmerge-rendering.toml @@ -20,6 +20,8 @@ rendering-correctness-kitless-legacy = [ "source/isaaclab_tasks/test/core/test_rendering_franka_cloth_kitless.py::test_rendering_franka_cloth_kitless[legacy-newton-ovrtx_renderer-static]", "source/isaaclab_tasks/test/core/test_rendering_franka_soft_kitless.py::test_rendering_franka_soft_kitless[legacy-newton-ovrtx_renderer-static]", "source/isaaclab_tasks/test/core/test_rendering_franka_cable_kitless.py::test_rendering_franka_cable_kitless[legacy-newton-ovrtx_renderer-static]", + "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[legacy-newton-ovrtx-async]", + "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[legacy-ovphysx-ovrtx-async]", ] rendering-correctness-kitless-ovstage = [ diff --git a/docs/source/concepts/renderers.rst b/docs/source/concepts/renderers.rst index 6ed4ae96fcf5..bf45759848ff 100644 --- a/docs/source/concepts/renderers.rst +++ b/docs/source/concepts/renderers.rst @@ -438,8 +438,8 @@ rendering, then read each camera's output. An empty sequence performs no renderi :meth:`~isaaclab.renderers.BaseRenderer.render` continues to accept a single render-data object. The default ``render_batch()`` implementation calls ``render()`` for each entry, so existing -custom renderers and single-camera callers need no changes. OVRTX overrides ``render_batch()`` -to submit the requested camera products in one native renderer step. +custom renderers and single-camera callers need no changes. OVRTX steps all registered render +products together but publishes only the requested cameras' observations. With eager sensor updates (``scene.cfg.lazy_sensor_update=False``), ``scene.update()`` advances sensor clocks in scene order and collects batch-capable sensors. After the loop, the camera @@ -508,6 +508,55 @@ Or install the public ``ovrtx`` package directly from PyPI: The :class:`~isaaclab.renderers.BaseRenderer` class is under active development and may change without notice. +.. _renderers-async-data-flow: + +Asynchronous Rendering +---------------------- + +:attr:`~isaaclab_ov.renderers.OVRTXRendererCfg.async_rendering` returns a camera's previous capture +while rendering its next image. This allows rendering to overlap simulation and Python work. +The first capture, including after reset, waits for a fresh image. The ovstage path stays synchronous. + +Latency is one capture, not necessarily one physics or control step. State observations remain +current, so enabling this option changes the policy's observation timing; training may need to +account for that delay. Physics and policy computation may also use the GPU. + +For a camera capturing once per environment step, steady-state operation is: + +.. code-block:: text + + Synchronous: physics k -> submit image k -> wait/read image k -> policy + Asynchronous: physics k -> submit image k -> wait/read image k-1 -> policy + | | + +---- image k renders ------------+--> physics k+1 + +The first image is returned immediately after priming and repeated once on the next capture. + +Each camera owns its pending observations. Reading camera A never changes camera B's pixels or +metadata, even when they share a native render submission. Reset discards that camera's pending +observations. A partial environment reset re-primes the whole tiled camera product, without +invalidating other cameras. + +Live fields such as ``camera.data.pos_w`` and ``camera.data.intrinsic_matrices`` remain current. +Use the capture metadata when pairing delayed pixels with a pose or calibration, for example +when deprojecting depth: + +.. code-block:: python + + data = camera.data + depth = data.output["distance_to_image_plane"] + capture = data.info["distance_to_image_plane"]["capture"] + position, orientation = capture["pos_w"], capture["quat_w_world"] + intrinsics, frame = capture["intrinsic_matrices"], capture["frame"] + +Capture metadata remains unchanged on later captures. Direct renderer callers use +:meth:`~isaaclab.renderers.BaseRenderer.prepare_capture` before submitting a capture and +:meth:`~isaaclab.renderers.BaseRenderer.read_output` to publish it. + +SDP fills renderer-owned transform and geometry buffers, including deformable meshes, particles, +and cables. Two buffers alternate per write and are reused only after OVRTX consumes them. +Physics can therefore update its live arrays while the previous capture renders. + See Also -------- diff --git a/docs/source/developer-tools/benchmarking/run_benchmarks.rst b/docs/source/developer-tools/benchmarking/run_benchmarks.rst index c679cfe1c68c..06cce3e6ec17 100644 --- a/docs/source/developer-tools/benchmarking/run_benchmarks.rst +++ b/docs/source/developer-tools/benchmarking/run_benchmarks.rst @@ -66,6 +66,24 @@ From a source installation, run: --output_path ./benchmark_results \ physics=isaacsim_physx +OVRTX camera tasks can compare synchronous against asynchronous (pipelined) rendering through a +Hydra override on the camera's renderer configuration: + +.. code-block:: bash + + ./isaaclab.sh benchmark runtime \ + --task Isaac-Cartpole-Camera-Direct \ + --num_envs 256 \ + --warmup_steps 30 \ + --num_steps 200 \ + --benchmark_formatter schema,omniperf \ + --output_path ./results/ovrtx_async \ + physics=newton_mjwarp renderer=ovrtx presets=rgb \ + env.scene.tiled_camera.renderer_cfg.async_rendering=True + +Only the OVRTX renderer implements asynchronous rendering. +See :ref:`renderers-async-data-flow` for how states, observations, and frames line up in each mode. + Read the result ~~~~~~~~~~~~~~~ diff --git a/skills/user/select-backends/SKILL.md b/skills/user/select-backends/SKILL.md index 77d3c07e5b74..642e0a9ae034 100644 --- a/skills/user/select-backends/SKILL.md +++ b/skills/user/select-backends/SKILL.md @@ -60,6 +60,7 @@ Keep this skill synchronized with `docs/source/concepts/physics_backends.rst`, ` - [Physics backends](../../../docs/source/concepts/physics_backends.rst) - [Backend architecture](../../../docs/source/concepts/backend_architecture.rst) - [Backends and presets](../../../docs/source/concepts/backends_and_presets.rst) +- [Renderers, including asynchronous rendering](../../../docs/source/concepts/renderers.rst) - [Schema cfgs](../../../docs/source/concepts/schema_cfgs.rst) - [Installation](../../../docs/source/setup/installation/index.rst) - [Task examples](../../../source/isaaclab_tasks/isaaclab_tasks) diff --git a/skills/user/use-sensors-actuators/SKILL.md b/skills/user/use-sensors-actuators/SKILL.md index ac85e1ece23c..b8920cb240cf 100644 --- a/skills/user/use-sensors-actuators/SKILL.md +++ b/skills/user/use-sensors-actuators/SKILL.md @@ -59,6 +59,8 @@ renderers, update the docs or source examples first. - [Evaluations](evaluations.md) - [Examples](examples.md) - [Add sensors on robot tutorial](../../../docs/source/how-to/add_sensors_on_robot.rst) +- [Camera docs](../../../docs/source/concepts/sensors/camera.rst) +- [Renderers, including asynchronous rendering](../../../docs/source/concepts/renderers.rst) - [Contact sensor docs](../../../docs/source/concepts/sensors/contact_sensor.rst) - [Joint wrench sensor docs](../../../docs/source/concepts/sensors/joint_wrench_sensor.rst) - [Actuators docs](../../../docs/source/concepts/actuators.rst) diff --git a/source/isaaclab/changelog.d/ovrtx-async-capture-metadata.minor.rst b/source/isaaclab/changelog.d/ovrtx-async-capture-metadata.minor.rst new file mode 100644 index 000000000000..63b9ab93c79a --- /dev/null +++ b/source/isaaclab/changelog.d/ovrtx-async-capture-metadata.minor.rst @@ -0,0 +1,5 @@ +Added +^^^^^ + +* Attached capture poses, calibration, and frame indices to delayed OVRTX images through + ``CameraData.info[output_name]["capture"]`` while preserving the live camera fields. diff --git a/source/isaaclab/changelog.d/pv-ovrtx-async-frame-announcements.minor.rst b/source/isaaclab/changelog.d/pv-ovrtx-async-frame-announcements.minor.rst new file mode 100644 index 000000000000..1c7e84c49008 --- /dev/null +++ b/source/isaaclab/changelog.d/pv-ovrtx-async-frame-announcements.minor.rst @@ -0,0 +1,7 @@ +Added +^^^^^ + +* Added :meth:`~isaaclab.renderers.BaseRenderer.prepare_capture` and + :meth:`~isaaclab.renderers.BaseRenderer.reset` hooks for delayed camera observations. + Asynchronous renderers can publish matching image metadata through ``CameraData.info`` and + discard pending observations on reset without changing live camera fields. diff --git a/source/isaaclab/isaaclab/renderers/base_renderer.py b/source/isaaclab/isaaclab/renderers/base_renderer.py index ba79dc6d81e8..3cea8c3f1db2 100644 --- a/source/isaaclab/isaaclab/renderers/base_renderer.py +++ b/source/isaaclab/isaaclab/renderers/base_renderer.py @@ -116,6 +116,19 @@ def set_outputs(self, render_data: Any, output_data: dict[str, ProxyArray]) -> N """ pass + def prepare_capture(self, render_data: Any, camera_data: CameraData, frame: ProxyArray) -> None: + """Snapshot metadata for the next capture when its image will be delivered asynchronously. + + Synchronous renderers need no snapshot. Delayed captures publish their pose, calibration, + and frame indices through ``camera_data.info[output_name]["capture"]`` without changing + the live camera fields. + + Args: + render_data: Renderer-owned camera resources. + camera_data: Current camera pose and calibration. + frame: Current per-environment capture indices, shape (N,), dtype ``wp.int64``. + """ + @abstractmethod def update_transforms(self) -> None: """Update scene transforms before rendering. @@ -176,7 +189,7 @@ def update_camera_intrinsics(self, render_data: Any, intrinsics: wp.array, param @abstractmethod def render(self, render_data: Any) -> None: - """Perform rendering and write to output buffers. + """Submit a capture; :meth:`read_output` publishes its camera's available observation. Args: render_data: The render data object from :meth:`create_render_data`. @@ -184,7 +197,7 @@ def render(self, render_data: Any) -> None: pass def render_batch(self, render_data: Sequence[Any]) -> None: - """Render a collection of cameras into their bound output buffers. + """Submit captures for a collection of cameras. All camera poses and shared scene state must be prepared before calling this method. An empty sequence is a no-op. Each object must belong to this renderer and appear once. @@ -201,6 +214,9 @@ def render_batch(self, render_data: Sequence[Any]) -> None: def read_output(self, render_data: Any, camera_data: CameraData) -> None: """Read rendered outputs from the renderer into the camera data container. + Asynchronous renderers may return the previous capture with its matching metadata in + ``camera_data.info``. Publishing one camera must not change another camera's observations. + Args: render_data: The render data object from :meth:`create_render_data`. camera_data: The :class:`~isaaclab.sensors.camera.camera_data.CameraData` @@ -217,6 +233,19 @@ def cleanup(self, render_data: Any) -> None: """ pass + def reset(self, render_data: Any, env_ids: Sequence[int] | None = None) -> None: + """Reset the renderer-owned state of a camera when its environments reset. + + A renderer implementation drops whatever per-camera state must not survive a reset. + Examples are pending asynchronous observations and accumulated temporal render history. + The default does nothing. + + Args: + render_data: The render data object from :meth:`create_render_data`. + env_ids: Environments being reset, or ``None`` for all. An implementation may reset + more than the given environments when its state is not separable per environment. + """ + def close(self) -> None: """Release resources owned by the renderer itself rather than by a render data. diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 5d1c98e9d1bc..18280035e2ec 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -606,6 +606,11 @@ def set_world_poses_from_view( def reset(self, env_ids: Sequence[int] | slice | None = None, env_mask: wp.array | None = None): if not self._is_initialized: raise RuntimeError("Camera could not be initialized. Check the renderer and simulation logs for details.") + # The hook takes plain id sequences, so a slice resolves to its range. A mask-only call + # passes None, which means all environments: the hook permits resetting more than asked. + if isinstance(env_ids, slice): + env_ids = range(*env_ids.indices(self._num_envs)) + self._renderer.reset(self._render_data, env_ids) # reset the timestamps super().reset(env_ids, env_mask) # reset the data @@ -704,6 +709,7 @@ def _prepare_camera(self, env_mask: wp.array) -> None: self._update_poses(env_mask=env_mask, frame_op=1) else: self._update_camera_state(env_mask=env_mask, frame_op=1) + self._renderer.prepare_capture(self._render_data, self._data, self._frame) def _update_buffers_impl(self, env_mask: wp.array): if not self._env_mask_has_any(env_mask): diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_data.py b/source/isaaclab/isaaclab/sensors/camera/camera_data.py index 2f6da47efc4b..e094990b6f0b 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_data.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_data.py @@ -43,8 +43,10 @@ def __init__(self): """The retrieved sensor info with sensor types as key. This contains extra information provided by the sensor such as semantic segmentation label mapping, prim paths. - For semantic-based data, this corresponds to the ``"info"`` key in the output of the sensor. For other sensor - types, the info is empty. + For semantic-based data, this corresponds to the ``"info"`` key in the output of the sensor. + Delayed OVRTX images also include ``"capture"``: snapshots of ``pos_w``, ``quat_w_world``, + ``intrinsic_matrices``, and per-environment ``frame`` indices matching the returned pixels. + Live pose and calibration fields are unchanged. Other outputs may have empty info. """ ## diff --git a/source/isaaclab/test/renderers/test_simulation_render_context.py b/source/isaaclab/test/renderers/test_simulation_render_context.py index daa1accc268a..1c4dba459ef1 100644 --- a/source/isaaclab/test/renderers/test_simulation_render_context.py +++ b/source/isaaclab/test/renderers/test_simulation_render_context.py @@ -576,9 +576,13 @@ def test_camera_batch_respects_period_partial_reset_and_updated_pose(camera_batc assert batches == [[("fast", 0.0), ("slow", 0.0)], [("fast", 0.0)]] np.testing.assert_array_equal(slow.frame.warp.numpy(), [1, 1]) np.testing.assert_allclose(slow._timestamp_last_update.numpy(), [0.1, 0.1]) + assert renderer.prepare_capture.call_args_list == [ + call(camera._render_data, camera._data, camera._frame) for camera in (fast, slow, fast) + ] slow.pose = 3.0 slow.reset(env_mask=wp.array([False, True], dtype=wp.bool, device="cpu")) + renderer.reset.assert_called_once_with(slow._render_data, None) InteractiveScene.update(scene, 0.1) assert batches[-1] == [("fast", 0.0), ("slow", 3.0)] assert slow.data.info["pose"] == 3.0 diff --git a/source/isaaclab_ov/changelog.d/ovrtx-async-operation-ownership.rst b/source/isaaclab_ov/changelog.d/ovrtx-async-operation-ownership.rst new file mode 100644 index 000000000000..0e601fdd46cc --- /dev/null +++ b/source/isaaclab_ov/changelog.d/ovrtx-async-operation-ownership.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Unified retention of asynchronous OVRTX camera, transform, and geometry buffers until native writes completed, + isolated camera captures across resets, and avoided extracting priming images twice. diff --git a/source/isaaclab_ov/changelog.d/pv-ovrtx-async-slot-buffers.minor.rst b/source/isaaclab_ov/changelog.d/pv-ovrtx-async-slot-buffers.minor.rst new file mode 100644 index 000000000000..827a57a5339b --- /dev/null +++ b/source/isaaclab_ov/changelog.d/pv-ovrtx-async-slot-buffers.minor.rst @@ -0,0 +1,9 @@ +Added +^^^^^ + +* Added an opt-in asynchronous OVRTX render path controlled by + :attr:`~isaaclab_ov.renderers.OVRTXRendererCfg.async_rendering`. + ``True`` returns each camera's previous capture while rendering the next image. The first + capture and the first capture after reset wait for a fresh image. Matching capture poses, + calibration, and frame indices are available in ``camera.data.info[output_name]["capture"]``; + the live camera fields remain current. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index 4cca89fb44fa..19c68646b57c 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -10,7 +10,9 @@ - **ovrtx_renderer.py** (this file): Orchestrates USD loading/cloning, camera and object bindings, and output buffers, borrowing native handles from OVRTXBackend. Each frame it: updates camera/object transforms (using kernels), steps the renderer, then extracts - tiles from the tiled framebuffer (kernels). + tiles from the tiled framebuffer (kernels). Under asynchronous rendering, each camera keeps + its own pending ``step_async`` render operation and publishes it at the next ``read_output``. + The ovrtx binding write operations retain their input buffers until ovrtx has consumed them. - **ovrtx_renderer_kernels.py**: Warp GPU kernels for OVRTX rendering pipeline. @@ -26,7 +28,9 @@ import os import sys import weakref -from collections.abc import Iterator, Sequence +from builtins import ExceptionGroup +from collections import deque +from collections.abc import Iterable, Iterator, Sequence from pathlib import Path from typing import TYPE_CHECKING, Any, NoReturn, cast @@ -54,6 +58,7 @@ PrimMode, Renderer, RendererConfig, + RenderProductSetOutputs, Semantic, TextureStreamingMode, ) @@ -71,7 +76,7 @@ from isaaclab.renderers import BaseRenderer, RenderBufferKind, RenderBufferSpec from isaaclab.scene_data import SceneDataFormat from isaaclab.sim import SimulationContext -from isaaclab.utils.buffers import TimestampedBuffer +from isaaclab.utils.warp import ProxyArray from isaaclab.utils.warp.warp_math import convert_camera_frame_orientation_convention_wp from isaaclab_ov.renderers.ovrtx_annotator_utils import ( @@ -106,15 +111,15 @@ if TYPE_CHECKING: from isaaclab_ppisp import PpispPipeline - from ovrtx import AttributeBinding, RenderProductSetOutputs + from ovrtx import AttributeBinding, Operation, PendingFetch, RenderProductSetOutputs from isaaclab.renderers.base_renderer import VisualMaterialBatch from isaaclab.sensors.camera.camera_data import CameraData - from isaaclab.utils.warp import ProxyArray from isaaclab.renderers.camera_render_spec import CameraRenderSpec _RENDER_VAR_PRIM_NAMES = render_var_prim_names_by_source() +_RENDER_DELTA_TIME = 1.0 / 60.0 _LDR_COLOR_VAR = "LdrColor" _CAMERA_INTRINSIC_ATTRIBUTES = ( "focalLength", @@ -142,6 +147,7 @@ "DistanceToCameraSD": ("distance_to_camera",), } + _PPISP_IMPORT_ERROR_MESSAGE = ( "isaaclab_ppisp is required when CameraCfg.isp_cfg is set. " "It ships with the Isaac Lab wheel (`pip install isaaclab`); otherwise install the " @@ -335,6 +341,10 @@ def __init__(self, spec: CameraRenderSpec, device, render_scope_name: str): self.num_rows = math.ceil(self.num_envs / self.num_cols) self.warp_buffers: dict[str, wp.array] = {} self.intrinsic_bindings: list[AttributeBinding] = [] + self.camera_writes = _AsyncWriteBuffers() + self.pending: tuple[Operation[PendingFetch[RenderProductSetOutputs]], dict | None] | None = None + self.ready: tuple[Operation[PendingFetch[RenderProductSetOutputs]], dict | None] | None = None + self.capture: dict[str, ProxyArray] = {} # Per-output metadata collected during render() and copied into CameraData.info by read_output(). # Populated for "semantic_segmentation" (with an "idToLabels" mapping) and # "instance_segmentation" (with "idToLabels" and "idToSemantics" mappings). @@ -357,8 +367,11 @@ def cleanup(self) -> None: is closed. The product path remains as an identifier for renderer bookkeeping. """ try: - self.resources.close() + with self.resources: + self.camera_writes.close() finally: + self.pending = self.ready = None + self.capture = {} self.camera_xform_binding = None self.camera_xform_query = None self.intrinsic_bindings.clear() @@ -391,10 +404,13 @@ def __init__(self, cfg: OVRTXRendererCfg): self._camera_render_data: list[OVRTXCameraRenderData] = [] self._next_camera_id = 0 self._sdp = SimulationContext.instance().get_scene_data_provider() - self._transforms = TimestampedBuffer(SceneDataFormat.TransposedMatrix44d()) + self._transforms_timestamp = -1 + self._transform_writes = _AsyncWriteBuffers(SceneDataFormat.TransposedMatrix44d() for _ in range(2)) self._object_scales: wp.array | None = None self._object_scales_by_path: dict[str, tuple[float, float, float]] = {} self._geometry_paths: list[str] = [] + self._geometry_offsets: dict[str, int] = {} + self._geometry_writes = _AsyncWriteBuffers() self._geometry_timestamp = -1 self._initialized_scene = False self._exported_usd_string: str | None = None @@ -405,6 +421,8 @@ def __init__(self, cfg: OVRTXRendererCfg): # Selected once at construction so every dispatch method below sees a stable path for the # lifetime of the renderer, even if the environment variable changes mid-process. self._use_ovstage = ovrtx_use_ovstage_enabled() + if cfg.async_rendering and self._use_ovstage: + logger.warning("Asynchronous OVRTX rendering is unsupported with ovstage; rendering synchronously.") self.backend: OVRTXBackend = SimulationContext.instance().get_or_create_backend( OVRTXBackendCfg( renderer_cfg=cfg, use_ovstage=self._use_ovstage, read_gpu_transforms=_read_gpu_transforms_enabled() @@ -722,9 +740,16 @@ def _setup_xform_bindings_legacy(self): def _setup_geometry_bindings_legacy(self) -> None: """Bind SDP's authored point prims without depending on their physics representation.""" - self._geometry_paths = list(self._sdp.get_geometry_points()) + points = self._sdp.get_geometry_points() + self._geometry_paths = list(points) if not self._geometry_paths: return + if self.cfg.async_rendering: + count = 0 + for path, array in points.items(): + self._geometry_offsets[path] = count + count += len(array) + self._geometry_writes = _AsyncWriteBuffers(wp.empty(count, wp.vec3f, device=self._device) for _ in range(2)) prim_count = len(self._geometry_paths) # Published points are world-space; do not apply inherited transforms a second time. self.backend.renderer.write_attribute( @@ -782,6 +807,10 @@ def create_render_data(self, spec: CameraRenderSpec) -> OVRTXCameraRenderData: else: self._register_camera(spec, render_data) if not self._use_ovstage: + render_data.camera_writes = _AsyncWriteBuffers( + tuple(wp.empty(spec.num_instances, dtype, device=self._device) for dtype in (wp.quatf, wp.mat44d)) + for _ in range(2 if self.cfg.async_rendering else 1) + ) for name in _CAMERA_INTRINSIC_ATTRIBUTES: binding = self.backend.renderer.bind_attribute( prim_paths=camera_paths, @@ -801,6 +830,11 @@ def create_render_data(self, spec: CameraRenderSpec) -> OVRTXCameraRenderData: def _register_camera(self, spec: CameraRenderSpec, render_data: OVRTXCameraRenderData) -> None: """Add another tiled product and camera binding without reloading the shared scene.""" + # Registration mutates the shared scene (a new product and camera bindings), so any + # in-flight render must finish first. + errors = self.drain_pending_renders() + if errors: + raise ExceptionGroup("OVRTX renders failed before camera registration", errors) camera_paths = _get_cloned_camera_paths(spec.camera_prim_paths[0], spec.num_instances) if not camera_paths: raise ValueError("OVRTX cameras must be under /World/envs/env_0/.") @@ -919,58 +953,54 @@ def _update_camera_legacy( intrinsics: ProxyArray, ) -> None: """Update camera transforms in OVRTX binding.""" - num_envs = positions.shape[0] - converted_wp = wp.empty(num_envs, dtype=wp.quatf, device=self._device) - convert_camera_frame_orientation_convention_wp( - src=orientations.warp, - dst=converted_wp, - origin="world", - target="opengl", - device=self._device, - ) - camera_transforms = wp.zeros(num_envs, dtype=wp.mat44d, device=self._device) + if render_data.camera_xform_binding is None: + return + stream = self._warp_device.stream + converted, matrices = render_data.camera_writes.acquire(stream) + convert_camera_frame_orientation_convention_wp(orientations, converted, "world", "opengl", device=self._device) wp.launch( - kernel=create_camera_transforms_kernel, - dim=num_envs, - inputs=[positions, converted_wp, camera_transforms], - device=self._device, + create_camera_transforms_kernel, len(matrices), inputs=[positions, converted, matrices], device=self._device ) - if render_data.camera_xform_binding is not None: - render_data.camera_xform_binding.write( - camera_transforms, - data_access=DataAccess.ASYNC, - cuda_stream=self._warp_device.stream.cuda_stream, - ) + operation = render_data.camera_writes.submit(render_data.camera_xform_binding, matrices, stream) + if not self.cfg.async_rendering: + operation.wait() + + def prepare_capture(self, render_data: OVRTXCameraRenderData, camera_data: CameraData, frame: ProxyArray) -> None: + """Snapshot delayed-image metadata without changing live camera state.""" + if self.cfg.async_rendering and not self._use_ovstage: + render_data.capture = { + name: ProxyArray(wp.clone(value.warp)) + for name, value in ( + ("pos_w", camera_data.pos_w), + ("quat_w_world", camera_data.quat_w_world), + ("intrinsic_matrices", camera_data.intrinsic_matrices), + ("frame", frame), + ) + } def read_output( self, render_data: OVRTXCameraRenderData, camera_data: CameraData, ) -> None: - """Forward per-output metadata collected during :meth:`render` into ``camera_data.info``. - - This is a *replace*, not a *merge*: every seeded output key is reset to this frame's metadata, - which is ``None`` when its render var was absent. Because :meth:`render` rebuilds ``renderer_info`` - from scratch each frame (see the ``renderer_info.clear()`` in :meth:`_process_render_frame`), a render - var that disappears on a later frame (e.g. a missing ``SemanticIdMap``) must clear the corresponding - ``camera_data.info`` entry too, or downstream consumers would keep reading stale labels. ``renderer_info`` - only ever holds a subset of the outputs, so iterating ``camera_data.info`` both preserves its - ``output``-mirroring key set and resets any dropped metadata to ``None``. - - Present entries are stored by reference (a shallow assignment, not a deep copy): ``camera_data.info`` - shares the same metadata dict objects as ``render_data.renderer_info`` (e.g. the semantic - ``idToLabels`` mapping). Those references stay valid even after ``renderer_info`` is cleared or - rebuilt, and each render builds a fresh metadata dict, so no aliased object is mutated in place. - - Pixel data needs no handling here: :meth:`set_outputs` wraps each ``camera_data.output`` tensor as a - zero-copy warp array stored in ``render_data.warp_buffers``, and :meth:`render` writes the rendered - tiles directly into those warp arrays. - - See :meth:`~isaaclab.renderers.base_renderer.BaseRenderer.read_output`. - """ + """Publish only this camera's image and its matching metadata.""" assert camera_data.info is not None, "CameraData.info should be created in CameraData.allocate" + if self.cfg.async_rendering and not self._use_ovstage and render_data.ready is None: + return + capture = {} + if render_data.ready is not None: + operation, capture = render_data.ready + if capture is None: + render_data.ready = None + return + self._process_render_products((render_data,), operation.wait().fetch()) + if render_data.ready is render_data.pending: + # Retain the native operation, but do not publish this priming image twice. + render_data.pending = (operation, None) + render_data.ready = None for output_name in camera_data.info: - camera_data.info[output_name] = render_data.renderer_info.get(output_name) + info = render_data.renderer_info.get(output_name) + camera_data.info[output_name] = {**(info or {}), "capture": capture} if capture else info def _generate_random_colors_from_ids(self, input_ids: wp.array, output_colors: wp.array | None) -> wp.array: """Generate pseudo-random RGBA colors from uint32 IDs into a reusable output buffer. @@ -1359,16 +1389,56 @@ def _render_legacy(self, render_data: Sequence[OVRTXCameraRenderData]) -> None: try: if material_writer is not None: material_writer.publish() - products = self.backend.renderer.step( - render_products=set(self._render_product_paths), - delta_time=1.0 / 60.0, - ) + products = set(self._render_product_paths) + if self.cfg.async_rendering: + unread = {data.ready[0] for data in render_data if data.ready is not None} + operation = self.backend.renderer.step_async(render_products=products, delta_time=_RENDER_DELTA_TIME) + for data in render_data: + previous = data.pending + data.pending = (operation, data.capture) + data.ready = previous if previous is not None else data.pending + # Repeated submissions may replace unread results, but must still complete them. + for previous in unread: + previous.wait().fetch() + else: + result = self.backend.renderer.step(render_products=products, delta_time=_RENDER_DELTA_TIME) + self._process_render_products(render_data, result) finally: if material_writer is not None: - drain_errors = contextlib.nullcontext() if sys.exc_info()[0] is None else contextlib.suppress(Exception) - with drain_errors: + # When another exception is already propagating, log the drain failure instead of + # replacing it. Losing it silently would hide a failed material write. + primary_active = sys.exc_info()[0] is not None + try: material_writer.drain() - self._process_render_products(render_data, products) + except Exception as e: + if not primary_active: + raise + logger.warning("Error draining material writes after a failed render: %s", e, exc_info=True) + + def drain_pending_renders(self, cameras: Sequence[OVRTXCameraRenderData] | None = None) -> list[Exception]: + """Complete native work without publishing observations into any camera.""" + cameras = self._camera_render_data if cameras is None else cameras + operations = {frame[0] for data in cameras for frame in (data.pending, data.ready) if frame is not None} + errors = [] + for operation in operations: + try: + operation.wait().fetch() + except Exception as error: + errors.append(error) + return errors + + def reset(self, render_data: OVRTXCameraRenderData, env_ids: Sequence[int] | None = None) -> None: + """Retire pre-reset frames. The next capture primes this whole tiled product. + + Pending captures cover every environment tile, so they are not separable per environment + and ``env_ids`` is not consulted. + """ + del env_ids + errors = self.drain_pending_renders((render_data,)) + render_data.pending = render_data.ready = None + render_data.capture = {} + if errors: + raise ExceptionGroup("OVRTX renders failed during camera reset", errors) def _process_render_products( self, render_data: Sequence[OVRTXCameraRenderData], products: RenderProductSetOutputs @@ -1426,14 +1496,20 @@ def _initialize_camera_render_data_from_spec( def update_transforms(self) -> None: """Write changed SDP transforms to OVRTX.""" binding = self._object_xform_query if self._use_ovstage else self._object_xform_binding - if binding is None or not self._sdp.get_transforms(self._transforms.data, scales=self._object_scales): + if binding is None: return timestamp = self._sdp.backend.transforms_timestamp - if self._transforms.timestamp == timestamp: + if self._transforms_timestamp == timestamp: + return + stream = self._warp_device.stream + asynchronous = self.cfg.async_rendering and not self._use_ovstage + output = self._transform_writes.acquire(stream) + if not self._sdp.get_transforms(output, allow_passthrough=not asynchronous, scales=self._object_scales): return - matrices = self._transforms.data.matrices - # Both writes wait for consumption; the producing CUDA stream orders access to the borrowed buffer. + matrices = output.matrices if self._use_ovstage: + # The write waits for consumption; the producing CUDA stream orders access to the + # borrowed buffer. self.backend.stage.write_attribute( binding, "omni:xform", @@ -1443,20 +1519,30 @@ def update_transforms(self) -> None: semantic=ovstage.AttributeSemantic.MATRIX, cuda_stream=self._warp_device.stream.cuda_stream, ).wait() + elif asynchronous: + self._transform_writes.submit(binding, matrices, stream) else: - binding.write(matrices, data_access=DataAccess.ASYNC, cuda_stream=self._warp_device.stream.cuda_stream) - self._transforms.timestamp = timestamp + binding.write(matrices, data_access=DataAccess.ASYNC, cuda_stream=stream.cuda_stream) + self._transforms_timestamp = timestamp def update_geometries(self) -> None: """Write changed SDP geometry to OVRTX.""" binding = self._geometry_points_query if self._use_ovstage else self._geometry_points_binding if binding is None: return - points = self._sdp.get_geometry_points() + stream = self._warp_device.stream + asynchronous = self.cfg.async_rendering and not self._use_ovstage + if asynchronous: + output = self._geometry_writes.acquire(stream) + self._sdp.get_geometry_points(output=output, offsets=self._geometry_offsets) + offsets = list(self._geometry_offsets.values()) + points = [output[start:end] for start, end in zip(offsets, [*offsets[1:], len(output)], strict=True)] + else: + points = self._sdp.get_geometry_points() + points = [points[path] for path in self._geometry_paths] timestamp = self._sdp.backend.geometry_timestamp if self._geometry_timestamp == timestamp: return - points = [points[path] for path in self._geometry_paths] if self._use_ovstage: self.backend.stage.write_attribute( binding, @@ -1467,10 +1553,10 @@ def update_geometries(self) -> None: semantic=ovstage.AttributeSemantic.POINT, cuda_stream=self._warp_device.stream.cuda_stream, ).wait() + elif asynchronous: + self._geometry_writes.submit(binding, points, stream) else: - binding.write( - cast(Any, points), data_access=DataAccess.ASYNC, cuda_stream=self._warp_device.stream.cuda_stream - ) + binding.write(cast(Any, points), data_access=DataAccess.ASYNC, cuda_stream=stream.cuda_stream) self._geometry_timestamp = timestamp def update_camera( @@ -1488,6 +1574,11 @@ def update_camera( def update_camera_intrinsics(self, render_data: OVRTXCameraRenderData, intrinsics: wp.array, parameters: wp.array): """Publish calibration columns from GPU memory into the renderer-owned scene.""" + # A runtime calibration change is a scene write for this camera's bindings, so its + # in-flight renders must finish first. Other cameras keep their pipelines. + errors = self.drain_pending_renders((render_data,)) + if errors: + raise ExceptionGroup("OVRTX renders failed before calibration update", errors) stream = wp.get_stream(parameters.device).cuda_stream if self._use_ovstage: self.backend.stage.write_attributes( @@ -1510,7 +1601,7 @@ def update_camera_intrinsics(self, render_data: OVRTXCameraRenderData, intrinsic operation.wait() def render(self, render_data: OVRTXCameraRenderData) -> None: - """Render one camera product into its bound output buffers.""" + """Submit one camera capture; :meth:`read_output` publishes the available image.""" self.render_batch((render_data,)) def render_batch(self, render_data: Sequence[OVRTXCameraRenderData]) -> None: @@ -1543,11 +1634,16 @@ def cleanup(self, render_data: OVRTXCameraRenderData | None) -> None: """ if render_data is None: return - render_data.cleanup() - if render_data in self._camera_render_data: - self._camera_render_data.remove(render_data) - if render_data.render_product_path in self._render_product_paths: - self._render_product_paths.remove(render_data.render_product_path) + try: + self.reset(render_data) + finally: + try: + render_data.cleanup() + finally: + if render_data in self._camera_render_data: + self._camera_render_data.remove(render_data) + if render_data.render_product_path in self._render_product_paths: + self._render_product_paths.remove(render_data.render_product_path) def _remove_camera_reference(self, reference: int) -> None: """Publish removal of a camera's USD reference at the shared stage's current ordinal.""" @@ -1555,20 +1651,24 @@ def _remove_camera_reference(self, reference: int) -> None: ovstage.population.apply_usd_changes(self.backend.stage, ordinal=self._current_ordinal) def close(self) -> None: - """Release this renderer's bindings; the registry closes shared native resources at simulation shutdown.""" - for render_data in tuple(self._camera_render_data): - self.cleanup(render_data) - if self._use_ovstage: - self._close_ovstage() - else: - self._close_legacy() - self._geometry_paths = [] - self._geometry_timestamp = -1 - self._transforms = TimestampedBuffer(SceneDataFormat.TransposedMatrix44d()) - self._render_product_paths.clear() - self._output_id_color_buffers.clear() - self._initialized_scene = False - self._visual_material_writer_ref = None + """Complete borrowed-buffer reads and release bindings; the registry owns the native engine.""" + try: + with contextlib.ExitStack() as resources: + resources.callback(self._close_ovstage if self._use_ovstage else self._close_legacy) + for render_data in tuple(self._camera_render_data): + resources.callback(self.cleanup, render_data) + resources.callback(self._transform_writes.close) + resources.callback(self._geometry_writes.close) + finally: + self._transform_writes = _AsyncWriteBuffers(SceneDataFormat.TransposedMatrix44d() for _ in range(2)) + self._geometry_offsets.clear() + self._geometry_paths = [] + self._geometry_timestamp = -1 + self._transforms_timestamp = -1 + self._render_product_paths.clear() + self._output_id_color_buffers.clear() + self._initialized_scene = False + self._visual_material_writer_ref = None # --------------------------------------------------------------------------- # ovstage implementation @@ -1787,12 +1887,20 @@ def _render_ovstage(self, render_data: Sequence[OVRTXCameraRenderData]) -> None: self.backend.stage.advance_write_floor(ordinal=self._current_ordinal).wait() finally: if material_writer is not None: - drain_errors = contextlib.nullcontext() if sys.exc_info()[0] is None else contextlib.suppress(Exception) - with drain_errors: + # When another exception is already propagating, log the drain failure instead of + # replacing it. Losing it silently would hide a failed material write. + primary_active = sys.exc_info()[0] is not None + try: material_writer.drain() + except Exception as e: + if not primary_active: + raise + logger.warning("Error draining material writes after a failed render: %s", e, exc_info=True) + # The ovstage path always renders synchronously. The ordinal advances before consumption: a + # failed consumption must not leave the ordinal at the write floor set above. products = self.backend.renderer.step( render_products=set(self._render_product_paths), - delta_time=1.0 / 60.0, + delta_time=_RENDER_DELTA_TIME, ordinal=self._current_ordinal, ) self._current_ordinal += 1 @@ -1835,3 +1943,35 @@ def _safe_destroy_path_list(path_list, name: str) -> None: self._object_scales = None self._object_scales_by_path = {} self._current_ordinal = 0 + + +class _AsyncWriteBuffers: + """Retain GPU buffers until native writes finish; reuse them in submission order.""" + + def __init__(self, buffers: Iterable[Any] = ()): + self._writes = deque((buffer, None, None) for buffer in buffers) + + def acquire(self, stream: wp.Stream) -> Any: + """Wait until the next buffer is writable on the given stream.""" + buffer, operation, producer = self._writes[0] + if operation is not None: + operation.wait() + if producer != stream: + stream.wait_stream(producer) + return buffer + + def submit(self, binding: AttributeBinding, values: Any, stream: wp.Stream) -> Operation: + """Submit a write and retain its inputs; a failed submission does not advance the buffers.""" + operation = binding.write_async(values, data_access=DataAccess.ASYNC, cuda_stream=stream.cuda_stream) + self._writes[0] = (self._writes[0][0], operation, stream) + self._writes.rotate(-1) + return operation + + def close(self) -> None: + """Complete every pending write before releasing storage, including on failure.""" + with contextlib.ExitStack() as writes: + writes.callback(self._writes.clear) + for _, operation, producer in self._writes: + if operation is not None: + writes.callback(wp.synchronize_stream, producer) + writes.callback(operation.wait) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py index 8b376db9a154..6ad6dbeb59a2 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py @@ -56,6 +56,16 @@ class OVRTXRendererCfg(RendererCfg): launcher_type: ClassVar[str] = "isaaclab_ov.app:OvrtxLauncher" """The launcher that registers the OVRTX runtime before the stage is created.""" + async_rendering: bool = False + """Trade one capture of camera latency for pipelined rendering. Defaults to False (synchronous). + + Each camera returns its previous capture while rendering its next image. The first capture + and the first capture after reset wait for a fresh image. Live pose and calibration fields + stay current. ``camera.data.info[output_name]["capture"]`` carries the image's matching + ``pos_w``, ``quat_w_world``, ``intrinsic_matrices``, and per-environment ``frame`` indices. + The ovstage path does not support pipelining and renders synchronously. + """ + temp_usd_dir: str | None = None """Directory for temporary USD debug dumps written during OVRTX stage preparation. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py b/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py index 01b19f3a4818..4d72d8f06d9f 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py @@ -8,6 +8,7 @@ from __future__ import annotations import itertools +import logging import weakref from typing import TYPE_CHECKING, Any @@ -20,6 +21,8 @@ from .ovrtx_renderer import OVRTXRenderer +logger = logging.getLogger(__name__) + class OVRTXVisualMaterialWriter: """Compile OVRTX material addresses once and publish dirty device buffers.""" @@ -56,7 +59,8 @@ def __init__(self, renderer: OVRTXRenderer, batches: tuple[VisualMaterialBatch, (batch.channel, f"inputs:{input_name}", list(batch.shader_paths[rows]), rows, dtype, shape) ) start = end - self._event = wp.Event(device=str(device)) + self._device = str(device) + self._event = wp.Event(device=self._device) try: for channel, attribute_name, shader_paths, rows, dtype, shape in groups: if renderer._use_ovstage: @@ -109,10 +113,15 @@ def publish(self) -> None: cuda_event=self._event.cuda_event, ) else: + # The event orders the read after the producers (access sync). The stream + # receives the done fence: work later enqueued on it waits for the read, so + # the next refill of these zero-copy buffers cannot race it. Fills and + # refills run on this device's current Warp stream. operation = address.write_async( self._buffers[channel][rows], data_access=DataAccess.ASYNC, cuda_event=self._event.cuda_event, + cuda_stream=wp.get_stream(self._device).cuda_stream, ) operations.append(operation) finally: @@ -120,10 +129,20 @@ def publish(self) -> None: channels.clear() def drain(self) -> None: - """Complete submitted writes before their scene buffers may change.""" + """Complete submitted writes before their scene buffers may change. + + Every op is waited even when an earlier one fails. Skipping the rest would leave their + writes pending with no remaining reference to wait on. + """ operations, self._operations = self._operations, () + errors = [] for operation in operations: - operation.wait() + try: + operation.wait() + except Exception as e: + errors.append(e) + if errors: + raise RuntimeError(f"{len(errors)} OVRTX material write(s) failed to complete") from errors[0] def _release_backend_addresses(self, renderer: OVRTXRenderer) -> None: for _channel, address, path_list, _attribute_name, _rows in self._addresses: @@ -141,6 +160,12 @@ def close(self) -> None: finally: renderer = self._renderer_ref() if renderer is not None: + # RenderContext.close() closes writers before the renderer, so an asynchronous + # render can still be in flight here and still read these bindings. Deliver every + # queued render before the release below. One failed render must not leave the + # others in flight while their bindings are released. + for error in renderer.drain_pending_renders(): + logger.warning("Error draining in-flight render before material release: %s", error) self._release_backend_addresses(renderer) self._dirty_channels.clear() self._buffers.clear() diff --git a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py index 767ef38aca08..aa881bf16396 100644 --- a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py +++ b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py @@ -331,6 +331,7 @@ def __str__(self) -> str: return "cuda:1" monkeypatch.setattr(ovrtx_renderer_module.wp, "get_device", lambda device: _FakeWarpDevice()) + monkeypatch.setattr(ovrtx_renderer_module.wp, "empty", Mock()) renderer.create_render_data(_make_camera_render_spec(num_envs=1, device="cuda:1")) combined_text = (tmp_path / _OVRTX_STAGE_FILE).read_text(encoding="utf-8") diff --git a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py index 9f48c53472d2..8eeea479e252 100644 --- a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py +++ b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py @@ -15,8 +15,7 @@ import pytest import warp as wp -from isaaclab.scene_data import SceneDataFormat -from isaaclab.utils.buffers import TimestampedBuffer +from isaaclab.scene_data import SceneDataFormat, SceneDataProvider _REQUIRED_MODULES = ("isaaclab_ov", "ovrtx", "pxr") _MISSING_MODULES = [module for module in _REQUIRED_MODULES if importlib.util.find_spec(module) is None] @@ -25,7 +24,7 @@ if not _MISSING_MODULES: import isaaclab_ov.renderers.ovrtx_renderer as ovrtx_renderer_module from isaaclab_ov.renderers import OVRTXRendererCfg - from isaaclab_ov.renderers.ovrtx_renderer import OVRTXRenderer + from isaaclab_ov.renderers.ovrtx_renderer import OVRTXRenderer, _AsyncWriteBuffers from ovrtx import DataAccess @@ -36,16 +35,22 @@ def _make_renderer_without_backend() -> tuple[OVRTXRenderer, MagicMock]: renderer._device = "cpu" renderer._geometry_paths = [] renderer._geometry_timestamp = -1 - renderer._transforms = TimestampedBuffer(SceneDataFormat.TransposedMatrix44d()) + renderer._geometry_offsets = {} + renderer._geometry_writes = _AsyncWriteBuffers() + renderer._transforms_timestamp = -1 renderer._use_ovstage = False renderer._init_fields_legacy() + renderer._transform_writes = _AsyncWriteBuffers(SceneDataFormat.TransposedMatrix44d() for _ in range(2)) + renderer._warp_device = SimpleNamespace(ordinal=0, stream=SimpleNamespace(cuda_stream=99)) return renderer, renderer.backend.renderer -@pytest.mark.parametrize("use_ovstage", [False, True]) -def test_geometry_bindings_borrow_mixed_sdp_points_and_follow_pointer_swaps(use_ovstage): +@pytest.mark.parametrize("mode", ["sync", "async", "ovstage"]) +def test_geometry_bindings_follow_mixed_sdp_points_and_pointer_swaps(mode): """Mesh, particle and curve points share one binding and retain publication ordering.""" renderer, native = _make_renderer_without_backend() + renderer.cfg.async_rendering = mode == "async" + use_ovstage = mode == "ovstage" assert not renderer.cfg.cloning_contexts renderer._use_ovstage = use_ovstage renderer._warp_device = SimpleNamespace(stream=SimpleNamespace(cuda_stream=42)) @@ -61,14 +66,21 @@ def test_geometry_bindings_borrow_mixed_sdp_points_and_follow_pointer_swaps(use_ publication = SimpleNamespace(points=points, geometry_timestamp=0) last_points = points - def read_points(): + def read_points(_format): nonlocal last_points if publication.points is not last_points: publication.geometry_timestamp += 1 last_points = publication.points - return publication.points - - renderer._sdp = SimpleNamespace(backend=publication, get_geometry_points=read_points) + batches = [] + for path in points if publication.points else (): + array = publication.points[path] + source = SceneDataFormat.Points() + source.points = array + batches.append((source, {path: (0, len(array))})) + return batches + + publication.get_geometry_batches = read_points + renderer._sdp = SceneDataProvider(publication) if use_ovstage: renderer._init_fields_ovstage() renderer._current_ordinal = 7 @@ -80,6 +92,7 @@ def read_points(): renderer.update_geometries() renderer.backend.stage.write_attribute.assert_not_called() publication.points = points + renderer._sdp = SceneDataProvider(publication) renderer._setup_geometry_bindings_ovstage() assert renderer._geometry_points_query == list(points) write = renderer.backend.stage.write_attribute @@ -92,6 +105,7 @@ def read_points(): renderer.update_geometries() native.bind_array_attribute.assert_not_called() publication.points = points + renderer._sdp = SceneDataProvider(publication) renderer._setup_geometry_bindings_legacy() assert native.bind_array_attribute.call_args.kwargs["prim_paths"] == list(points) np.testing.assert_array_equal( @@ -100,7 +114,8 @@ def read_points(): np.testing.assert_array_equal( native.write_attribute.call_args_list[1].kwargs["tensor"], np.tile(np.eye(4), (len(points), 1, 1)) ) - write = renderer._geometry_points_binding.write + binding = renderer._geometry_points_binding + write = binding.write_async if mode == "async" else binding.write write.side_effect = RuntimeError("native write failed") with pytest.raises(RuntimeError, match="native write failed"): @@ -111,7 +126,12 @@ def read_points(): renderer.update_geometries() assert write.call_count == 1 data = write.call_args.kwargs["tensors"] if use_ovstage else write.call_args.args[0] - assert [item.data if use_ovstage else item.ptr for item in data] == [array.ptr for array in points.values()] + for item, array in zip(data, points.values(), strict=True): + if mode == "async": + assert item.ptr != array.ptr + np.testing.assert_array_equal(item.numpy(), array.numpy()) + else: + assert (item.data if use_ovstage else item.ptr) == array.ptr kwargs = write.call_args.kwargs assert kwargs["cuda_stream"] == 42 if use_ovstage: @@ -125,17 +145,35 @@ def read_points(): renderer.update_geometries() assert write.call_count == 2 data = write.call_args.kwargs["tensors"] if use_ovstage else write.call_args.args[0] - assert [item.data if use_ovstage else item.ptr for item in data] == [ - publication.points[path].ptr for path in points - ] + for item, path in zip(data, points, strict=True): + array = publication.points[path] + if mode == "async": + np.testing.assert_array_equal(item.numpy(), array.numpy()) + else: + assert (item.data if use_ovstage else item.ptr) == array.ptr + if mode == "async": + # Mutating live physics storage cannot change either queued snapshot. + snapshots = [call.args[0] for call in write.call_args_list] + for array in publication.points.values(): + array.fill_(wp.vec3f(-1)) + for snapshot in snapshots: + for item, array in zip(snapshot, points.values(), strict=True): + np.testing.assert_array_equal(item.numpy(), array.numpy()) + publication.geometry_timestamp += 1 + renderer.update_geometries() + write.return_value.wait.assert_called() + assert write.call_args.args[0][0].ptr == snapshots[0][0].ptr + binding.write.assert_not_called() -@pytest.mark.parametrize("use_ovstage", [False, True]) -def test_update_transforms_consumes_sdp_matrices_once_per_publication(monkeypatch, use_ovstage): - """Both OVRTX paths bind published bodies and consume SDP's scaled, transposed matrices.""" +@pytest.mark.parametrize("mode", ["sync", "async", "ovstage"]) +def test_update_transforms_consumes_sdp_matrices_once_per_publication(monkeypatch, mode): + """Borrow synchronous publications or convert directly into retained asynchronous write buffers.""" from isaaclab.scene_data import SceneDataFormat, SceneDataProvider renderer, _ = _make_renderer_without_backend() + renderer.cfg.async_rendering = mode == "async" + use_ovstage = mode == "ovstage" paths = ["/World/Shared", "/World/envs/env_1/Object"] poses = np.array([[1, 2, 3, 0, 0, 0, 1], [4, 5, 6, 0, 0, 0, 1]], dtype=np.float32) transforms = SceneDataFormat.Transform() @@ -165,6 +203,10 @@ def test_update_transforms_consumes_sdp_matrices_once_per_publication(monkeypatc renderer._setup_xform_bindings_legacy() assert renderer.backend.renderer.bind_attribute.call_args.kwargs["prim_paths"] == paths renderer._object_xform_binding.write = lambda matrices, **kwargs: writes.append((None, matrices, kwargs)) + operation = MagicMock() + renderer._object_xform_binding.write_async = lambda matrices, **kwargs: ( + writes.append((None, matrices, kwargs)) or operation + ) renderer.update_transforms() renderer.update_transforms() @@ -186,6 +228,14 @@ def test_update_transforms_consumes_sdp_matrices_once_per_publication(monkeypatc renderer.update_transforms() assert len(writes) == 2 updated = writes[1][2]["tensors"] if use_ovstage else writes[1][1] - assert updated is matrices + assert (updated is matrices) == (mode != "async") expected[:, 3, :3] = poses[:, :3] np.testing.assert_array_equal(updated.numpy(), expected) + if mode == "async": + # The new capture has not overwritten the previous borrowed input. + np.testing.assert_array_equal(matrices.numpy()[:, 3, 0], poses[:, 0] - 10) + operation.wait.assert_not_called() + backend.transforms_timestamp += 1 + renderer.update_transforms() + operation.wait.assert_called_once_with() + assert writes[2][1] is matrices diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index 53cf3edddf4e..c8799f163f95 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -10,6 +10,7 @@ import importlib.util import sys import types +from builtins import ExceptionGroup from unittest.mock import MagicMock import numpy as np @@ -22,6 +23,7 @@ from isaaclab.sensors.camera.camera_data import CameraData, RenderBufferKind, RenderBufferSpec from isaaclab.sim import PinholeCameraCfg, SimulationContext from isaaclab.utils import replace +from isaaclab.utils.warp import ProxyArray _REQUIRED_MODULES = ("isaaclab_ov", "ovrtx") _MISSING_MODULES = [module for module in _REQUIRED_MODULES if importlib.util.find_spec(module) is None] @@ -41,6 +43,7 @@ OVRTXBackend, OVRTXCameraRenderData, OVRTXRenderer, + _AsyncWriteBuffers, _gpu_side_render_var_sync_enabled, ovrtx_use_ovstage_enabled, ) @@ -85,6 +88,10 @@ def _make_ovrtx_renderer_without_backend() -> OVRTXRenderer: renderer.backend._resources = contextlib.ExitStack() SimulationContext.instance()._backend_registry.append((cfg, renderer.backend)) renderer._camera_render_data = [] + renderer._transform_writes = _AsyncWriteBuffers() + renderer._geometry_writes = _AsyncWriteBuffers() + renderer._geometry_offsets = {} + renderer._use_ovstage = False return renderer @@ -240,19 +247,22 @@ def test_ovrtx_render_batch_empty_sequence_does_not_require_initialized_backend( @pytest.mark.integration @pytest.mark.rendering -@pytest.mark.parametrize("use_ovstage", [False, True]) -def test_ovrtx_multiple_cameras_render_independent_views(monkeypatch, use_ovstage, tmp_path): - """Check independent camera batches and save RGB/depth frames under pytest's temporary directory. - - Use ``--basetemp=/tmp/ovrtx-camera-frames`` to choose where pytest writes the captures. - Depth PNGs use a shared 0-8 m range (near is white); NPY files retain the raw depths [m]. - """ - from PIL import Image - +@pytest.mark.parametrize( + "use_ovstage, asynchronous, geometry", + [ + (False, False, "rigid"), + (True, False, "rigid"), + *[(False, True, kind) for kind in ("mesh", "particles", "cable")], + ], + ids=["legacy", "ovstage", "async-mesh", "async-particles", "async-cable"], +) +def test_ovrtx_multiple_cameras_render_independent_views(monkeypatch, use_ovstage, asynchronous, geometry): + """Cameras preserve independent captures, GPU input lifetimes, and reset/cleanup boundaries.""" from pxr import Gf, Usd, UsdGeom, UsdLux from isaaclab.cloner import make_clone_plan from isaaclab.renderers.camera_render_spec import CameraRenderSpec + from isaaclab.scene_data import SceneDataFormat, SceneDataProvider from isaaclab.utils.math import convert_camera_frame_orientation_convention from isaaclab.utils.warp import ProxyArray @@ -266,23 +276,52 @@ def test_ovrtx_multiple_cameras_render_independent_views(monkeypatch, use_ovstag UsdGeom.Xform.Define(stage, "/World/envs/env_0") if not use_ovstage: UsdGeom.Xform.Define(stage, "/World/envs/env_1") + batches, initial_points = [], [] for index, x in enumerate((0.0, 2.0)): camera = UsdGeom.Camera.Define(stage, f"/World/envs/env_0/cam{index}") camera.CreateProjectionAttr("orthographic") camera.CreateHorizontalApertureAttr(20.0) camera.CreateVerticalApertureAttr(20.0) camera.AddTranslateOp().Set(Gf.Vec3d(x, 0, 5)) - cube = UsdGeom.Cube.Define(stage, f"/World/envs/env_0/cube{index}") - cube.CreateSizeAttr(1.0) - cube.CreateDisplayColorAttr([(0.8, 0.1, 0.1) if index == 0 else (0.1, 0.8, 0.1)]) - cube.AddTranslateOp().Set(Gf.Vec3d(x, 0, index)) - - renderer = OVRTXRenderer(OVRTXRendererCfg()) + path = f"/World/envs/env_0/object{index}" + if geometry == "rigid": + cube = UsdGeom.Cube.Define(stage, path) + cube.CreateSizeAttr(1.0) + cube.CreateDisplayColorAttr([(0.8, 0.1, 0.1) if index == 0 else (0.1, 0.8, 0.1)]) + cube.AddTranslateOp().Set(Gf.Vec3d(x, 0, index)) + continue + if geometry == "mesh": + prim = UsdGeom.Mesh.Define(stage, path) + prim.CreateFaceVertexCountsAttr([4]) + prim.CreateFaceVertexIndicesAttr([0, 1, 2, 3]) + vertices = [(x + dx, dy, index + 0.5) for dx, dy in ((-0.5, -0.5), (0.5, -0.5), (0.5, 0.5), (-0.5, 0.5))] + elif geometry == "particles": + prim = UsdGeom.Points.Define(stage, path) + prim.CreateWidthsAttr([1.0]) + vertices = [(x, 0, index)] + else: + prim = UsdGeom.BasisCurves.Define(stage, path) + prim.CreateTypeAttr("linear") + prim.CreateWrapAttr("nonperiodic") + prim.CreateCurveVertexCountsAttr([3]) + prim.CreateWidthsAttr([1.0]) + prim.SetWidthsInterpolation("constant") + vertices = [(x, y, index) for y in (-0.5, 0, 0.5)] + prim.CreatePointsAttr(vertices) + initial_points.append(np.asarray(vertices, dtype=np.float32)) + source = SceneDataFormat.Points() + source.points = wp.array(vertices, dtype=wp.vec3f, device="cuda:0") + batches.append((source, {path.replace("env_0", f"env_{world}"): (0, len(vertices)) for world in range(2)})) + + renderer = OVRTXRenderer(OVRTXRendererCfg(async_rendering=asynchronous)) + publication = types.SimpleNamespace( + transform_paths=[], geometry_timestamp=0, get_geometry_batches=lambda _format: batches + ) + renderer._sdp = SceneDataProvider(publication) renderer._exported_usd_string = stage.ExportToString() - plan = make_clone_plan( + renderer._clone_plan = make_clone_plan( (AssetBaseCfg(prim_path="/World/envs/env_[^/]+"),), ((0,),), 2, positions=np.zeros((2, 3), dtype=np.float32) ) - renderer._clone_plan = plan cameras = [] def camera_scope_exists(rd): @@ -295,16 +334,8 @@ def camera_scope_exists(rd): return query.result().total_prim_count > 0 return any(path.startswith(scope) for path in renderer.backend.renderer.query_prims()) - def check_depth(rd, data, expected, label): + def check_depth(rd, data, expected): depth = data.output["distance_to_image_plane"].torch - for env_id in range(rd.num_envs): - prefix = tmp_path / f"{label}_env{env_id}" - rgb = data.output["rgb"].torch[env_id].cpu().numpy() - depth_m = depth[env_id, ..., 0].cpu().numpy() - Image.fromarray(rgb).save(f"{prefix}_rgb.png") - np.save(f"{prefix}_depth.npy", depth_m) - preview = ((1.0 - np.clip(depth_m / 8.0, 0.0, 1.0)) * 255).astype(np.uint8) - Image.fromarray(preview).save(f"{prefix}_depth.png") assert depth.shape == (2, rd.height, rd.width, 1) torch.testing.assert_close( depth[:, rd.height // 2, rd.width // 2, 0], @@ -338,8 +369,10 @@ def check_depth(rd, data, expected, label): renderer.set_outputs(rd, data.output) cameras.append((rd, data)) # Register the next camera after rendering has already started. + renderer.update_geometries() renderer.render(rd) - check_depth(rd, data, 5.0 - index - 0.5, f"initial_cam{index}") + renderer.read_output(rd, data) + check_depth(rd, data, 5.0 - index - 0.5) if index == 1: normals = data.output["normals"].torch[:, height // 2, width // 2, :3] torch.testing.assert_close( @@ -354,13 +387,55 @@ def check_depth(rd, data, expected, label): orientations = ProxyArray(wp.from_torch(quats, dtype=wp.quatf)) renderer.update_camera(cameras[1][0], positions, orientations, cameras[1][1].intrinsic_matrices) renderer.render_batch([rd for rd, _ in cameras]) - check_depth(*cameras[0], 4.5, "after_move_cam0") - check_depth(*cameras[1], 5.5, "after_move_cam1") + for rd, data in cameras: + renderer.read_output(rd, data) + check_depth(*cameras[0], 4.5) + check_depth(*cameras[1], 3.5 if asynchronous else 5.5) + + if asynchronous: + renderer.render_batch([rd for rd, _ in cameras]) + for rd, data in cameras: + renderer.read_output(rd, data) + check_depth(*cameras[1], 5.5) + # Reuse camera buffers across producer streams without advancing the other camera. + for height in (6.0, 8.0, 9.0): + with wp.ScopedStream(wp.Stream("cuda:0")): + positions = ProxyArray(wp.array([[0.0, 0, height]] * 2, dtype=wp.vec3f, device="cuda:0")) + renderer.update_camera(cameras[0][0], positions, orientations, cameras[0][1].intrinsic_matrices) + renderer.render(cameras[0][0]) + renderer.read_output(*cameras[0]) + wp.synchronize_stream() + check_depth(*cameras[1], 5.5) + check_depth(*cameras[0], 7.5) + renderer.reset(cameras[0][0]) + renderer.render(cameras[0][0]) + renderer.read_output(*cameras[0]) + check_depth(*cameras[0], 8.5) + for height in (1.0, 2.0, 3.0): + with wp.ScopedStream(wp.Stream("cuda:0")): + for (source, _), vertices in zip(batches, initial_points, strict=True): + source.points.assign(vertices + (0, 0, height)) + publication.geometry_timestamp += 1 + renderer.update_geometries() + # Physics may overwrite its storage immediately after SDP snapshots it. + for source, _ in batches: + source.points.fill_(wp.vec3f(-100)) + renderer.render_batch([rd for rd, _ in cameras]) + for rd, data in cameras: + renderer.read_output(rd, data) + wp.synchronize_stream() + check_depth(*cameras[0], 8.5 - (height - 1)) + check_depth(*cameras[1], 5.5 - (height - 1)) + renderer.reset(cameras[1][0], [0]) + renderer.render(cameras[1][0]) + renderer.read_output(*cameras[1]) + check_depth(*cameras[1], 2.5) assert all(camera_scope_exists(rd) for rd, _ in cameras) renderer.cleanup(cameras[0][0]) assert not camera_scope_exists(cameras[0][0]) renderer.render(cameras[1][0]) - check_depth(*cameras[1], 5.5, "after_cleanup_cam1") + renderer.read_output(*cameras[1]) + check_depth(*cameras[1], 2.5 if asynchronous else 5.5) renderer.cleanup(cameras[1][0]) renderer.cleanup(cameras[1][0]) assert not camera_scope_exists(cameras[1][0]) @@ -369,6 +444,93 @@ def check_depth(rd, data, expected, label): SimulationContext.instance().close_backend(renderer.backend) +@pytest.mark.parametrize("batch", [False, True], ids=["lazy", "batched"]) +def test_async_cameras_publish_independently_with_capture_metadata_and_reset(monkeypatch, batch): + """Completed batches do not update another camera; reset retires only that camera's history.""" + renderer = _make_ovrtx_renderer_without_backend() + renderer.cfg.async_rendering = True + renderer._initialized_scene = True + renderer._visual_material_writer_ref = None + cameras = [_make_ovrtx_camera_render_data() for _ in range(2)] + renderer._camera_render_data = cameras.copy() + camera_data = [] + for index, camera in enumerate(cameras): + camera.render_product_path = f"/Render/Camera{index}" + data = CameraData.allocate(["rgb"], 2, 2, 2, "cpu", renderer.supported_output_types()) + data.create_buffers(2, "cpu") + renderer.set_outputs(camera, data.output) + camera_data.append(data) + renderer._render_product_paths = [camera.render_product_path for camera in cameras] + frame = ProxyArray(wp.zeros(2, dtype=wp.int64, device="cpu")) + operations = [] + ordinal = 0 + + def submit(render_products, delta_time): + operation = MagicMock() + operation.wait.return_value.fetch.return_value = { + path: types.SimpleNamespace(frames=[ordinal]) for path in render_products + } + operations.append(operation) + return operation + + def consume(camera, value, buffers): + buffers["rgba"].fill_(value) + + renderer.backend.renderer = types.SimpleNamespace(step_async=submit) + consume = MagicMock(side_effect=consume) + monkeypatch.setattr(renderer, "_process_render_frame", consume) + + def capture(value, indices): + nonlocal ordinal + ordinal = value + frame.warp.fill_(value) + for index in indices: + data = camera_data[index] + data.pos_w.warp.fill_(wp.vec3f(value)) + data.intrinsic_matrices.warp.fill_(wp.mat33f(value)) + renderer.prepare_capture(cameras[index], data, frame) + if batch: + renderer.render_batch([cameras[index] for index in indices]) + else: + for index in indices: + renderer.render(cameras[index]) + for index in indices: + renderer.read_output(cameras[index], camera_data[index]) + + capture(1, (0, 1)) + capture(2, (0, 1)) + capture(3, (0,)) + assert consume.call_count == 3 # Priming images are not extracted again on the next capture. + for index, expected in enumerate((2, 1)): + data = camera_data[index] + np.testing.assert_array_equal(data.output["rgb"].warp.numpy(), expected) + for name in ("frame", "pos_w", "intrinsic_matrices"): + np.testing.assert_array_equal(data.info["rgb"]["capture"][name].warp.numpy(), expected) + np.testing.assert_array_equal(camera_data[0].pos_w.warp.numpy(), 3) + saved_capture = camera_data[0].info["rgb"]["capture"] + renderer.read_output(cameras[0], camera_data[0]) + assert camera_data[0].info["rgb"]["capture"] is saved_capture + assert renderer.drain_pending_renders() == [] + np.testing.assert_array_equal(camera_data[1].output["rgb"].warp.numpy(), 1) + + renderer.reset(cameras[0]) + capture(4, (0,)) + np.testing.assert_array_equal(camera_data[0].output["rgb"].warp.numpy(), 4) + np.testing.assert_array_equal(saved_capture["pos_w"].warp.numpy(), 2) + capture(5, (0, 1)) + assert consume.call_count == 5 + np.testing.assert_array_equal(camera_data[0].output["rgb"].warp.numpy(), 4) + np.testing.assert_array_equal(camera_data[1].output["rgb"].warp.numpy(), 2) + renderer.cleanup(cameras[0]) + + monkeypatch.setattr(renderer, "_process_render_frame", MagicMock(side_effect=RuntimeError("output extraction"))) + with pytest.raises(RuntimeError, match="output extraction"): + capture(6, (1,)) + renderer.cleanup(cameras[1]) + assert all(operation.wait.called for operation in operations) + assert renderer._camera_render_data == [] + + def test_ovrtx_set_outputs_wraps_caller_torch_zero_copy(): """OVRTXRenderer.set_outputs publishes warp views over the caller's warp storage.""" renderer = _make_ovrtx_renderer_without_backend() @@ -698,10 +860,13 @@ def test_ovrtx_map_render_var_orders_the_read_against_render_completion(monkeypa assert render_var.ordering == [expected] -@pytest.mark.parametrize("cleanup_directly", [False, True]) -@pytest.mark.parametrize("use_ovstage", [False, True]) -def test_ovrtx_cleanup_releases_only_the_given_render_data(cleanup_directly, use_ovstage): - """Release one camera's pose and calibration resources once, keeping other cameras usable.""" +@pytest.mark.parametrize( + "use_ovstage, cleanup_directly, failure", + [(stage, direct, None) for stage in (False, True) for direct in (False, True)] + + [(False, False, "render"), (False, False, "write")], +) +def test_ovrtx_cleanup_releases_only_the_given_render_data(monkeypatch, cleanup_directly, use_ovstage, failure): + """Release only the given camera, even if its pending native operations fail.""" events = [] renderer = ( _make_ovstage_renderer_with_backend(events) if use_ovstage else _make_legacy_renderer_with_backend(events) @@ -724,6 +889,24 @@ def test_ovrtx_cleanup_releases_only_the_given_render_data(cleanup_directly, use render_data.intrinsic_bindings = [_RecordingBinding(events, "intrinsics")] render_data.resources.callback(render_data.intrinsic_bindings[0].unbind) + operations = [MagicMock(), MagicMock()] + operations[-1].wait.side_effect = RuntimeError("device lost") + if failure == "render": + render_data.pending = (operations[-1], {}) + elif failure == "write": + render_data.camera_writes = _AsyncWriteBuffers((object(), object())) + binding = MagicMock() + binding.write_async.side_effect = operations + stream = types.SimpleNamespace(cuda_stream=0) + for _ in operations: + render_data.camera_writes.submit(binding, object(), stream) + monkeypatch.setattr(wp, "synchronize_stream", lambda _stream: None) + if failure: + with pytest.raises(ExceptionGroup if failure == "render" else RuntimeError): + renderer.cleanup(render_data) + if failure == "write": + operations[0].wait.assert_called_once_with() + if cleanup_directly: render_data.cleanup() renderer.cleanup(None) @@ -743,6 +926,7 @@ def test_ovrtx_cleanup_releases_only_the_given_render_data(cleanup_directly, use assert render_data.warp_buffers == {} assert render_data.renderer_info == {} assert render_data.ppisp_pipeline is None + assert render_data.pending is render_data.ready is None assert renderer._render_product_paths == ["/RenderCamera_0/RenderProduct_camera"] assert renderer._initialized_scene is True diff --git a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py index 82c8eb2cb1d4..fa13f55445cb 100644 --- a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py +++ b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py @@ -20,6 +20,7 @@ if not _MISSING_MODULES: from isaaclab_ov.renderers.ovrtx_renderer import OVRTXRenderer + from isaaclab_ov.renderers.ovrtx_renderer_cfg import OVRTXRendererCfg from ovrtx import DataAccess from isaaclab.renderers.base_renderer import VisualMaterialBatch @@ -118,6 +119,8 @@ def _renderer(*, use_ovstage: bool = False): renderer._use_ovstage = use_ovstage renderer.backend.renderer = _NativeRecorder(events) renderer._visual_material_writer_ref = None + renderer.cfg = OVRTXRendererCfg() + renderer._camera_render_data = [] if use_ovstage: renderer.backend.stage = _OvstageRecorder(events) renderer.backend.paths = _PathRecorder() diff --git a/source/isaaclab_tasks/changelog.d/pv-ovrtx-async-slot-buffers.skip b/source/isaaclab_tasks/changelog.d/pv-ovrtx-async-slot-buffers.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py index 3cabdd67325f..ff2367b550db 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py @@ -21,9 +21,15 @@ pytestmark = pytest.mark.arm_ci -_RENDERING_PARAMS = group_rendering_params( - make_kitless_rendering_params(MINIMAL_KITLESS_PHYSICS_RENDERER_AOV_COMBINATIONS) -) +_RENDERING_PARAMS = [ + pytest.param(*param.values, False, marks=param.marks, id=param.id) + for param in group_rendering_params( + make_kitless_rendering_params(MINIMAL_KITLESS_PHYSICS_RENDERER_AOV_COMBINATIONS) + ) +] + [ + pytest.param("legacy", physics, "ovrtx_renderer", ["rgb"], True, id=f"legacy-{physics}-ovrtx-async") + for physics in ("newton", "ovphysx") +] _COMPARISON_SCORES: list[dict] = [] _determinism_fixture = make_determinism_fixture() @@ -33,8 +39,10 @@ @pytest.mark.parametrize( - "ovstage_variant,physics_backend,renderer,data_types", _RENDERING_PARAMS, indirect=["ovstage_variant"] + "ovstage_variant,physics_backend,renderer,data_types,async_rendering", + _RENDERING_PARAMS, + indirect=["ovstage_variant"], ) -def test_rendering_cartpole_kitless(ovstage_variant, physics_backend, renderer, data_types): +def test_rendering_cartpole_kitless(ovstage_variant, physics_backend, renderer, data_types, async_rendering): """Camera output must match golden images (Cartpole camera presets env).""" - rendering_test_cartpole(physics_backend, renderer, data_types, _COMPARISON_SCORES) + rendering_test_cartpole(physics_backend, renderer, data_types, _COMPARISON_SCORES, async_rendering=async_rendering) diff --git a/source/isaaclab_tasks/test/rendering_test_utils.py b/source/isaaclab_tasks/test/rendering_test_utils.py index 936a42de0311..6d1f9774d0c3 100644 --- a/source/isaaclab_tasks/test/rendering_test_utils.py +++ b/source/isaaclab_tasks/test/rendering_test_utils.py @@ -1706,6 +1706,7 @@ def rendering_test_cartpole( comparison_scores: list[dict], *, compare_golden: bool = False, + async_rendering: bool = False, ) -> None: for data_type in data_types: _skip_if_newton_motion_vectors(physics_backend, data_type) @@ -1766,6 +1767,8 @@ class _CartpoleCameraTestEnvCfg(CartpoleCameraEnvCfg): env_cfg.scene.num_envs = 4 env_cfg.scene.tiled_camera.data_types = data_types + if async_rendering: + env_cfg.scene.tiled_camera.renderer_cfg.async_rendering = True if getattr(env_cfg.scene.tiled_camera.renderer_cfg, "renderer_type", None) == "newton_warp": env_cfg.scene.tiled_camera.renderer_cfg.render_order = "pixel_priority"