From 41af2a846c7caefb262c7e1741c6a7e0e956f3e6 Mon Sep 17 00:00:00 2001 From: Xinjie Yao Date: Thu, 22 Oct 2026 13:58:08 -0700 Subject: [PATCH 01/11] Add single-body SimReady Central asset search Signed-off-by: Xinjie Yao --- .../cli_runner.rst | 10 ++ .../simready_asset_search.py | 12 +- .../simready_central.py | 121 ++++++++++++++++++ isaaclab_arena/tests/test_simready_central.py | 121 ++++++++++++++++++ .../review_gui/generation_panel.py | 6 +- pyproject.toml | 1 + 6 files changed, 267 insertions(+), 4 deletions(-) create mode 100644 isaaclab_arena/agentic_environment_generation/simready_central.py create mode 100644 isaaclab_arena/tests/test_simready_central.py diff --git a/docs/pages/concepts/agentic_environment_generation/cli_runner.rst b/docs/pages/concepts/agentic_environment_generation/cli_runner.rst index 849e12d1de..e8c3632da0 100644 --- a/docs/pages/concepts/agentic_environment_generation/cli_runner.rst +++ b/docs/pages/concepts/agentic_environment_generation/cli_runner.rst @@ -110,6 +110,7 @@ These options are defined by the environment generation runner: - Agent * - ``--simready_source SOURCE`` - Chooses the SimReady search backend. Default: ``isaac-sim-ga``. + Use ``simready-central`` for single-rigid-body assets on Hugging Face. - Agent * - ``--simready_s3_url URL`` - Overrides the S3 root for S3-based SimReady sources. @@ -124,6 +125,15 @@ These options are defined by the environment generation runner: - Sets the number of zero-action simulation steps. Default: ``20``. - Evaluation smoke test +With ``--enable_simready_search --simready_source simready-central``, search +requires indexed rigid-body count equal to one. Selected asset folders are cached +from their pinned Hugging Face dataset revision, preserving relative dependencies. +Before registration, the composed USD must have exactly one enabled rigid body, +no articulation root, and no unresolved composition errors. A local wrapper selects +PhysX variants; the generated spec references that wrapper and needs the same cache +when run elsewhere. Assets with missing count metadata or unsupported packaging are +skipped. Articulated assets and multi-body assemblies are not imported. + Build and simulation options ---------------------------- diff --git a/isaaclab_arena/agentic_environment_generation/simready_asset_search.py b/isaaclab_arena/agentic_environment_generation/simready_asset_search.py index 5a7e2e7f77..65e90ef98f 100644 --- a/isaaclab_arena/agentic_environment_generation/simready_asset_search.py +++ b/isaaclab_arena/agentic_environment_generation/simready_asset_search.py @@ -37,6 +37,7 @@ class SimReadySourceKind(str, Enum): ISAAC_SIM_GA = "isaac-sim-ga" S3 = "s3" SERVICE = "service" + CENTRAL = "simready-central" @dataclass @@ -88,15 +89,20 @@ def search_simready_objects(object_phrases: list[str], config: SimReadySearchCon if not phrases: return SimReadyCandidateCatalogue() - library = _configure_asset_library(config) - if library is None: + library = None if config.source == SimReadySourceKind.CENTRAL else _configure_asset_library(config) + if library is None and config.source != SimReadySourceKind.CENTRAL: return SimReadyCandidateCatalogue() candidates: list[SimReadyObjectCandidate] = [] unmatched_phrases: list[str] = [] for phrase in phrases: try: - hits = _search_phrase(library, phrase, max_results=config.max_results_per_object) + if config.source == SimReadySourceKind.CENTRAL: + from isaaclab_arena.agentic_environment_generation.simready_central import search_central_objects + + hits = search_central_objects(phrase, max_results=config.max_results_per_object) + else: + hits = _search_phrase(library, phrase, max_results=config.max_results_per_object) except Exception as exc: _logger.warning("simready search failed for %r: %s", phrase, exc) hits = [] diff --git a/isaaclab_arena/agentic_environment_generation/simready_central.py b/isaaclab_arena/agentic_environment_generation/simready_central.py new file mode 100644 index 0000000000..3b2a6432d2 --- /dev/null +++ b/isaaclab_arena/agentic_environment_generation/simready_central.py @@ -0,0 +1,121 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Discover and prepare single-body props from Hugging Face SimReady Central.""" + +from __future__ import annotations + +import hashlib +import logging +import re +from pathlib import Path, PurePosixPath + +import requests + +from isaaclab_arena.agentic_environment_generation.simready_asset_search import ( + MIN_INSPECTED_MATCHES_PER_PHRASE, + SimReadyObjectCandidate, +) + +CENTRAL_SEARCH_URL = "https://nvidia-simready-central.hf.space/search_hybrid" +_logger = logging.getLogger(__name__) + + +def search_central_objects(phrase: str, *, max_results: int) -> list[SimReadyObjectCandidate]: + """Search Central and retain only locally validated single-body props.""" + assert max_results > 0, "max_results must be positive" + response = requests.post( + CENTRAL_SEARCH_URL, + headers={"x-usdsearch-storage-backend": "usdsearch-dev"}, + json={ + "hybrid_text_query": phrase, + "file_extension_include": "usd*", + "filter_by_properties_numeric": "simready_metadata_rigid_body_count=1", + "return_usd_properties": True, + "limit": max(MIN_INSPECTED_MATCHES_PER_PHRASE, max_results), + }, + timeout=30, + ) + response.raise_for_status() + candidates = [] + for hit in response.json()["hits"]: + try: + source = hit["source"] + # Fail closed even if a backend ignores the numeric filter. + counts = [ + prop.get("value_numeric", prop.get("value")) + for prop in source.get("usd_properties", []) + if prop.get("name") == "simready_metadata_rigid_body_count" + ] + if not counts or any(float(count) != 1 for count in counts): + continue + usd_path = _prepare_central_asset(source["path"]) + candidates.append( + SimReadyObjectCandidate( + search_phrase=phrase, + usd_path=usd_path, + tags=("sim-ready", "single-rigid-body"), + relevance_score=hit.get("rrf_score"), + ) + ) + except Exception as exc: + _logger.info("Skipping Central asset %s: %s", hit.get("id"), exc) + continue + if len(candidates) >= max_results: + break + return candidates + + +def _central_asset_location(url: str) -> tuple[str, str, str, str]: + """Parse a revision-pinned Central dataset URL and its asset package folder.""" + match = re.fullmatch(r"(?:hf|https)://huggingface\.co/datasets/([^/]+/[^/]+)/resolve/([0-9a-f]{40})/(.+)", url) + assert match, "expected a pinned Hugging Face dataset URL" + repo_id, revision, filename = match.groups() + path = PurePosixPath(filename) + assert not path.is_absolute() and ".." not in path.parts, "invalid asset path" + assert not any(char in filename for char in "*?[]\\"), "invalid asset path" + assert path.suffix in (".usd", ".usda", ".usdc", ".usdz"), "expected USD asset" + # Central packages place the entrypoint in /simready_usd/. Fetch its + # siblings too (physics layers, materials, textures), never the entire dataset. + assert "simready_usd" in path.parts, "unsupported Central package layout" + folder = PurePosixPath(*path.parts[: path.parts.index("simready_usd")]) + assert str(folder) != ".", "asset must have a package folder" + return repo_id, revision, filename, str(folder) + + +def _prepare_central_asset(url: str) -> str: + """Cache a pinned asset package and write a validated PhysX reference wrapper.""" + from huggingface_hub import snapshot_download + from pxr import Usd, UsdGeom, UsdPhysics + + from isaaclab_arena.utils.usd.rigid_bodies import apply_usd_variant_selections + + repo_id, revision, filename, folder = _central_asset_location(url) + snapshot = snapshot_download( + repo_id=repo_id, repo_type="dataset", revision=revision, allow_patterns=[f"{folder}/*"] + ) + asset_path = str(Path(snapshot) / filename) + asset_stage = Usd.Stage.Open(asset_path) + assert asset_stage and asset_stage.GetDefaultPrim(), "asset requires a default prim" + stage = Usd.Stage.CreateInMemory() + UsdGeom.SetStageMetersPerUnit(stage, UsdGeom.GetStageMetersPerUnit(asset_stage)) + UsdGeom.SetStageUpAxis(stage, UsdGeom.GetStageUpAxis(asset_stage)) + root = stage.DefinePrim("/Asset") + root.GetReferences().AddReference(asset_path) + stage.SetDefaultPrim(root) + apply_usd_variant_selections(stage, {"Physics": "physics", "PhysX": "Enabled"}) + assert not stage.GetCompositionErrors(), "USD has unresolved composition errors" + bodies = [] + for prim in Usd.PrimRange.Stage(stage, Usd.TraverseInstanceProxies()): + assert not prim.HasAPI(UsdPhysics.ArticulationRootAPI), "asset has an articulation root" + if prim.HasAPI(UsdPhysics.RigidBodyAPI): + bodies.append(prim) + assert len(bodies) == 1, f"expected one rigid body, found {len(bodies)}" + assert UsdPhysics.RigidBodyAPI(bodies[0]).GetRigidBodyEnabledAttr().Get(), "rigid body is disabled" + cache = Path.home() / ".cache" / "isaaclab_arena" / "simready-central" + cache.mkdir(parents=True, exist_ok=True) + wrapper = cache / f"{hashlib.sha256(url.encode()).hexdigest()}.usda" + stage.GetRootLayer().Export(str(wrapper)) + return str(wrapper) diff --git a/isaaclab_arena/tests/test_simready_central.py b/isaaclab_arena/tests/test_simready_central.py new file mode 100644 index 0000000000..7fac4a4622 --- /dev/null +++ b/isaaclab_arena/tests/test_simready_central.py @@ -0,0 +1,121 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Central search filtering and pinned asset preparation tests.""" + +from unittest.mock import Mock, patch + +import pytest + +from isaaclab_arena.agentic_environment_generation.simready_asset_search import ( + SimReadySearchConfig, + SimReadySourceKind, + search_simready_objects, +) +from isaaclab_arena.agentic_environment_generation.simready_central import ( + _central_asset_location, + search_central_objects, +) + +MODULE = "isaaclab_arena.agentic_environment_generation.simready_central" +REVISION = "a" * 40 +URL = f"hf://huggingface.co/datasets/nvidia/simready-assets/resolve/{REVISION}/pallet/simready_usd/pallet.usd" + + +def _hit(count, path=URL): + properties = [] if count is None else [{"name": "simready_metadata_rigid_body_count", "value_numeric": count}] + return {"source": {"path": path, "usd_properties": properties}, "rrf_score": 0.5} + + +def test_central_requires_single_body_metadata_and_skips_invalid_usd(): + response = Mock() + response.json.return_value = {"hits": [_hit(None), _hit(2), _hit(1), _hit(1)]} + with ( + patch(f"{MODULE}.requests.post", return_value=response) as post, + patch( + f"{MODULE}._prepare_central_asset", side_effect=[ValueError("articulation"), "/cache/pallet.usda"] + ) as prepare, + ): + candidates = search_central_objects("pallet", max_results=1) + assert [candidate.usd_path for candidate in candidates] == ["/cache/pallet.usda"] + assert prepare.call_count == 2 + assert post.call_args.kwargs["json"]["filter_by_properties_numeric"] == "simready_metadata_rigid_body_count=1" + assert post.call_args.kwargs["timeout"] == 30 + + +def test_central_failure_leaves_phrase_unmatched(): + with patch(f"{MODULE}.requests.post", side_effect=TimeoutError("offline")): + catalogue = search_simready_objects(["pallet"], SimReadySearchConfig(source=SimReadySourceKind.CENTRAL)) + assert catalogue.candidates == [] + assert catalogue.unmatched_phrases == ["pallet"] + + +def test_central_package_location_preserves_revision_and_folder(): + assert _central_asset_location(URL) == ( + "nvidia/simready-assets", + REVISION, + "pallet/simready_usd/pallet.usd", + "pallet", + ) + + +@pytest.mark.parametrize( + "url", + [ + URL.replace(REVISION, "main"), + URL.replace("pallet/simready_usd", "../simready_usd"), + URL.replace("pallet/simready_usd/", ""), + ], +) +def test_central_rejects_unpinned_or_unbounded_packages(url): + with pytest.raises(AssertionError): + _central_asset_location(url) + + +def _test_central_usd_validation(simulation_app): + import tempfile + from pathlib import Path + + from pxr import Usd, UsdGeom, UsdPhysics + + from isaaclab_arena.agentic_environment_generation.simready_central import _prepare_central_asset + + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + asset = folder / "pallet/simready_usd/pallet.usd" + asset.parent.mkdir(parents=True) + stage = Usd.Stage.CreateNew(str(asset)) + root = UsdGeom.Xform.Define(stage, "/Pallet").GetPrim() + stage.SetDefaultPrim(root) + body = UsdPhysics.RigidBodyAPI.Apply(root) + body.CreateRigidBodyEnabledAttr(True) + stage.GetRootLayer().Save() + with ( + patch("huggingface_hub.snapshot_download", return_value=directory), + patch(f"{MODULE}.Path.home", return_value=folder), + ): + wrapper = _prepare_central_asset(URL) + assert Usd.Stage.Open(wrapper).GetDefaultPrim().HasAPI(UsdPhysics.RigidBodyAPI) + body.GetRigidBodyEnabledAttr().Set(False) + stage.GetRootLayer().Save() + with pytest.raises(AssertionError, match="disabled"): + _prepare_central_asset(URL) + body.GetRigidBodyEnabledAttr().Set(True) + UsdPhysics.ArticulationRootAPI.Apply(root) + stage.GetRootLayer().Save() + with pytest.raises(AssertionError, match="articulation root"): + _prepare_central_asset(URL) + root.RemoveAPI(UsdPhysics.ArticulationRootAPI) + UsdPhysics.RigidBodyAPI.Apply(UsdGeom.Xform.Define(stage, "/Pallet/Other").GetPrim()) + stage.GetRootLayer().Save() + with pytest.raises(AssertionError, match="found 2"): + _prepare_central_asset(URL) + return True + + +def test_central_usd_validation(): + from isaaclab_arena.tests.utils.persistent_simulation_app import run_function_with_persistent_simulation_app + + assert run_function_with_persistent_simulation_app(_test_central_usd_validation) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py index d06945c9a5..4750ada308 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py @@ -318,7 +318,7 @@ def render_generation_panel() -> None: st.session_state["enable_simready_search"] = st.checkbox( "Enable SimReady search", value=bool(st.session_state.get("enable_simready_search", False)), - help="Search Isaac Sim GA SimReady props for objects the Arena asset catalog does not cover.", + help="Search SimReady for objects the Arena asset catalog does not cover.", ) source_options = [kind.value for kind in SimReadySourceKind] st.session_state["simready_source"] = st.selectbox( @@ -341,6 +341,10 @@ def render_generation_panel() -> None: value=st.session_state.get("simready_s3_url", ""), placeholder="Leave empty for the Isaac Sim 6.0 GA SimReady bucket", ) + elif source == SimReadySourceKind.CENTRAL.value: + st.caption( + "Searches Hugging Face SimReady Central; accepts only single rigid bodies without articulations." + ) elif source == SimReadySourceKind.SERVICE.value: st.session_state["simready_service_url"] = st.text_input( "Service URL", diff --git a/pyproject.toml b/pyproject.toml index 752f0fb778..de86d5cb68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dev = [ # its own: it adds one wheel, since Isaac Sim already brings everything it depends on, and an # extra only bought a way to install the review GUI with its asset search missing. "simready-search>=2026.4.2", + "huggingface-hub>=0.36", ] # PEP 735 dependency groups for the native uv install. pip (used by the Docker From 847ba2eb6fc9762b4673f35b0407000464957419 Mon Sep 17 00:00:00 2001 From: Xinjie Yao Date: Thu, 22 Oct 2026 14:52:11 -0700 Subject: [PATCH 02/11] Note future Newton support for SimReady Central Signed-off-by: Xinjie Yao --- .../agentic_environment_generation/simready_central.py | 1 + 1 file changed, 1 insertion(+) diff --git a/isaaclab_arena/agentic_environment_generation/simready_central.py b/isaaclab_arena/agentic_environment_generation/simready_central.py index 3b2a6432d2..210f4e48d8 100644 --- a/isaaclab_arena/agentic_environment_generation/simready_central.py +++ b/isaaclab_arena/agentic_environment_generation/simready_central.py @@ -105,6 +105,7 @@ def _prepare_central_asset(url: str) -> str: root = stage.DefinePrim("/Asset") root.GetReferences().AddReference(asset_path) stage.SetDefaultPrim(root) + # TODO(xinjieyao, 20261022): Support Newton. apply_usd_variant_selections(stage, {"Physics": "physics", "PhysX": "Enabled"}) assert not stage.GetCompositionErrors(), "USD has unresolved composition errors" bodies = [] From d291faf24efb1f3350c50a12ea68c00eeb8f3051 Mon Sep 17 00:00:00 2001 From: Xinjie Yao Date: Thu, 22 Oct 2026 17:43:19 -0700 Subject: [PATCH 03/11] Add SimReady source lists and Central scene example Signed-off-by: Xinjie Yao --- .../cli_runner.rst | 21 +++- .../agentic_env_gen/central_egg_tray.rst | 101 ++++++++++++++++++ .../agentic_env_gen/index.rst | 2 + .../environment_generation_agent.py | 27 ++++- .../simready_asset_search.py | 46 ++++++-- .../simready_central.py | 62 +++++++++-- .../test_environment_generation_agent.py | 15 +++ .../tests/test_simready_asset_search.py | 84 ++++++++++++++- isaaclab_arena/tests/test_simready_central.py | 32 +++++- .../central_egg_tray_example.py | 24 +++++ .../cli_runner.py | 43 +++++++- .../review_gui/generation_panel.py | 69 +++++++++--- .../review_gui/streamlit_ui.py | 11 ++ 13 files changed, 490 insertions(+), 47 deletions(-) create mode 100644 docs/pages/example_workflows/agentic_env_gen/central_egg_tray.rst create mode 100644 isaaclab_arena_examples/agentic_environment_generation/central_egg_tray_example.py diff --git a/docs/pages/concepts/agentic_environment_generation/cli_runner.rst b/docs/pages/concepts/agentic_environment_generation/cli_runner.rst index e8c3632da0..90faf66f7e 100644 --- a/docs/pages/concepts/agentic_environment_generation/cli_runner.rst +++ b/docs/pages/concepts/agentic_environment_generation/cli_runner.rst @@ -108,10 +108,16 @@ These options are defined by the environment generation runner: * - ``--enable_simready_search`` - Searches SimReady for prompt objects missing from Arena's asset catalog. - Agent - * - ``--simready_source SOURCE`` - - Chooses the SimReady search backend. Default: ``isaac-sim-ga``. + * - ``--simready_sources SOURCE [SOURCE ...]`` + - Chooses one or more SimReady search backends. Default: ``isaac-sim-ga``. Use ``simready-central`` for single-rigid-body assets on Hugging Face. - Agent + * - ``--simready_asset_url URL`` + - Selects an exact pinned Central asset; repeat for multiple assets. + - Agent + * - ``--example central-egg-tray`` + - Loads the egg/tray prompt and four pinned Palatial Central assets. + - Agent * - ``--simready_s3_url URL`` - Overrides the S3 root for S3-based SimReady sources. - Agent @@ -183,3 +189,14 @@ relevant to ``build`` and ``full`` are: Run the script with ``--help`` for the complete launcher-specific option list, including device and livestream settings supplied by Isaac Lab. + +Combined SimReady search +------------------------ + +Use ``--enable_simready_search --simready_sources isaac-sim-ga simready-central`` to search both the +Isaac Sim GA asset library and SimReady Central by object name. The result +limit applies per object per backend, so the default returns up to two +candidates per object. Identical USD paths are deduplicated per phrase. +If one backend fails, results from the other remain available. In the GUI, +select the desired backends under **SimReady sources**. Pinned Central URLs remain an +exact-selection mode under ``simready-central``. diff --git a/docs/pages/example_workflows/agentic_env_gen/central_egg_tray.rst b/docs/pages/example_workflows/agentic_env_gen/central_egg_tray.rst new file mode 100644 index 0000000000..f178f5ec21 --- /dev/null +++ b/docs/pages/example_workflows/agentic_env_gen/central_egg_tray.rst @@ -0,0 +1,101 @@ +SimReady Central: Egg and Tray +============================= + +Generate a Franka pick-and-place environment using four single-rigid-body assets +from SimReady Central. The egg starts on the maple table; its destination is the +egg tray. A nut and pizza are distractors. Complete the shared +:ref:`agentic-env-gen-prerequisites` first. + +Pinned assets +------------- + +All four assets come from ``PalatialSim/palatial-nv-asset-library`` at revision +``69a87329036cec0440403351c12230970c4e67d8`` and use the CC-BY-4.0 license. +Credit Palatial Platforms when sharing rendered examples or derived assets. + +.. list-table:: + :header-rows: 1 + + * - Asset + - Dataset entry + - Role + * - Egg Tray + - `D00290 `_ + - Destination + * - Egg + - `D00287 `_ + - Pick-up object + * - Nut + - `D00554 `_ + - Distractor + * - Pizza + - `D00802 `_ + - Distractor + +The preset looks up these exact revisions in Central and requires indexed body +count equal to one. It then downloads each asset package and verifies the USD +contains one enabled rigid body and no articulation root. An unavailable or +incompatible pinned asset fails generation rather than selecting a substitute. + +CLI +--- + +.. code-block:: bash + + python isaaclab_arena_examples/agentic_environment_generation/cli_runner.py \ + --mode resolve --example central-egg-tray \ + --out_dir /tmp/central-egg-tray + +Select a configured inference endpoint with ``--inference_endpoint`` if needed. +The runner prints the returned assets with their source URLs, local USD paths, +and ``used_in_spec`` flags. It saves the generated YAML and a sibling +``.assets.json`` report. Check that all four entries have ``used_in_spec: true``. +The LLM determines the final layout, so review the generated spec before use. + +Build the generated environment and run a short zero-action check: + +.. code-block:: bash + + python isaaclab_arena_examples/agentic_environment_generation/cli_runner.py \ + --mode build --env_spec /tmp/central-egg-tray/central_egg_tray.yaml \ + --num_envs 1 --num_steps 20 + +For a different pinned asset list, use ``--enable_simready_search`` with +``--simready_source simready-central`` and repeat ``--simready_asset_url URL``. +Both revision-pinned Hugging Face ``blob`` and ``resolve`` URLs are accepted. + +GUI +--- + +.. code-block:: bash + + python isaaclab_arena_examples/agentic_environment_generation/gui_runner.py + +Open the displayed browser URL, click **Load Central egg-tray example**, choose +your inference endpoint, and click **Generate spec**. The preset fills the prompt, +enables Central search, and populates the four pinned URLs. + +The **Returned SimReady assets** table shows the source URLs and cached USD paths. +Use **Download asset report** to save the same report format as the CLI. The +generated YAML appears in the editor; the visualization panel renders the assets. +Use the simulation-preview controls to inspect the resulting scene. + +To reopen the CLI result with its saved asset report: + +.. code-block:: bash + + python isaaclab_arena_examples/agentic_environment_generation/gui_runner.py \ + --env_spec /tmp/central-egg-tray/central_egg_tray.yaml + +Cache and reproducibility +------------------------- + +Packages use the Hugging Face cache, and the generated spec references local +wrapper files in ``~/.cache/isaaclab_arena/simready-central``. Keep those files +and the package cache available when reopening the YAML. Regenerate on a new +machine from the pinned preset rather than copying cache paths. If a Docker host +cache is a symlink to an unmounted directory, set ``HF_HOME`` to a writable, +mounted location before running either interface. + +This example uses PhysX. A zero-action run checks construction and stepping; it +does not establish successful egg placement by a policy. diff --git a/docs/pages/example_workflows/agentic_env_gen/index.rst b/docs/pages/example_workflows/agentic_env_gen/index.rst index 681b3c9092..438f86c6a8 100644 --- a/docs/pages/example_workflows/agentic_env_gen/index.rst +++ b/docs/pages/example_workflows/agentic_env_gen/index.rst @@ -17,6 +17,7 @@ In this section, we will walk through the following example environment generati - :doc:`tabletop_pnp_homogenous_object/index` - :doc:`tabletop_pnp_heterogeneous_object/index` - :doc:`tabletop_pnp_composite_task/index` + - :doc:`central_egg_tray` - Kitchen Pick and Place task @@ -156,5 +157,6 @@ and can be used directly for policy evaluation: tabletop_pnp_homogenous_object/index tabletop_pnp_heterogeneous_object/index tabletop_pnp_composite_task/index + central_egg_tray kitchen_pick_and_place/index kitchen_open_door/index diff --git a/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py b/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py index 4f4ab4a51a..378ecfcb0d 100644 --- a/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py +++ b/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py @@ -8,7 +8,7 @@ from __future__ import annotations import logging -from dataclasses import replace +from dataclasses import asdict, replace from typing import Any from isaaclab_arena.agentic_environment_generation.catalogues import ( @@ -22,6 +22,7 @@ from isaaclab_arena.agentic_environment_generation.missing_object_inference import MissingObjectInference from isaaclab_arena.agentic_environment_generation.prim_path_inference import PrimPathInference from isaaclab_arena.agentic_environment_generation.simready_asset_search import ( + SimReadyObjectCandidate, SimReadySearchConfig, search_simready_objects, ) @@ -88,6 +89,7 @@ def __init__( self._traces: list[str] = [] self._unavailable_objects: list[str] = [] self._simready_usd_paths: dict[str, str] = {} + self._simready_candidates: list[SimReadyObjectCandidate] = [] @property def traces(self) -> tuple[str, ...]: @@ -109,6 +111,14 @@ def unavailable_objects(self) -> tuple[str, ...]: """ return tuple(self._unavailable_objects) + def simready_asset_report(self, spec: ArenaEnvGraphSpec | None) -> list[dict[str, Any]]: + """Return discovered assets, their source URLs, and whether the spec uses them.""" + used_paths = {obj.params.get("usd_path") for obj in spec.objects} if spec is not None else set() + return [ + {**asdict(candidate), "used_in_spec": candidate.usd_path in used_paths} + for candidate in self._simready_candidates + ] + def generate_spec( self, prompt: str, @@ -136,6 +146,7 @@ def generate_spec( self._traces = [] self._unavailable_objects = [] self._simready_usd_paths = {} + self._simready_candidates = [] asset_catalog = asset_catalog or build_asset_catalogue() relation_catalog = relation_catalog or build_relation_catalogue() task_catalog = task_catalog or build_task_catalogue() @@ -202,11 +213,17 @@ def _extend_catalogue_with_simready(self, prompt: str, asset_catalog: AssetCatal # import pxr, and a pxr import before SimulationApp starts breaks the unit tests. from isaaclab_arena.assets.simready_object_library import register_searched_simready_object - phrases = self.missing_object_inference.infer(prompt, asset_catalog) - if not phrases: - return asset_catalog - search_result = search_simready_objects(phrases, self.simready_config) + if self.simready_config.central_asset_urls: + from isaaclab_arena.agentic_environment_generation.simready_central import load_pinned_central_objects + + search_result = load_pinned_central_objects(self.simready_config.central_asset_urls) + else: + phrases = self.missing_object_inference.infer(prompt, asset_catalog) + if not phrases: + return asset_catalog + search_result = search_simready_objects(phrases, self.simready_config) self._unavailable_objects = list(search_result.unmatched_phrases) + self._simready_candidates = list(search_result.candidates) if not search_result.candidates: return asset_catalog extended = replace(asset_catalog, objects=list(asset_catalog.objects)) diff --git a/isaaclab_arena/agentic_environment_generation/simready_asset_search.py b/isaaclab_arena/agentic_environment_generation/simready_asset_search.py index 65e90ef98f..e15ce203f7 100644 --- a/isaaclab_arena/agentic_environment_generation/simready_asset_search.py +++ b/isaaclab_arena/agentic_environment_generation/simready_asset_search.py @@ -44,10 +44,11 @@ class SimReadySourceKind(str, Enum): class SimReadySearchConfig: """Where to search for SimReady objects and how many hits to keep.""" - source: SimReadySourceKind = SimReadySourceKind.ISAAC_SIM_GA + sources: list[SimReadySourceKind] = field(default_factory=lambda: [SimReadySourceKind.ISAAC_SIM_GA]) s3_url: str = ISAAC_SIMREADY_GA_S3_URL service_url: str = DEFAULT_SIMREADY_SERVICE_URL max_results_per_object: int = 1 + central_asset_urls: tuple[str, ...] = () @dataclass(frozen=True) @@ -58,6 +59,7 @@ class SimReadyObjectCandidate: usd_path: str tags: tuple[str, ...] = ("sim-ready",) relevance_score: float | None = None + source_url: str | None = None @property def registry_name(self) -> str: @@ -89,15 +91,39 @@ def search_simready_objects(object_phrases: list[str], config: SimReadySearchCon if not phrases: return SimReadyCandidateCatalogue() - library = None if config.source == SimReadySourceKind.CENTRAL else _configure_asset_library(config) - if library is None and config.source != SimReadySourceKind.CENTRAL: + candidates = [] + seen = set() + for source in dict.fromkeys(config.sources): + try: + result = _search_simready_source(phrases, config, source) + except Exception as exc: + _logger.warning("simready backend %s failed: %s", source, exc) + continue + for candidate in result.candidates: + key = (candidate.search_phrase, candidate.usd_path) + if key not in seen: + seen.add(key) + candidates.append(candidate) + matched = {candidate.search_phrase for candidate in candidates} + return SimReadyCandidateCatalogue( + candidates=candidates, + unmatched_phrases=[phrase for phrase in phrases if phrase not in matched], + ) + + +def _search_simready_source( + phrases: list[str], config: SimReadySearchConfig, source: SimReadySourceKind +) -> SimReadyCandidateCatalogue: + """Search one source, retaining the configured number of hits per phrase.""" + library = None if source == SimReadySourceKind.CENTRAL else _configure_asset_library(config, source) + if library is None and source != SimReadySourceKind.CENTRAL: return SimReadyCandidateCatalogue() candidates: list[SimReadyObjectCandidate] = [] unmatched_phrases: list[str] = [] for phrase in phrases: try: - if config.source == SimReadySourceKind.CENTRAL: + if source == SimReadySourceKind.CENTRAL: from isaaclab_arena.agentic_environment_generation.simready_central import search_central_objects hits = search_central_objects(phrase, max_results=config.max_results_per_object) @@ -116,14 +142,14 @@ def search_simready_objects(object_phrases: list[str], config: SimReadySearchCon def simready_search_config_from_cli( - source: str, + sources: list[str], s3_url: str | None, service_url: str | None, max_results_per_object: int, ) -> SimReadySearchConfig: """Build a search configuration from CLI or GUI arguments, filling in the default URLs.""" return SimReadySearchConfig( - source=SimReadySourceKind(source), + sources=[SimReadySourceKind(source) for source in dict.fromkeys(sources)], s3_url=s3_url or ISAAC_SIMREADY_GA_S3_URL, service_url=service_url or DEFAULT_SIMREADY_SERVICE_URL, max_results_per_object=max_results_per_object, @@ -134,10 +160,10 @@ def simready_search_config_from_cli( _NON_ALPHANUMERIC = re.compile(r"[^a-z0-9]+") -def _configure_asset_library(config: SimReadySearchConfig) -> Any | None: +def _configure_asset_library(config: SimReadySearchConfig, source: SimReadySourceKind) -> Any | None: """Build the SimReady asset library for the configured source, or None if the source is unknown.""" - if config.source not in tuple(SimReadySourceKind): - _logger.error("unknown simready source: %s", config.source) + if source not in tuple(SimReadySourceKind): + _logger.error("unknown simready source: %s", source) return None # Imported here rather than at module scope only to keep the AWS stack out of the import path @@ -145,7 +171,7 @@ def _configure_asset_library(config: SimReadySearchConfig) -> Any | None: from simready.search import AssetLibrary library = AssetLibrary() - if config.source in (SimReadySourceKind.ISAAC_SIM_GA, SimReadySourceKind.S3): + if source in (SimReadySourceKind.ISAAC_SIM_GA, SimReadySourceKind.S3): asyncio.run(library.add_s3_source(config.s3_url or ISAAC_SIMREADY_GA_S3_URL)) else: library.add_service_source(config.service_url or DEFAULT_SIMREADY_SERVICE_URL) diff --git a/isaaclab_arena/agentic_environment_generation/simready_central.py b/isaaclab_arena/agentic_environment_generation/simready_central.py index 210f4e48d8..4de6b57e1f 100644 --- a/isaaclab_arena/agentic_environment_generation/simready_central.py +++ b/isaaclab_arena/agentic_environment_generation/simready_central.py @@ -16,6 +16,7 @@ from isaaclab_arena.agentic_environment_generation.simready_asset_search import ( MIN_INSPECTED_MATCHES_PER_PHRASE, + SimReadyCandidateCatalogue, SimReadyObjectCandidate, ) @@ -23,6 +24,44 @@ _logger = logging.getLogger(__name__) +def load_pinned_central_objects(urls: tuple[str, ...]) -> SimReadyCandidateCatalogue: + """Look up exact pinned Central assets and validate each before offering them to the agent.""" + catalogue = SimReadyCandidateCatalogue() + for url in dict.fromkeys(urls): + repo_id, revision, filename, _ = _central_asset_location(url) + source_url = f"https://huggingface.co/datasets/{repo_id}/resolve/{revision}/{filename}" + response = requests.post( + CENTRAL_SEARCH_URL, + headers={"x-usdsearch-storage-backend": "usdsearch-dev"}, + json={ + "filter_url_regexp": re.escape(source_url.replace("https://", "hf://", 1)), + "filter_by_properties_numeric": "simready_metadata_rigid_body_count=1", + "return_usd_properties": True, + "limit": 1, + }, + timeout=30, + ) + response.raise_for_status() + hits = response.json()["hits"] + assert hits, f"Pinned asset is not indexed as a single rigid body in Central: {source_url}" + source = hits[0]["source"] + assert source["path"].replace("hf://", "https://", 1) == source_url, "Central returned a different asset" + properties = {prop["name"]: prop for prop in source.get("usd_properties", [])} + count = properties.get("simready_metadata_rigid_body_count", {}) + assert float(count.get("value_numeric", count.get("value", 0))) == 1, "expected one indexed rigid body" + name = properties.get("simready_metadata_asset_name", {}).get("value") + assert name, f"Central asset has no descriptive name: {source_url}" + catalogue.candidates.append( + SimReadyObjectCandidate( + search_phrase=name, + usd_path=_prepare_central_asset(source_url), + source_url=source_url, + tags=("sim-ready", "single-rigid-body", name.lower()), + ) + ) + return catalogue + + def search_central_objects(phrase: str, *, max_results: int) -> list[SimReadyObjectCandidate]: """Search Central and retain only locally validated single-body props.""" assert max_results > 0, "max_results must be positive" @@ -58,6 +97,7 @@ def search_central_objects(phrase: str, *, max_results: int) -> list[SimReadyObj usd_path=usd_path, tags=("sim-ready", "single-rigid-body"), relevance_score=hit.get("rrf_score"), + source_url=source["path"].replace("hf://", "https://", 1), ) ) except Exception as exc: @@ -70,7 +110,9 @@ def search_central_objects(phrase: str, *, max_results: int) -> list[SimReadyObj def _central_asset_location(url: str) -> tuple[str, str, str, str]: """Parse a revision-pinned Central dataset URL and its asset package folder.""" - match = re.fullmatch(r"(?:hf|https)://huggingface\.co/datasets/([^/]+/[^/]+)/resolve/([0-9a-f]{40})/(.+)", url) + match = re.fullmatch( + r"(?:hf|https)://huggingface\.co/datasets/([^/]+/[^/]+)/(?:resolve|blob)/([0-9a-f]{40})/(.+)", url + ) assert match, "expected a pinned Hugging Face dataset URL" repo_id, revision, filename = match.groups() path = PurePosixPath(filename) @@ -85,18 +127,24 @@ def _central_asset_location(url: str) -> tuple[str, str, str, str]: return repo_id, revision, filename, str(folder) -def _prepare_central_asset(url: str) -> str: - """Cache a pinned asset package and write a validated PhysX reference wrapper.""" +def _download_central_asset_to_cache(url: str) -> str: + """Download the pinned asset package and return its cached USD entrypoint.""" from huggingface_hub import snapshot_download - from pxr import Usd, UsdGeom, UsdPhysics - - from isaaclab_arena.utils.usd.rigid_bodies import apply_usd_variant_selections repo_id, revision, filename, folder = _central_asset_location(url) snapshot = snapshot_download( repo_id=repo_id, repo_type="dataset", revision=revision, allow_patterns=[f"{folder}/*"] ) - asset_path = str(Path(snapshot) / filename) + return str(Path(snapshot) / filename) + + +def _prepare_central_asset(url: str) -> str: + """Write a validated PhysX reference wrapper for a cached Central asset.""" + from pxr import Usd, UsdGeom, UsdPhysics + + from isaaclab_arena.utils.usd.rigid_bodies import apply_usd_variant_selections + + asset_path = _download_central_asset_to_cache(url) asset_stage = Usd.Stage.Open(asset_path) assert asset_stage and asset_stage.GetDefaultPrim(), "asset requires a default prim" stage = Usd.Stage.CreateInMemory() diff --git a/isaaclab_arena/tests/test_environment_generation_agent.py b/isaaclab_arena/tests/test_environment_generation_agent.py index d5243a4026..f5b1e24b92 100644 --- a/isaaclab_arena/tests/test_environment_generation_agent.py +++ b/isaaclab_arena/tests/test_environment_generation_agent.py @@ -269,6 +269,20 @@ def test_a_searched_simready_object_reaches_the_spec_with_its_usd_path(self, moc # the generic SimReady asset every process has, carrying the path the search found. assert searched.registry_name == SIMREADY_USD_OBJECT_REGISTRY_NAME assert searched.params == {"usd_path": "s3://bucket/trash_can.usd"} + report = agent_obj.simready_asset_report(spec) + assert len(report) == 1 + assert report[0]["used_in_spec"] is True + assert report[0]["usd_path"] == "s3://bucket/trash_can.usd" + assert agent_obj.simready_asset_report(None)[0]["used_in_spec"] is False + agent_obj.enable_simready_search = False + client.chat.completions.create.side_effect = [chat_response(content=json.dumps(minimal_spec_dict()))] + next_spec, _ = agent_obj.generate_spec( + "p", + asset_catalog=catalog("catalog"), + relation_catalog=relation_catalog("RELATIONS"), + task_catalog=make_task_catalog("TASKS"), + ) + assert agent_obj.simready_asset_report(next_spec) == [] # Tags in params would be a duplicate keyword argument at build time. assert all("tags" not in asset["params"] for asset in spec.to_dict()["objects"]) @@ -306,6 +320,7 @@ def test_a_catalogue_object_keeps_its_own_registry_name(self, mock_search, agent assert all(asset.registry_name != SIMREADY_USD_OBJECT_REGISTRY_NAME for asset in spec.objects) assert all("usd_path" not in asset.params for asset in spec.objects) assert "simready_assets" not in spec.to_dict() + assert agent_obj.simready_asset_report(spec)[0]["used_in_spec"] is False @patch("isaaclab_arena.agentic_environment_generation.environment_generation_agent.search_simready_objects") def test_generate_spec_builds_the_spec_without_an_object_no_asset_was_found_for(self, mock_search, agent): diff --git a/isaaclab_arena/tests/test_simready_asset_search.py b/isaaclab_arena/tests/test_simready_asset_search.py index 8cea746cfd..82cb3c0a26 100644 --- a/isaaclab_arena/tests/test_simready_asset_search.py +++ b/isaaclab_arena/tests/test_simready_asset_search.py @@ -154,12 +154,12 @@ def test_search_reports_a_phrase_with_only_rejected_hits_as_unmatched(mock_confi def test_simready_search_config_from_cli_defaults(): config = simready_search_config_from_cli( - source=SimReadySourceKind.ISAAC_SIM_GA.value, + sources=[SimReadySourceKind.ISAAC_SIM_GA.value], s3_url=None, service_url=None, max_results_per_object=2, ) - assert config.source == SimReadySourceKind.ISAAC_SIM_GA + assert config.sources == [SimReadySourceKind.ISAAC_SIM_GA] assert config.max_results_per_object == 2 @@ -177,7 +177,7 @@ def test_search_simready_objects_returns_candidates(mock_configure, mock_search_ usd_path="s3://bucket/bowl.usd", ) ] - catalog = search_simready_objects(["ceramic bowl"], SimReadySearchConfig(source=SimReadySourceKind.ISAAC_SIM_GA)) + catalog = search_simready_objects(["ceramic bowl"], SimReadySearchConfig(sources=[SimReadySourceKind.ISAAC_SIM_GA])) assert len(catalog.candidates) == 1 assert catalog.candidates[0].usd_path == "s3://bucket/bowl.usd" assert catalog.candidates[0].registry_name == SIMREADY_USD_OBJECT_REGISTRY_NAME @@ -187,9 +187,8 @@ def test_configure_asset_library_records_an_unknown_source(caplog): from isaaclab_arena.agentic_environment_generation.simready_asset_search import _configure_asset_library config = SimReadySearchConfig() - config.source = "not-a-source" with caplog.at_level(logging.ERROR, logger=SEARCH_LOGGER): - assert _configure_asset_library(config) is None + assert _configure_asset_library(config, "not-a-source") is None assert "unknown simready source" in caplog.text @@ -216,3 +215,78 @@ def test_search_simready_objects_records_no_matches(mock_configure, mock_search_ assert catalog.candidates == [] assert catalog.unmatched_phrases == ["missing object"] assert "no usable asset" in caplog.text + + +def test_multiple_sources_merge_and_deduplicate_candidates(): + shared = SimReadyObjectCandidate(search_phrase="egg", usd_path="shared.usd") + central = SimReadyObjectCandidate(search_phrase="egg", usd_path="central.usd") + with ( + patch(f"{SEARCH_LOGGER}._configure_asset_library", return_value=object()), + patch(f"{SEARCH_LOGGER}._search_phrase", side_effect=[[shared], []]), + patch( + "isaaclab_arena.agentic_environment_generation.simready_central.search_central_objects", + side_effect=[[shared, central], []], + ), + ): + result = search_simready_objects( + ["egg", "missing"], + SimReadySearchConfig(sources=[SimReadySourceKind.ISAAC_SIM_GA, SimReadySourceKind.CENTRAL]), + ) + assert result.candidates == [shared, central] + assert result.unmatched_phrases == ["missing"] + + +def test_multiple_sources_keep_central_when_library_initialization_fails(): + central = SimReadyObjectCandidate(search_phrase="egg", usd_path="central.usd") + with ( + patch(f"{SEARCH_LOGGER}._configure_asset_library", side_effect=RuntimeError("unavailable")), + patch( + "isaaclab_arena.agentic_environment_generation.simready_central.search_central_objects", + return_value=[central], + ), + ): + result = search_simready_objects( + ["egg"], SimReadySearchConfig(sources=[SimReadySourceKind.ISAAC_SIM_GA, SimReadySourceKind.CENTRAL]) + ) + assert result.candidates == [central] + assert not result.unmatched_phrases + + +def test_multiple_sources_keep_library_when_central_fails(): + candidate = SimReadyObjectCandidate(search_phrase="egg", usd_path="library.usd") + with ( + patch(f"{SEARCH_LOGGER}._configure_asset_library", return_value=object()), + patch(f"{SEARCH_LOGGER}._search_phrase", return_value=[candidate]), + patch( + "isaaclab_arena.agentic_environment_generation.simready_central.search_central_objects", + side_effect=RuntimeError("unavailable"), + ), + ): + result = search_simready_objects( + ["egg"], SimReadySearchConfig(sources=[SimReadySourceKind.ISAAC_SIM_GA, SimReadySourceKind.CENTRAL]) + ) + assert result.candidates == [candidate] + assert not result.unmatched_phrases + + +def test_source_list_searches_selected_backends_once(): + config = simready_search_config_from_cli( + sources=["service", "simready-central", "service"], + s3_url=None, + service_url="http://asset-service", + max_results_per_object=2, + ) + assert config.sources == [SimReadySourceKind.SERVICE, SimReadySourceKind.CENTRAL] + with ( + patch(f"{SEARCH_LOGGER}._configure_asset_library", return_value=object()) as configure, + patch(f"{SEARCH_LOGGER}._search_phrase", return_value=[]) as library_search, + patch( + "isaaclab_arena.agentic_environment_generation.simready_central.search_central_objects", + return_value=[], + ) as central_search, + ): + result = search_simready_objects(["egg"], config) + configure.assert_called_once_with(config, SimReadySourceKind.SERVICE) + library_search.assert_called_once() + central_search.assert_called_once_with("egg", max_results=2) + assert result.unmatched_phrases == ["egg"] diff --git a/isaaclab_arena/tests/test_simready_central.py b/isaaclab_arena/tests/test_simready_central.py index 7fac4a4622..a98473ab80 100644 --- a/isaaclab_arena/tests/test_simready_central.py +++ b/isaaclab_arena/tests/test_simready_central.py @@ -16,6 +16,7 @@ ) from isaaclab_arena.agentic_environment_generation.simready_central import ( _central_asset_location, + load_pinned_central_objects, search_central_objects, ) @@ -47,7 +48,7 @@ def test_central_requires_single_body_metadata_and_skips_invalid_usd(): def test_central_failure_leaves_phrase_unmatched(): with patch(f"{MODULE}.requests.post", side_effect=TimeoutError("offline")): - catalogue = search_simready_objects(["pallet"], SimReadySearchConfig(source=SimReadySourceKind.CENTRAL)) + catalogue = search_simready_objects(["pallet"], SimReadySearchConfig(sources=[SimReadySourceKind.CENTRAL])) assert catalogue.candidates == [] assert catalogue.unmatched_phrases == ["pallet"] @@ -61,6 +62,35 @@ def test_central_package_location_preserves_revision_and_folder(): ) +def test_pinned_central_assets_accept_blob_urls_and_preserve_identity(): + hit = _hit(1) + hit["source"]["usd_properties"].append({"name": "simready_metadata_asset_name", "value": "Egg Tray"}) + response = Mock() + response.json.return_value = {"hits": [hit]} + blob_url = URL.replace("hf://", "https://").replace("/resolve/", "/blob/") + with ( + patch(f"{MODULE}.requests.post", return_value=response), + patch(f"{MODULE}._prepare_central_asset", return_value="/cache/tray.usda") as prepare, + ): + catalogue = load_pinned_central_objects((blob_url,)) + assert catalogue.candidates[0].search_phrase == "Egg Tray" + assert catalogue.candidates[0].source_url == URL.replace("hf://", "https://") + prepare.assert_called_once_with(URL.replace("hf://", "https://")) + + +@pytest.mark.parametrize("hits", [[], [_hit(2)], [_hit(1, path=URL.replace("pallet.usd", "other.usd"))]]) +def test_pinned_central_assets_never_substitute_or_accept_multiple_bodies(hits): + response = Mock() + response.json.return_value = {"hits": hits} + with ( + patch(f"{MODULE}.requests.post", return_value=response), + patch(f"{MODULE}._prepare_central_asset") as prepare, + pytest.raises(AssertionError), + ): + load_pinned_central_objects((URL,)) + prepare.assert_not_called() + + @pytest.mark.parametrize( "url", [ diff --git a/isaaclab_arena_examples/agentic_environment_generation/central_egg_tray_example.py b/isaaclab_arena_examples/agentic_environment_generation/central_egg_tray_example.py new file mode 100644 index 0000000000..369dafea3b --- /dev/null +++ b/isaaclab_arena_examples/agentic_environment_generation/central_egg_tray_example.py @@ -0,0 +1,24 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Pinned SimReady Central egg-and-tray example shared by CLI and GUI.""" + +CENTRAL_EGG_TRAY_PROMPT = ( + "Create an environment named central_egg_tray. Franka picks up the SimReady egg from the maple table " + "and places it into the SimReady egg tray. Include the SimReady nut and SimReady pizza as separate " + "distractor objects on the table. Use all four supplied SimReady assets: simready_egg, simready_egg_tray, " + "simready_nut, simready_pizza. Keep the egg outside the tray initially. " + "Use PickAndPlaceTask for the egg and tray. Do not use object sets." +) + +_DATASET_REVISION_URL = ( + "https://huggingface.co/datasets/PalatialSim/palatial-nv-asset-library/resolve/" + "69a87329036cec0440403351c12230970c4e67d8" +) + +CENTRAL_EGG_TRAY_ASSET_URLS = tuple( + f"{_DATASET_REVISION_URL}/{asset_id}/simready_usd/{asset_id}.usd" + for asset_id in ("D00290", "D00287", "D00554", "D00802") +) diff --git a/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py b/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py index ce1958689b..5b77d32983 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py +++ b/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py @@ -73,6 +73,11 @@ def add_agentic_env_gen_runner_cli_args(parser: argparse.ArgumentParser) -> None "'schema', 'catalog', and 'prim_tree' make no agent call." ), ) + group.add_argument( + "--example", + choices=("central-egg-tray",), + help="Load a prompt and pinned asset selection for a maintained example.", + ) group.add_argument( "--prompt", type=str, @@ -119,11 +124,19 @@ def add_agentic_env_gen_runner_cli_args(parser: argparse.ArgumentParser) -> None help="Search SimReady for objects the Arena asset catalog does not cover.", ) group.add_argument( + "--simready_sources", "--simready_source", + nargs="+", type=str, choices=tuple(kind.value for kind in SimReadySourceKind), - default="isaac-sim-ga", - help="SimReady search backend (default: isaac-sim-ga Isaac Sim 6.0 GA props).", + default=["isaac-sim-ga"], + help="SimReady backends to search. Result limits apply per backend.", + ) + group.add_argument( + "--simready_asset_url", + action="append", + default=[], + help="Pinned Central asset URL; repeat to use an exact asset list instead of natural-language search.", ) group.add_argument( "--simready_s3_url", @@ -165,11 +178,16 @@ def resolve_env_spec(args_cli: argparse.Namespace) -> Path | None: task_catalog = build_task_catalogue() simready_config = simready_search_config_from_cli( - source=args_cli.simready_source, + sources=args_cli.simready_sources, s3_url=args_cli.simready_s3_url, service_url=args_cli.simready_service_url, max_results_per_object=args_cli.simready_max_results_per_object, ) + if args_cli.simready_asset_url: + assert args_cli.enable_simready_search and args_cli.simready_sources == [ + "simready-central" + ], "--simready_asset_url requires --enable_simready_search --simready_source simready-central" + simready_config.central_asset_urls = tuple(args_cli.simready_asset_url) agent_kwargs: dict = { "temperature": args_cli.temperature, "enable_simready_search": args_cli.enable_simready_search, @@ -186,6 +204,9 @@ def resolve_env_spec(args_cli: argparse.Namespace) -> Path | None: relation_catalog=relation_catalog, task_catalog=task_catalog, ) + asset_report = agent.simready_asset_report(env_graph_spec) + if asset_report: + print(f"\n[runner] SimReady assets:\n{json.dumps(asset_report, indent=2)}", flush=True) # agent.traces holds one line per failure, e.g. # "embodiment.registry_name: Unknown asset registry_name 'not_a_real_asset'" # "Task 'PickAndPlaceTask' is missing required param 'pick_up_object'" @@ -216,6 +237,10 @@ def resolve_env_spec(args_cli: argparse.Namespace) -> Path | None: ) path = write_env_graph_spec(env_graph_spec, args_cli.out_dir) print(f"[runner] wrote environment graph spec → {path}", flush=True) + if args_cli.enable_simready_search: + asset_report_path = path.with_suffix(".assets.json") + asset_report_path.write_text(json.dumps(asset_report, indent=2) + "\n", encoding="utf-8") + print(f"[runner] wrote SimReady asset report → {asset_report_path}", flush=True) return path @@ -355,6 +380,18 @@ def main() -> int: parser = get_isaaclab_arena_cli_parser() add_agentic_env_gen_runner_cli_args(parser) args_cli = parser.parse_args() + if args_cli.example == "central-egg-tray": + from isaaclab_arena_examples.agentic_environment_generation.central_egg_tray_example import ( + CENTRAL_EGG_TRAY_ASSET_URLS, + CENTRAL_EGG_TRAY_PROMPT, + ) + + if args_cli.prompt == DEFAULT_PROMPT: + args_cli.prompt = CENTRAL_EGG_TRAY_PROMPT + args_cli.enable_simready_search = True + args_cli.simready_sources = ["simready-central"] + if not args_cli.simready_asset_url: + args_cli.simready_asset_url = list(CENTRAL_EGG_TRAY_ASSET_URLS) if args_cli.mode == "schema": print_schema() diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py index 4750ada308..036d999be2 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py @@ -5,6 +5,7 @@ from __future__ import annotations +import json import os import traceback import yaml @@ -63,12 +64,17 @@ def get_catalogue_bundle() -> CatalogueBundle: def _simready_config_from_session() -> SimReadySearchConfig: """Read the SimReady search settings from Streamlit session state.""" - return simready_search_config_from_cli( - source=st.session_state.get("simready_source", SimReadySourceKind.ISAAC_SIM_GA.value), + config = simready_search_config_from_cli( + sources=st.session_state.get("simready_sources", [SimReadySourceKind.ISAAC_SIM_GA.value]), s3_url=st.session_state.get("simready_s3_url") or None, service_url=st.session_state.get("simready_service_url") or None, max_results_per_object=int(st.session_state.get("simready_max_results_per_object", 1)), ) + if config.sources == [SimReadySourceKind.CENTRAL]: + config.central_asset_urls = tuple( + line.strip() for line in st.session_state.get("simready_asset_urls", "").splitlines() if line.strip() + ) + return config def available_inference_endpoints() -> list[str]: @@ -111,10 +117,11 @@ def _generation_agent_cache_key( "generation_agent", endpoint_name, simready_enabled, - simready_config.source.value, + tuple(source.value for source in simready_config.sources), simready_config.s3_url, simready_config.service_url, simready_config.max_results_per_object, + simready_config.central_asset_urls, ) @@ -211,6 +218,7 @@ def _finish(severity: str, message: str) -> tuple[bool, str]: def run_generation_pipeline(prompt: str) -> tuple[bool, str]: """Call the LLM and load the returned environment graph spec YAML into the editor.""" + st.session_state.pop("simready_asset_report", None) prompt = prompt.strip() if not prompt: return False, "Enter a prompt describing the environment." @@ -234,6 +242,8 @@ def run_generation_pipeline(prompt: str) -> tuple[bool, str]: except Exception: return False, traceback.format_exc() + st.session_state["simready_asset_report"] = agent.simready_asset_report(spec) + try: yaml_text = yaml.safe_dump( spec.to_dict() if spec is not None else (data or {}), @@ -302,6 +312,18 @@ def render_generation_panel() -> None: endpoint = INFERENCE_ENDPOINTS[endpoint_name] st.caption(f"Model: `{endpoint.model}` at `{endpoint.base_url}`") + if st.button("Load Central egg-tray example"): + from isaaclab_arena_examples.agentic_environment_generation.central_egg_tray_example import ( + CENTRAL_EGG_TRAY_ASSET_URLS, + CENTRAL_EGG_TRAY_PROMPT, + ) + + st.session_state["generation_prompt"] = CENTRAL_EGG_TRAY_PROMPT + st.session_state["enable_simready_search"] = True + st.session_state["simready_sources"] = [SimReadySourceKind.CENTRAL.value] + st.session_state["simready_asset_urls"] = "\n".join(CENTRAL_EGG_TRAY_ASSET_URLS) + st.session_state.pop("simready_asset_report", None) + prompt = st.text_area( "Prompt", value=st.session_state.get("generation_prompt", DEFAULT_GENERATION_PROMPT), @@ -321,10 +343,11 @@ def render_generation_panel() -> None: help="Search SimReady for objects the Arena asset catalog does not cover.", ) source_options = [kind.value for kind in SimReadySourceKind] - st.session_state["simready_source"] = st.selectbox( - "SimReady source", + st.session_state["simready_sources"] = st.multiselect( + "SimReady sources", options=source_options, - index=source_options.index(st.session_state.get("simready_source", SimReadySourceKind.ISAAC_SIM_GA.value)), + help="Search each selected backend; the result limit applies to each backend.", + default=st.session_state.get("simready_sources", [SimReadySourceKind.ISAAC_SIM_GA.value]), ) st.session_state["simready_max_results_per_object"] = st.number_input( "Max results per object", @@ -332,29 +355,36 @@ def render_generation_panel() -> None: max_value=10, value=int(st.session_state.get("simready_max_results_per_object", 1)), ) - # Each source reads exactly one location, so only that one is offered. Showing both - # invites filling in a field the search then ignores. - source = st.session_state["simready_source"] - if source == SimReadySourceKind.S3.value: + sources = st.session_state["simready_sources"] + if SimReadySourceKind.S3.value in sources: st.session_state["simready_s3_url"] = st.text_input( "S3 URL", value=st.session_state.get("simready_s3_url", ""), placeholder="Leave empty for the Isaac Sim 6.0 GA SimReady bucket", ) - elif source == SimReadySourceKind.CENTRAL.value: + if sources == [SimReadySourceKind.CENTRAL.value]: + st.text_area( + "Pinned Central asset URLs (optional, one per line)", + key="simready_asset_urls", + help=( + "Use these exact assets instead of natural-language search. Only single rigid bodies are accepted." + ), + ) st.caption( "Searches Hugging Face SimReady Central; accepts only single rigid bodies without articulations." ) - elif source == SimReadySourceKind.SERVICE.value: + if SimReadySourceKind.SERVICE.value in sources: st.session_state["simready_service_url"] = st.text_input( "Service URL", value=st.session_state.get("simready_service_url", ""), placeholder=DEFAULT_SIMREADY_SERVICE_URL, ) - else: + if SimReadySourceKind.ISAAC_SIM_GA.value in sources: st.caption("Searches the Isaac Sim 6.0 GA SimReady library — no further configuration needed.") - generate_disabled = not available + generate_disabled = not available or ( + st.session_state.get("enable_simready_search", False) and not st.session_state.get("simready_sources") + ) if st.button("Generate spec", type="primary", width="stretch", disabled=generate_disabled): with st.spinner("Generating spec (LLM call)…"): ok, message = run_generation_pipeline(st.session_state["generation_prompt"]) @@ -364,3 +394,14 @@ def render_generation_panel() -> None: st.rerun() else: st.error(f"Generation failed\n\n```\n{message}\n```", icon="🛑") + + asset_report = st.session_state.get("simready_asset_report", []) + if asset_report: + with st.expander("Returned SimReady assets", expanded=True): + st.dataframe(asset_report, hide_index=True, width="stretch") + st.download_button( + "Download asset report", + data=json.dumps(asset_report, indent=2), + file_name="simready_assets.json", + mime="application/json", + ) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py index 1281acd349..324e4f9e73 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py @@ -11,6 +11,7 @@ from __future__ import annotations import argparse +import json from pathlib import Path import streamlit as st @@ -68,6 +69,16 @@ def initialize_state(yaml_path: Path | None, out_dir: Path) -> None: st.session_state["out_dir"] = str(out_dir.resolve()) st.session_state.pop("_validation_text", None) st.session_state.pop("_validation_result", None) + st.session_state.pop("simready_asset_report", None) + if yaml_path is not None: + report_path = yaml_path.with_suffix(".assets.json") + if report_path.is_file(): + try: + report = json.loads(report_path.read_text(encoding="utf-8")) + if isinstance(report, list) and all(isinstance(row, dict) for row in report): + st.session_state["simready_asset_report"] = report + except (OSError, ValueError) as exc: + st.warning(f"Could not load asset report: {exc}") if yaml_path is None: st.session_state["original_text"] = "" From 11082085e4887572c4ce7baa5cd6e60e0eca722a Mon Sep 17 00:00:00 2001 From: Xinjie Yao Date: Thu, 22 Oct 2026 21:00:35 -0700 Subject: [PATCH 04/11] Define the tabletop egg-tray scene in YAML Signed-off-by: Xinjie Yao --- .../cli_runner.rst | 3 - .../agentic_env_gen/central_egg_tray.rst | 101 ------------------ .../agentic_env_gen/index.rst | 4 +- .../tabletop_pick_place_egg_tray.rst | 71 ++++++++++++ .../assets/simready_object_library.py | 4 + .../tabletop_pick_place_egg_tray.yaml | 82 ++++++++++++++ .../central_egg_tray_example.py | 24 ----- .../cli_runner.py | 18 ---- .../review_gui/generation_panel.py | 12 --- 9 files changed, 159 insertions(+), 160 deletions(-) delete mode 100644 docs/pages/example_workflows/agentic_env_gen/central_egg_tray.rst create mode 100644 docs/pages/example_workflows/agentic_env_gen/tabletop_pick_place_egg_tray.rst create mode 100644 isaaclab_arena_environments/maple_table_top/tabletop_pick_place_egg_tray.yaml delete mode 100644 isaaclab_arena_examples/agentic_environment_generation/central_egg_tray_example.py diff --git a/docs/pages/concepts/agentic_environment_generation/cli_runner.rst b/docs/pages/concepts/agentic_environment_generation/cli_runner.rst index 90faf66f7e..942abb74a4 100644 --- a/docs/pages/concepts/agentic_environment_generation/cli_runner.rst +++ b/docs/pages/concepts/agentic_environment_generation/cli_runner.rst @@ -115,9 +115,6 @@ These options are defined by the environment generation runner: * - ``--simready_asset_url URL`` - Selects an exact pinned Central asset; repeat for multiple assets. - Agent - * - ``--example central-egg-tray`` - - Loads the egg/tray prompt and four pinned Palatial Central assets. - - Agent * - ``--simready_s3_url URL`` - Overrides the S3 root for S3-based SimReady sources. - Agent diff --git a/docs/pages/example_workflows/agentic_env_gen/central_egg_tray.rst b/docs/pages/example_workflows/agentic_env_gen/central_egg_tray.rst deleted file mode 100644 index f178f5ec21..0000000000 --- a/docs/pages/example_workflows/agentic_env_gen/central_egg_tray.rst +++ /dev/null @@ -1,101 +0,0 @@ -SimReady Central: Egg and Tray -============================= - -Generate a Franka pick-and-place environment using four single-rigid-body assets -from SimReady Central. The egg starts on the maple table; its destination is the -egg tray. A nut and pizza are distractors. Complete the shared -:ref:`agentic-env-gen-prerequisites` first. - -Pinned assets -------------- - -All four assets come from ``PalatialSim/palatial-nv-asset-library`` at revision -``69a87329036cec0440403351c12230970c4e67d8`` and use the CC-BY-4.0 license. -Credit Palatial Platforms when sharing rendered examples or derived assets. - -.. list-table:: - :header-rows: 1 - - * - Asset - - Dataset entry - - Role - * - Egg Tray - - `D00290 `_ - - Destination - * - Egg - - `D00287 `_ - - Pick-up object - * - Nut - - `D00554 `_ - - Distractor - * - Pizza - - `D00802 `_ - - Distractor - -The preset looks up these exact revisions in Central and requires indexed body -count equal to one. It then downloads each asset package and verifies the USD -contains one enabled rigid body and no articulation root. An unavailable or -incompatible pinned asset fails generation rather than selecting a substitute. - -CLI ---- - -.. code-block:: bash - - python isaaclab_arena_examples/agentic_environment_generation/cli_runner.py \ - --mode resolve --example central-egg-tray \ - --out_dir /tmp/central-egg-tray - -Select a configured inference endpoint with ``--inference_endpoint`` if needed. -The runner prints the returned assets with their source URLs, local USD paths, -and ``used_in_spec`` flags. It saves the generated YAML and a sibling -``.assets.json`` report. Check that all four entries have ``used_in_spec: true``. -The LLM determines the final layout, so review the generated spec before use. - -Build the generated environment and run a short zero-action check: - -.. code-block:: bash - - python isaaclab_arena_examples/agentic_environment_generation/cli_runner.py \ - --mode build --env_spec /tmp/central-egg-tray/central_egg_tray.yaml \ - --num_envs 1 --num_steps 20 - -For a different pinned asset list, use ``--enable_simready_search`` with -``--simready_source simready-central`` and repeat ``--simready_asset_url URL``. -Both revision-pinned Hugging Face ``blob`` and ``resolve`` URLs are accepted. - -GUI ---- - -.. code-block:: bash - - python isaaclab_arena_examples/agentic_environment_generation/gui_runner.py - -Open the displayed browser URL, click **Load Central egg-tray example**, choose -your inference endpoint, and click **Generate spec**. The preset fills the prompt, -enables Central search, and populates the four pinned URLs. - -The **Returned SimReady assets** table shows the source URLs and cached USD paths. -Use **Download asset report** to save the same report format as the CLI. The -generated YAML appears in the editor; the visualization panel renders the assets. -Use the simulation-preview controls to inspect the resulting scene. - -To reopen the CLI result with its saved asset report: - -.. code-block:: bash - - python isaaclab_arena_examples/agentic_environment_generation/gui_runner.py \ - --env_spec /tmp/central-egg-tray/central_egg_tray.yaml - -Cache and reproducibility -------------------------- - -Packages use the Hugging Face cache, and the generated spec references local -wrapper files in ``~/.cache/isaaclab_arena/simready-central``. Keep those files -and the package cache available when reopening the YAML. Regenerate on a new -machine from the pinned preset rather than copying cache paths. If a Docker host -cache is a symlink to an unmounted directory, set ``HF_HOME`` to a writable, -mounted location before running either interface. - -This example uses PhysX. A zero-action run checks construction and stepping; it -does not establish successful egg placement by a policy. diff --git a/docs/pages/example_workflows/agentic_env_gen/index.rst b/docs/pages/example_workflows/agentic_env_gen/index.rst index 438f86c6a8..7a3c700879 100644 --- a/docs/pages/example_workflows/agentic_env_gen/index.rst +++ b/docs/pages/example_workflows/agentic_env_gen/index.rst @@ -17,7 +17,7 @@ In this section, we will walk through the following example environment generati - :doc:`tabletop_pnp_homogenous_object/index` - :doc:`tabletop_pnp_heterogeneous_object/index` - :doc:`tabletop_pnp_composite_task/index` - - :doc:`central_egg_tray` + - :doc:`tabletop_pick_place_egg_tray` - Kitchen Pick and Place task @@ -157,6 +157,6 @@ and can be used directly for policy evaluation: tabletop_pnp_homogenous_object/index tabletop_pnp_heterogeneous_object/index tabletop_pnp_composite_task/index - central_egg_tray + tabletop_pick_place_egg_tray kitchen_pick_and_place/index kitchen_open_door/index diff --git a/docs/pages/example_workflows/agentic_env_gen/tabletop_pick_place_egg_tray.rst b/docs/pages/example_workflows/agentic_env_gen/tabletop_pick_place_egg_tray.rst new file mode 100644 index 0000000000..24cebcdc35 --- /dev/null +++ b/docs/pages/example_workflows/agentic_env_gen/tabletop_pick_place_egg_tray.rst @@ -0,0 +1,71 @@ +Tabletop Pick and Place: Egg and Tray +==================================== + +Load ``isaaclab_arena_environments/maple_table_top/tabletop_pick_place_egg_tray.yaml`` +to build a Franka pick-and-place environment. Three eggs start on the maple table. The task moves ``egg`` into the egg tray; +``egg_2``, ``egg_3``, a nut, and pizza are distractors. +The environment is defined directly in YAML and does not require an LLM call. +Complete the shared :ref:`agentic-env-gen-prerequisites` first. + +Pinned assets +------------- + +All four assets come from ``PalatialSim/palatial-nv-asset-library`` at revision +``69a87329036cec0440403351c12230970c4e67d8`` and use the CC-BY-4.0 license. +Credit Palatial Platforms when sharing rendered examples or derived assets. + +.. list-table:: + :header-rows: 1 + + * - Asset + - Dataset entry + - Role + * - Egg Tray + - `D00290 `_ + - Destination + * - Egg + - `D00287 `_ + - Pick-up object + * - Nut + - `D00554 `_ + - Distractor + * - Pizza + - `D00802 `_ + - Distractor + +The YAML contains revision-pinned asset URLs. When building the environment, +``SimReadyUsdObject`` downloads each asset package and creates a cached PhysX +wrapper, requiring exactly one enabled rigid body and no articulation root. + +CLI +--- + +.. code-block:: bash + + python isaaclab_arena_examples/agentic_environment_generation/cli_runner.py \ + --mode build \ + --env_spec isaaclab_arena_environments/maple_table_top/tabletop_pick_place_egg_tray.yaml \ + --num_envs 1 --num_steps 20 + +GUI +--- + +.. code-block:: bash + + python isaaclab_arena_examples/agentic_environment_generation/gui_runner.py \ + --env_spec isaaclab_arena_environments/maple_table_top/tabletop_pick_place_egg_tray.yaml + +The YAML opens in the editor. Use the visualization and simulation-preview +controls to inspect the scene. + +Cache and reproducibility +------------------------- + +Packages use the Hugging Face cache. Validated wrapper files are stored in +``~/.cache/isaaclab_arena/simready-central``. The checked-in YAML keeps the +pinned URLs, so it can be opened on another machine without editing cache paths. +If a Docker host cache points to an unmounted directory, set ``HF_HOME`` to a +writable, mounted location before running either interface. + +This example uses PhysX. A zero-action run checks construction and stepping; +it does not establish successful egg placement by a policy. diff --git a/isaaclab_arena/assets/simready_object_library.py b/isaaclab_arena/assets/simready_object_library.py index 1bbb214381..6eb59ef092 100644 --- a/isaaclab_arena/assets/simready_object_library.py +++ b/isaaclab_arena/assets/simready_object_library.py @@ -46,6 +46,10 @@ def __init__( **kwargs: Any, ): assert usd_path, "simready_usd_object requires params.usd_path" + if usd_path.startswith(("https://huggingface.co/datasets/", "hf://huggingface.co/datasets/")): + from isaaclab_arena.agentic_environment_generation.simready_central import _prepare_central_asset + + usd_path = _prepare_central_asset(usd_path) spawn_cfg_addon = dict(kwargs.pop("spawn_cfg_addon", None) or {}) spawn_cfg_addon.setdefault("variants", dict(SIMREADY_PHYSICS_VARIANTS)) super().__init__( diff --git a/isaaclab_arena_environments/maple_table_top/tabletop_pick_place_egg_tray.yaml b/isaaclab_arena_environments/maple_table_top/tabletop_pick_place_egg_tray.yaml new file mode 100644 index 0000000000..0e3386d443 --- /dev/null +++ b/isaaclab_arena_environments/maple_table_top/tabletop_pick_place_egg_tray.yaml @@ -0,0 +1,82 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +env_name: tabletop_pick_place_egg_tray +embodiment: + id: franka + registry_name: franka_ik + params: {} +background: + id: maple_table + registry_name: maple_table_robolab + params: {} +objects: +- id: egg + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/PalatialSim/palatial-nv-asset-library/resolve/69a87329036cec0440403351c12230970c4e67d8/D00287/simready_usd/D00287.usd +- id: egg_2 + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/PalatialSim/palatial-nv-asset-library/resolve/69a87329036cec0440403351c12230970c4e67d8/D00287/simready_usd/D00287.usd +- id: egg_3 + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/PalatialSim/palatial-nv-asset-library/resolve/69a87329036cec0440403351c12230970c4e67d8/D00287/simready_usd/D00287.usd +- id: egg_tray + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/PalatialSim/palatial-nv-asset-library/resolve/69a87329036cec0440403351c12230970c4e67d8/D00290/simready_usd/D00290.usd +- id: nut + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/PalatialSim/palatial-nv-asset-library/resolve/69a87329036cec0440403351c12230970c4e67d8/D00554/simready_usd/D00554.usd +- id: pizza + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/PalatialSim/palatial-nv-asset-library/resolve/69a87329036cec0440403351c12230970c4e67d8/D00802/simready_usd/D00802.usd +relations: +- kind: is_anchor + subject: maple_table + params: {} +- kind: 'on' + subject: egg + reference: maple_table + params: {} +- kind: 'on' + subject: egg_2 + reference: maple_table + params: {} +- kind: 'on' + subject: egg_3 + reference: maple_table + params: {} +- kind: 'on' + subject: egg_tray + reference: maple_table + params: {} +- kind: 'on' + subject: nut + reference: maple_table + params: {} +- kind: 'on' + subject: pizza + reference: maple_table + params: {} +- kind: next_to + subject: egg + reference: egg_tray + params: + side: positive_x +task: + composition: atomic + description: Franka picks up the egg from the maple table and places it into the + egg tray, leaving two additional eggs, the nut, and pizza as distractors. + subtasks: + - kind: PickAndPlaceTask + params: + pick_up_object: egg + destination_location: egg_tray + background_scene: maple_table diff --git a/isaaclab_arena_examples/agentic_environment_generation/central_egg_tray_example.py b/isaaclab_arena_examples/agentic_environment_generation/central_egg_tray_example.py deleted file mode 100644 index 369dafea3b..0000000000 --- a/isaaclab_arena_examples/agentic_environment_generation/central_egg_tray_example.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Pinned SimReady Central egg-and-tray example shared by CLI and GUI.""" - -CENTRAL_EGG_TRAY_PROMPT = ( - "Create an environment named central_egg_tray. Franka picks up the SimReady egg from the maple table " - "and places it into the SimReady egg tray. Include the SimReady nut and SimReady pizza as separate " - "distractor objects on the table. Use all four supplied SimReady assets: simready_egg, simready_egg_tray, " - "simready_nut, simready_pizza. Keep the egg outside the tray initially. " - "Use PickAndPlaceTask for the egg and tray. Do not use object sets." -) - -_DATASET_REVISION_URL = ( - "https://huggingface.co/datasets/PalatialSim/palatial-nv-asset-library/resolve/" - "69a87329036cec0440403351c12230970c4e67d8" -) - -CENTRAL_EGG_TRAY_ASSET_URLS = tuple( - f"{_DATASET_REVISION_URL}/{asset_id}/simready_usd/{asset_id}.usd" - for asset_id in ("D00290", "D00287", "D00554", "D00802") -) diff --git a/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py b/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py index 5b77d32983..d6b841943f 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py +++ b/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py @@ -73,11 +73,6 @@ def add_agentic_env_gen_runner_cli_args(parser: argparse.ArgumentParser) -> None "'schema', 'catalog', and 'prim_tree' make no agent call." ), ) - group.add_argument( - "--example", - choices=("central-egg-tray",), - help="Load a prompt and pinned asset selection for a maintained example.", - ) group.add_argument( "--prompt", type=str, @@ -380,19 +375,6 @@ def main() -> int: parser = get_isaaclab_arena_cli_parser() add_agentic_env_gen_runner_cli_args(parser) args_cli = parser.parse_args() - if args_cli.example == "central-egg-tray": - from isaaclab_arena_examples.agentic_environment_generation.central_egg_tray_example import ( - CENTRAL_EGG_TRAY_ASSET_URLS, - CENTRAL_EGG_TRAY_PROMPT, - ) - - if args_cli.prompt == DEFAULT_PROMPT: - args_cli.prompt = CENTRAL_EGG_TRAY_PROMPT - args_cli.enable_simready_search = True - args_cli.simready_sources = ["simready-central"] - if not args_cli.simready_asset_url: - args_cli.simready_asset_url = list(CENTRAL_EGG_TRAY_ASSET_URLS) - if args_cli.mode == "schema": print_schema() return 0 diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py index 036d999be2..f25350658f 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py @@ -312,18 +312,6 @@ def render_generation_panel() -> None: endpoint = INFERENCE_ENDPOINTS[endpoint_name] st.caption(f"Model: `{endpoint.model}` at `{endpoint.base_url}`") - if st.button("Load Central egg-tray example"): - from isaaclab_arena_examples.agentic_environment_generation.central_egg_tray_example import ( - CENTRAL_EGG_TRAY_ASSET_URLS, - CENTRAL_EGG_TRAY_PROMPT, - ) - - st.session_state["generation_prompt"] = CENTRAL_EGG_TRAY_PROMPT - st.session_state["enable_simready_search"] = True - st.session_state["simready_sources"] = [SimReadySourceKind.CENTRAL.value] - st.session_state["simready_asset_urls"] = "\n".join(CENTRAL_EGG_TRAY_ASSET_URLS) - st.session_state.pop("simready_asset_report", None) - prompt = st.text_area( "Prompt", value=st.session_state.get("generation_prompt", DEFAULT_GENERATION_PROMPT), From 801fed46c632f1840284e80f91336ec7e4523b99 Mon Sep 17 00:00:00 2001 From: Xinjie Yao Date: Thu, 22 Oct 2026 21:19:45 -0700 Subject: [PATCH 05/11] Remove pinned asset inputs from environment generation Signed-off-by: Xinjie Yao --- .../cli_runner.rst | 6 +- .../agentic_env_gen/index.rst | 2 - .../tabletop_pick_place_egg_tray.rst | 71 ------------------- .../environment_generation_agent.py | 13 ++-- .../simready_asset_search.py | 14 ++-- .../simready_central.py | 39 ---------- .../tests/test_simready_asset_search.py | 4 +- isaaclab_arena/tests/test_simready_central.py | 30 -------- .../cli_runner.py | 11 --- .../review_gui/generation_panel.py | 14 +--- 10 files changed, 17 insertions(+), 187 deletions(-) delete mode 100644 docs/pages/example_workflows/agentic_env_gen/tabletop_pick_place_egg_tray.rst diff --git a/docs/pages/concepts/agentic_environment_generation/cli_runner.rst b/docs/pages/concepts/agentic_environment_generation/cli_runner.rst index 942abb74a4..131b31b87d 100644 --- a/docs/pages/concepts/agentic_environment_generation/cli_runner.rst +++ b/docs/pages/concepts/agentic_environment_generation/cli_runner.rst @@ -112,9 +112,6 @@ These options are defined by the environment generation runner: - Chooses one or more SimReady search backends. Default: ``isaac-sim-ga``. Use ``simready-central`` for single-rigid-body assets on Hugging Face. - Agent - * - ``--simready_asset_url URL`` - - Selects an exact pinned Central asset; repeat for multiple assets. - - Agent * - ``--simready_s3_url URL`` - Overrides the S3 root for S3-based SimReady sources. - Agent @@ -195,5 +192,4 @@ Isaac Sim GA asset library and SimReady Central by object name. The result limit applies per object per backend, so the default returns up to two candidates per object. Identical USD paths are deduplicated per phrase. If one backend fails, results from the other remain available. In the GUI, -select the desired backends under **SimReady sources**. Pinned Central URLs remain an -exact-selection mode under ``simready-central``. +select the desired backends under **SimReady sources**. diff --git a/docs/pages/example_workflows/agentic_env_gen/index.rst b/docs/pages/example_workflows/agentic_env_gen/index.rst index 7a3c700879..681b3c9092 100644 --- a/docs/pages/example_workflows/agentic_env_gen/index.rst +++ b/docs/pages/example_workflows/agentic_env_gen/index.rst @@ -17,7 +17,6 @@ In this section, we will walk through the following example environment generati - :doc:`tabletop_pnp_homogenous_object/index` - :doc:`tabletop_pnp_heterogeneous_object/index` - :doc:`tabletop_pnp_composite_task/index` - - :doc:`tabletop_pick_place_egg_tray` - Kitchen Pick and Place task @@ -157,6 +156,5 @@ and can be used directly for policy evaluation: tabletop_pnp_homogenous_object/index tabletop_pnp_heterogeneous_object/index tabletop_pnp_composite_task/index - tabletop_pick_place_egg_tray kitchen_pick_and_place/index kitchen_open_door/index diff --git a/docs/pages/example_workflows/agentic_env_gen/tabletop_pick_place_egg_tray.rst b/docs/pages/example_workflows/agentic_env_gen/tabletop_pick_place_egg_tray.rst deleted file mode 100644 index 24cebcdc35..0000000000 --- a/docs/pages/example_workflows/agentic_env_gen/tabletop_pick_place_egg_tray.rst +++ /dev/null @@ -1,71 +0,0 @@ -Tabletop Pick and Place: Egg and Tray -==================================== - -Load ``isaaclab_arena_environments/maple_table_top/tabletop_pick_place_egg_tray.yaml`` -to build a Franka pick-and-place environment. Three eggs start on the maple table. The task moves ``egg`` into the egg tray; -``egg_2``, ``egg_3``, a nut, and pizza are distractors. -The environment is defined directly in YAML and does not require an LLM call. -Complete the shared :ref:`agentic-env-gen-prerequisites` first. - -Pinned assets -------------- - -All four assets come from ``PalatialSim/palatial-nv-asset-library`` at revision -``69a87329036cec0440403351c12230970c4e67d8`` and use the CC-BY-4.0 license. -Credit Palatial Platforms when sharing rendered examples or derived assets. - -.. list-table:: - :header-rows: 1 - - * - Asset - - Dataset entry - - Role - * - Egg Tray - - `D00290 `_ - - Destination - * - Egg - - `D00287 `_ - - Pick-up object - * - Nut - - `D00554 `_ - - Distractor - * - Pizza - - `D00802 `_ - - Distractor - -The YAML contains revision-pinned asset URLs. When building the environment, -``SimReadyUsdObject`` downloads each asset package and creates a cached PhysX -wrapper, requiring exactly one enabled rigid body and no articulation root. - -CLI ---- - -.. code-block:: bash - - python isaaclab_arena_examples/agentic_environment_generation/cli_runner.py \ - --mode build \ - --env_spec isaaclab_arena_environments/maple_table_top/tabletop_pick_place_egg_tray.yaml \ - --num_envs 1 --num_steps 20 - -GUI ---- - -.. code-block:: bash - - python isaaclab_arena_examples/agentic_environment_generation/gui_runner.py \ - --env_spec isaaclab_arena_environments/maple_table_top/tabletop_pick_place_egg_tray.yaml - -The YAML opens in the editor. Use the visualization and simulation-preview -controls to inspect the scene. - -Cache and reproducibility -------------------------- - -Packages use the Hugging Face cache. Validated wrapper files are stored in -``~/.cache/isaaclab_arena/simready-central``. The checked-in YAML keeps the -pinned URLs, so it can be opened on another machine without editing cache paths. -If a Docker host cache points to an unmounted directory, set ``HF_HOME`` to a -writable, mounted location before running either interface. - -This example uses PhysX. A zero-action run checks construction and stepping; -it does not establish successful egg placement by a policy. diff --git a/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py b/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py index 378ecfcb0d..2852f9a03c 100644 --- a/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py +++ b/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py @@ -213,15 +213,10 @@ def _extend_catalogue_with_simready(self, prompt: str, asset_catalog: AssetCatal # import pxr, and a pxr import before SimulationApp starts breaks the unit tests. from isaaclab_arena.assets.simready_object_library import register_searched_simready_object - if self.simready_config.central_asset_urls: - from isaaclab_arena.agentic_environment_generation.simready_central import load_pinned_central_objects - - search_result = load_pinned_central_objects(self.simready_config.central_asset_urls) - else: - phrases = self.missing_object_inference.infer(prompt, asset_catalog) - if not phrases: - return asset_catalog - search_result = search_simready_objects(phrases, self.simready_config) + phrases = self.missing_object_inference.infer(prompt, asset_catalog) + if not phrases: + return asset_catalog + search_result = search_simready_objects(phrases, self.simready_config) self._unavailable_objects = list(search_result.unmatched_phrases) self._simready_candidates = list(search_result.candidates) if not search_result.candidates: diff --git a/isaaclab_arena/agentic_environment_generation/simready_asset_search.py b/isaaclab_arena/agentic_environment_generation/simready_asset_search.py index e15ce203f7..fbbbe99257 100644 --- a/isaaclab_arena/agentic_environment_generation/simready_asset_search.py +++ b/isaaclab_arena/agentic_environment_generation/simready_asset_search.py @@ -48,7 +48,6 @@ class SimReadySearchConfig: s3_url: str = ISAAC_SIMREADY_GA_S3_URL service_url: str = DEFAULT_SIMREADY_SERVICE_URL max_results_per_object: int = 1 - central_asset_urls: tuple[str, ...] = () @dataclass(frozen=True) @@ -115,7 +114,11 @@ def _search_simready_source( phrases: list[str], config: SimReadySearchConfig, source: SimReadySourceKind ) -> SimReadyCandidateCatalogue: """Search one source, retaining the configured number of hits per phrase.""" - library = None if source == SimReadySourceKind.CENTRAL else _configure_asset_library(config, source) + library = ( + None + if source == SimReadySourceKind.CENTRAL + else _configure_asset_library(source, s3_url=config.s3_url, service_url=config.service_url) + ) if library is None and source != SimReadySourceKind.CENTRAL: return SimReadyCandidateCatalogue() @@ -128,6 +131,7 @@ def _search_simready_source( hits = search_central_objects(phrase, max_results=config.max_results_per_object) else: + # Query the configured asset library through the simready-search API. hits = _search_phrase(library, phrase, max_results=config.max_results_per_object) except Exception as exc: _logger.warning("simready search failed for %r: %s", phrase, exc) @@ -160,7 +164,7 @@ def simready_search_config_from_cli( _NON_ALPHANUMERIC = re.compile(r"[^a-z0-9]+") -def _configure_asset_library(config: SimReadySearchConfig, source: SimReadySourceKind) -> Any | None: +def _configure_asset_library(source: SimReadySourceKind, *, s3_url: str, service_url: str) -> Any | None: """Build the SimReady asset library for the configured source, or None if the source is unknown.""" if source not in tuple(SimReadySourceKind): _logger.error("unknown simready source: %s", source) @@ -172,9 +176,9 @@ def _configure_asset_library(config: SimReadySearchConfig, source: SimReadySourc library = AssetLibrary() if source in (SimReadySourceKind.ISAAC_SIM_GA, SimReadySourceKind.S3): - asyncio.run(library.add_s3_source(config.s3_url or ISAAC_SIMREADY_GA_S3_URL)) + asyncio.run(library.add_s3_source(s3_url or ISAAC_SIMREADY_GA_S3_URL)) else: - library.add_service_source(config.service_url or DEFAULT_SIMREADY_SERVICE_URL) + library.add_service_source(service_url or DEFAULT_SIMREADY_SERVICE_URL) return library diff --git a/isaaclab_arena/agentic_environment_generation/simready_central.py b/isaaclab_arena/agentic_environment_generation/simready_central.py index 4de6b57e1f..84ebcae0a7 100644 --- a/isaaclab_arena/agentic_environment_generation/simready_central.py +++ b/isaaclab_arena/agentic_environment_generation/simready_central.py @@ -16,7 +16,6 @@ from isaaclab_arena.agentic_environment_generation.simready_asset_search import ( MIN_INSPECTED_MATCHES_PER_PHRASE, - SimReadyCandidateCatalogue, SimReadyObjectCandidate, ) @@ -24,44 +23,6 @@ _logger = logging.getLogger(__name__) -def load_pinned_central_objects(urls: tuple[str, ...]) -> SimReadyCandidateCatalogue: - """Look up exact pinned Central assets and validate each before offering them to the agent.""" - catalogue = SimReadyCandidateCatalogue() - for url in dict.fromkeys(urls): - repo_id, revision, filename, _ = _central_asset_location(url) - source_url = f"https://huggingface.co/datasets/{repo_id}/resolve/{revision}/{filename}" - response = requests.post( - CENTRAL_SEARCH_URL, - headers={"x-usdsearch-storage-backend": "usdsearch-dev"}, - json={ - "filter_url_regexp": re.escape(source_url.replace("https://", "hf://", 1)), - "filter_by_properties_numeric": "simready_metadata_rigid_body_count=1", - "return_usd_properties": True, - "limit": 1, - }, - timeout=30, - ) - response.raise_for_status() - hits = response.json()["hits"] - assert hits, f"Pinned asset is not indexed as a single rigid body in Central: {source_url}" - source = hits[0]["source"] - assert source["path"].replace("hf://", "https://", 1) == source_url, "Central returned a different asset" - properties = {prop["name"]: prop for prop in source.get("usd_properties", [])} - count = properties.get("simready_metadata_rigid_body_count", {}) - assert float(count.get("value_numeric", count.get("value", 0))) == 1, "expected one indexed rigid body" - name = properties.get("simready_metadata_asset_name", {}).get("value") - assert name, f"Central asset has no descriptive name: {source_url}" - catalogue.candidates.append( - SimReadyObjectCandidate( - search_phrase=name, - usd_path=_prepare_central_asset(source_url), - source_url=source_url, - tags=("sim-ready", "single-rigid-body", name.lower()), - ) - ) - return catalogue - - def search_central_objects(phrase: str, *, max_results: int) -> list[SimReadyObjectCandidate]: """Search Central and retain only locally validated single-body props.""" assert max_results > 0, "max_results must be positive" diff --git a/isaaclab_arena/tests/test_simready_asset_search.py b/isaaclab_arena/tests/test_simready_asset_search.py index 82cb3c0a26..824418f3c9 100644 --- a/isaaclab_arena/tests/test_simready_asset_search.py +++ b/isaaclab_arena/tests/test_simready_asset_search.py @@ -188,7 +188,7 @@ def test_configure_asset_library_records_an_unknown_source(caplog): config = SimReadySearchConfig() with caplog.at_level(logging.ERROR, logger=SEARCH_LOGGER): - assert _configure_asset_library(config, "not-a-source") is None + assert _configure_asset_library("not-a-source", s3_url=config.s3_url, service_url=config.service_url) is None assert "unknown simready source" in caplog.text @@ -286,7 +286,7 @@ def test_source_list_searches_selected_backends_once(): ) as central_search, ): result = search_simready_objects(["egg"], config) - configure.assert_called_once_with(config, SimReadySourceKind.SERVICE) + configure.assert_called_once_with(SimReadySourceKind.SERVICE, s3_url=config.s3_url, service_url=config.service_url) library_search.assert_called_once() central_search.assert_called_once_with("egg", max_results=2) assert result.unmatched_phrases == ["egg"] diff --git a/isaaclab_arena/tests/test_simready_central.py b/isaaclab_arena/tests/test_simready_central.py index a98473ab80..e66f60a144 100644 --- a/isaaclab_arena/tests/test_simready_central.py +++ b/isaaclab_arena/tests/test_simready_central.py @@ -16,7 +16,6 @@ ) from isaaclab_arena.agentic_environment_generation.simready_central import ( _central_asset_location, - load_pinned_central_objects, search_central_objects, ) @@ -62,35 +61,6 @@ def test_central_package_location_preserves_revision_and_folder(): ) -def test_pinned_central_assets_accept_blob_urls_and_preserve_identity(): - hit = _hit(1) - hit["source"]["usd_properties"].append({"name": "simready_metadata_asset_name", "value": "Egg Tray"}) - response = Mock() - response.json.return_value = {"hits": [hit]} - blob_url = URL.replace("hf://", "https://").replace("/resolve/", "/blob/") - with ( - patch(f"{MODULE}.requests.post", return_value=response), - patch(f"{MODULE}._prepare_central_asset", return_value="/cache/tray.usda") as prepare, - ): - catalogue = load_pinned_central_objects((blob_url,)) - assert catalogue.candidates[0].search_phrase == "Egg Tray" - assert catalogue.candidates[0].source_url == URL.replace("hf://", "https://") - prepare.assert_called_once_with(URL.replace("hf://", "https://")) - - -@pytest.mark.parametrize("hits", [[], [_hit(2)], [_hit(1, path=URL.replace("pallet.usd", "other.usd"))]]) -def test_pinned_central_assets_never_substitute_or_accept_multiple_bodies(hits): - response = Mock() - response.json.return_value = {"hits": hits} - with ( - patch(f"{MODULE}.requests.post", return_value=response), - patch(f"{MODULE}._prepare_central_asset") as prepare, - pytest.raises(AssertionError), - ): - load_pinned_central_objects((URL,)) - prepare.assert_not_called() - - @pytest.mark.parametrize( "url", [ diff --git a/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py b/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py index d6b841943f..890c6a3fcb 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py +++ b/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py @@ -127,12 +127,6 @@ def add_agentic_env_gen_runner_cli_args(parser: argparse.ArgumentParser) -> None default=["isaac-sim-ga"], help="SimReady backends to search. Result limits apply per backend.", ) - group.add_argument( - "--simready_asset_url", - action="append", - default=[], - help="Pinned Central asset URL; repeat to use an exact asset list instead of natural-language search.", - ) group.add_argument( "--simready_s3_url", type=str, @@ -178,11 +172,6 @@ def resolve_env_spec(args_cli: argparse.Namespace) -> Path | None: service_url=args_cli.simready_service_url, max_results_per_object=args_cli.simready_max_results_per_object, ) - if args_cli.simready_asset_url: - assert args_cli.enable_simready_search and args_cli.simready_sources == [ - "simready-central" - ], "--simready_asset_url requires --enable_simready_search --simready_source simready-central" - simready_config.central_asset_urls = tuple(args_cli.simready_asset_url) agent_kwargs: dict = { "temperature": args_cli.temperature, "enable_simready_search": args_cli.enable_simready_search, diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py index f25350658f..ce94d3c8e7 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py @@ -70,10 +70,6 @@ def _simready_config_from_session() -> SimReadySearchConfig: service_url=st.session_state.get("simready_service_url") or None, max_results_per_object=int(st.session_state.get("simready_max_results_per_object", 1)), ) - if config.sources == [SimReadySourceKind.CENTRAL]: - config.central_asset_urls = tuple( - line.strip() for line in st.session_state.get("simready_asset_urls", "").splitlines() if line.strip() - ) return config @@ -121,7 +117,6 @@ def _generation_agent_cache_key( simready_config.s3_url, simready_config.service_url, simready_config.max_results_per_object, - simready_config.central_asset_urls, ) @@ -350,14 +345,7 @@ def render_generation_panel() -> None: value=st.session_state.get("simready_s3_url", ""), placeholder="Leave empty for the Isaac Sim 6.0 GA SimReady bucket", ) - if sources == [SimReadySourceKind.CENTRAL.value]: - st.text_area( - "Pinned Central asset URLs (optional, one per line)", - key="simready_asset_urls", - help=( - "Use these exact assets instead of natural-language search. Only single rigid bodies are accepted." - ), - ) + if SimReadySourceKind.CENTRAL.value in sources: st.caption( "Searches Hugging Face SimReady Central; accepts only single rigid bodies without articulations." ) From 18b1efc7d3df78cda204f727850ff62e01cb907a Mon Sep 17 00:00:00 2001 From: Xinjie Yao Date: Thu, 22 Oct 2026 21:31:48 -0700 Subject: [PATCH 06/11] Name SimReady helpers by their operations Signed-off-by: Xinjie Yao --- .../simready_asset_search.py | 47 ++++---- .../simready_central.py | 14 +-- .../assets/simready_object_library.py | 4 +- .../tests/test_simready_asset_search.py | 103 +++++++++--------- isaaclab_arena/tests/test_simready_central.py | 18 +-- 5 files changed, 98 insertions(+), 88 deletions(-) diff --git a/isaaclab_arena/agentic_environment_generation/simready_asset_search.py b/isaaclab_arena/agentic_environment_generation/simready_asset_search.py index fbbbe99257..fdbee0f37c 100644 --- a/isaaclab_arena/agentic_environment_generation/simready_asset_search.py +++ b/isaaclab_arena/agentic_environment_generation/simready_asset_search.py @@ -117,7 +117,7 @@ def _search_simready_source( library = ( None if source == SimReadySourceKind.CENTRAL - else _configure_asset_library(source, s3_url=config.s3_url, service_url=config.service_url) + else _create_asset_library_for_source(source, s3_url=config.s3_url, service_url=config.service_url) ) if library is None and source != SimReadySourceKind.CENTRAL: return SimReadyCandidateCatalogue() @@ -132,7 +132,7 @@ def _search_simready_source( hits = search_central_objects(phrase, max_results=config.max_results_per_object) else: # Query the configured asset library through the simready-search API. - hits = _search_phrase(library, phrase, max_results=config.max_results_per_object) + hits = _search_library_rigid_body_candidates(library, phrase, max_results=config.max_results_per_object) except Exception as exc: _logger.warning("simready search failed for %r: %s", phrase, exc) hits = [] @@ -164,7 +164,7 @@ def simready_search_config_from_cli( _NON_ALPHANUMERIC = re.compile(r"[^a-z0-9]+") -def _configure_asset_library(source: SimReadySourceKind, *, s3_url: str, service_url: str) -> Any | None: +def _create_asset_library_for_source(source: SimReadySourceKind, *, s3_url: str, service_url: str) -> Any | None: """Build the SimReady asset library for the configured source, or None if the source is unknown.""" if source not in tuple(SimReadySourceKind): _logger.error("unknown simready source: %s", source) @@ -182,8 +182,11 @@ def _configure_asset_library(source: SimReadySourceKind, *, s3_url: str, service return library -def _search_phrase(library: AssetLibrary, phrase: str, *, max_results: int) -> list[SimReadyObjectCandidate]: - matches = _keep_whole_word_matches(library.search(include_any=_phrase_path_filters(phrase)), phrase) +def _search_library_rigid_body_candidates( + library: AssetLibrary, phrase: str, *, max_results: int +) -> list[SimReadyObjectCandidate]: + """Search the library by phrase and retain ranked assets with exactly one rigid body.""" + matches = _filter_and_rank_path_matches(library.search(include_any=_build_path_filters_from_phrase(phrase)), phrase) # Inspect a few more hits than are wanted back, so a rejected one can be replaced rather than # costing the phrase a result. Past that, give up and let the agent fall back to the Arena @@ -195,7 +198,7 @@ def _search_phrase(library: AssetLibrary, phrase: str, *, max_results: int) -> l candidates: list[SimReadyObjectCandidate] = [] for match in ranked: usd_path = str(match.asset_path) - is_valid, rejection_reason = _is_valid_isaaclab_rigidbody(usd_path) + is_valid, rejection_reason = _has_single_rigid_body(usd_path) if is_valid: candidates.append(_build_simready_object_candidate_from_match(match, phrase)) if len(candidates) >= max_results: @@ -205,38 +208,42 @@ def _search_phrase(library: AssetLibrary, phrase: str, *, max_results: int) -> l return candidates -def _phrase_path_filters(phrase: str) -> list[Any]: +def _build_path_filters_from_phrase(phrase: str) -> list[Any]: """Build one path filter per word of the phrase, so a hit only has to match one word.""" from simready.search import SearchFilterPathContains - return [SearchFilterPathContains(word) for word in _phrase_words(phrase)] + return [SearchFilterPathContains(word) for word in _split_phrase_into_words(phrase)] -def _keep_whole_word_matches(matches: list[Any], phrase: str) -> list[Any]: +def _filter_and_rank_path_matches(matches: list[Any], phrase: str) -> list[Any]: """Keep only the hits named after the object being asked for, best match first.""" - words = _phrase_words(phrase) + words = _split_phrase_into_words(phrase) assert words, "a search phrase needs at least one word" object_word = words[-1] - kept = [match for match in matches if _is_word_in_path(object_word, _split_path_into_words(str(match.asset_path)))] + kept = [ + match + for match in matches + if _path_words_match_singular_or_plural(object_word, _split_path_into_words(str(match.asset_path))) + ] # The sort is stable, so hits that tie on both counts keep the order the search gave them. kept.sort( key=lambda match: ( getattr(match, "relevance_score", None) or 0.0, - _count_matching_words(phrase, str(match.asset_path)), + _count_phrase_words_in_path(phrase, str(match.asset_path)), ), reverse=True, ) return kept -def _is_valid_isaaclab_rigidbody(usd_path: str) -> tuple[bool, str]: - """Say whether a SimReady asset can be spawned as an Isaac Lab rigid object. +def _has_single_rigid_body(usd_path: str) -> tuple[bool, str]: + """Check that a readable USD contains exactly one rigid body with the SimReady variants selected. Args: usd_path: The asset to look at, local or remote. Returns: - Whether the asset is usable, and a phrase naming the problem when it is not, such as + Whether exactly one rigid body was found, and a reason when the check fails, such as "it has no rigid body". The phrase is empty when the asset is usable. """ try: @@ -255,12 +262,12 @@ def _build_simready_object_candidate_from_match(match: Any, phrase: str) -> SimR return SimReadyObjectCandidate( search_phrase=phrase, usd_path=str(match.asset_path), - tags=tuple(dict.fromkeys(("sim-ready", *_phrase_words(phrase)))), + tags=tuple(dict.fromkeys(("sim-ready", *_split_phrase_into_words(phrase)))), relevance_score=getattr(match, "relevance_score", None), ) -def _phrase_words(phrase: str) -> list[str]: +def _split_phrase_into_words(phrase: str) -> list[str]: """Split a search phrase into its lowercase words, in order.""" return [word for word in _NON_ALPHANUMERIC.split(phrase.lower()) if word] @@ -272,14 +279,14 @@ def _split_path_into_words(asset_path: str) -> frozenset[str]: return frozenset(word for word in _NON_ALPHANUMERIC.split(spaced.lower()) if word) -def _is_word_in_path(word: str, path_words: frozenset[str]) -> bool: +def _path_words_match_singular_or_plural(word: str, path_words: frozenset[str]) -> bool: """True if the word is one of the path words. A trailing "s" on either side is ignored.""" if word in path_words or f"{word}s" in path_words: return True return word.endswith("s") and word[:-1] in path_words -def _count_matching_words(phrase: str, asset_path: str) -> int: +def _count_phrase_words_in_path(phrase: str, asset_path: str) -> int: """Count how many words of the phrase appear as whole words in the asset path.""" path_words = _split_path_into_words(asset_path) - return sum(1 for word in _phrase_words(phrase) if _is_word_in_path(word, path_words)) + return sum(1 for word in _split_phrase_into_words(phrase) if _path_words_match_singular_or_plural(word, path_words)) diff --git a/isaaclab_arena/agentic_environment_generation/simready_central.py b/isaaclab_arena/agentic_environment_generation/simready_central.py index 84ebcae0a7..126e399c6e 100644 --- a/isaaclab_arena/agentic_environment_generation/simready_central.py +++ b/isaaclab_arena/agentic_environment_generation/simready_central.py @@ -51,7 +51,7 @@ def search_central_objects(phrase: str, *, max_results: int) -> list[SimReadyObj ] if not counts or any(float(count) != 1 for count in counts): continue - usd_path = _prepare_central_asset(source["path"]) + usd_path = _download_and_wrap_central_asset(source["path"]) candidates.append( SimReadyObjectCandidate( search_phrase=phrase, @@ -69,7 +69,7 @@ def search_central_objects(phrase: str, *, max_results: int) -> list[SimReadyObj return candidates -def _central_asset_location(url: str) -> tuple[str, str, str, str]: +def _parse_central_asset_url(url: str) -> tuple[str, str, str, str]: """Parse a revision-pinned Central dataset URL and its asset package folder.""" match = re.fullmatch( r"(?:hf|https)://huggingface\.co/datasets/([^/]+/[^/]+)/(?:resolve|blob)/([0-9a-f]{40})/(.+)", url @@ -88,24 +88,24 @@ def _central_asset_location(url: str) -> tuple[str, str, str, str]: return repo_id, revision, filename, str(folder) -def _download_central_asset_to_cache(url: str) -> str: +def _download_central_asset_package_to_cache(url: str) -> str: """Download the pinned asset package and return its cached USD entrypoint.""" from huggingface_hub import snapshot_download - repo_id, revision, filename, folder = _central_asset_location(url) + repo_id, revision, filename, folder = _parse_central_asset_url(url) snapshot = snapshot_download( repo_id=repo_id, repo_type="dataset", revision=revision, allow_patterns=[f"{folder}/*"] ) return str(Path(snapshot) / filename) -def _prepare_central_asset(url: str) -> str: - """Write a validated PhysX reference wrapper for a cached Central asset.""" +def _download_and_wrap_central_asset(url: str) -> str: + """Download a Central package, validate its rigid body, and write a cached PhysX USD wrapper.""" from pxr import Usd, UsdGeom, UsdPhysics from isaaclab_arena.utils.usd.rigid_bodies import apply_usd_variant_selections - asset_path = _download_central_asset_to_cache(url) + asset_path = _download_central_asset_package_to_cache(url) asset_stage = Usd.Stage.Open(asset_path) assert asset_stage and asset_stage.GetDefaultPrim(), "asset requires a default prim" stage = Usd.Stage.CreateInMemory() diff --git a/isaaclab_arena/assets/simready_object_library.py b/isaaclab_arena/assets/simready_object_library.py index 6eb59ef092..e6168f6f6f 100644 --- a/isaaclab_arena/assets/simready_object_library.py +++ b/isaaclab_arena/assets/simready_object_library.py @@ -47,9 +47,9 @@ def __init__( ): assert usd_path, "simready_usd_object requires params.usd_path" if usd_path.startswith(("https://huggingface.co/datasets/", "hf://huggingface.co/datasets/")): - from isaaclab_arena.agentic_environment_generation.simready_central import _prepare_central_asset + from isaaclab_arena.agentic_environment_generation.simready_central import _download_and_wrap_central_asset - usd_path = _prepare_central_asset(usd_path) + usd_path = _download_and_wrap_central_asset(usd_path) spawn_cfg_addon = dict(kwargs.pop("spawn_cfg_addon", None) or {}) spawn_cfg_addon.setdefault("variants", dict(SIMREADY_PHYSICS_VARIANTS)) super().__init__( diff --git a/isaaclab_arena/tests/test_simready_asset_search.py b/isaaclab_arena/tests/test_simready_asset_search.py index 824418f3c9..b611009b0b 100644 --- a/isaaclab_arena/tests/test_simready_asset_search.py +++ b/isaaclab_arena/tests/test_simready_asset_search.py @@ -14,9 +14,9 @@ SimReadyObjectCandidate, SimReadySearchConfig, SimReadySourceKind, - _count_matching_words, - _is_valid_isaaclab_rigidbody, - _keep_whole_word_matches, + _count_phrase_words_in_path, + _filter_and_rank_path_matches, + _has_single_rigid_body, _split_path_into_words, search_simready_objects, simready_search_config_from_cli, @@ -31,15 +31,15 @@ TRASH_CAN_PATH = "SimReady/Residential/Garage/sm_trashCan_wheeled_green_a01_01.usd" PLAIN_TRASH_CAN_PATH = "SimReady/Residential/Kitchen/sm_trashCan_a01_01.usd" -RIGID_BODY_CHECK_TARGET = ( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._is_valid_isaaclab_rigidbody" -) +RIGID_BODY_CHECK_TARGET = "isaaclab_arena.agentic_environment_generation.simready_asset_search._has_single_rigid_body" # Patched where it is looked up, not where it is defined: simready_asset_search binds the name at # import time, so patching the defining module would leave that binding untouched. RIGID_BODY_PATHS_TARGET = ( "isaaclab_arena.agentic_environment_generation.simready_asset_search.read_asset_rigid_body_paths" ) -PATH_FILTERS_TARGET = "isaaclab_arena.agentic_environment_generation.simready_asset_search._phrase_path_filters" +PATH_FILTERS_TARGET = ( + "isaaclab_arena.agentic_environment_generation.simready_asset_search._build_path_filters_from_phrase" +) # Stubbed wherever the fake library stands in for a real one: building a path filter needs the # simready package, which only ships with Isaac Sim, and the fake library ignores the filters. @@ -65,66 +65,66 @@ def test_split_path_into_words_splits_camel_case_and_separators(): assert "trashcan" not in words -def test_count_matching_words_ignores_substring_hits(): +def test_count_phrase_words_in_path_ignores_substring_hits(): # "bin" only appears inside "Cabinets", which is how a cabinet got picked for a grey bin. - assert _count_matching_words("grey bin", CABINET_PATH) == 0 - assert _count_matching_words("kitchen cabinet", CABINET_PATH) == 2 + assert _count_phrase_words_in_path("grey bin", CABINET_PATH) == 0 + assert _count_phrase_words_in_path("kitchen cabinet", CABINET_PATH) == 2 -def test_count_matching_words_tolerates_plurals(): - assert _count_matching_words("cabinet", CABINET_PATH) == 1 - assert _count_matching_words("cabinets", CABINET_PATH) == 1 +def test_count_phrase_words_in_path_tolerates_plurals(): + assert _count_phrase_words_in_path("cabinet", CABINET_PATH) == 1 + assert _count_phrase_words_in_path("cabinets", CABINET_PATH) == 1 -def test_keep_whole_word_matches_filters_and_ranks(): +def test_filter_and_rank_path_matches_filters_and_ranks(): cabinet = _FakeMatch(CABINET_PATH) trash_can = _FakeMatch(TRASH_CAN_PATH) - assert _keep_whole_word_matches([cabinet], "grey bin") == [] + assert _filter_and_rank_path_matches([cabinet], "grey bin") == [] # "green trash can" matches three words in the trash can and none in the cabinet. - assert _keep_whole_word_matches([cabinet, trash_can], "green trash can") == [trash_can] + assert _filter_and_rank_path_matches([cabinet, trash_can], "green trash can") == [trash_can] -def test_keep_whole_word_matches_orders_by_word_overlap(): +def test_filter_and_rank_path_matches_orders_by_word_overlap(): green_can = _FakeMatch(TRASH_CAN_PATH) plain_can = _FakeMatch(PLAIN_TRASH_CAN_PATH) - ranked = _keep_whole_word_matches([plain_can, green_can], "green trash can") + ranked = _filter_and_rank_path_matches([plain_can, green_can], "green trash can") assert ranked == [green_can, plain_can] -def test_keep_whole_word_matches_rejects_an_asset_that_only_shares_a_describing_word(): +def test_filter_and_rank_path_matches_rejects_an_asset_that_only_shares_a_describing_word(): grey_cabinet = _FakeMatch(GREY_CABINET_PATH) # A grey cabinet is not a grey bin: the colour matches but the object itself does not, and # handing back the cabinet is how a bin became a cabinet in a generated environment. - assert _keep_whole_word_matches([grey_cabinet], "grey bin") == [] - assert _keep_whole_word_matches([grey_cabinet], "grey cabinet") == [grey_cabinet] + assert _filter_and_rank_path_matches([grey_cabinet], "grey bin") == [] + assert _filter_and_rank_path_matches([grey_cabinet], "grey cabinet") == [grey_cabinet] -def test_is_valid_isaaclab_rigidbody_accepts_a_single_rigid_body(): +def test_has_single_rigid_body_accepts_a_single_rigid_body(): with patch(RIGID_BODY_PATHS_TARGET, return_value=["/Asset/bottle"]): - assert _is_valid_isaaclab_rigidbody("s3://bucket/bottle.usd") == (True, "") + assert _has_single_rigid_body("s3://bucket/bottle.usd") == (True, "") -def test_is_valid_isaaclab_rigidbody_turns_down_several_rigid_bodies(): +def test_has_single_rigid_body_turns_down_several_rigid_bodies(): # A bottle and its cap are two bodies even though a fixed joint holds them together. with patch(RIGID_BODY_PATHS_TARGET, return_value=["/Asset/bottle", "/Asset/cap"]): - assert _is_valid_isaaclab_rigidbody("s3://bucket/bottle.usd") == (False, "it has 2 rigid bodies") + assert _has_single_rigid_body("s3://bucket/bottle.usd") == (False, "it has 2 rigid bodies") -def test_is_valid_isaaclab_rigidbody_turns_down_an_asset_without_physics(): +def test_has_single_rigid_body_turns_down_an_asset_without_physics(): with patch(RIGID_BODY_PATHS_TARGET, return_value=[]): - assert _is_valid_isaaclab_rigidbody("s3://bucket/decoration.usd") == (False, "it has no rigid body") + assert _has_single_rigid_body("s3://bucket/decoration.usd") == (False, "it has no rigid body") -def test_is_valid_isaaclab_rigidbody_turns_down_an_unreadable_asset(): +def test_has_single_rigid_body_turns_down_an_unreadable_asset(): with patch(RIGID_BODY_PATHS_TARGET, side_effect=FileNotFoundError("no such file")): - is_valid, reason = _is_valid_isaaclab_rigidbody("s3://bucket/missing.usd") + is_valid, reason = _has_single_rigid_body("s3://bucket/missing.usd") assert not is_valid assert "could not be read" in reason @patch(PATH_FILTERS_TARGET, return_value=[]) @patch( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._configure_asset_library", + "isaaclab_arena.agentic_environment_generation.simready_asset_search._create_asset_library_for_source", ) def test_search_falls_back_to_the_next_hit_when_one_is_rejected(mock_configure, mock_path_filters, caplog): mock_configure.return_value = _FakeLibrary([_FakeMatch(TRASH_CAN_PATH), _FakeMatch(PLAIN_TRASH_CAN_PATH)]) @@ -140,7 +140,7 @@ def test_search_falls_back_to_the_next_hit_when_one_is_rejected(mock_configure, @patch(PATH_FILTERS_TARGET, return_value=[]) @patch( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._configure_asset_library", + "isaaclab_arena.agentic_environment_generation.simready_asset_search._create_asset_library_for_source", ) def test_search_reports_a_phrase_with_only_rejected_hits_as_unmatched(mock_configure, mock_path_filters, caplog): mock_configure.return_value = _FakeLibrary([_FakeMatch(CABINET_PATH)]) @@ -164,14 +164,14 @@ def test_simready_search_config_from_cli_defaults(): @patch( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._search_phrase", + "isaaclab_arena.agentic_environment_generation.simready_asset_search._search_library_rigid_body_candidates", ) @patch( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._configure_asset_library", + "isaaclab_arena.agentic_environment_generation.simready_asset_search._create_asset_library_for_source", ) -def test_search_simready_objects_returns_candidates(mock_configure, mock_search_phrase): +def test_search_simready_objects_returns_candidates(mock_configure, mock_search_library_rigid_body_candidates): mock_configure.return_value = _FakeLibrary() - mock_search_phrase.return_value = [ + mock_search_library_rigid_body_candidates.return_value = [ SimReadyObjectCandidate( search_phrase="ceramic bowl", usd_path="s3://bucket/bowl.usd", @@ -183,17 +183,20 @@ def test_search_simready_objects_returns_candidates(mock_configure, mock_search_ assert catalog.candidates[0].registry_name == SIMREADY_USD_OBJECT_REGISTRY_NAME -def test_configure_asset_library_records_an_unknown_source(caplog): - from isaaclab_arena.agentic_environment_generation.simready_asset_search import _configure_asset_library +def test_create_asset_library_for_source_records_an_unknown_source(caplog): + from isaaclab_arena.agentic_environment_generation.simready_asset_search import _create_asset_library_for_source config = SimReadySearchConfig() with caplog.at_level(logging.ERROR, logger=SEARCH_LOGGER): - assert _configure_asset_library("not-a-source", s3_url=config.s3_url, service_url=config.service_url) is None + assert ( + _create_asset_library_for_source("not-a-source", s3_url=config.s3_url, service_url=config.service_url) + is None + ) assert "unknown simready source" in caplog.text @patch( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._configure_asset_library", + "isaaclab_arena.agentic_environment_generation.simready_asset_search._create_asset_library_for_source", ) def test_search_simready_objects_returns_empty_when_library_unavailable(mock_configure): mock_configure.return_value = None @@ -202,14 +205,14 @@ def test_search_simready_objects_returns_empty_when_library_unavailable(mock_con @patch( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._search_phrase", + "isaaclab_arena.agentic_environment_generation.simready_asset_search._search_library_rigid_body_candidates", ) @patch( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._configure_asset_library", + "isaaclab_arena.agentic_environment_generation.simready_asset_search._create_asset_library_for_source", ) -def test_search_simready_objects_records_no_matches(mock_configure, mock_search_phrase, caplog): +def test_search_simready_objects_records_no_matches(mock_configure, mock_search_library_rigid_body_candidates, caplog): mock_configure.return_value = _FakeLibrary() - mock_search_phrase.return_value = [] + mock_search_library_rigid_body_candidates.return_value = [] with caplog.at_level(logging.INFO, logger=SEARCH_LOGGER): catalog = search_simready_objects(["missing object"], SimReadySearchConfig()) assert catalog.candidates == [] @@ -221,8 +224,8 @@ def test_multiple_sources_merge_and_deduplicate_candidates(): shared = SimReadyObjectCandidate(search_phrase="egg", usd_path="shared.usd") central = SimReadyObjectCandidate(search_phrase="egg", usd_path="central.usd") with ( - patch(f"{SEARCH_LOGGER}._configure_asset_library", return_value=object()), - patch(f"{SEARCH_LOGGER}._search_phrase", side_effect=[[shared], []]), + patch(f"{SEARCH_LOGGER}._create_asset_library_for_source", return_value=object()), + patch(f"{SEARCH_LOGGER}._search_library_rigid_body_candidates", side_effect=[[shared], []]), patch( "isaaclab_arena.agentic_environment_generation.simready_central.search_central_objects", side_effect=[[shared, central], []], @@ -239,7 +242,7 @@ def test_multiple_sources_merge_and_deduplicate_candidates(): def test_multiple_sources_keep_central_when_library_initialization_fails(): central = SimReadyObjectCandidate(search_phrase="egg", usd_path="central.usd") with ( - patch(f"{SEARCH_LOGGER}._configure_asset_library", side_effect=RuntimeError("unavailable")), + patch(f"{SEARCH_LOGGER}._create_asset_library_for_source", side_effect=RuntimeError("unavailable")), patch( "isaaclab_arena.agentic_environment_generation.simready_central.search_central_objects", return_value=[central], @@ -255,8 +258,8 @@ def test_multiple_sources_keep_central_when_library_initialization_fails(): def test_multiple_sources_keep_library_when_central_fails(): candidate = SimReadyObjectCandidate(search_phrase="egg", usd_path="library.usd") with ( - patch(f"{SEARCH_LOGGER}._configure_asset_library", return_value=object()), - patch(f"{SEARCH_LOGGER}._search_phrase", return_value=[candidate]), + patch(f"{SEARCH_LOGGER}._create_asset_library_for_source", return_value=object()), + patch(f"{SEARCH_LOGGER}._search_library_rigid_body_candidates", return_value=[candidate]), patch( "isaaclab_arena.agentic_environment_generation.simready_central.search_central_objects", side_effect=RuntimeError("unavailable"), @@ -278,8 +281,8 @@ def test_source_list_searches_selected_backends_once(): ) assert config.sources == [SimReadySourceKind.SERVICE, SimReadySourceKind.CENTRAL] with ( - patch(f"{SEARCH_LOGGER}._configure_asset_library", return_value=object()) as configure, - patch(f"{SEARCH_LOGGER}._search_phrase", return_value=[]) as library_search, + patch(f"{SEARCH_LOGGER}._create_asset_library_for_source", return_value=object()) as configure, + patch(f"{SEARCH_LOGGER}._search_library_rigid_body_candidates", return_value=[]) as library_search, patch( "isaaclab_arena.agentic_environment_generation.simready_central.search_central_objects", return_value=[], diff --git a/isaaclab_arena/tests/test_simready_central.py b/isaaclab_arena/tests/test_simready_central.py index e66f60a144..538b699592 100644 --- a/isaaclab_arena/tests/test_simready_central.py +++ b/isaaclab_arena/tests/test_simready_central.py @@ -15,7 +15,7 @@ search_simready_objects, ) from isaaclab_arena.agentic_environment_generation.simready_central import ( - _central_asset_location, + _parse_central_asset_url, search_central_objects, ) @@ -35,7 +35,7 @@ def test_central_requires_single_body_metadata_and_skips_invalid_usd(): with ( patch(f"{MODULE}.requests.post", return_value=response) as post, patch( - f"{MODULE}._prepare_central_asset", side_effect=[ValueError("articulation"), "/cache/pallet.usda"] + f"{MODULE}._download_and_wrap_central_asset", side_effect=[ValueError("articulation"), "/cache/pallet.usda"] ) as prepare, ): candidates = search_central_objects("pallet", max_results=1) @@ -53,7 +53,7 @@ def test_central_failure_leaves_phrase_unmatched(): def test_central_package_location_preserves_revision_and_folder(): - assert _central_asset_location(URL) == ( + assert _parse_central_asset_url(URL) == ( "nvidia/simready-assets", REVISION, "pallet/simready_usd/pallet.usd", @@ -71,7 +71,7 @@ def test_central_package_location_preserves_revision_and_folder(): ) def test_central_rejects_unpinned_or_unbounded_packages(url): with pytest.raises(AssertionError): - _central_asset_location(url) + _parse_central_asset_url(url) def _test_central_usd_validation(simulation_app): @@ -80,7 +80,7 @@ def _test_central_usd_validation(simulation_app): from pxr import Usd, UsdGeom, UsdPhysics - from isaaclab_arena.agentic_environment_generation.simready_central import _prepare_central_asset + from isaaclab_arena.agentic_environment_generation.simready_central import _download_and_wrap_central_asset with tempfile.TemporaryDirectory() as directory: folder = Path(directory) @@ -96,22 +96,22 @@ def _test_central_usd_validation(simulation_app): patch("huggingface_hub.snapshot_download", return_value=directory), patch(f"{MODULE}.Path.home", return_value=folder), ): - wrapper = _prepare_central_asset(URL) + wrapper = _download_and_wrap_central_asset(URL) assert Usd.Stage.Open(wrapper).GetDefaultPrim().HasAPI(UsdPhysics.RigidBodyAPI) body.GetRigidBodyEnabledAttr().Set(False) stage.GetRootLayer().Save() with pytest.raises(AssertionError, match="disabled"): - _prepare_central_asset(URL) + _download_and_wrap_central_asset(URL) body.GetRigidBodyEnabledAttr().Set(True) UsdPhysics.ArticulationRootAPI.Apply(root) stage.GetRootLayer().Save() with pytest.raises(AssertionError, match="articulation root"): - _prepare_central_asset(URL) + _download_and_wrap_central_asset(URL) root.RemoveAPI(UsdPhysics.ArticulationRootAPI) UsdPhysics.RigidBodyAPI.Apply(UsdGeom.Xform.Define(stage, "/Pallet/Other").GetPrim()) stage.GetRootLayer().Save() with pytest.raises(AssertionError, match="found 2"): - _prepare_central_asset(URL) + _download_and_wrap_central_asset(URL) return True From 3cd7a772ef599fc48748f79ce10f90e71ede8222 Mon Sep 17 00:00:00 2001 From: Xinjie Yao Date: Thu, 22 Oct 2026 22:17:50 -0700 Subject: [PATCH 07/11] Remove SimReady report files and CLI doc changes Signed-off-by: Xinjie Yao --- .../cli_runner.rst | 24 ++----------------- .../cli_runner.py | 4 ---- .../review_gui/generation_panel.py | 7 ------ .../review_gui/streamlit_ui.py | 11 --------- 4 files changed, 2 insertions(+), 44 deletions(-) diff --git a/docs/pages/concepts/agentic_environment_generation/cli_runner.rst b/docs/pages/concepts/agentic_environment_generation/cli_runner.rst index 131b31b87d..849e12d1de 100644 --- a/docs/pages/concepts/agentic_environment_generation/cli_runner.rst +++ b/docs/pages/concepts/agentic_environment_generation/cli_runner.rst @@ -108,9 +108,8 @@ These options are defined by the environment generation runner: * - ``--enable_simready_search`` - Searches SimReady for prompt objects missing from Arena's asset catalog. - Agent - * - ``--simready_sources SOURCE [SOURCE ...]`` - - Chooses one or more SimReady search backends. Default: ``isaac-sim-ga``. - Use ``simready-central`` for single-rigid-body assets on Hugging Face. + * - ``--simready_source SOURCE`` + - Chooses the SimReady search backend. Default: ``isaac-sim-ga``. - Agent * - ``--simready_s3_url URL`` - Overrides the S3 root for S3-based SimReady sources. @@ -125,15 +124,6 @@ These options are defined by the environment generation runner: - Sets the number of zero-action simulation steps. Default: ``20``. - Evaluation smoke test -With ``--enable_simready_search --simready_source simready-central``, search -requires indexed rigid-body count equal to one. Selected asset folders are cached -from their pinned Hugging Face dataset revision, preserving relative dependencies. -Before registration, the composed USD must have exactly one enabled rigid body, -no articulation root, and no unresolved composition errors. A local wrapper selects -PhysX variants; the generated spec references that wrapper and needs the same cache -when run elsewhere. Assets with missing count metadata or unsupported packaging are -skipped. Articulated assets and multi-body assemblies are not imported. - Build and simulation options ---------------------------- @@ -183,13 +173,3 @@ relevant to ``build`` and ``full`` are: Run the script with ``--help`` for the complete launcher-specific option list, including device and livestream settings supplied by Isaac Lab. - -Combined SimReady search ------------------------- - -Use ``--enable_simready_search --simready_sources isaac-sim-ga simready-central`` to search both the -Isaac Sim GA asset library and SimReady Central by object name. The result -limit applies per object per backend, so the default returns up to two -candidates per object. Identical USD paths are deduplicated per phrase. -If one backend fails, results from the other remain available. In the GUI, -select the desired backends under **SimReady sources**. diff --git a/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py b/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py index 890c6a3fcb..bb3a5f0bf9 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py +++ b/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py @@ -221,10 +221,6 @@ def resolve_env_spec(args_cli: argparse.Namespace) -> Path | None: ) path = write_env_graph_spec(env_graph_spec, args_cli.out_dir) print(f"[runner] wrote environment graph spec → {path}", flush=True) - if args_cli.enable_simready_search: - asset_report_path = path.with_suffix(".assets.json") - asset_report_path.write_text(json.dumps(asset_report, indent=2) + "\n", encoding="utf-8") - print(f"[runner] wrote SimReady asset report → {asset_report_path}", flush=True) return path diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py index ce94d3c8e7..91d2d8055a 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py @@ -5,7 +5,6 @@ from __future__ import annotations -import json import os import traceback import yaml @@ -375,9 +374,3 @@ def render_generation_panel() -> None: if asset_report: with st.expander("Returned SimReady assets", expanded=True): st.dataframe(asset_report, hide_index=True, width="stretch") - st.download_button( - "Download asset report", - data=json.dumps(asset_report, indent=2), - file_name="simready_assets.json", - mime="application/json", - ) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py index 324e4f9e73..374e469770 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py @@ -11,7 +11,6 @@ from __future__ import annotations import argparse -import json from pathlib import Path import streamlit as st @@ -70,16 +69,6 @@ def initialize_state(yaml_path: Path | None, out_dir: Path) -> None: st.session_state.pop("_validation_text", None) st.session_state.pop("_validation_result", None) st.session_state.pop("simready_asset_report", None) - if yaml_path is not None: - report_path = yaml_path.with_suffix(".assets.json") - if report_path.is_file(): - try: - report = json.loads(report_path.read_text(encoding="utf-8")) - if isinstance(report, list) and all(isinstance(row, dict) for row in report): - st.session_state["simready_asset_report"] = report - except (OSError, ValueError) as exc: - st.warning(f"Could not load asset report: {exc}") - if yaml_path is None: st.session_state["original_text"] = "" st.session_state["edited_text"] = "" From 48da72d1d55f6f3725687be25f0e8e5610fd237b Mon Sep 17 00:00:00 2001 From: Xinjie Yao Date: Fri, 23 Oct 2026 17:05:25 -0700 Subject: [PATCH 08/11] Split SimReady backends and update Central example Signed-off-by: Xinjie Yao --- ..._pick_place_screwdriver_bearing_wrench.png | 3 + .../environment_generation_agent.py | 4 +- .../simready_asset_search.py | 259 ++---------------- .../simready_central_assets.py | 83 ++++++ .../simready_ga_assets.py | 183 +++++++++++++ .../simready_search_interface.py | 87 ++++++ .../simready_central_asset_loader.py} | 87 ++---- .../assets/simready_object_library.py | 4 +- .../test_environment_generation_agent.py | 2 +- .../tests/test_simready_asset_search.py | 70 +++-- isaaclab_arena/tests/test_simready_central.py | 52 +++- .../tabletop_pick_place_egg_tray.yaml | 82 ------ ...pick_place_screwdriver_bearing_wrench.yaml | 55 ++++ .../cli_runner.py | 4 +- .../review_gui/generation_panel.py | 2 +- .../tests/test_review_gui.py | 2 +- 16 files changed, 539 insertions(+), 440 deletions(-) create mode 100644 docs/images/agentic_environment_generation/tabletop_pick_place_screwdriver_bearing_wrench.png create mode 100644 isaaclab_arena/agentic_environment_generation/simready_central_assets.py create mode 100644 isaaclab_arena/agentic_environment_generation/simready_ga_assets.py create mode 100644 isaaclab_arena/agentic_environment_generation/simready_search_interface.py rename isaaclab_arena/{agentic_environment_generation/simready_central.py => assets/simready_central_asset_loader.py} (56%) delete mode 100644 isaaclab_arena_environments/maple_table_top/tabletop_pick_place_egg_tray.yaml create mode 100644 isaaclab_arena_environments/maple_table_top/tabletop_pick_place_screwdriver_bearing_wrench.yaml diff --git a/docs/images/agentic_environment_generation/tabletop_pick_place_screwdriver_bearing_wrench.png b/docs/images/agentic_environment_generation/tabletop_pick_place_screwdriver_bearing_wrench.png new file mode 100644 index 0000000000..29bb63d538 --- /dev/null +++ b/docs/images/agentic_environment_generation/tabletop_pick_place_screwdriver_bearing_wrench.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1659641612e828317dd1032d230d3dbbafa6b11fcb2c77f1e5b23365f0c02aca +size 839360 diff --git a/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py b/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py index 2852f9a03c..3f76f0c73d 100644 --- a/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py +++ b/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py @@ -21,10 +21,10 @@ ) from isaaclab_arena.agentic_environment_generation.missing_object_inference import MissingObjectInference from isaaclab_arena.agentic_environment_generation.prim_path_inference import PrimPathInference -from isaaclab_arena.agentic_environment_generation.simready_asset_search import ( +from isaaclab_arena.agentic_environment_generation.simready_asset_search import search_simready_objects +from isaaclab_arena.agentic_environment_generation.simready_search_interface import ( SimReadyObjectCandidate, SimReadySearchConfig, - search_simready_objects, ) from isaaclab_arena.agentic_environment_generation.spec_inference import SpecInference from isaaclab_arena.assets.simready_constants import SIMREADY_USD_OBJECT_REGISTRY_NAME diff --git a/isaaclab_arena/agentic_environment_generation/simready_asset_search.py b/isaaclab_arena/agentic_environment_generation/simready_asset_search.py index fdbee0f37c..127b11ad37 100644 --- a/isaaclab_arena/agentic_environment_generation/simready_asset_search.py +++ b/isaaclab_arena/agentic_environment_generation/simready_asset_search.py @@ -3,77 +3,20 @@ # # SPDX-License-Identifier: Apache-2.0 -"""SimReady asset search for agentic environment generation.""" - +"""Coordinate selected SimReady backends and merge their search results.""" from __future__ import annotations -import asyncio import logging -import re -from dataclasses import dataclass, field -from enum import Enum -from typing import TYPE_CHECKING, Any -from isaaclab_arena.assets.simready_constants import ( - DEFAULT_SIMREADY_SERVICE_URL, - ISAAC_SIMREADY_GA_S3_URL, - SIMREADY_PHYSICS_VARIANTS, - SIMREADY_USD_OBJECT_REGISTRY_NAME, +from isaaclab_arena.agentic_environment_generation.simready_search_interface import ( + SimReadyCandidateCatalogue, + SimReadySearchBackend, + SimReadySearchConfig, + SimReadySourceKind, ) -from isaaclab_arena.utils.usd.rigid_bodies import read_asset_rigid_body_paths - -if TYPE_CHECKING: - from simready.search import AssetLibrary _logger = logging.getLogger(__name__) -MIN_INSPECTED_MATCHES_PER_PHRASE = 5 -"""How many hits per phrase to inspect for rigid bodies, unless more results than this were asked for.""" - - -class SimReadySourceKind(str, Enum): - """Which SimReady backend to search.""" - - ISAAC_SIM_GA = "isaac-sim-ga" - S3 = "s3" - SERVICE = "service" - CENTRAL = "simready-central" - - -@dataclass -class SimReadySearchConfig: - """Where to search for SimReady objects and how many hits to keep.""" - - sources: list[SimReadySourceKind] = field(default_factory=lambda: [SimReadySourceKind.ISAAC_SIM_GA]) - s3_url: str = ISAAC_SIMREADY_GA_S3_URL - service_url: str = DEFAULT_SIMREADY_SERVICE_URL - max_results_per_object: int = 1 - - -@dataclass(frozen=True) -class SimReadyObjectCandidate: - """One SimReady search hit exposed to spec inference.""" - - search_phrase: str - usd_path: str - tags: tuple[str, ...] = ("sim-ready",) - relevance_score: float | None = None - source_url: str | None = None - - @property - def registry_name(self) -> str: - return SIMREADY_USD_OBJECT_REGISTRY_NAME - - -@dataclass -class SimReadyCandidateCatalogue: - """The SimReady hits found for one prompt, handed to spec inference.""" - - candidates: list[SimReadyObjectCandidate] = field(default_factory=list) - - unmatched_phrases: list[str] = field(default_factory=list) - """Objects the search found no usable asset for, so they are left out of the spec.""" - def search_simready_objects(object_phrases: list[str], config: SimReadySearchConfig) -> SimReadyCandidateCatalogue: """Search SimReady for an asset for each object phrase. A hit that cannot be spawned as a rigid object is turned down, and the next hit is tried in @@ -94,11 +37,12 @@ def search_simready_objects(object_phrases: list[str], config: SimReadySearchCon seen = set() for source in dict.fromkeys(config.sources): try: - result = _search_simready_source(phrases, config, source) + backend = _create_search_backend(source, config) + result = backend.search(phrases, config.max_results_per_object) except Exception as exc: _logger.warning("simready backend %s failed: %s", source, exc) continue - for candidate in result.candidates: + for candidate in result: key = (candidate.search_phrase, candidate.usd_path) if key not in seen: seen.add(key) @@ -110,183 +54,12 @@ def search_simready_objects(object_phrases: list[str], config: SimReadySearchCon ) -def _search_simready_source( - phrases: list[str], config: SimReadySearchConfig, source: SimReadySourceKind -) -> SimReadyCandidateCatalogue: - """Search one source, retaining the configured number of hits per phrase.""" - library = ( - None - if source == SimReadySourceKind.CENTRAL - else _create_asset_library_for_source(source, s3_url=config.s3_url, service_url=config.service_url) - ) - if library is None and source != SimReadySourceKind.CENTRAL: - return SimReadyCandidateCatalogue() - - candidates: list[SimReadyObjectCandidate] = [] - unmatched_phrases: list[str] = [] - for phrase in phrases: - try: - if source == SimReadySourceKind.CENTRAL: - from isaaclab_arena.agentic_environment_generation.simready_central import search_central_objects - - hits = search_central_objects(phrase, max_results=config.max_results_per_object) - else: - # Query the configured asset library through the simready-search API. - hits = _search_library_rigid_body_candidates(library, phrase, max_results=config.max_results_per_object) - except Exception as exc: - _logger.warning("simready search failed for %r: %s", phrase, exc) - hits = [] - if hits: - candidates.extend(hits) - else: - _logger.info("simready search found no usable asset for %r", phrase) - unmatched_phrases.append(phrase) - - return SimReadyCandidateCatalogue(candidates=candidates, unmatched_phrases=unmatched_phrases) - - -def simready_search_config_from_cli( - sources: list[str], - s3_url: str | None, - service_url: str | None, - max_results_per_object: int, -) -> SimReadySearchConfig: - """Build a search configuration from CLI or GUI arguments, filling in the default URLs.""" - return SimReadySearchConfig( - sources=[SimReadySourceKind(source) for source in dict.fromkeys(sources)], - s3_url=s3_url or ISAAC_SIMREADY_GA_S3_URL, - service_url=service_url or DEFAULT_SIMREADY_SERVICE_URL, - max_results_per_object=max_results_per_object, - ) - - -_CAMEL_CASE_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") -_NON_ALPHANUMERIC = re.compile(r"[^a-z0-9]+") - - -def _create_asset_library_for_source(source: SimReadySourceKind, *, s3_url: str, service_url: str) -> Any | None: - """Build the SimReady asset library for the configured source, or None if the source is unknown.""" - if source not in tuple(SimReadySourceKind): - _logger.error("unknown simready source: %s", source) - return None - - # Imported here rather than at module scope only to keep the AWS stack out of the import path - # of every caller. - from simready.search import AssetLibrary - - library = AssetLibrary() - if source in (SimReadySourceKind.ISAAC_SIM_GA, SimReadySourceKind.S3): - asyncio.run(library.add_s3_source(s3_url or ISAAC_SIMREADY_GA_S3_URL)) - else: - library.add_service_source(service_url or DEFAULT_SIMREADY_SERVICE_URL) - return library - - -def _search_library_rigid_body_candidates( - library: AssetLibrary, phrase: str, *, max_results: int -) -> list[SimReadyObjectCandidate]: - """Search the library by phrase and retain ranked assets with exactly one rigid body.""" - matches = _filter_and_rank_path_matches(library.search(include_any=_build_path_filters_from_phrase(phrase)), phrase) - - # Inspect a few more hits than are wanted back, so a rejected one can be replaced rather than - # costing the phrase a result. Past that, give up and let the agent fall back to the Arena - # asset registry: every further hit inspected is another asset downloaded. - ranked = matches[: max(MIN_INSPECTED_MATCHES_PER_PHRASE, max_results)] - if len(ranked) < len(matches): - _logger.info("simready search checked the best %d of %d hits for %r", len(ranked), len(matches), phrase) - - candidates: list[SimReadyObjectCandidate] = [] - for match in ranked: - usd_path = str(match.asset_path) - is_valid, rejection_reason = _has_single_rigid_body(usd_path) - if is_valid: - candidates.append(_build_simready_object_candidate_from_match(match, phrase)) - if len(candidates) >= max_results: - break - else: - _logger.info("simready rejected %s for %r: %s", usd_path, phrase, rejection_reason) - return candidates - - -def _build_path_filters_from_phrase(phrase: str) -> list[Any]: - """Build one path filter per word of the phrase, so a hit only has to match one word.""" - from simready.search import SearchFilterPathContains - - return [SearchFilterPathContains(word) for word in _split_phrase_into_words(phrase)] - - -def _filter_and_rank_path_matches(matches: list[Any], phrase: str) -> list[Any]: - """Keep only the hits named after the object being asked for, best match first.""" - words = _split_phrase_into_words(phrase) - assert words, "a search phrase needs at least one word" - object_word = words[-1] - kept = [ - match - for match in matches - if _path_words_match_singular_or_plural(object_word, _split_path_into_words(str(match.asset_path))) - ] - # The sort is stable, so hits that tie on both counts keep the order the search gave them. - kept.sort( - key=lambda match: ( - getattr(match, "relevance_score", None) or 0.0, - _count_phrase_words_in_path(phrase, str(match.asset_path)), - ), - reverse=True, - ) - return kept - - -def _has_single_rigid_body(usd_path: str) -> tuple[bool, str]: - """Check that a readable USD contains exactly one rigid body with the SimReady variants selected. - - Args: - usd_path: The asset to look at, local or remote. - - Returns: - Whether exactly one rigid body was found, and a reason when the check fails, such as - "it has no rigid body". The phrase is empty when the asset is usable. - """ - try: - rigid_body_paths = read_asset_rigid_body_paths(usd_path, SIMREADY_PHYSICS_VARIANTS) - except Exception as exc: - return False, f"its USD could not be read: {exc}" - # Only one RigidBodyAPI is allowed on a USD asset. - if len(rigid_body_paths) == 1: - return True, "" - if not rigid_body_paths: - return False, "it has no rigid body" - return False, f"it has {len(rigid_body_paths)} rigid bodies" - - -def _build_simready_object_candidate_from_match(match: Any, phrase: str) -> SimReadyObjectCandidate: - return SimReadyObjectCandidate( - search_phrase=phrase, - usd_path=str(match.asset_path), - tags=tuple(dict.fromkeys(("sim-ready", *_split_phrase_into_words(phrase)))), - relevance_score=getattr(match, "relevance_score", None), - ) - - -def _split_phrase_into_words(phrase: str) -> list[str]: - """Split a search phrase into its lowercase words, in order.""" - return [word for word in _NON_ALPHANUMERIC.split(phrase.lower()) if word] - - -def _split_path_into_words(asset_path: str) -> frozenset[str]: - """Split an asset path into lowercase words, also splitting camelCase.""" - # e.g. trashcan_wheeled_green_a01_01.usd -> trashcan wheeled green a01 01 - spaced = _CAMEL_CASE_BOUNDARY.sub(" ", asset_path) - return frozenset(word for word in _NON_ALPHANUMERIC.split(spaced.lower()) if word) - - -def _path_words_match_singular_or_plural(word: str, path_words: frozenset[str]) -> bool: - """True if the word is one of the path words. A trailing "s" on either side is ignored.""" - if word in path_words or f"{word}s" in path_words: - return True - return word.endswith("s") and word[:-1] in path_words +def _create_search_backend(source: SimReadySourceKind, config: SimReadySearchConfig) -> SimReadySearchBackend: + """Construct the implementation for one selected source.""" + if source == SimReadySourceKind.CENTRAL: + from isaaclab_arena.agentic_environment_generation.simready_central_assets import CentralSearchBackend + return CentralSearchBackend() + from isaaclab_arena.agentic_environment_generation.simready_ga_assets import AssetLibrarySearchBackend -def _count_phrase_words_in_path(phrase: str, asset_path: str) -> int: - """Count how many words of the phrase appear as whole words in the asset path.""" - path_words = _split_path_into_words(asset_path) - return sum(1 for word in _split_phrase_into_words(phrase) if _path_words_match_singular_or_plural(word, path_words)) + return AssetLibrarySearchBackend(source, s3_url=config.s3_url, service_url=config.service_url) diff --git a/isaaclab_arena/agentic_environment_generation/simready_central_assets.py b/isaaclab_arena/agentic_environment_generation/simready_central_assets.py new file mode 100644 index 0000000000..0283cfd3e6 --- /dev/null +++ b/isaaclab_arena/agentic_environment_generation/simready_central_assets.py @@ -0,0 +1,83 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Search SimReady Central through its specialized REST API.""" +from __future__ import annotations + +import logging + +import requests + +from isaaclab_arena.agentic_environment_generation.simready_search_interface import ( + MIN_INSPECTED_MATCHES_PER_PHRASE, + SimReadyObjectCandidate, +) +from isaaclab_arena.assets.simready_central_asset_loader import download_and_wrap_central_asset + +CENTRAL_SEARCH_URL = "https://nvidia-simready-central.hf.space/search_hybrid" +_logger = logging.getLogger(__name__) + + +class CentralSearchBackend: + """Search Central and validate downloaded single-rigid-body packages.""" + + def search(self, phrases: list[str], max_results: int) -> list[SimReadyObjectCandidate]: + """Search each phrase independently so one request failure does not discard other hits.""" + candidates = [] + for phrase in phrases: + try: + hits = search_central_objects(phrase, max_results=max_results) + if not hits: + _logger.info("Central found no usable asset for %r", phrase) + candidates.extend(hits) + except Exception as exc: + _logger.warning("Central search failed for %r: %s", phrase, exc) + return candidates + + +def search_central_objects(phrase: str, *, max_results: int) -> list[SimReadyObjectCandidate]: + """Search Central and retain only locally validated single-body props.""" + assert max_results > 0, "max_results must be positive" + response = requests.post( + CENTRAL_SEARCH_URL, + headers={"x-usdsearch-storage-backend": "usdsearch-dev"}, + json={ + "hybrid_text_query": phrase, + "file_extension_include": "usd*", + "filter_by_properties_numeric": "simready_metadata_rigid_body_count=1", + "return_usd_properties": True, + "limit": max(MIN_INSPECTED_MATCHES_PER_PHRASE, max_results), + }, + timeout=30, + ) + response.raise_for_status() + candidates = [] + for hit in response.json()["hits"]: + try: + source = hit["source"] + # Fail closed even if a backend ignores the numeric filter. + counts = [ + prop.get("value_numeric", prop.get("value")) + for prop in source.get("usd_properties", []) + if prop.get("name") == "simready_metadata_rigid_body_count" + ] + if not counts or any(float(count) != 1 for count in counts): + continue + usd_path = download_and_wrap_central_asset(source["path"]) + candidates.append( + SimReadyObjectCandidate( + search_phrase=phrase, + usd_path=usd_path, + tags=("sim-ready", "single-rigid-body"), + relevance_score=hit.get("rrf_score"), + source_url=source["path"].replace("hf://", "https://", 1), + ) + ) + except Exception as exc: + _logger.info("Skipping Central asset %s: %s", hit.get("id"), exc) + continue + if len(candidates) >= max_results: + break + return candidates diff --git a/isaaclab_arena/agentic_environment_generation/simready_ga_assets.py b/isaaclab_arena/agentic_environment_generation/simready_ga_assets.py new file mode 100644 index 0000000000..8b86e4e9cf --- /dev/null +++ b/isaaclab_arena/agentic_environment_generation/simready_ga_assets.py @@ -0,0 +1,183 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Search GA, S3, and service assets through the simready-search AssetLibrary API.""" +from __future__ import annotations + +import asyncio +import logging +import re +from typing import TYPE_CHECKING, Any + +from isaaclab_arena.agentic_environment_generation.simready_search_interface import ( + MIN_INSPECTED_MATCHES_PER_PHRASE, + SimReadyObjectCandidate, + SimReadySourceKind, +) +from isaaclab_arena.assets.simready_constants import ( + DEFAULT_SIMREADY_SERVICE_URL, + ISAAC_SIMREADY_GA_S3_URL, + SIMREADY_PHYSICS_VARIANTS, +) +from isaaclab_arena.utils.usd.rigid_bodies import read_asset_rigid_body_paths + +if TYPE_CHECKING: + from simready.search import AssetLibrary +_logger = logging.getLogger(__name__) + + +class AssetLibrarySearchBackend: + """Search a GA, S3, or service source through AssetLibrary.""" + + def __init__(self, source: SimReadySourceKind, *, s3_url: str, service_url: str): + self.library = _create_asset_library_for_source(source, s3_url=s3_url, service_url=service_url) + + def search(self, phrases: list[str], max_results: int) -> list[SimReadyObjectCandidate]: + """Search each phrase and retain single-rigid-body candidates.""" + if self.library is None: + return [] + candidates = [] + for phrase in phrases: + try: + # Query the simready-search API through its AssetLibrary backend. + hits = _search_library_rigid_body_candidates(self.library, phrase, max_results=max_results) + if not hits: + _logger.info("simready found no usable asset for %r", phrase) + candidates.extend(hits) + except Exception as exc: + _logger.warning("simready search failed for %r: %s", phrase, exc) + return candidates + + +_CAMEL_CASE_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") +_NON_ALPHANUMERIC = re.compile(r"[^a-z0-9]+") + + +def _create_asset_library_for_source(source: SimReadySourceKind, *, s3_url: str, service_url: str) -> Any | None: + """Build the SimReady asset library for the configured source, or None if the source is unknown.""" + if source not in (SimReadySourceKind.ISAAC_SIM_GA, SimReadySourceKind.S3, SimReadySourceKind.SERVICE): + _logger.error("unknown simready source: %s", source) + return None + + # Imported here rather than at module scope only to keep the AWS stack out of the import path + # of every caller. + from simready.search import AssetLibrary + + library = AssetLibrary() + if source in (SimReadySourceKind.ISAAC_SIM_GA, SimReadySourceKind.S3): + asyncio.run(library.add_s3_source(s3_url or ISAAC_SIMREADY_GA_S3_URL)) + else: + library.add_service_source(service_url or DEFAULT_SIMREADY_SERVICE_URL) + return library + + +def _search_library_rigid_body_candidates( + library: AssetLibrary, phrase: str, *, max_results: int +) -> list[SimReadyObjectCandidate]: + """Search the library by phrase and retain ranked assets with exactly one rigid body.""" + matches = _filter_and_rank_path_matches(library.search(include_any=_build_path_filters_from_phrase(phrase)), phrase) + + # Inspect a few more hits than are wanted back, so a rejected one can be replaced rather than + # costing the phrase a result. Past that, give up and let the agent fall back to the Arena + # asset registry: every further hit inspected is another asset downloaded. + ranked = matches[: max(MIN_INSPECTED_MATCHES_PER_PHRASE, max_results)] + if len(ranked) < len(matches): + _logger.info("simready search checked the best %d of %d hits for %r", len(ranked), len(matches), phrase) + + candidates: list[SimReadyObjectCandidate] = [] + for match in ranked: + usd_path = str(match.asset_path) + is_valid, rejection_reason = _has_single_rigid_body(usd_path) + if is_valid: + candidates.append(_build_simready_object_candidate_from_match(match, phrase)) + if len(candidates) >= max_results: + break + else: + _logger.info("simready rejected %s for %r: %s", usd_path, phrase, rejection_reason) + return candidates + + +def _build_path_filters_from_phrase(phrase: str) -> list[Any]: + """Build one path filter per word of the phrase, so a hit only has to match one word.""" + from simready.search import SearchFilterPathContains + + return [SearchFilterPathContains(word) for word in _split_phrase_into_words(phrase)] + + +def _filter_and_rank_path_matches(matches: list[Any], phrase: str) -> list[Any]: + """Keep only the hits named after the object being asked for, best match first.""" + words = _split_phrase_into_words(phrase) + assert words, "a search phrase needs at least one word" + object_word = words[-1] + kept = [ + match + for match in matches + if _path_words_match_singular_or_plural(object_word, _split_path_into_words(str(match.asset_path))) + ] + # The sort is stable, so hits that tie on both counts keep the order the search gave them. + kept.sort( + key=lambda match: ( + getattr(match, "relevance_score", None) or 0.0, + _count_phrase_words_in_path(phrase, str(match.asset_path)), + ), + reverse=True, + ) + return kept + + +def _has_single_rigid_body(usd_path: str) -> tuple[bool, str]: + """Check that a readable USD contains exactly one rigid body with the SimReady variants selected. + + Args: + usd_path: The asset to look at, local or remote. + + Returns: + Whether exactly one rigid body was found, and a reason when the check fails, such as + "it has no rigid body". The phrase is empty when the asset is usable. + """ + try: + rigid_body_paths = read_asset_rigid_body_paths(usd_path, SIMREADY_PHYSICS_VARIANTS) + except Exception as exc: + return False, f"its USD could not be read: {exc}" + # Only one RigidBodyAPI is allowed on a USD asset. + if len(rigid_body_paths) == 1: + return True, "" + if not rigid_body_paths: + return False, "it has no rigid body" + return False, f"it has {len(rigid_body_paths)} rigid bodies" + + +def _build_simready_object_candidate_from_match(match: Any, phrase: str) -> SimReadyObjectCandidate: + return SimReadyObjectCandidate( + search_phrase=phrase, + usd_path=str(match.asset_path), + tags=tuple(dict.fromkeys(("sim-ready", *_split_phrase_into_words(phrase)))), + relevance_score=getattr(match, "relevance_score", None), + ) + + +def _split_phrase_into_words(phrase: str) -> list[str]: + """Split a search phrase into its lowercase words, in order.""" + return [word for word in _NON_ALPHANUMERIC.split(phrase.lower()) if word] + + +def _split_path_into_words(asset_path: str) -> frozenset[str]: + """Split an asset path into lowercase words, also splitting camelCase.""" + # e.g. trashcan_wheeled_green_a01_01.usd -> trashcan wheeled green a01 01 + spaced = _CAMEL_CASE_BOUNDARY.sub(" ", asset_path) + return frozenset(word for word in _NON_ALPHANUMERIC.split(spaced.lower()) if word) + + +def _path_words_match_singular_or_plural(word: str, path_words: frozenset[str]) -> bool: + """True if the word is one of the path words. A trailing "s" on either side is ignored.""" + if word in path_words or f"{word}s" in path_words: + return True + return word.endswith("s") and word[:-1] in path_words + + +def _count_phrase_words_in_path(phrase: str, asset_path: str) -> int: + """Count how many words of the phrase appear as whole words in the asset path.""" + path_words = _split_path_into_words(asset_path) + return sum(1 for word in _split_phrase_into_words(phrase) if _path_words_match_singular_or_plural(word, path_words)) diff --git a/isaaclab_arena/agentic_environment_generation/simready_search_interface.py b/isaaclab_arena/agentic_environment_generation/simready_search_interface.py new file mode 100644 index 0000000000..b119281d5d --- /dev/null +++ b/isaaclab_arena/agentic_environment_generation/simready_search_interface.py @@ -0,0 +1,87 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Shared types and protocol for SimReady search backends.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Protocol + +from isaaclab_arena.assets.simready_constants import ( + DEFAULT_SIMREADY_SERVICE_URL, + ISAAC_SIMREADY_GA_S3_URL, + SIMREADY_USD_OBJECT_REGISTRY_NAME, +) + +MIN_INSPECTED_MATCHES_PER_PHRASE = 5 +"""How many hits per phrase to inspect for rigid bodies, unless more results than this were asked for.""" + + +class SimReadySourceKind(str, Enum): + """Which SimReady backend to search.""" + + ISAAC_SIM_GA = "isaac-sim-ga" + S3 = "s3" + SERVICE = "service" + CENTRAL = "simready-central" + + +@dataclass +class SimReadySearchConfig: + """Where to search for SimReady objects and how many hits to keep.""" + + sources: list[SimReadySourceKind] = field(default_factory=lambda: [SimReadySourceKind.ISAAC_SIM_GA]) + s3_url: str = ISAAC_SIMREADY_GA_S3_URL + service_url: str = DEFAULT_SIMREADY_SERVICE_URL + max_results_per_object: int = 1 + + +@dataclass(frozen=True) +class SimReadyObjectCandidate: + """One SimReady search hit exposed to spec inference.""" + + search_phrase: str + usd_path: str + tags: tuple[str, ...] = ("sim-ready",) + relevance_score: float | None = None + source_url: str | None = None + + @property + def registry_name(self) -> str: + return SIMREADY_USD_OBJECT_REGISTRY_NAME + + +@dataclass +class SimReadyCandidateCatalogue: + """The SimReady hits found for one prompt, handed to spec inference.""" + + candidates: list[SimReadyObjectCandidate] = field(default_factory=list) + + unmatched_phrases: list[str] = field(default_factory=list) + """Objects the search found no usable asset for, so they are left out of the spec.""" + + +class SimReadySearchBackend(Protocol): + """Return validated candidates for object phrases from one backend.""" + + def search(self, phrases: list[str], max_results: int) -> list[SimReadyObjectCandidate]: + """Search each phrase, retaining at most max_results candidates per phrase.""" + ... + + +def simready_search_config_from_cli( + sources: list[str], + s3_url: str | None, + service_url: str | None, + max_results_per_object: int, +) -> SimReadySearchConfig: + """Build a search configuration from CLI or GUI arguments, filling in the default URLs.""" + return SimReadySearchConfig( + sources=[SimReadySourceKind(source) for source in dict.fromkeys(sources)], + s3_url=s3_url or ISAAC_SIMREADY_GA_S3_URL, + service_url=service_url or DEFAULT_SIMREADY_SERVICE_URL, + max_results_per_object=max_results_per_object, + ) diff --git a/isaaclab_arena/agentic_environment_generation/simready_central.py b/isaaclab_arena/assets/simready_central_asset_loader.py similarity index 56% rename from isaaclab_arena/agentic_environment_generation/simready_central.py rename to isaaclab_arena/assets/simready_central_asset_loader.py index 126e399c6e..dd6b3d3cb8 100644 --- a/isaaclab_arena/agentic_environment_generation/simready_central.py +++ b/isaaclab_arena/assets/simready_central_asset_loader.py @@ -3,71 +3,13 @@ # # SPDX-License-Identifier: Apache-2.0 -"""Discover and prepare single-body props from Hugging Face SimReady Central.""" - +"""Download Central asset packages and create validated PhysX USD wrappers.""" from __future__ import annotations import hashlib -import logging import re from pathlib import Path, PurePosixPath -import requests - -from isaaclab_arena.agentic_environment_generation.simready_asset_search import ( - MIN_INSPECTED_MATCHES_PER_PHRASE, - SimReadyObjectCandidate, -) - -CENTRAL_SEARCH_URL = "https://nvidia-simready-central.hf.space/search_hybrid" -_logger = logging.getLogger(__name__) - - -def search_central_objects(phrase: str, *, max_results: int) -> list[SimReadyObjectCandidate]: - """Search Central and retain only locally validated single-body props.""" - assert max_results > 0, "max_results must be positive" - response = requests.post( - CENTRAL_SEARCH_URL, - headers={"x-usdsearch-storage-backend": "usdsearch-dev"}, - json={ - "hybrid_text_query": phrase, - "file_extension_include": "usd*", - "filter_by_properties_numeric": "simready_metadata_rigid_body_count=1", - "return_usd_properties": True, - "limit": max(MIN_INSPECTED_MATCHES_PER_PHRASE, max_results), - }, - timeout=30, - ) - response.raise_for_status() - candidates = [] - for hit in response.json()["hits"]: - try: - source = hit["source"] - # Fail closed even if a backend ignores the numeric filter. - counts = [ - prop.get("value_numeric", prop.get("value")) - for prop in source.get("usd_properties", []) - if prop.get("name") == "simready_metadata_rigid_body_count" - ] - if not counts or any(float(count) != 1 for count in counts): - continue - usd_path = _download_and_wrap_central_asset(source["path"]) - candidates.append( - SimReadyObjectCandidate( - search_phrase=phrase, - usd_path=usd_path, - tags=("sim-ready", "single-rigid-body"), - relevance_score=hit.get("rrf_score"), - source_url=source["path"].replace("hf://", "https://", 1), - ) - ) - except Exception as exc: - _logger.info("Skipping Central asset %s: %s", hit.get("id"), exc) - continue - if len(candidates) >= max_results: - break - return candidates - def _parse_central_asset_url(url: str) -> tuple[str, str, str, str]: """Parse a revision-pinned Central dataset URL and its asset package folder.""" @@ -90,16 +32,31 @@ def _parse_central_asset_url(url: str) -> tuple[str, str, str, str]: def _download_central_asset_package_to_cache(url: str) -> str: """Download the pinned asset package and return its cached USD entrypoint.""" - from huggingface_hub import snapshot_download + from concurrent.futures import ThreadPoolExecutor + + from huggingface_hub import HfApi, hf_hub_download + from huggingface_hub.hf_api import RepoFile repo_id, revision, filename, folder = _parse_central_asset_url(url) - snapshot = snapshot_download( - repo_id=repo_id, repo_type="dataset", revision=revision, allow_patterns=[f"{folder}/*"] - ) - return str(Path(snapshot) / filename) + # List only this package: a dataset-wide snapshot scan is expensive for large libraries. + files = [ + entry.path + for entry in HfApi().list_repo_tree( + repo_id=repo_id, repo_type="dataset", revision=revision, path_in_repo=folder, recursive=True + ) + if isinstance(entry, RepoFile) + ] + assert filename in files, "asset package is missing its USD entrypoint" + + def download_file(path: str) -> str: + return hf_hub_download(repo_id=repo_id, repo_type="dataset", revision=revision, filename=path) + + with ThreadPoolExecutor(max_workers=8) as executor: + cached_paths = dict(zip(files, executor.map(download_file, files), strict=True)) + return cached_paths[filename] -def _download_and_wrap_central_asset(url: str) -> str: +def download_and_wrap_central_asset(url: str) -> str: """Download a Central package, validate its rigid body, and write a cached PhysX USD wrapper.""" from pxr import Usd, UsdGeom, UsdPhysics diff --git a/isaaclab_arena/assets/simready_object_library.py b/isaaclab_arena/assets/simready_object_library.py index e6168f6f6f..bef856fb5d 100644 --- a/isaaclab_arena/assets/simready_object_library.py +++ b/isaaclab_arena/assets/simready_object_library.py @@ -47,9 +47,9 @@ def __init__( ): assert usd_path, "simready_usd_object requires params.usd_path" if usd_path.startswith(("https://huggingface.co/datasets/", "hf://huggingface.co/datasets/")): - from isaaclab_arena.agentic_environment_generation.simready_central import _download_and_wrap_central_asset + from isaaclab_arena.assets.simready_central_asset_loader import download_and_wrap_central_asset - usd_path = _download_and_wrap_central_asset(usd_path) + usd_path = download_and_wrap_central_asset(usd_path) spawn_cfg_addon = dict(kwargs.pop("spawn_cfg_addon", None) or {}) spawn_cfg_addon.setdefault("variants", dict(SIMREADY_PHYSICS_VARIANTS)) super().__init__( diff --git a/isaaclab_arena/tests/test_environment_generation_agent.py b/isaaclab_arena/tests/test_environment_generation_agent.py index f5b1e24b92..39a6f9bd0f 100644 --- a/isaaclab_arena/tests/test_environment_generation_agent.py +++ b/isaaclab_arena/tests/test_environment_generation_agent.py @@ -18,7 +18,7 @@ build_task_catalogue, ) from isaaclab_arena.agentic_environment_generation.environment_generation_agent import EnvironmentGenerationAgent -from isaaclab_arena.agentic_environment_generation.simready_asset_search import ( +from isaaclab_arena.agentic_environment_generation.simready_search_interface import ( SimReadyCandidateCatalogue, SimReadyObjectCandidate, ) diff --git a/isaaclab_arena/tests/test_simready_asset_search.py b/isaaclab_arena/tests/test_simready_asset_search.py index b611009b0b..64555c69ed 100644 --- a/isaaclab_arena/tests/test_simready_asset_search.py +++ b/isaaclab_arena/tests/test_simready_asset_search.py @@ -10,20 +10,24 @@ import logging from unittest.mock import patch -from isaaclab_arena.agentic_environment_generation.simready_asset_search import ( - SimReadyObjectCandidate, - SimReadySearchConfig, - SimReadySourceKind, +import pytest + +from isaaclab_arena.agentic_environment_generation.simready_asset_search import search_simready_objects +from isaaclab_arena.agentic_environment_generation.simready_ga_assets import ( _count_phrase_words_in_path, _filter_and_rank_path_matches, _has_single_rigid_body, _split_path_into_words, - search_simready_objects, +) +from isaaclab_arena.agentic_environment_generation.simready_search_interface import ( + SimReadyObjectCandidate, + SimReadySearchConfig, + SimReadySourceKind, simready_search_config_from_cli, ) from isaaclab_arena.assets.simready_constants import SIMREADY_USD_OBJECT_REGISTRY_NAME -SEARCH_LOGGER = "isaaclab_arena.agentic_environment_generation.simready_asset_search" +SEARCH_LOGGER = "isaaclab_arena.agentic_environment_generation.simready_ga_assets" """Where the search reports its progress, now that the agent's traces carry only errors.""" CABINET_PATH = "SimReady/Residential/Kitchen/Cabinets/Cabinet_D01/sm_fixture_cabinet_d01_01.usd" @@ -31,15 +35,11 @@ TRASH_CAN_PATH = "SimReady/Residential/Garage/sm_trashCan_wheeled_green_a01_01.usd" PLAIN_TRASH_CAN_PATH = "SimReady/Residential/Kitchen/sm_trashCan_a01_01.usd" -RIGID_BODY_CHECK_TARGET = "isaaclab_arena.agentic_environment_generation.simready_asset_search._has_single_rigid_body" -# Patched where it is looked up, not where it is defined: simready_asset_search binds the name at +RIGID_BODY_CHECK_TARGET = "isaaclab_arena.agentic_environment_generation.simready_ga_assets._has_single_rigid_body" +# Patched where it is looked up, not where it is defined: simready_ga_assets binds the name at # import time, so patching the defining module would leave that binding untouched. -RIGID_BODY_PATHS_TARGET = ( - "isaaclab_arena.agentic_environment_generation.simready_asset_search.read_asset_rigid_body_paths" -) -PATH_FILTERS_TARGET = ( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._build_path_filters_from_phrase" -) +RIGID_BODY_PATHS_TARGET = "isaaclab_arena.agentic_environment_generation.simready_ga_assets.read_asset_rigid_body_paths" +PATH_FILTERS_TARGET = "isaaclab_arena.agentic_environment_generation.simready_ga_assets._build_path_filters_from_phrase" # Stubbed wherever the fake library stands in for a real one: building a path filter needs the # simready package, which only ships with Isaac Sim, and the fake library ignores the filters. @@ -124,7 +124,7 @@ def test_has_single_rigid_body_turns_down_an_unreadable_asset(): @patch(PATH_FILTERS_TARGET, return_value=[]) @patch( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._create_asset_library_for_source", + "isaaclab_arena.agentic_environment_generation.simready_ga_assets._create_asset_library_for_source", ) def test_search_falls_back_to_the_next_hit_when_one_is_rejected(mock_configure, mock_path_filters, caplog): mock_configure.return_value = _FakeLibrary([_FakeMatch(TRASH_CAN_PATH), _FakeMatch(PLAIN_TRASH_CAN_PATH)]) @@ -140,7 +140,7 @@ def test_search_falls_back_to_the_next_hit_when_one_is_rejected(mock_configure, @patch(PATH_FILTERS_TARGET, return_value=[]) @patch( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._create_asset_library_for_source", + "isaaclab_arena.agentic_environment_generation.simready_ga_assets._create_asset_library_for_source", ) def test_search_reports_a_phrase_with_only_rejected_hits_as_unmatched(mock_configure, mock_path_filters, caplog): mock_configure.return_value = _FakeLibrary([_FakeMatch(CABINET_PATH)]) @@ -164,10 +164,10 @@ def test_simready_search_config_from_cli_defaults(): @patch( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._search_library_rigid_body_candidates", + "isaaclab_arena.agentic_environment_generation.simready_ga_assets._search_library_rigid_body_candidates", ) @patch( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._create_asset_library_for_source", + "isaaclab_arena.agentic_environment_generation.simready_ga_assets._create_asset_library_for_source", ) def test_search_simready_objects_returns_candidates(mock_configure, mock_search_library_rigid_body_candidates): mock_configure.return_value = _FakeLibrary() @@ -184,7 +184,7 @@ def test_search_simready_objects_returns_candidates(mock_configure, mock_search_ def test_create_asset_library_for_source_records_an_unknown_source(caplog): - from isaaclab_arena.agentic_environment_generation.simready_asset_search import _create_asset_library_for_source + from isaaclab_arena.agentic_environment_generation.simready_ga_assets import _create_asset_library_for_source config = SimReadySearchConfig() with caplog.at_level(logging.ERROR, logger=SEARCH_LOGGER): @@ -196,7 +196,7 @@ def test_create_asset_library_for_source_records_an_unknown_source(caplog): @patch( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._create_asset_library_for_source", + "isaaclab_arena.agentic_environment_generation.simready_ga_assets._create_asset_library_for_source", ) def test_search_simready_objects_returns_empty_when_library_unavailable(mock_configure): mock_configure.return_value = None @@ -205,10 +205,10 @@ def test_search_simready_objects_returns_empty_when_library_unavailable(mock_con @patch( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._search_library_rigid_body_candidates", + "isaaclab_arena.agentic_environment_generation.simready_ga_assets._search_library_rigid_body_candidates", ) @patch( - "isaaclab_arena.agentic_environment_generation.simready_asset_search._create_asset_library_for_source", + "isaaclab_arena.agentic_environment_generation.simready_ga_assets._create_asset_library_for_source", ) def test_search_simready_objects_records_no_matches(mock_configure, mock_search_library_rigid_body_candidates, caplog): mock_configure.return_value = _FakeLibrary() @@ -227,7 +227,7 @@ def test_multiple_sources_merge_and_deduplicate_candidates(): patch(f"{SEARCH_LOGGER}._create_asset_library_for_source", return_value=object()), patch(f"{SEARCH_LOGGER}._search_library_rigid_body_candidates", side_effect=[[shared], []]), patch( - "isaaclab_arena.agentic_environment_generation.simready_central.search_central_objects", + "isaaclab_arena.agentic_environment_generation.simready_central_assets.search_central_objects", side_effect=[[shared, central], []], ), ): @@ -244,7 +244,7 @@ def test_multiple_sources_keep_central_when_library_initialization_fails(): with ( patch(f"{SEARCH_LOGGER}._create_asset_library_for_source", side_effect=RuntimeError("unavailable")), patch( - "isaaclab_arena.agentic_environment_generation.simready_central.search_central_objects", + "isaaclab_arena.agentic_environment_generation.simready_central_assets.search_central_objects", return_value=[central], ), ): @@ -261,7 +261,7 @@ def test_multiple_sources_keep_library_when_central_fails(): patch(f"{SEARCH_LOGGER}._create_asset_library_for_source", return_value=object()), patch(f"{SEARCH_LOGGER}._search_library_rigid_body_candidates", return_value=[candidate]), patch( - "isaaclab_arena.agentic_environment_generation.simready_central.search_central_objects", + "isaaclab_arena.agentic_environment_generation.simready_central_assets.search_central_objects", side_effect=RuntimeError("unavailable"), ), ): @@ -284,7 +284,7 @@ def test_source_list_searches_selected_backends_once(): patch(f"{SEARCH_LOGGER}._create_asset_library_for_source", return_value=object()) as configure, patch(f"{SEARCH_LOGGER}._search_library_rigid_body_candidates", return_value=[]) as library_search, patch( - "isaaclab_arena.agentic_environment_generation.simready_central.search_central_objects", + "isaaclab_arena.agentic_environment_generation.simready_central_assets.search_central_objects", return_value=[], ) as central_search, ): @@ -293,3 +293,21 @@ def test_source_list_searches_selected_backends_once(): library_search.assert_called_once() central_search.assert_called_once_with("egg", max_results=2) assert result.unmatched_phrases == ["egg"] + + +@pytest.mark.parametrize("source", [SimReadySourceKind.ISAAC_SIM_GA, SimReadySourceKind.CENTRAL]) +def test_backend_keeps_other_phrase_results_when_one_phrase_fails(source): + candidate = SimReadyObjectCandidate(search_phrase="wrench", usd_path="wrench.usd") + target = ( + f"{SEARCH_LOGGER}._search_library_rigid_body_candidates" + if source == SimReadySourceKind.ISAAC_SIM_GA + else "isaaclab_arena.agentic_environment_generation.simready_central_assets.search_central_objects" + ) + with ( + patch(f"{SEARCH_LOGGER}._create_asset_library_for_source", return_value=object()), + patch(target, side_effect=[RuntimeError("unavailable"), [candidate]]) as search, + ): + result = search_simready_objects(["missing", "wrench"], SimReadySearchConfig(sources=[source])) + assert search.call_count == 2 + assert result.candidates == [candidate] + assert result.unmatched_phrases == ["missing"] diff --git a/isaaclab_arena/tests/test_simready_central.py b/isaaclab_arena/tests/test_simready_central.py index 538b699592..21d061538f 100644 --- a/isaaclab_arena/tests/test_simready_central.py +++ b/isaaclab_arena/tests/test_simready_central.py @@ -9,17 +9,16 @@ import pytest -from isaaclab_arena.agentic_environment_generation.simready_asset_search import ( +from isaaclab_arena.agentic_environment_generation.simready_asset_search import search_simready_objects +from isaaclab_arena.agentic_environment_generation.simready_central_assets import search_central_objects +from isaaclab_arena.agentic_environment_generation.simready_search_interface import ( SimReadySearchConfig, SimReadySourceKind, - search_simready_objects, -) -from isaaclab_arena.agentic_environment_generation.simready_central import ( - _parse_central_asset_url, - search_central_objects, ) +from isaaclab_arena.assets.simready_central_asset_loader import _parse_central_asset_url -MODULE = "isaaclab_arena.agentic_environment_generation.simready_central" +MODULE = "isaaclab_arena.agentic_environment_generation.simready_central_assets" +LOADER = "isaaclab_arena.assets.simready_central_asset_loader" REVISION = "a" * 40 URL = f"hf://huggingface.co/datasets/nvidia/simready-assets/resolve/{REVISION}/pallet/simready_usd/pallet.usd" @@ -35,7 +34,7 @@ def test_central_requires_single_body_metadata_and_skips_invalid_usd(): with ( patch(f"{MODULE}.requests.post", return_value=response) as post, patch( - f"{MODULE}._download_and_wrap_central_asset", side_effect=[ValueError("articulation"), "/cache/pallet.usda"] + f"{MODULE}.download_and_wrap_central_asset", side_effect=[ValueError("articulation"), "/cache/pallet.usda"] ) as prepare, ): candidates = search_central_objects("pallet", max_results=1) @@ -80,7 +79,7 @@ def _test_central_usd_validation(simulation_app): from pxr import Usd, UsdGeom, UsdPhysics - from isaaclab_arena.agentic_environment_generation.simready_central import _download_and_wrap_central_asset + from isaaclab_arena.assets.simready_central_asset_loader import download_and_wrap_central_asset with tempfile.TemporaryDirectory() as directory: folder = Path(directory) @@ -93,25 +92,25 @@ def _test_central_usd_validation(simulation_app): body.CreateRigidBodyEnabledAttr(True) stage.GetRootLayer().Save() with ( - patch("huggingface_hub.snapshot_download", return_value=directory), - patch(f"{MODULE}.Path.home", return_value=folder), + patch(f"{LOADER}._download_central_asset_package_to_cache", return_value=str(asset)), + patch(f"{LOADER}.Path.home", return_value=folder), ): - wrapper = _download_and_wrap_central_asset(URL) + wrapper = download_and_wrap_central_asset(URL) assert Usd.Stage.Open(wrapper).GetDefaultPrim().HasAPI(UsdPhysics.RigidBodyAPI) body.GetRigidBodyEnabledAttr().Set(False) stage.GetRootLayer().Save() with pytest.raises(AssertionError, match="disabled"): - _download_and_wrap_central_asset(URL) + download_and_wrap_central_asset(URL) body.GetRigidBodyEnabledAttr().Set(True) UsdPhysics.ArticulationRootAPI.Apply(root) stage.GetRootLayer().Save() with pytest.raises(AssertionError, match="articulation root"): - _download_and_wrap_central_asset(URL) + download_and_wrap_central_asset(URL) root.RemoveAPI(UsdPhysics.ArticulationRootAPI) UsdPhysics.RigidBodyAPI.Apply(UsdGeom.Xform.Define(stage, "/Pallet/Other").GetPrim()) stage.GetRootLayer().Save() with pytest.raises(AssertionError, match="found 2"): - _download_and_wrap_central_asset(URL) + download_and_wrap_central_asset(URL) return True @@ -119,3 +118,26 @@ def test_central_usd_validation(): from isaaclab_arena.tests.utils.persistent_simulation_app import run_function_with_persistent_simulation_app assert run_function_with_persistent_simulation_app(_test_central_usd_validation) + + +def test_central_download_lists_only_the_selected_package(): + from huggingface_hub.hf_api import RepoFile + + from isaaclab_arena.assets.simready_central_asset_loader import _download_central_asset_package_to_cache + + filenames = ["pallet/simready_usd/pallet.usd", "pallet/textures/color.png"] + entries = [RepoFile(path=path, size=1, oid="a" * 40) for path in filenames] + with ( + patch("huggingface_hub.HfApi.list_repo_tree", return_value=iter(entries)) as list_tree, + patch("huggingface_hub.hf_hub_download", side_effect=lambda **kw: "/cache/" + kw["filename"]) as download, + ): + assert _download_central_asset_package_to_cache(URL) == "/cache/" + filenames[0] + list_tree.assert_called_once_with( + repo_id="nvidia/simready-assets", + repo_type="dataset", + revision=REVISION, + path_in_repo="pallet", + recursive=True, + ) + assert {call.kwargs["filename"] for call in download.call_args_list} == set(filenames) + assert all(call.kwargs["revision"] == REVISION for call in download.call_args_list) diff --git a/isaaclab_arena_environments/maple_table_top/tabletop_pick_place_egg_tray.yaml b/isaaclab_arena_environments/maple_table_top/tabletop_pick_place_egg_tray.yaml deleted file mode 100644 index 0e3386d443..0000000000 --- a/isaaclab_arena_environments/maple_table_top/tabletop_pick_place_egg_tray.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -env_name: tabletop_pick_place_egg_tray -embodiment: - id: franka - registry_name: franka_ik - params: {} -background: - id: maple_table - registry_name: maple_table_robolab - params: {} -objects: -- id: egg - registry_name: simready_usd_object - params: - usd_path: https://huggingface.co/datasets/PalatialSim/palatial-nv-asset-library/resolve/69a87329036cec0440403351c12230970c4e67d8/D00287/simready_usd/D00287.usd -- id: egg_2 - registry_name: simready_usd_object - params: - usd_path: https://huggingface.co/datasets/PalatialSim/palatial-nv-asset-library/resolve/69a87329036cec0440403351c12230970c4e67d8/D00287/simready_usd/D00287.usd -- id: egg_3 - registry_name: simready_usd_object - params: - usd_path: https://huggingface.co/datasets/PalatialSim/palatial-nv-asset-library/resolve/69a87329036cec0440403351c12230970c4e67d8/D00287/simready_usd/D00287.usd -- id: egg_tray - registry_name: simready_usd_object - params: - usd_path: https://huggingface.co/datasets/PalatialSim/palatial-nv-asset-library/resolve/69a87329036cec0440403351c12230970c4e67d8/D00290/simready_usd/D00290.usd -- id: nut - registry_name: simready_usd_object - params: - usd_path: https://huggingface.co/datasets/PalatialSim/palatial-nv-asset-library/resolve/69a87329036cec0440403351c12230970c4e67d8/D00554/simready_usd/D00554.usd -- id: pizza - registry_name: simready_usd_object - params: - usd_path: https://huggingface.co/datasets/PalatialSim/palatial-nv-asset-library/resolve/69a87329036cec0440403351c12230970c4e67d8/D00802/simready_usd/D00802.usd -relations: -- kind: is_anchor - subject: maple_table - params: {} -- kind: 'on' - subject: egg - reference: maple_table - params: {} -- kind: 'on' - subject: egg_2 - reference: maple_table - params: {} -- kind: 'on' - subject: egg_3 - reference: maple_table - params: {} -- kind: 'on' - subject: egg_tray - reference: maple_table - params: {} -- kind: 'on' - subject: nut - reference: maple_table - params: {} -- kind: 'on' - subject: pizza - reference: maple_table - params: {} -- kind: next_to - subject: egg - reference: egg_tray - params: - side: positive_x -task: - composition: atomic - description: Franka picks up the egg from the maple table and places it into the - egg tray, leaving two additional eggs, the nut, and pizza as distractors. - subtasks: - - kind: PickAndPlaceTask - params: - pick_up_object: egg - destination_location: egg_tray - background_scene: maple_table diff --git a/isaaclab_arena_environments/maple_table_top/tabletop_pick_place_screwdriver_bearing_wrench.yaml b/isaaclab_arena_environments/maple_table_top/tabletop_pick_place_screwdriver_bearing_wrench.yaml new file mode 100644 index 0000000000..c2e243c9ec --- /dev/null +++ b/isaaclab_arena_environments/maple_table_top/tabletop_pick_place_screwdriver_bearing_wrench.yaml @@ -0,0 +1,55 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +# Assets: ForgeCAD/simready-assets and nvidia/simready-assets, CC-BY-4.0. +# Credit ForgeCAD (screwdriver) and NVIDIA (bearing and wrench). +env_name: tabletop_pick_place_screwdriver_bearing_wrench +embodiment: + id: franka + registry_name: franka_ik + params: {} +background: + id: maple_table + registry_name: maple_table_robolab + params: {} +objects: +- id: screwdriver + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/ForgeCAD/simready-assets/resolve/828771cf28d7fc2c46a7a29eadfcf7bfcc6958f8/screwdriver/simready_usd/screwdriver.usda +- id: ball_bearing + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/nvidia/simready-assets/resolve/653d91e182011f357cf1644013cc94ddf194d349/consumable_bearing_ball_a01/simready_usd/sm_consumable_bearing_ball_a01_01.usd +- id: combination_wrench + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/nvidia/simready-assets/resolve/653d91e182011f357cf1644013cc94ddf194d349/handTool_wrench_combination_a01/simready_usd/sm_handTool_wrench_combination_a01_01.usd +relations: +- kind: is_anchor + subject: maple_table + params: {} +- kind: 'on' + subject: screwdriver + reference: maple_table + params: {} +- kind: 'on' + subject: ball_bearing + reference: maple_table + params: {} +- kind: 'on' + subject: combination_wrench + reference: maple_table + params: {} +task: + composition: atomic + description: Franka picks up the screwdriver and places it back on the maple + table, leaving the ball bearing and combination wrench as distractors. + subtasks: + - kind: PickAndPlaceTask + params: + pick_up_object: screwdriver + destination_location: maple_table + background_scene: maple_table diff --git a/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py b/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py index bb3a5f0bf9..f26a26040e 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py +++ b/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py @@ -56,7 +56,7 @@ def add_agentic_env_gen_runner_cli_args(parser: argparse.ArgumentParser) -> None: - from isaaclab_arena.agentic_environment_generation.simready_asset_search import SimReadySourceKind + from isaaclab_arena.agentic_environment_generation.simready_search_interface import SimReadySourceKind group = parser.add_argument_group("Agentic Environment Generation Runner") group.add_argument( @@ -155,7 +155,7 @@ def resolve_env_spec(args_cli: argparse.Namespace) -> Path | None: build_task_catalogue, ) from isaaclab_arena.agentic_environment_generation.environment_generation_agent import EnvironmentGenerationAgent - from isaaclab_arena.agentic_environment_generation.simready_asset_search import simready_search_config_from_cli + from isaaclab_arena.agentic_environment_generation.simready_search_interface import simready_search_config_from_cli # The generation passes log what they searched for and found; this is a console tool, so show # it. A no-op once something else owns the root logger, such as Kit in the modes that start it. diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py index 91d2d8055a..36610624f9 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py @@ -22,7 +22,7 @@ build_task_catalogue, ) from isaaclab_arena.agentic_environment_generation.environment_generation_agent import EnvironmentGenerationAgent -from isaaclab_arena.agentic_environment_generation.simready_asset_search import ( +from isaaclab_arena.agentic_environment_generation.simready_search_interface import ( SimReadySearchConfig, SimReadySourceKind, simready_search_config_from_cli, diff --git a/isaaclab_arena_examples/tests/test_review_gui.py b/isaaclab_arena_examples/tests/test_review_gui.py index 763ba8880f..a4b65e7b7e 100644 --- a/isaaclab_arena_examples/tests/test_review_gui.py +++ b/isaaclab_arena_examples/tests/test_review_gui.py @@ -13,7 +13,7 @@ import pytest -from isaaclab_arena.agentic_environment_generation.simready_asset_search import SimReadySearchConfig +from isaaclab_arena.agentic_environment_generation.simready_search_interface import SimReadySearchConfig from isaaclab_arena.agentic_environment_generation.spec_io import env_graph_spec_path, write_env_graph_spec from isaaclab_arena.assets.object_type import ObjectType from isaaclab_arena.environment_spec.arena_env_graph_spec import ArenaEnvGraphSpec From 512bebfabf05460d3fdcaba1ba4500a2df176e5f Mon Sep 17 00:00:00 2001 From: Xinjie Yao Date: Fri, 23 Oct 2026 21:08:43 -0700 Subject: [PATCH 09/11] Remove Central scene capture from documentation Signed-off-by: Xinjie Yao --- .../tabletop_pick_place_screwdriver_bearing_wrench.png | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 docs/images/agentic_environment_generation/tabletop_pick_place_screwdriver_bearing_wrench.png diff --git a/docs/images/agentic_environment_generation/tabletop_pick_place_screwdriver_bearing_wrench.png b/docs/images/agentic_environment_generation/tabletop_pick_place_screwdriver_bearing_wrench.png deleted file mode 100644 index 29bb63d538..0000000000 --- a/docs/images/agentic_environment_generation/tabletop_pick_place_screwdriver_bearing_wrench.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1659641612e828317dd1032d230d3dbbafa6b11fcb2c77f1e5b23365f0c02aca -size 839360 From 240d7f9e701d22cb56acec120d16823499303dfe Mon Sep 17 00:00:00 2001 From: Xinjie Yao Date: Fri, 23 Oct 2026 21:16:38 -0700 Subject: [PATCH 10/11] Update lock for Hugging Face Hub dependency Signed-off-by: Xinjie Yao --- uv.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uv.lock b/uv.lock index 95ce179f2b..2729adccbf 100644 --- a/uv.lock +++ b/uv.lock @@ -1573,6 +1573,7 @@ dependencies = [ [package.optional-dependencies] dev = [ { name = "debugpy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "huggingface-hub", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "jupyter", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "simready-search", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "streamlit", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, @@ -1613,6 +1614,7 @@ openpi = [ requires-dist = [ { name = "debugpy", marker = "extra == 'dev'" }, { name = "decorator", specifier = ">=4.0.2,<5" }, + { name = "huggingface-hub", marker = "extra == 'dev'", specifier = ">=0.36" }, { name = "imageio" }, { name = "imageio-ffmpeg" }, { name = "isaacteleop", extras = ["retargeters", "ui", "cloudxr"], specifier = "==1.4.126rc1", index = "https://pypi.org/simple" }, From 7edc7df8f61bc802b9ec02bb62c442c88154be18 Mon Sep 17 00:00:00 2001 From: Xinjie Yao Date: Sun, 25 Oct 2026 15:08:59 -0700 Subject: [PATCH 11/11] Fix SimReady selection and update spray-gun scene Signed-off-by: Xinjie Yao --- .../environment_generation_agent.py | 23 ++--- .../simready_ga_assets.py | 3 +- .../assets/simready_central_asset_loader.py | 16 +++- .../assets/simready_object_library.py | 19 ++-- .../test_environment_generation_agent.py | 73 +++++++++++---- .../tests/test_simready_asset_search.py | 14 ++- isaaclab_arena/tests/test_simready_central.py | 23 ++++- .../tests/test_simready_object_library.py | 20 +++-- ...pick_place_screwdriver_bearing_wrench.yaml | 55 ------------ ..._place_spray_gun_stand_bearing_wrench.yaml | 88 +++++++++++++++++++ .../cli_runner.py | 3 - .../review_gui/generation_panel.py | 8 -- .../review_gui/streamlit_ui.py | 1 - 13 files changed, 221 insertions(+), 125 deletions(-) delete mode 100644 isaaclab_arena_environments/maple_table_top/tabletop_pick_place_screwdriver_bearing_wrench.yaml create mode 100644 isaaclab_arena_environments/maple_table_top/tabletop_pick_place_spray_gun_stand_bearing_wrench.yaml diff --git a/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py b/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py index 3f76f0c73d..0930e95a9e 100644 --- a/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py +++ b/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py @@ -8,7 +8,7 @@ from __future__ import annotations import logging -from dataclasses import asdict, replace +from dataclasses import replace from typing import Any from isaaclab_arena.agentic_environment_generation.catalogues import ( @@ -22,10 +22,7 @@ from isaaclab_arena.agentic_environment_generation.missing_object_inference import MissingObjectInference from isaaclab_arena.agentic_environment_generation.prim_path_inference import PrimPathInference from isaaclab_arena.agentic_environment_generation.simready_asset_search import search_simready_objects -from isaaclab_arena.agentic_environment_generation.simready_search_interface import ( - SimReadyObjectCandidate, - SimReadySearchConfig, -) +from isaaclab_arena.agentic_environment_generation.simready_search_interface import SimReadySearchConfig from isaaclab_arena.agentic_environment_generation.spec_inference import SpecInference from isaaclab_arena.assets.simready_constants import SIMREADY_USD_OBJECT_REGISTRY_NAME from isaaclab_arena.environment_spec.arena_env_graph_spec import ArenaEnvGraphSpec @@ -89,7 +86,6 @@ def __init__( self._traces: list[str] = [] self._unavailable_objects: list[str] = [] self._simready_usd_paths: dict[str, str] = {} - self._simready_candidates: list[SimReadyObjectCandidate] = [] @property def traces(self) -> tuple[str, ...]: @@ -111,14 +107,6 @@ def unavailable_objects(self) -> tuple[str, ...]: """ return tuple(self._unavailable_objects) - def simready_asset_report(self, spec: ArenaEnvGraphSpec | None) -> list[dict[str, Any]]: - """Return discovered assets, their source URLs, and whether the spec uses them.""" - used_paths = {obj.params.get("usd_path") for obj in spec.objects} if spec is not None else set() - return [ - {**asdict(candidate), "used_in_spec": candidate.usd_path in used_paths} - for candidate in self._simready_candidates - ] - def generate_spec( self, prompt: str, @@ -146,7 +134,6 @@ def generate_spec( self._traces = [] self._unavailable_objects = [] self._simready_usd_paths = {} - self._simready_candidates = [] asset_catalog = asset_catalog or build_asset_catalogue() relation_catalog = relation_catalog or build_relation_catalogue() task_catalog = task_catalog or build_task_catalogue() @@ -218,18 +205,18 @@ def _extend_catalogue_with_simready(self, prompt: str, asset_catalog: AssetCatal return asset_catalog search_result = search_simready_objects(phrases, self.simready_config) self._unavailable_objects = list(search_result.unmatched_phrases) - self._simready_candidates = list(search_result.candidates) if not search_result.candidates: return asset_catalog extended = replace(asset_catalog, objects=list(asset_catalog.objects)) for candidate in search_result.candidates: - asset_cls = register_searched_simready_object(candidate.search_phrase, candidate.usd_path, candidate.tags) + usd_path = candidate.source_url or candidate.usd_path + asset_cls = register_searched_simready_object(candidate.search_phrase, usd_path, candidate.tags) tags = [tag for tag in asset_cls.tags if tag != "object"] extended.objects.append({ "name": asset_cls.name, "tags": tags, "object_type": asset_cls.object_type.value, }) - self._simready_usd_paths[asset_cls.name] = candidate.usd_path + self._simready_usd_paths[asset_cls.name] = usd_path _logger.info("catalogued %r for %r: %s", asset_cls.name, candidate.search_phrase, candidate.usd_path) return extended diff --git a/isaaclab_arena/agentic_environment_generation/simready_ga_assets.py b/isaaclab_arena/agentic_environment_generation/simready_ga_assets.py index 8b86e4e9cf..cc3bd5beaa 100644 --- a/isaaclab_arena/agentic_environment_generation/simready_ga_assets.py +++ b/isaaclab_arena/agentic_environment_generation/simready_ga_assets.py @@ -67,7 +67,8 @@ def _create_asset_library_for_source(source: SimReadySourceKind, *, s3_url: str, library = AssetLibrary() if source in (SimReadySourceKind.ISAAC_SIM_GA, SimReadySourceKind.S3): - asyncio.run(library.add_s3_source(s3_url or ISAAC_SIMREADY_GA_S3_URL)) + url = ISAAC_SIMREADY_GA_S3_URL if source == SimReadySourceKind.ISAAC_SIM_GA else s3_url + asyncio.run(library.add_s3_source(url or ISAAC_SIMREADY_GA_S3_URL)) else: library.add_service_source(service_url or DEFAULT_SIMREADY_SERVICE_URL) return library diff --git a/isaaclab_arena/assets/simready_central_asset_loader.py b/isaaclab_arena/assets/simready_central_asset_loader.py index dd6b3d3cb8..3cd8f9b8bb 100644 --- a/isaaclab_arena/assets/simready_central_asset_loader.py +++ b/isaaclab_arena/assets/simready_central_asset_loader.py @@ -9,13 +9,23 @@ import hashlib import re from pathlib import Path, PurePosixPath +from urllib.parse import urlsplit + + +def is_central_asset_url(url: str) -> bool: + """Identify Hugging Face dataset URLs handled by the Central asset loader.""" + parsed = urlsplit(url) + return ( + parsed.scheme in ("https", "hf") and parsed.netloc == "huggingface.co" and parsed.path.startswith("/datasets/") + ) def _parse_central_asset_url(url: str) -> tuple[str, str, str, str]: """Parse a revision-pinned Central dataset URL and its asset package folder.""" - match = re.fullmatch( - r"(?:hf|https)://huggingface\.co/datasets/([^/]+/[^/]+)/(?:resolve|blob)/([0-9a-f]{40})/(.+)", url - ) + assert is_central_asset_url(url), "expected a Hugging Face dataset URL" + parsed = urlsplit(url) + assert not parsed.query and not parsed.fragment, "expected a plain asset URL" + match = re.fullmatch(r"/datasets/([^/]+/[^/]+)/(?:resolve|blob)/([0-9a-f]{40})/(.+)", parsed.path) assert match, "expected a pinned Hugging Face dataset URL" repo_id, revision, filename = match.groups() path = PurePosixPath(filename) diff --git a/isaaclab_arena/assets/simready_object_library.py b/isaaclab_arena/assets/simready_object_library.py index bef856fb5d..09e541e0cf 100644 --- a/isaaclab_arena/assets/simready_object_library.py +++ b/isaaclab_arena/assets/simready_object_library.py @@ -7,6 +7,7 @@ from __future__ import annotations +import hashlib import re from collections.abc import Sequence from typing import Any @@ -46,9 +47,12 @@ def __init__( **kwargs: Any, ): assert usd_path, "simready_usd_object requires params.usd_path" - if usd_path.startswith(("https://huggingface.co/datasets/", "hf://huggingface.co/datasets/")): - from isaaclab_arena.assets.simready_central_asset_loader import download_and_wrap_central_asset + from isaaclab_arena.assets.simready_central_asset_loader import ( + download_and_wrap_central_asset, + is_central_asset_url, + ) + if is_central_asset_url(usd_path): usd_path = download_and_wrap_central_asset(usd_path) spawn_cfg_addon = dict(kwargs.pop("spawn_cfg_addon", None) or {}) spawn_cfg_addon.setdefault("variants", dict(SIMREADY_PHYSICS_VARIANTS)) @@ -65,11 +69,12 @@ def __init__( ) -def simready_search_registry_name(search_phrase: str) -> str: - """Return the catalogue name used to register the SimReady asset found for a search phrase.""" +def simready_search_registry_name(search_phrase: str, usd_path: str) -> str: + """Return a stable catalogue name for a phrase and asset path.""" slug = _NON_IDENTIFIER_CHARACTERS.sub("_", search_phrase.strip().lower()).strip("_") assert slug, f"search phrase {search_phrase!r} has no characters usable in a registry name" - return f"{SIMREADY_SEARCH_REGISTRY_PREFIX}{slug}" + digest = hashlib.sha256(usd_path.encode()).hexdigest()[:16] + return f"{SIMREADY_SEARCH_REGISTRY_PREFIX}{slug}_{digest}" def register_searched_simready_object( @@ -80,14 +85,14 @@ def register_searched_simready_object( """Register a searched SimReady asset so it can be picked like any other catalogue object. Args: - search_phrase: Phrase the asset was found for; it determines the catalogue name. + search_phrase: Phrase the asset was found for; used with its path to name the catalogue entry. usd_path: USD path or URL of the found asset. tags: Extra catalogue tags to expose alongside the SimReady ones. Returns: The registered asset class. """ - registry_name = simready_search_registry_name(search_phrase) + registry_name = simready_search_registry_name(search_phrase, usd_path) registry = AssetRegistry() if registry.is_registered(registry_name, ensure_loaded=False): return registry.get_asset_by_name(registry_name) diff --git a/isaaclab_arena/tests/test_environment_generation_agent.py b/isaaclab_arena/tests/test_environment_generation_agent.py index 39a6f9bd0f..f18d70d336 100644 --- a/isaaclab_arena/tests/test_environment_generation_agent.py +++ b/isaaclab_arena/tests/test_environment_generation_agent.py @@ -215,6 +215,8 @@ def _generate_with_simready(self, agent_obj, asset_catalog=None): @patch("isaaclab_arena.agentic_environment_generation.environment_generation_agent.search_simready_objects") def test_generate_spec_catalogues_what_the_search_finds_before_inferring_the_spec(self, mock_search, agent): + from isaaclab_arena.assets.simready_object_library import simready_search_registry_name + agent_obj, client = agent mock_search.return_value = SimReadyCandidateCatalogue( candidates=[ @@ -236,12 +238,16 @@ def test_generate_spec_catalogues_what_the_search_finds_before_inferring_the_spe # The found asset reaches spec inference as an ordinary OBJECTS entry, so the model picks # it by name and the spec needs no usd_path of its own. spec_user_msg = client.chat.completions.create.call_args.kwargs["messages"][1]["content"] - assert "simready_green_trash_can" in spec_user_msg + assert simready_search_registry_name("green trash can", "s3://bucket/trash_can.usd") in spec_user_msg assert "s3://bucket/trash_can.usd" not in spec_user_msg - assert AssetRegistry().is_registered("simready_green_trash_can") + assert AssetRegistry().is_registered( + simready_search_registry_name("green trash can", "s3://bucket/trash_can.usd") + ) @patch("isaaclab_arena.agentic_environment_generation.environment_generation_agent.search_simready_objects") def test_a_searched_simready_object_reaches_the_spec_with_its_usd_path(self, mock_search, agent): + from isaaclab_arena.assets.simready_object_library import simready_search_registry_name + agent_obj, client = agent mock_search.return_value = SimReadyCandidateCatalogue( candidates=[ @@ -255,7 +261,7 @@ def test_a_searched_simready_object_reaches_the_spec_with_its_usd_path(self, moc spec_dict = minimal_spec_dict() spec_dict["objects"].append({ "id": "trash_can", - "registry_name": "simready_green_trash_can", + "registry_name": simready_search_registry_name("green trash can", "s3://bucket/trash_can.usd"), "params": {}, }) client.chat.completions.create.side_effect = [ @@ -269,31 +275,61 @@ def test_a_searched_simready_object_reaches_the_spec_with_its_usd_path(self, moc # the generic SimReady asset every process has, carrying the path the search found. assert searched.registry_name == SIMREADY_USD_OBJECT_REGISTRY_NAME assert searched.params == {"usd_path": "s3://bucket/trash_can.usd"} - report = agent_obj.simready_asset_report(spec) - assert len(report) == 1 - assert report[0]["used_in_spec"] is True - assert report[0]["usd_path"] == "s3://bucket/trash_can.usd" - assert agent_obj.simready_asset_report(None)[0]["used_in_spec"] is False - agent_obj.enable_simready_search = False - client.chat.completions.create.side_effect = [chat_response(content=json.dumps(minimal_spec_dict()))] - next_spec, _ = agent_obj.generate_spec( - "p", - asset_catalog=catalog("catalog"), - relation_catalog=relation_catalog("RELATIONS"), - task_catalog=make_task_catalog("TASKS"), - ) - assert agent_obj.simready_asset_report(next_spec) == [] # Tags in params would be a duplicate keyword argument at build time. assert all("tags" not in asset["params"] for asset in spec.to_dict()["objects"]) + @patch("isaaclab_arena.agentic_environment_generation.environment_generation_agent.search_simready_objects") + def test_multiple_candidates_export_distinct_portable_paths(self, mock_search, agent): + from isaaclab_arena.assets.simready_object_library import simready_search_registry_name + + agent_obj, client = agent + central_url = ( + "https://huggingface.co/datasets/nvidia/simready-assets/resolve/" + "a" * 40 + "/can/simready_usd/can.usd" + ) + paths = ["s3://bucket/can.usd", central_url] + mock_search.return_value = SimReadyCandidateCatalogue( + candidates=[ + SimReadyObjectCandidate(search_phrase="can", usd_path=paths[0], tags=("ga-can",)), + SimReadyObjectCandidate( + search_phrase="can", + usd_path="/other-machine/cache/can.usda", + source_url=central_url, + tags=("central-can",), + ), + ] + ) + spec_dict = minimal_spec_dict() + for index, path in enumerate(paths): + spec_dict["objects"].append( + {"id": f"can_{index}", "registry_name": simready_search_registry_name("can", path), "params": {}} + ) + client.chat.completions.create.side_effect = [ + self._missing_objects_response("can"), + chat_response(content=json.dumps(spec_dict)), + ] + spec, _ = self._generate_with_simready(agent_obj) + assert isinstance(spec, ArenaEnvGraphSpec) + exported = spec.to_dict() + selected = [obj for obj in exported["objects"] if obj["id"].startswith("can_")] + assert [obj["params"]["usd_path"] for obj in selected] == paths + assert all(obj["registry_name"] == SIMREADY_USD_OBJECT_REGISTRY_NAME for obj in selected) + assert "/other-machine/" not in json.dumps(exported) + for path, tag in zip(paths, ["ga-can", "central-can"], strict=True): + cls = AssetRegistry().get_asset_by_name(simready_search_registry_name("can", path)) + assert tag in cls.tags + @patch("isaaclab_arena.agentic_environment_generation.environment_generation_agent.search_simready_objects") def test_a_searched_simready_object_in_an_object_set_is_rejected(self, mock_search, agent): + from isaaclab_arena.assets.simready_object_library import simready_search_registry_name + agent_obj, client = agent mock_search.return_value = SimReadyCandidateCatalogue( candidates=[SimReadyObjectCandidate(search_phrase="green trash can", usd_path="s3://bucket/trash_can.usd")] ) spec_dict = minimal_spec_dict() - spec_dict["object_sets"] = [{"id": "bins", "members": ["simready_green_trash_can"]}] + spec_dict["object_sets"] = [ + {"id": "bins", "members": [simready_search_registry_name("green trash can", "s3://bucket/trash_can.usd")]} + ] client.chat.completions.create.side_effect = [ self._missing_objects_response("green trash can"), chat_response(content=json.dumps(spec_dict)), @@ -320,7 +356,6 @@ def test_a_catalogue_object_keeps_its_own_registry_name(self, mock_search, agent assert all(asset.registry_name != SIMREADY_USD_OBJECT_REGISTRY_NAME for asset in spec.objects) assert all("usd_path" not in asset.params for asset in spec.objects) assert "simready_assets" not in spec.to_dict() - assert agent_obj.simready_asset_report(spec)[0]["used_in_spec"] is False @patch("isaaclab_arena.agentic_environment_generation.environment_generation_agent.search_simready_objects") def test_generate_spec_builds_the_spec_without_an_object_no_asset_was_found_for(self, mock_search, agent): diff --git a/isaaclab_arena/tests/test_simready_asset_search.py b/isaaclab_arena/tests/test_simready_asset_search.py index 64555c69ed..494794190a 100644 --- a/isaaclab_arena/tests/test_simready_asset_search.py +++ b/isaaclab_arena/tests/test_simready_asset_search.py @@ -8,7 +8,7 @@ from __future__ import annotations import logging -from unittest.mock import patch +from unittest.mock import AsyncMock, patch import pytest @@ -311,3 +311,15 @@ def test_backend_keeps_other_phrase_results_when_one_phrase_fails(source): assert search.call_count == 2 assert result.candidates == [candidate] assert result.unmatched_phrases == ["missing"] + + +@pytest.mark.parametrize("source", [SimReadySourceKind.ISAAC_SIM_GA, SimReadySourceKind.S3]) +def test_ga_ignores_custom_s3_url(source): + from isaaclab_arena.agentic_environment_generation.simready_ga_assets import _create_asset_library_for_source + from isaaclab_arena.assets.simready_constants import ISAAC_SIMREADY_GA_S3_URL + + with patch("simready.search.AssetLibrary") as library: + library.return_value.add_s3_source = AsyncMock() + _create_asset_library_for_source(source, s3_url="s3://custom/assets", service_url="") + expected = ISAAC_SIMREADY_GA_S3_URL if source == SimReadySourceKind.ISAAC_SIM_GA else "s3://custom/assets" + library.return_value.add_s3_source.assert_awaited_once_with(expected) diff --git a/isaaclab_arena/tests/test_simready_central.py b/isaaclab_arena/tests/test_simready_central.py index 21d061538f..ae085fb2c4 100644 --- a/isaaclab_arena/tests/test_simready_central.py +++ b/isaaclab_arena/tests/test_simready_central.py @@ -15,7 +15,7 @@ SimReadySearchConfig, SimReadySourceKind, ) -from isaaclab_arena.assets.simready_central_asset_loader import _parse_central_asset_url +from isaaclab_arena.assets.simready_central_asset_loader import _parse_central_asset_url, is_central_asset_url MODULE = "isaaclab_arena.agentic_environment_generation.simready_central_assets" LOADER = "isaaclab_arena.assets.simready_central_asset_loader" @@ -141,3 +141,24 @@ def test_central_download_lists_only_the_selected_package(): ) assert {call.kwargs["filename"] for call in download.call_args_list} == set(filenames) assert all(call.kwargs["revision"] == REVISION for call in download.call_args_list) + + +@pytest.mark.parametrize("scheme", ["https", "hf"]) +def test_central_url_recognition_and_parsing_share_supported_schemes(scheme): + url = URL.replace("hf://", f"{scheme}://") + assert is_central_asset_url(url) + assert _parse_central_asset_url(url) == _parse_central_asset_url(URL) + + +@pytest.mark.parametrize( + "url", + [ + "/cache/pallet.usda", + "s3://bucket/pallet.usd", + URL.replace("huggingface.co", "huggingface.co.example.com"), + URL.replace("/datasets/", "/models/"), + URL.replace("hf://", "http://"), + ], +) +def test_central_url_recognition_leaves_other_asset_paths_unchanged(url): + assert not is_central_asset_url(url) diff --git a/isaaclab_arena/tests/test_simready_object_library.py b/isaaclab_arena/tests/test_simready_object_library.py index a2f624ecb2..1b7b4fe88e 100644 --- a/isaaclab_arena/tests/test_simready_object_library.py +++ b/isaaclab_arena/tests/test_simready_object_library.py @@ -74,8 +74,8 @@ def test_simready_usd_object_enables_physics_variant_by_default(tmp_path): def test_simready_search_registry_name_derives_an_identifier_from_the_phrase(): from isaaclab_arena.assets.simready_object_library import simready_search_registry_name - assert simready_search_registry_name("green trash can") == "simready_green_trash_can" - assert simready_search_registry_name(" Chrome/Watering-Can ") == "simready_chrome_watering_can" + assert simready_search_registry_name("green trash can", "a.usd").startswith("simready_green_trash_can_") + assert simready_search_registry_name(" Chrome/Watering-Can ", "a.usd").startswith("simready_chrome_watering_can_") def test_register_searched_simready_object_makes_it_a_catalogue_entry(): @@ -86,8 +86,8 @@ def test_register_searched_simready_object_makes_it_a_catalogue_entry(): asset_cls = register_searched_simready_object( "green trash can", "s3://bucket/trash_can.usd", ("sim-ready", "green") ) - assert asset_cls.name == "simready_green_trash_can" - assert AssetRegistry().get_asset_by_name("simready_green_trash_can") is asset_cls + assert asset_cls.name.startswith("simready_green_trash_can_") + assert AssetRegistry().get_asset_by_name(asset_cls.name) is asset_cls # Rigid, so the entry can also be an object_set member; the search only accepts one rigid body. assert asset_cls.object_type is ObjectType.RIGID assert "green" in asset_cls.tags @@ -97,16 +97,20 @@ def test_a_searched_asset_resolves_its_usd_path_without_spec_params(): from isaaclab_arena.assets.simready_object_library import register_searched_simready_object ensure_assets_registered() - register_searched_simready_object("chrome watering can", "s3://bucket/watering_can.usd") + asset_cls = register_searched_simready_object("chrome watering can", "s3://bucket/watering_can.usd") # The path is baked into the registered class, so a spec naming it carries no params at all. - spec = AssetSpec(id="watering_can", registry_name="simready_chrome_watering_can") + spec = AssetSpec(id="watering_can", registry_name=asset_cls.name) assert spec.resolve_usd_path() == "s3://bucket/watering_can.usd" -def test_registering_the_same_phrase_twice_keeps_the_first_asset(): +def test_registering_distinct_assets_for_the_same_phrase_preserves_both(): from isaaclab_arena.assets.simready_object_library import register_searched_simready_object ensure_assets_registered() first = register_searched_simready_object("steel kettle", "s3://bucket/kettle_a.usd") second = register_searched_simready_object("steel kettle", "s3://bucket/kettle_b.usd") - assert second is first + assert second is not first + assert first.name != second.name + assert AssetSpec(id="first", registry_name=first.name).resolve_usd_path() == "s3://bucket/kettle_a.usd" + assert AssetSpec(id="second", registry_name=second.name).resolve_usd_path() == "s3://bucket/kettle_b.usd" + assert register_searched_simready_object("steel kettle", "s3://bucket/kettle_a.usd") is first diff --git a/isaaclab_arena_environments/maple_table_top/tabletop_pick_place_screwdriver_bearing_wrench.yaml b/isaaclab_arena_environments/maple_table_top/tabletop_pick_place_screwdriver_bearing_wrench.yaml deleted file mode 100644 index c2e243c9ec..0000000000 --- a/isaaclab_arena_environments/maple_table_top/tabletop_pick_place_screwdriver_bearing_wrench.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -# Assets: ForgeCAD/simready-assets and nvidia/simready-assets, CC-BY-4.0. -# Credit ForgeCAD (screwdriver) and NVIDIA (bearing and wrench). -env_name: tabletop_pick_place_screwdriver_bearing_wrench -embodiment: - id: franka - registry_name: franka_ik - params: {} -background: - id: maple_table - registry_name: maple_table_robolab - params: {} -objects: -- id: screwdriver - registry_name: simready_usd_object - params: - usd_path: https://huggingface.co/datasets/ForgeCAD/simready-assets/resolve/828771cf28d7fc2c46a7a29eadfcf7bfcc6958f8/screwdriver/simready_usd/screwdriver.usda -- id: ball_bearing - registry_name: simready_usd_object - params: - usd_path: https://huggingface.co/datasets/nvidia/simready-assets/resolve/653d91e182011f357cf1644013cc94ddf194d349/consumable_bearing_ball_a01/simready_usd/sm_consumable_bearing_ball_a01_01.usd -- id: combination_wrench - registry_name: simready_usd_object - params: - usd_path: https://huggingface.co/datasets/nvidia/simready-assets/resolve/653d91e182011f357cf1644013cc94ddf194d349/handTool_wrench_combination_a01/simready_usd/sm_handTool_wrench_combination_a01_01.usd -relations: -- kind: is_anchor - subject: maple_table - params: {} -- kind: 'on' - subject: screwdriver - reference: maple_table - params: {} -- kind: 'on' - subject: ball_bearing - reference: maple_table - params: {} -- kind: 'on' - subject: combination_wrench - reference: maple_table - params: {} -task: - composition: atomic - description: Franka picks up the screwdriver and places it back on the maple - table, leaving the ball bearing and combination wrench as distractors. - subtasks: - - kind: PickAndPlaceTask - params: - pick_up_object: screwdriver - destination_location: maple_table - background_scene: maple_table diff --git a/isaaclab_arena_environments/maple_table_top/tabletop_pick_place_spray_gun_stand_bearing_wrench.yaml b/isaaclab_arena_environments/maple_table_top/tabletop_pick_place_spray_gun_stand_bearing_wrench.yaml new file mode 100644 index 0000000000..ba30fdd830 --- /dev/null +++ b/isaaclab_arena_environments/maple_table_top/tabletop_pick_place_spray_gun_stand_bearing_wrench.yaml @@ -0,0 +1,88 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +# Assets: NVIDIA SimReady Central (nvidia/simready-assets), CC-BY-4.0. +env_name: tabletop_pick_place_spray_gun_stand_bearing_wrench +embodiment: + id: franka + registry_name: franka_ik + params: {} +background: + id: maple_table + registry_name: maple_table_robolab + params: {} +objects: +- id: spray_gun + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/nvidia/simready-assets/resolve/653d91e182011f357cf1644013cc94ddf194d349/paint_supplies_spraygun_b01/simready_usd/sm_paint_supplies_spraygun_b01_01.usd +- id: spray_gun_stand + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/nvidia/simready-assets/resolve/653d91e182011f357cf1644013cc94ddf194d349/serviceTool_sprayGunStand_benchMount_a01/simready_usd/sm_serviceTool_sprayGunStand_benchMount_a01_01.usd +- id: ball_bearing + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/nvidia/simready-assets/resolve/653d91e182011f357cf1644013cc94ddf194d349/consumable_bearing_ball_a01/simready_usd/sm_consumable_bearing_ball_a01_01.usd +- id: combination_wrench + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/nvidia/simready-assets/resolve/653d91e182011f357cf1644013cc94ddf194d349/handTool_wrench_combination_a01/simready_usd/sm_handTool_wrench_combination_a01_01.usd +- id: paint_jar + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/nvidia/simready-assets/resolve/653d91e182011f357cf1644013cc94ddf194d349/paint_container_jar_orange_a01/simready_usd/sm_paint_container_jar_orange_a01_01.usd +- id: paint_brush + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/nvidia/simready-assets/resolve/653d91e182011f357cf1644013cc94ddf194d349/paint_supplies_brush_a08/simready_usd/sm_paint_supplies_brush_a08_01.usd +- id: paint_tube + registry_name: simready_usd_object + params: + usd_path: https://huggingface.co/datasets/nvidia/simready-assets/resolve/653d91e182011f357cf1644013cc94ddf194d349/paint_container_tube_d02/simready_usd/sm_paint_container_tube_d02_01.usd +relations: +- kind: is_anchor + subject: maple_table + params: {} +- kind: 'on' + subject: spray_gun + reference: maple_table + params: {} +- kind: 'on' + subject: spray_gun_stand + reference: maple_table + params: {} +- kind: 'on' + subject: ball_bearing + reference: maple_table + params: {} +- kind: 'on' + subject: combination_wrench + reference: maple_table + params: {} +- kind: 'on' + subject: paint_jar + reference: maple_table + params: {} +- kind: 'on' + subject: paint_brush + reference: maple_table + params: {} +- kind: 'on' + subject: paint_tube + reference: maple_table + params: {} +task: + composition: atomic + description: Franka picks up the spray gun from the maple table and places it on + the bench stand, leaving the paint jar, brush, paint tube, ball bearing, and + combination wrench as distractors. + subtasks: + - kind: PickAndPlaceTask + params: + pick_up_object: spray_gun + destination_location: spray_gun_stand + placement_consecutive_steps: 10 + background_scene: maple_table diff --git a/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py b/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py index f26a26040e..c8ef4eff32 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py +++ b/isaaclab_arena_examples/agentic_environment_generation/cli_runner.py @@ -188,9 +188,6 @@ def resolve_env_spec(args_cli: argparse.Namespace) -> Path | None: relation_catalog=relation_catalog, task_catalog=task_catalog, ) - asset_report = agent.simready_asset_report(env_graph_spec) - if asset_report: - print(f"\n[runner] SimReady assets:\n{json.dumps(asset_report, indent=2)}", flush=True) # agent.traces holds one line per failure, e.g. # "embodiment.registry_name: Unknown asset registry_name 'not_a_real_asset'" # "Task 'PickAndPlaceTask' is missing required param 'pick_up_object'" diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py index 36610624f9..f359e7491d 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/generation_panel.py @@ -212,7 +212,6 @@ def _finish(severity: str, message: str) -> tuple[bool, str]: def run_generation_pipeline(prompt: str) -> tuple[bool, str]: """Call the LLM and load the returned environment graph spec YAML into the editor.""" - st.session_state.pop("simready_asset_report", None) prompt = prompt.strip() if not prompt: return False, "Enter a prompt describing the environment." @@ -236,8 +235,6 @@ def run_generation_pipeline(prompt: str) -> tuple[bool, str]: except Exception: return False, traceback.format_exc() - st.session_state["simready_asset_report"] = agent.simready_asset_report(spec) - try: yaml_text = yaml.safe_dump( spec.to_dict() if spec is not None else (data or {}), @@ -369,8 +366,3 @@ def render_generation_panel() -> None: st.rerun() else: st.error(f"Generation failed\n\n```\n{message}\n```", icon="🛑") - - asset_report = st.session_state.get("simready_asset_report", []) - if asset_report: - with st.expander("Returned SimReady assets", expanded=True): - st.dataframe(asset_report, hide_index=True, width="stretch") diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py index 374e469770..b693fbc337 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py @@ -68,7 +68,6 @@ def initialize_state(yaml_path: Path | None, out_dir: Path) -> None: st.session_state["out_dir"] = str(out_dir.resolve()) st.session_state.pop("_validation_text", None) st.session_state.pop("_validation_result", None) - st.session_state.pop("simready_asset_report", None) if yaml_path is None: st.session_state["original_text"] = "" st.session_state["edited_text"] = ""