From f542bf5ddc78bb4c8a27b8549327cf488e9f3d3d Mon Sep 17 00:00:00 2001 From: Mustafa Haiderbhai Date: Mon, 5 Oct 2026 13:32:12 -0700 Subject: [PATCH 1/4] Publish validated release wheels for uvx demos --- .github/workflows/wheel.yml | 94 ++++++- docs/source/setup/demos.rst | 69 +++++ .../changelog.d/qa-uvx-demos.fixed.rst | 2 + source/isaaclab/isaaclab/app/sim_launcher.py | 1 + tools/qa_uvx.py | 235 ++++++++++++++++++ tools/test/test_qa_uvx.py | 62 +++++ 6 files changed, 460 insertions(+), 3 deletions(-) create mode 100644 source/isaaclab/changelog.d/qa-uvx-demos.fixed.rst create mode 100644 tools/qa_uvx.py create mode 100644 tools/test/test_qa_uvx.py diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 56fe3ed057f8..3349e7e3804d 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -6,6 +6,8 @@ name: Build PIP Wheel on: + release: + types: [published] pull_request: types: [opened, synchronize, reopened] push: @@ -27,6 +29,9 @@ jobs: name: Build Wheel runs-on: ubuntu-latest timeout-minutes: 20 + outputs: + artifact_name: ${{ steps.meta.outputs.artifact_name }} + version: ${{ steps.meta.outputs.version }} steps: - name: Detect wheel-relevant changes @@ -49,6 +54,7 @@ jobs: $'^VERSION$\tPackage version' $'^source/\tPython packages' $'^tools/wheel_builder/\tWheel build tooling' + $'^tools/qa_uvx\.py$\tInstalled uvx validation' $'^\.github/workflows/wheel\.yml$\tThis workflow file' ) @@ -155,19 +161,46 @@ jobs: - name: Compute wheel metadata if: steps.changes.outputs.run_build == 'true' id: meta + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} run: | set -euo pipefail version=$(cat VERSION) sha_slug="${GITHUB_SHA:0:7}" - echo "artifact_name=isaaclab-${version}-build${{ github.run_number }}-${sha_slug}" >> "$GITHUB_OUTPUT" + if [ "$GITHUB_EVENT_NAME" = "release" ]; then + if [ "$RELEASE_TAG" != "v${version}" ]; then + echo "::error::Release tag must match VERSION (expected v${version}, got ${RELEASE_TAG})" + exit 1 + fi + echo "artifact_name=isaaclab-${version}" >> "$GITHUB_OUTPUT" + else + echo "artifact_name=isaaclab-${version}-build${{ github.run_number }}-${sha_slug}" >> "$GITHUB_OUTPUT" + fi + echo "version=${version}" >> "$GITHUB_OUTPUT" - name: Build wheel if: steps.changes.outputs.run_build == 'true' env: - WHEEL_BUILD_NUMBER: ${{ github.run_number }} - WHEEL_SHA: ${{ github.sha }} + # PyPI releases use VERSION without the local build suffix reserved for CI artifacts. + WHEEL_BUILD_NUMBER: ${{ github.event_name != 'release' && github.run_number || '' }} + WHEEL_SHA: ${{ github.event_name != 'release' && github.sha || '' }} run: bash tools/wheel_builder/build.sh + - name: Test wheel through uvx + if: steps.changes.outputs.run_build == 'true' + run: | + set -euo pipefail + wheels=(tools/wheel_builder/build/dist/isaaclab-*.whl) + uv run --no-project python tools/qa_uvx.py --package "${wheels[0]}" --examples --output /tmp/uvx-qa-wheel + + - name: Upload uvx QA results + if: always() && steps.changes.outputs.run_build == 'true' + uses: actions/upload-artifact@v7 + with: + name: uvx-qa-wheel + path: /tmp/uvx-qa-wheel + if-no-files-found: ignore + - name: Upload wheel artifact if: steps.changes.outputs.run_build == 'true' uses: actions/upload-artifact@v7 @@ -216,3 +249,58 @@ jobs: --extra-index-url https://pypi.nvidia.com \ --index-strategy unsafe-best-match \ "${wheel}[isaacsim]" + + publish-wheel: + name: Publish release wheel to PyPI + needs: build-wheel + if: github.event_name == 'release' && github.repository == 'isaac-sim/IsaacLab' + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: + name: pypi + url: https://pypi.org/project/isaaclab/ + permissions: + contents: read + id-token: write + steps: + - name: Set up uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 + + - name: Download validated wheel + uses: actions/download-artifact@v7 + with: + name: ${{ needs.build-wheel.outputs.artifact_name }} + path: dist + + - name: Publish wheel + run: uv publish --trusted-publishing always --publish-url https://upload.pypi.org/legacy/ dist/isaaclab-*.whl + + verify-release: + name: Verify PyPI release through uvx + needs: [build-wheel, publish-wheel] + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout release + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Set up uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 + + - name: Verify published wheel through uvx + env: + RELEASE_VERSION: ${{ needs.build-wheel.outputs.version }} + run: | + uv run --no-project python tools/qa_uvx.py \ + --package "isaaclab==${RELEASE_VERSION}" --examples --output /tmp/uvx-qa-pypi-release \ + --uvx-arg=--default-index=https://pypi.org/simple + uv run --no-project python tools/qa_uvx.py --examples --output /tmp/uvx-qa-pypi-default \ + --uvx-arg=--default-index=https://pypi.org/simple + + - name: Upload published-package QA results + if: always() + uses: actions/upload-artifact@v7 + with: + name: uvx-qa-pypi + path: /tmp/uvx-qa-pypi-* + if-no-files-found: ignore diff --git a/docs/source/setup/demos.rst b/docs/source/setup/demos.rst index ff737c3d34d4..d32f65a6d60a 100644 --- a/docs/source/setup/demos.rst +++ b/docs/source/setup/demos.rst @@ -120,3 +120,72 @@ Command Builder H1 locomotion uses a published policy. In the Newton viewer, press N to select a robot, I/J/L to walk forward or turn, K to stop, and C to toggle the follow camera. Pick and place requires Kit input. For autonomous H1 task playback, use ``isaaclab play``. + +QA with uvx +----------- + +Give QA ``tools/qa_uvx.py`` from the source checkout. It is a standalone Python script and can be copied +to a machine without Isaac Lab installed. The same commands work on Linux and Windows with uv on PATH: + +.. code-block:: bash + + # Test the default published package, demo catalogs, installed resources, and demo help. + uv run --no-project python tools/qa_uvx.py + + # Test the candidate wheel and also check example help. + uv run --no-project python tools/qa_uvx.py --package /path/to/isaaclab.whl --examples + + # Run Zoo for 60 steps with its default interactive viewer. + uv run --no-project python tools/qa_uvx.py --package /path/to/isaaclab.whl --run --demo zoo + + # Run a short simulation without a viewer on a GPU worker. + uv run --no-project python tools/qa_uvx.py --package /path/to/isaaclab.whl --run --demo zoo -- --viz none --device cuda:0 + +``--package`` also accepts a pinned requirement such as ``isaaclab==`` or +``isaaclab[isaacsim]==`` for Pick and Place. Add ``--index https://pypi.nvidia.com`` +when testing a release hosted on NVIDIA's index. The package must contain the redesigned ``demo`` command; +older releases, including ``3.0.0rc1``, do not. Validate both the candidate wheel and the default published +package before declaring the advertised ``uvx isaaclab demo `` path ready. + +The script launches the real ``uvx`` executable with Python 3.12, ignores uv configuration files and +already-installed tools, clears Python import overrides, and runs from a temporary directory outside the +checkout. It checks that every catalog entry resolves to a file inside the installed package. Default checks +do not simulate; ``--run`` launches the selected demos with ``--max_steps``. Repeat ``--demo`` to select +several demos, use ``--steps`` to change their duration, and pass backend or viewer options after ``--``. +Without a selection, all demos with their required modules available are checked. Missing optional modules +are recorded as skips; explicitly selected demos fail when their dependencies are missing. + +Each run writes command logs and ``report.json`` in a new ``uvx-qa-`` directory; ``--output`` +can choose a different new directory. The report records the OS, architecture, Python and package versions, +GPU/driver information when available, commands, exit codes, durations, and skips. Failures and timeouts +return a nonzero exit status. ``--timeout`` sets the deadline per command, including the initial package +installation (default: 1800 seconds). uv's download cache is reused; to test with a fresh cache, set +``UV_CACHE_DIR`` to a new directory before running the script. + +Run the candidate on each supported QA platform, including Linux x86_64, Linux aarch64, and Windows x86_64. +CLI checks need network access and disk space for the package dependencies. Simulation checks also need +the selected backend's supported hardware; viewer checks need a desktop session. Exercise Zoo and H1 for +rigid-body simulation, Block and Tackle for VBD, and Snowball Smash and Teapot Fill for CUDA MPM. Test +Pick and Place separately with the Isaac Sim extra. A successful simulation check establishes startup, +bounded execution, and shutdown; QA should also inspect rendering and keyboard/drag interactions in a +longer interactive run. Attach the result directory and any visual observations to the QA report. + +Release publication +^^^^^^^^^^^^^^^^^^^ + +The bare ``uvx isaaclab`` command resolves the package from PyPI. Wheels uploaded only as GitHub artifacts +or to NVIDIA's index do not make this command available. A Python 3.12 ABI error can mean the index still +contains only the older Python 3.10/3.11 packages; changing demo arguments or GPU drivers cannot fix that. + +``.github/workflows/wheel.yml`` validates candidate wheels through this QA script. On a published GitHub +release, it builds without the CI-only local version suffix and publishes the validated wheel to PyPI. +The release tag must be ``v`` and match the repository's ``VERSION`` file. After publishing, it +checks both the exact release requirement and the default ``uvx isaaclab`` path against the public index. + +Before enabling release publication, the PyPI ``isaaclab`` project owner must register the trusted publisher +with owner ``isaac-sim``, repository ``IsaacLab``, workflow ``wheel.yml``, and environment ``pypi``. See +`PyPI's trusted publisher setup `__. +The workflow requires this registration and the GitHub ``pypi`` environment; it does not use a stored API +token. Backport these changes to the release branch before tagging the release that supplies the corrected +wheel. Until that wheel is published, QA should use the candidate wheel and report the public-index failure +as unresolved. diff --git a/source/isaaclab/changelog.d/qa-uvx-demos.fixed.rst b/source/isaaclab/changelog.d/qa-uvx-demos.fixed.rst new file mode 100644 index 000000000000..72e1231c0c05 --- /dev/null +++ b/source/isaaclab/changelog.d/qa-uvx-demos.fixed.rst @@ -0,0 +1,2 @@ +* Fixed simulation startup failing with a missing ``sys`` import in the platform-specific device check, + including demos launched through ``uvx``. diff --git a/source/isaaclab/isaaclab/app/sim_launcher.py b/source/isaaclab/isaaclab/app/sim_launcher.py index 6cf87ca8bb0f..955737e67ea0 100644 --- a/source/isaaclab/isaaclab/app/sim_launcher.py +++ b/source/isaaclab/isaaclab/app/sim_launcher.py @@ -15,6 +15,7 @@ import argparse import logging import os +import sys import traceback from collections.abc import Generator from contextlib import contextmanager diff --git a/tools/qa_uvx.py b/tools/qa_uvx.py new file mode 100644 index 000000000000..3aeb5c3c5873 --- /dev/null +++ b/tools/qa_uvx.py @@ -0,0 +1,235 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Test released or candidate demos through uvx without a checkout or Isaac Lab installation. + +Run with ``uv run --no-project python tools/qa_uvx.py --help``. Only Python's standard library +and uvx are required. Additional arguments after ``--`` are forwarded to simulation runs. +""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import os +import platform +import shutil +import signal +import subprocess +import tempfile +import time +from datetime import datetime, timezone +from pathlib import Path + +_PROBE = """ +import importlib.metadata +import json +import sys +from pathlib import Path +import isaaclab +from isaaclab.programs import DEMOS, EXAMPLES + +package_root = Path(isaaclab.__file__).resolve().parent +catalogs = {} +for command, catalog in (("demo", DEMOS), ("example", EXAMPLES)): + if not catalog: + raise RuntimeError(f"Empty {command} catalog") + catalogs[command] = [] + for program in catalog: + path = program.path.resolve() + if not path.is_relative_to(package_root): + raise RuntimeError(f"Program outside installed package: {path}") + if not path.is_file(): + raise RuntimeError(f"Missing packaged program: {path}") + catalogs[command].append({ + "name": program.name, + "extras": program.extras, + "missing_modules": program.missing_modules(), + }) +Path(sys.argv[1]).write_text(json.dumps({ + "version": importlib.metadata.version("isaaclab"), + "package_root": str(package_root), + "python": sys.version, + "dependencies": {name: importlib.metadata.version(name) for name in ("torch", "newton", "warp-lang")}, + "catalogs": catalogs, +}), encoding="utf-8") +""" + + +def _run_check(name: str, command: list[str], directory: Path, env: dict[str, str], log: Path, timeout: float) -> dict: + """Run a check, saving output and killing its process tree on timeout or interruption.""" + started = time.monotonic() + result = {"name": name, "command": command, "log": str(log), "status": "failed"} + with log.open("w", encoding="utf-8") as output: + try: + process = subprocess.Popen( + command, + cwd=directory, + env=env, + stdin=subprocess.DEVNULL, + stdout=output, + stderr=subprocess.STDOUT, + start_new_session=os.name != "nt", + ) + except OSError as error: + result["reason"] = str(error) + else: + try: + result["returncode"] = process.wait(timeout=timeout) + if process.returncode == 0: + result["status"] = "passed" + else: + result["reason"] = f"Exit code {process.returncode}" + except (subprocess.TimeoutExpired, KeyboardInterrupt) as error: + if os.name == "nt": + subprocess.run( + ["taskkill", "/PID", str(process.pid), "/T", "/F"], + stdout=output, + stderr=subprocess.STDOUT, + check=False, + ) + else: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.wait() + if isinstance(error, KeyboardInterrupt): + raise + result["reason"] = f"Timed out after {timeout:g} seconds" + result["seconds"] = round(time.monotonic() - started, 2) + print(f"{result['status'].upper():7} {name}" + (f": {result['reason']}" if "reason" in result else "")) + return result + + +def main(argv: list[str] | None = None) -> int: + """Check uvx packaging, catalogs and help; optionally launch bounded demo simulations.""" + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--package", default="isaaclab", help="Requirement, wheel path/URL, or Git URL for uvx --from.") + parser.add_argument("--python", default="3.12", help="Python requested from uvx (default: 3.12).") + parser.add_argument("--index", action="append", default=[], help="Additional package index; repeat as needed.") + parser.add_argument("--uvx-arg", action="append", default=[], help="Extra uvx option, e.g. --uvx-arg=--offline.") + parser.add_argument("--demo", action="append", help="Demo to check; repeat to select several (default: catalog).") + parser.add_argument("--examples", action="store_true", help="Also check example help (does not simulate examples).") + parser.add_argument("--run", action="store_true", help="Run selected demos with --max_steps; default is CLI only.") + parser.add_argument("--steps", type=int, default=60, help="Simulation steps per demo (default: 60).") + parser.add_argument( + "--timeout", type=float, default=1800, help="Deadline per command, including install (seconds)." + ) + parser.add_argument("--output", type=Path, help="New directory for logs and report.json.") + parser.add_argument("demo_args", nargs=argparse.REMAINDER, help="Arguments after -- forwarded to demo runs.") + args = parser.parse_args(argv) + if args.steps < 1 or not 0 < args.timeout < float("inf"): + parser.error("--steps and --timeout must be positive and finite") + if args.demo_args and not args.run: + parser.error("arguments after -- require --run") + uvx = shutil.which("uvx") + if uvx is None: + parser.error("uvx is not on PATH; install uv first") + # Resolve local wheels before changing to a temporary working directory. + package = str(Path(args.package).resolve()) if Path(args.package).is_file() else args.package + output_dir = (args.output or Path(f"uvx-qa-{datetime.now(timezone.utc):%Y%m%dT%H%M%S%fZ}")).resolve() + try: + output_dir.mkdir(parents=True, exist_ok=False) + except OSError as error: + parser.error(f"cannot create output directory: {error}") + env = os.environ.copy() + for variable in ("PYTHONPATH", "PYTHONHOME", "VIRTUAL_ENV", "ISAAC_PATH", "LIVESTREAM"): + env.pop(variable, None) + env["PYTHONUNBUFFERED"] = "1" + base = [uvx, "--no-config", "--isolated", "--python", args.python] + if package != "isaaclab": + base.extend(["--from", package]) + for index in args.index: + base.extend(["--index", index]) + base.extend(args.uvx_arg) + base.append("isaaclab") + report = { + "started_utc": datetime.now(timezone.utc).isoformat(), + "platform": platform.platform(), + "machine": platform.machine(), + "package": package, + "requested_python": args.python, + "simulation": args.run, + "checks": [], + } + checks = report["checks"] + try: + with tempfile.TemporaryDirectory(prefix="isaaclab-uvx-qa-") as temporary: + directory = Path(temporary) + checks.append( + _run_check( + "uvx-version", [uvx, "--version"], directory, env, output_dir / "uvx-version.log", args.timeout + ) + ) + gpu_tool = shutil.which("nvidia-smi") + if gpu_tool: + try: + gpu = subprocess.run( + [gpu_tool, "--query-gpu=name,driver_version", "--format=csv,noheader"], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + report["gpu"] = {"returncode": gpu.returncode, "output": gpu.stdout, "error": gpu.stderr} + except (OSError, subprocess.TimeoutExpired) as error: + report["gpu"] = {"error": str(error)} + + def check(name: str, arguments: list[str]) -> dict: + result = _run_check(name, [*base, *arguments], directory, env, output_dir / f"{name}.log", args.timeout) + checks.append(result) + return result + + # Exercise the user-facing catalog before probing the installed wheel. + if check("demo-list", ["demo", "list"])["status"] != "passed": + return 1 + probe = directory / "probe.py" + probe.write_text(_PROBE, encoding="utf-8") + metadata = directory / "package.json" + if check("installed-package", ["-p", str(probe), str(metadata)])["status"] != "passed": + return 1 + report["installed"] = json.loads(metadata.read_text(encoding="utf-8")) + catalogs = report["installed"]["catalogs"] + if args.demo: + unknown = set(args.demo) - {program["name"] for program in catalogs["demo"]} + if unknown: + checks.append( + {"name": "demo-selection", "status": "failed", "reason": f"Unknown demos: {sorted(unknown)}"} + ) + return 1 + if args.examples: + check("example-list", ["example", "list"]) + for command in ("demo", "example") if args.examples else ("demo",): + for program in catalogs[command]: + name = program["name"] + if command == "demo" and args.demo and name not in args.demo: + continue + if program["missing_modules"] and not (command == "demo" and args.demo): + reason = f"Missing {program['missing_modules']}; requires extras {program['extras']}" + checks.append({"name": f"{command}-{name}", "status": "skipped", "reason": reason}) + print(f"SKIPPED {command}-{name}: {reason}") + continue + result = check(f"{command}-{name}-help", [command, name, "--help"]) + if result["status"] == "passed" and command == "demo" and args.run: + forwarded = args.demo_args[1:] if args.demo_args[:1] == ["--"] else args.demo_args + check(f"demo-{name}-run", [command, name, *forwarded, "--max_steps", str(args.steps), "--info"]) + except (OSError, ValueError, KeyError) as error: + checks.append({"name": "qa-runner", "status": "failed", "reason": str(error)}) + print(f"FAILED qa-runner: {error}") + except KeyboardInterrupt: + checks.append({"name": "qa-runner", "status": "failed", "reason": "Interrupted"}) + finally: + report["passed"] = bool(checks) and all(result["status"] != "failed" for result in checks) + report["counts"] = { + status: sum(result["status"] == status for result in checks) for status in ("passed", "failed", "skipped") + } + (output_dir / "report.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(f"Results: {report['counts']}") + print(f"Report: {output_dir / 'report.json'}") + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/test/test_qa_uvx.py b/tools/test/test_qa_uvx.py new file mode 100644 index 000000000000..5edad2e5ba7a --- /dev/null +++ b/tools/test/test_qa_uvx.py @@ -0,0 +1,62 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Check that the QA runner reports process failures and bounds hung commands.""" + +import importlib.util +import json +import os +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture +def qa(): + spec = importlib.util.spec_from_file_location("qa_uvx", Path(__file__).resolve().parents[1] / "qa_uvx.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_failed_command_preserves_output_and_exit_code(qa, tmp_path): + log = tmp_path / "failed.log" + result = qa._run_check( + "failure", + [sys.executable, "-c", "print('diagnostic'); raise SystemExit(7)"], + tmp_path, + os.environ.copy(), + log, + 10, + ) + assert result["status"] == "failed" + assert result["returncode"] == 7 + assert "diagnostic" in log.read_text() + + +def test_timeout_is_a_failure(qa, tmp_path): + result = qa._run_check( + "hang", + [sys.executable, "-c", "import time; time.sleep(30)"], + tmp_path, + os.environ.copy(), + tmp_path / "hang.log", + 0.2, + ) + assert result["status"] == "failed" + assert "Timed out" in result["reason"] + assert result["seconds"] < 10 + + +def test_cli_failure_writes_failed_report_and_returns_nonzero(qa, tmp_path, monkeypatch): + # Python rejects uvx's options, providing a real nonzero subprocess without network access. + monkeypatch.setattr(qa.shutil, "which", lambda name: sys.executable if name == "uvx" else None) + output = tmp_path / "results" + assert qa.main(["--output", str(output)]) == 1 + report = json.loads((output / "report.json").read_text()) + assert report["passed"] is False + assert report["counts"]["failed"] >= 1 + assert any(check["name"] == "demo-list" and check["status"] == "failed" for check in report["checks"]) From ceebd5230e6742f13f31547fe878d3cc36dfb81e Mon Sep 17 00:00:00 2001 From: Mustafa Haiderbhai Date: Mon, 5 Oct 2026 13:36:12 -0700 Subject: [PATCH 2/4] Remove publication automation and extra QA files --- .github/workflows/wheel.yml | 78 ++----------------------------------- docs/source/setup/demos.rst | 69 -------------------------------- tools/test/test_qa_uvx.py | 62 ----------------------------- 3 files changed, 3 insertions(+), 206 deletions(-) delete mode 100644 tools/test/test_qa_uvx.py diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 3349e7e3804d..84b3e96c3fc2 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -6,8 +6,6 @@ name: Build PIP Wheel on: - release: - types: [published] pull_request: types: [opened, synchronize, reopened] push: @@ -29,9 +27,6 @@ jobs: name: Build Wheel runs-on: ubuntu-latest timeout-minutes: 20 - outputs: - artifact_name: ${{ steps.meta.outputs.artifact_name }} - version: ${{ steps.meta.outputs.version }} steps: - name: Detect wheel-relevant changes @@ -161,29 +156,17 @@ jobs: - name: Compute wheel metadata if: steps.changes.outputs.run_build == 'true' id: meta - env: - RELEASE_TAG: ${{ github.event.release.tag_name }} run: | set -euo pipefail version=$(cat VERSION) sha_slug="${GITHUB_SHA:0:7}" - if [ "$GITHUB_EVENT_NAME" = "release" ]; then - if [ "$RELEASE_TAG" != "v${version}" ]; then - echo "::error::Release tag must match VERSION (expected v${version}, got ${RELEASE_TAG})" - exit 1 - fi - echo "artifact_name=isaaclab-${version}" >> "$GITHUB_OUTPUT" - else - echo "artifact_name=isaaclab-${version}-build${{ github.run_number }}-${sha_slug}" >> "$GITHUB_OUTPUT" - fi - echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "artifact_name=isaaclab-${version}-build${{ github.run_number }}-${sha_slug}" >> "$GITHUB_OUTPUT" - name: Build wheel if: steps.changes.outputs.run_build == 'true' env: - # PyPI releases use VERSION without the local build suffix reserved for CI artifacts. - WHEEL_BUILD_NUMBER: ${{ github.event_name != 'release' && github.run_number || '' }} - WHEEL_SHA: ${{ github.event_name != 'release' && github.sha || '' }} + WHEEL_BUILD_NUMBER: ${{ github.run_number }} + WHEEL_SHA: ${{ github.sha }} run: bash tools/wheel_builder/build.sh - name: Test wheel through uvx @@ -249,58 +232,3 @@ jobs: --extra-index-url https://pypi.nvidia.com \ --index-strategy unsafe-best-match \ "${wheel}[isaacsim]" - - publish-wheel: - name: Publish release wheel to PyPI - needs: build-wheel - if: github.event_name == 'release' && github.repository == 'isaac-sim/IsaacLab' - runs-on: ubuntu-latest - timeout-minutes: 20 - environment: - name: pypi - url: https://pypi.org/project/isaaclab/ - permissions: - contents: read - id-token: write - steps: - - name: Set up uv - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 - - - name: Download validated wheel - uses: actions/download-artifact@v7 - with: - name: ${{ needs.build-wheel.outputs.artifact_name }} - path: dist - - - name: Publish wheel - run: uv publish --trusted-publishing always --publish-url https://upload.pypi.org/legacy/ dist/isaaclab-*.whl - - verify-release: - name: Verify PyPI release through uvx - needs: [build-wheel, publish-wheel] - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - name: Checkout release - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - - name: Set up uv - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 - - - name: Verify published wheel through uvx - env: - RELEASE_VERSION: ${{ needs.build-wheel.outputs.version }} - run: | - uv run --no-project python tools/qa_uvx.py \ - --package "isaaclab==${RELEASE_VERSION}" --examples --output /tmp/uvx-qa-pypi-release \ - --uvx-arg=--default-index=https://pypi.org/simple - uv run --no-project python tools/qa_uvx.py --examples --output /tmp/uvx-qa-pypi-default \ - --uvx-arg=--default-index=https://pypi.org/simple - - - name: Upload published-package QA results - if: always() - uses: actions/upload-artifact@v7 - with: - name: uvx-qa-pypi - path: /tmp/uvx-qa-pypi-* - if-no-files-found: ignore diff --git a/docs/source/setup/demos.rst b/docs/source/setup/demos.rst index d32f65a6d60a..ff737c3d34d4 100644 --- a/docs/source/setup/demos.rst +++ b/docs/source/setup/demos.rst @@ -120,72 +120,3 @@ Command Builder H1 locomotion uses a published policy. In the Newton viewer, press N to select a robot, I/J/L to walk forward or turn, K to stop, and C to toggle the follow camera. Pick and place requires Kit input. For autonomous H1 task playback, use ``isaaclab play``. - -QA with uvx ------------ - -Give QA ``tools/qa_uvx.py`` from the source checkout. It is a standalone Python script and can be copied -to a machine without Isaac Lab installed. The same commands work on Linux and Windows with uv on PATH: - -.. code-block:: bash - - # Test the default published package, demo catalogs, installed resources, and demo help. - uv run --no-project python tools/qa_uvx.py - - # Test the candidate wheel and also check example help. - uv run --no-project python tools/qa_uvx.py --package /path/to/isaaclab.whl --examples - - # Run Zoo for 60 steps with its default interactive viewer. - uv run --no-project python tools/qa_uvx.py --package /path/to/isaaclab.whl --run --demo zoo - - # Run a short simulation without a viewer on a GPU worker. - uv run --no-project python tools/qa_uvx.py --package /path/to/isaaclab.whl --run --demo zoo -- --viz none --device cuda:0 - -``--package`` also accepts a pinned requirement such as ``isaaclab==`` or -``isaaclab[isaacsim]==`` for Pick and Place. Add ``--index https://pypi.nvidia.com`` -when testing a release hosted on NVIDIA's index. The package must contain the redesigned ``demo`` command; -older releases, including ``3.0.0rc1``, do not. Validate both the candidate wheel and the default published -package before declaring the advertised ``uvx isaaclab demo `` path ready. - -The script launches the real ``uvx`` executable with Python 3.12, ignores uv configuration files and -already-installed tools, clears Python import overrides, and runs from a temporary directory outside the -checkout. It checks that every catalog entry resolves to a file inside the installed package. Default checks -do not simulate; ``--run`` launches the selected demos with ``--max_steps``. Repeat ``--demo`` to select -several demos, use ``--steps`` to change their duration, and pass backend or viewer options after ``--``. -Without a selection, all demos with their required modules available are checked. Missing optional modules -are recorded as skips; explicitly selected demos fail when their dependencies are missing. - -Each run writes command logs and ``report.json`` in a new ``uvx-qa-`` directory; ``--output`` -can choose a different new directory. The report records the OS, architecture, Python and package versions, -GPU/driver information when available, commands, exit codes, durations, and skips. Failures and timeouts -return a nonzero exit status. ``--timeout`` sets the deadline per command, including the initial package -installation (default: 1800 seconds). uv's download cache is reused; to test with a fresh cache, set -``UV_CACHE_DIR`` to a new directory before running the script. - -Run the candidate on each supported QA platform, including Linux x86_64, Linux aarch64, and Windows x86_64. -CLI checks need network access and disk space for the package dependencies. Simulation checks also need -the selected backend's supported hardware; viewer checks need a desktop session. Exercise Zoo and H1 for -rigid-body simulation, Block and Tackle for VBD, and Snowball Smash and Teapot Fill for CUDA MPM. Test -Pick and Place separately with the Isaac Sim extra. A successful simulation check establishes startup, -bounded execution, and shutdown; QA should also inspect rendering and keyboard/drag interactions in a -longer interactive run. Attach the result directory and any visual observations to the QA report. - -Release publication -^^^^^^^^^^^^^^^^^^^ - -The bare ``uvx isaaclab`` command resolves the package from PyPI. Wheels uploaded only as GitHub artifacts -or to NVIDIA's index do not make this command available. A Python 3.12 ABI error can mean the index still -contains only the older Python 3.10/3.11 packages; changing demo arguments or GPU drivers cannot fix that. - -``.github/workflows/wheel.yml`` validates candidate wheels through this QA script. On a published GitHub -release, it builds without the CI-only local version suffix and publishes the validated wheel to PyPI. -The release tag must be ``v`` and match the repository's ``VERSION`` file. After publishing, it -checks both the exact release requirement and the default ``uvx isaaclab`` path against the public index. - -Before enabling release publication, the PyPI ``isaaclab`` project owner must register the trusted publisher -with owner ``isaac-sim``, repository ``IsaacLab``, workflow ``wheel.yml``, and environment ``pypi``. See -`PyPI's trusted publisher setup `__. -The workflow requires this registration and the GitHub ``pypi`` environment; it does not use a stored API -token. Backport these changes to the release branch before tagging the release that supplies the corrected -wheel. Until that wheel is published, QA should use the candidate wheel and report the public-index failure -as unresolved. diff --git a/tools/test/test_qa_uvx.py b/tools/test/test_qa_uvx.py deleted file mode 100644 index 5edad2e5ba7a..000000000000 --- a/tools/test/test_qa_uvx.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Check that the QA runner reports process failures and bounds hung commands.""" - -import importlib.util -import json -import os -import sys -from pathlib import Path - -import pytest - - -@pytest.fixture -def qa(): - spec = importlib.util.spec_from_file_location("qa_uvx", Path(__file__).resolve().parents[1] / "qa_uvx.py") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def test_failed_command_preserves_output_and_exit_code(qa, tmp_path): - log = tmp_path / "failed.log" - result = qa._run_check( - "failure", - [sys.executable, "-c", "print('diagnostic'); raise SystemExit(7)"], - tmp_path, - os.environ.copy(), - log, - 10, - ) - assert result["status"] == "failed" - assert result["returncode"] == 7 - assert "diagnostic" in log.read_text() - - -def test_timeout_is_a_failure(qa, tmp_path): - result = qa._run_check( - "hang", - [sys.executable, "-c", "import time; time.sleep(30)"], - tmp_path, - os.environ.copy(), - tmp_path / "hang.log", - 0.2, - ) - assert result["status"] == "failed" - assert "Timed out" in result["reason"] - assert result["seconds"] < 10 - - -def test_cli_failure_writes_failed_report_and_returns_nonzero(qa, tmp_path, monkeypatch): - # Python rejects uvx's options, providing a real nonzero subprocess without network access. - monkeypatch.setattr(qa.shutil, "which", lambda name: sys.executable if name == "uvx" else None) - output = tmp_path / "results" - assert qa.main(["--output", str(output)]) == 1 - report = json.loads((output / "report.json").read_text()) - assert report["passed"] is False - assert report["counts"]["failed"] >= 1 - assert any(check["name"] == "demo-list" and check["status"] == "failed" for check in report["checks"]) From 7092a84df78ceb87e3d70ae3893a4fdd81e47ac7 Mon Sep 17 00:00:00 2001 From: Mustafa Haiderbhai Date: Mon, 5 Oct 2026 21:21:06 -0700 Subject: [PATCH 3/4] Keep only the simulation launcher bug fix --- .github/workflows/wheel.yml | 16 -- ...ulation-launcher-platform-check.fixed.rst} | 0 tools/qa_uvx.py | 235 ------------------ 3 files changed, 251 deletions(-) rename source/isaaclab/changelog.d/{qa-uvx-demos.fixed.rst => fix-simulation-launcher-platform-check.fixed.rst} (100%) delete mode 100644 tools/qa_uvx.py diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 84b3e96c3fc2..56fe3ed057f8 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -49,7 +49,6 @@ jobs: $'^VERSION$\tPackage version' $'^source/\tPython packages' $'^tools/wheel_builder/\tWheel build tooling' - $'^tools/qa_uvx\.py$\tInstalled uvx validation' $'^\.github/workflows/wheel\.yml$\tThis workflow file' ) @@ -169,21 +168,6 @@ jobs: WHEEL_SHA: ${{ github.sha }} run: bash tools/wheel_builder/build.sh - - name: Test wheel through uvx - if: steps.changes.outputs.run_build == 'true' - run: | - set -euo pipefail - wheels=(tools/wheel_builder/build/dist/isaaclab-*.whl) - uv run --no-project python tools/qa_uvx.py --package "${wheels[0]}" --examples --output /tmp/uvx-qa-wheel - - - name: Upload uvx QA results - if: always() && steps.changes.outputs.run_build == 'true' - uses: actions/upload-artifact@v7 - with: - name: uvx-qa-wheel - path: /tmp/uvx-qa-wheel - if-no-files-found: ignore - - name: Upload wheel artifact if: steps.changes.outputs.run_build == 'true' uses: actions/upload-artifact@v7 diff --git a/source/isaaclab/changelog.d/qa-uvx-demos.fixed.rst b/source/isaaclab/changelog.d/fix-simulation-launcher-platform-check.fixed.rst similarity index 100% rename from source/isaaclab/changelog.d/qa-uvx-demos.fixed.rst rename to source/isaaclab/changelog.d/fix-simulation-launcher-platform-check.fixed.rst diff --git a/tools/qa_uvx.py b/tools/qa_uvx.py deleted file mode 100644 index 3aeb5c3c5873..000000000000 --- a/tools/qa_uvx.py +++ /dev/null @@ -1,235 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Test released or candidate demos through uvx without a checkout or Isaac Lab installation. - -Run with ``uv run --no-project python tools/qa_uvx.py --help``. Only Python's standard library -and uvx are required. Additional arguments after ``--`` are forwarded to simulation runs. -""" - -from __future__ import annotations - -import argparse -import contextlib -import json -import os -import platform -import shutil -import signal -import subprocess -import tempfile -import time -from datetime import datetime, timezone -from pathlib import Path - -_PROBE = """ -import importlib.metadata -import json -import sys -from pathlib import Path -import isaaclab -from isaaclab.programs import DEMOS, EXAMPLES - -package_root = Path(isaaclab.__file__).resolve().parent -catalogs = {} -for command, catalog in (("demo", DEMOS), ("example", EXAMPLES)): - if not catalog: - raise RuntimeError(f"Empty {command} catalog") - catalogs[command] = [] - for program in catalog: - path = program.path.resolve() - if not path.is_relative_to(package_root): - raise RuntimeError(f"Program outside installed package: {path}") - if not path.is_file(): - raise RuntimeError(f"Missing packaged program: {path}") - catalogs[command].append({ - "name": program.name, - "extras": program.extras, - "missing_modules": program.missing_modules(), - }) -Path(sys.argv[1]).write_text(json.dumps({ - "version": importlib.metadata.version("isaaclab"), - "package_root": str(package_root), - "python": sys.version, - "dependencies": {name: importlib.metadata.version(name) for name in ("torch", "newton", "warp-lang")}, - "catalogs": catalogs, -}), encoding="utf-8") -""" - - -def _run_check(name: str, command: list[str], directory: Path, env: dict[str, str], log: Path, timeout: float) -> dict: - """Run a check, saving output and killing its process tree on timeout or interruption.""" - started = time.monotonic() - result = {"name": name, "command": command, "log": str(log), "status": "failed"} - with log.open("w", encoding="utf-8") as output: - try: - process = subprocess.Popen( - command, - cwd=directory, - env=env, - stdin=subprocess.DEVNULL, - stdout=output, - stderr=subprocess.STDOUT, - start_new_session=os.name != "nt", - ) - except OSError as error: - result["reason"] = str(error) - else: - try: - result["returncode"] = process.wait(timeout=timeout) - if process.returncode == 0: - result["status"] = "passed" - else: - result["reason"] = f"Exit code {process.returncode}" - except (subprocess.TimeoutExpired, KeyboardInterrupt) as error: - if os.name == "nt": - subprocess.run( - ["taskkill", "/PID", str(process.pid), "/T", "/F"], - stdout=output, - stderr=subprocess.STDOUT, - check=False, - ) - else: - with contextlib.suppress(ProcessLookupError): - os.killpg(process.pid, signal.SIGKILL) - process.wait() - if isinstance(error, KeyboardInterrupt): - raise - result["reason"] = f"Timed out after {timeout:g} seconds" - result["seconds"] = round(time.monotonic() - started, 2) - print(f"{result['status'].upper():7} {name}" + (f": {result['reason']}" if "reason" in result else "")) - return result - - -def main(argv: list[str] | None = None) -> int: - """Check uvx packaging, catalogs and help; optionally launch bounded demo simulations.""" - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--package", default="isaaclab", help="Requirement, wheel path/URL, or Git URL for uvx --from.") - parser.add_argument("--python", default="3.12", help="Python requested from uvx (default: 3.12).") - parser.add_argument("--index", action="append", default=[], help="Additional package index; repeat as needed.") - parser.add_argument("--uvx-arg", action="append", default=[], help="Extra uvx option, e.g. --uvx-arg=--offline.") - parser.add_argument("--demo", action="append", help="Demo to check; repeat to select several (default: catalog).") - parser.add_argument("--examples", action="store_true", help="Also check example help (does not simulate examples).") - parser.add_argument("--run", action="store_true", help="Run selected demos with --max_steps; default is CLI only.") - parser.add_argument("--steps", type=int, default=60, help="Simulation steps per demo (default: 60).") - parser.add_argument( - "--timeout", type=float, default=1800, help="Deadline per command, including install (seconds)." - ) - parser.add_argument("--output", type=Path, help="New directory for logs and report.json.") - parser.add_argument("demo_args", nargs=argparse.REMAINDER, help="Arguments after -- forwarded to demo runs.") - args = parser.parse_args(argv) - if args.steps < 1 or not 0 < args.timeout < float("inf"): - parser.error("--steps and --timeout must be positive and finite") - if args.demo_args and not args.run: - parser.error("arguments after -- require --run") - uvx = shutil.which("uvx") - if uvx is None: - parser.error("uvx is not on PATH; install uv first") - # Resolve local wheels before changing to a temporary working directory. - package = str(Path(args.package).resolve()) if Path(args.package).is_file() else args.package - output_dir = (args.output or Path(f"uvx-qa-{datetime.now(timezone.utc):%Y%m%dT%H%M%S%fZ}")).resolve() - try: - output_dir.mkdir(parents=True, exist_ok=False) - except OSError as error: - parser.error(f"cannot create output directory: {error}") - env = os.environ.copy() - for variable in ("PYTHONPATH", "PYTHONHOME", "VIRTUAL_ENV", "ISAAC_PATH", "LIVESTREAM"): - env.pop(variable, None) - env["PYTHONUNBUFFERED"] = "1" - base = [uvx, "--no-config", "--isolated", "--python", args.python] - if package != "isaaclab": - base.extend(["--from", package]) - for index in args.index: - base.extend(["--index", index]) - base.extend(args.uvx_arg) - base.append("isaaclab") - report = { - "started_utc": datetime.now(timezone.utc).isoformat(), - "platform": platform.platform(), - "machine": platform.machine(), - "package": package, - "requested_python": args.python, - "simulation": args.run, - "checks": [], - } - checks = report["checks"] - try: - with tempfile.TemporaryDirectory(prefix="isaaclab-uvx-qa-") as temporary: - directory = Path(temporary) - checks.append( - _run_check( - "uvx-version", [uvx, "--version"], directory, env, output_dir / "uvx-version.log", args.timeout - ) - ) - gpu_tool = shutil.which("nvidia-smi") - if gpu_tool: - try: - gpu = subprocess.run( - [gpu_tool, "--query-gpu=name,driver_version", "--format=csv,noheader"], - capture_output=True, - text=True, - timeout=10, - check=False, - ) - report["gpu"] = {"returncode": gpu.returncode, "output": gpu.stdout, "error": gpu.stderr} - except (OSError, subprocess.TimeoutExpired) as error: - report["gpu"] = {"error": str(error)} - - def check(name: str, arguments: list[str]) -> dict: - result = _run_check(name, [*base, *arguments], directory, env, output_dir / f"{name}.log", args.timeout) - checks.append(result) - return result - - # Exercise the user-facing catalog before probing the installed wheel. - if check("demo-list", ["demo", "list"])["status"] != "passed": - return 1 - probe = directory / "probe.py" - probe.write_text(_PROBE, encoding="utf-8") - metadata = directory / "package.json" - if check("installed-package", ["-p", str(probe), str(metadata)])["status"] != "passed": - return 1 - report["installed"] = json.loads(metadata.read_text(encoding="utf-8")) - catalogs = report["installed"]["catalogs"] - if args.demo: - unknown = set(args.demo) - {program["name"] for program in catalogs["demo"]} - if unknown: - checks.append( - {"name": "demo-selection", "status": "failed", "reason": f"Unknown demos: {sorted(unknown)}"} - ) - return 1 - if args.examples: - check("example-list", ["example", "list"]) - for command in ("demo", "example") if args.examples else ("demo",): - for program in catalogs[command]: - name = program["name"] - if command == "demo" and args.demo and name not in args.demo: - continue - if program["missing_modules"] and not (command == "demo" and args.demo): - reason = f"Missing {program['missing_modules']}; requires extras {program['extras']}" - checks.append({"name": f"{command}-{name}", "status": "skipped", "reason": reason}) - print(f"SKIPPED {command}-{name}: {reason}") - continue - result = check(f"{command}-{name}-help", [command, name, "--help"]) - if result["status"] == "passed" and command == "demo" and args.run: - forwarded = args.demo_args[1:] if args.demo_args[:1] == ["--"] else args.demo_args - check(f"demo-{name}-run", [command, name, *forwarded, "--max_steps", str(args.steps), "--info"]) - except (OSError, ValueError, KeyError) as error: - checks.append({"name": "qa-runner", "status": "failed", "reason": str(error)}) - print(f"FAILED qa-runner: {error}") - except KeyboardInterrupt: - checks.append({"name": "qa-runner", "status": "failed", "reason": "Interrupted"}) - finally: - report["passed"] = bool(checks) and all(result["status"] != "failed" for result in checks) - report["counts"] = { - status: sum(result["status"] == status for result in checks) for status in ("passed", "failed", "skipped") - } - (output_dir / "report.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") - print(f"Results: {report['counts']}") - print(f"Report: {output_dir / 'report.json'}") - return 0 if report["passed"] else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) From 1f1b1f3519d7202a3a520aba842d99918558f244 Mon Sep 17 00:00:00 2001 From: Mustafa Haiderbhai Date: Mon, 5 Oct 2026 22:39:24 -0700 Subject: [PATCH 4/4] Embed benchmark snapshots in downloaded documentation --- docs/_extensions/isaaclab_docs.py | 18 ++++++ .../source/_static/css/environment-browser.js | 40 ++----------- docs/source/setup/environments.rst | 4 +- tools/test/test_doc_benchmarks.py | 57 +++++++++++++++++++ 4 files changed, 82 insertions(+), 37 deletions(-) create mode 100644 tools/test/test_doc_benchmarks.py diff --git a/docs/_extensions/isaaclab_docs.py b/docs/_extensions/isaaclab_docs.py index 0f6a61a92084..2de7ac4e2b63 100644 --- a/docs/_extensions/isaaclab_docs.py +++ b/docs/_extensions/isaaclab_docs.py @@ -7,6 +7,7 @@ from __future__ import annotations +import csv import inspect import json import posixpath @@ -84,6 +85,22 @@ def _parse_rst(directive: SphinxDirective, content: str) -> list[nodes.Node]: return container.children +class IsaacLabBenchmarkData(SphinxDirective): + """Embed benchmark snapshots so downloaded HTML works without an HTTP server.""" + + def run(self) -> list[nodes.Node]: + snapshots = {} + for channel in ("release", "develop"): + path = Path(self.env.srcdir) / "source/_static/benchmarks" / f"environment-performance-{channel}.csv" + self.env.note_dependency(str(path)) + with path.open(encoding="utf-8", newline="") as stream: + snapshots[channel] = list(csv.DictReader(stream)) + # Prevent CSV text from closing the script element, even inside a JSON string. + payload = json.dumps(snapshots).replace("<", "\\u003c") + html = f'' + return [nodes.raw("", html, format="html")] + + class IsaacLabCloneCommands(SphinxDirective): """Render SSH/HTTPS clone tabs using copy-friendly ``code-block`` directives.""" @@ -283,6 +300,7 @@ def setup(app): app.add_config_value("torchvision_version", "", "env") app.add_config_value("ovrtx_spec", "", "env") app.add_role("isaaclab-source", IsaacLabSourceLink()) + app.add_directive("isaaclab-benchmark-data", IsaacLabBenchmarkData) app.add_directive("isaaclab-clone-commands", IsaacLabCloneCommands) app.add_directive("isaaclab-clone-https", IsaacLabCloneHttps) app.add_directive("isaaclab-uv-isaacsim-wheel-install", IsaacLabUvIsaacSimWheelInstall) diff --git a/docs/source/_static/css/environment-browser.js b/docs/source/_static/css/environment-browser.js index 567d87999296..fdbdb2a292ee 100644 --- a/docs/source/_static/css/environment-browser.js +++ b/docs/source/_static/css/environment-browser.js @@ -642,32 +642,6 @@ updateSelection(); }; - const parseCsv = (contents) => { - const parseLine = (line) => { - const values = []; - let value = ""; - let quoted = false; - for (let index = 0; index < line.length; index += 1) { - const character = line[index]; - if (character === '"' && quoted && line[index + 1] === '"') { - value += '"'; - index += 1; - } else if (character === '"') { - quoted = !quoted; - } else if (character === "," && !quoted) { - values.push(value); - value = ""; - } else { - value += character; - } - } - values.push(value); - return values; - }; - const [header, ...lines] = contents.trim().split(/\r?\n/).map(parseLine); - return lines.map((line) => Object.fromEntries(header.map((name, index) => [name, line[index]]))); - }; - const formatFps = (value) => { if (value >= 1_000_000) { return `${Number((value / 1_000_000).toFixed(2))}M`; @@ -931,18 +905,14 @@ chart.replaceChildren(...(rows.length ? [renderBenchmarkChart(rows, maximum)] : [])); }; - const renderBenchmarks = async () => { + const renderBenchmarks = () => { if (!benchmarks) { return; } - benchmarkRows = (await Promise.all(["release", "develop"].map(async (channel) => { + const snapshots = JSON.parse(document.querySelector("[data-environment-benchmark-rows]").textContent); + benchmarkRows = ["release", "develop"].flatMap((channel) => { try { - const source = new URL(benchmarks.getAttribute(`data-benchmark-${channel}-source`), window.location.href); - const response = await fetch(source); - if (!response.ok) { - throw new Error(`Benchmark request failed with ${response.status}`); - } - return parseCsv(await response.text()) + return snapshots[channel] .filter((row) => row.data_origin === "measured" && row.workload === "training") .filter((row) => [row.collection_fps_mean, row.total_fps_mean] .every((value) => Number.isFinite(Number(value)) && Number(value) > 0)) @@ -955,7 +925,7 @@ console.error(error); return []; } - }))).flat(); + }); updatePreview(); }; diff --git a/docs/source/setup/environments.rst b/docs/source/setup/environments.rst index c19f7f280062..400287c4ef9b 100644 --- a/docs/source/setup/environments.rst +++ b/docs/source/setup/environments.rst @@ -102,11 +102,11 @@ Task Preview Benchmarks ---------- +.. isaaclab-benchmark-data:: + .. raw:: html
diff --git a/tools/test/test_doc_benchmarks.py b/tools/test/test_doc_benchmarks.py new file mode 100644 index 000000000000..b876d14e5e0d --- /dev/null +++ b/tools/test/test_doc_benchmarks.py @@ -0,0 +1,57 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for self-contained benchmark snapshots in built documentation.""" + +import csv +import json +import re +import subprocess +import sys +from pathlib import Path + + +def test_benchmark_snapshots_are_embedded_and_refresh_on_csv_changes(tmp_path): + """HTML preserves both CSV snapshots safely and tracks changes during incremental builds.""" + extensions = Path(__file__).resolve().parents[2] / "docs/_extensions" + source = tmp_path / "docs" + snapshots = source / "source/_static/benchmarks" + snapshots.mkdir(parents=True) + (source / "conf.py").write_text( + f"import sys\nsys.path.insert(0, {str(extensions)!r})\nextensions = ['isaaclab_docs']\n", + encoding="utf-8", + ) + (source / "index.rst").write_text("Benchmarks\n==========\n\n.. isaaclab-benchmark-data::\n", encoding="utf-8") + rows = { + "release": [{"task": "Isaac-Cartpole", "collection_fps_mean": "100", "notes": '\nquoted "value",'}], + "develop": [{"task": "Isaac-Cartpole", "collection_fps_mean": "200", "notes": "historical snapshot"}], + } + + def write_snapshot(channel): + with (snapshots / f"environment-performance-{channel}.csv").open("w", encoding="utf-8", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=["task", "collection_fps_mean", "notes"]) + writer.writeheader() + writer.writerows(rows[channel]) + + def build_snapshots(): + output = tmp_path / "html" + subprocess.run( + [sys.executable, "-m", "sphinx", "-W", "-q", str(source), str(output)], + check=True, + capture_output=True, + text=True, + ) + html = (output / "index.html").read_text(encoding="utf-8") + payload = re.search(r'', html) + assert payload is not None + return json.loads(payload.group(1)) + + for channel in rows: + write_snapshot(channel) + assert build_snapshots() == rows + + rows["develop"][0]["collection_fps_mean"] = "300" + write_snapshot("develop") + assert build_snapshots() == rows