From 32b923fda2140d73e51ca6731cf13825563b61af Mon Sep 17 00:00:00 2001 From: jichuanh Date: Tue, 13 Oct 2026 02:26:29 -0700 Subject: [PATCH 01/21] Run the test orchestrator on Windows Windows cannot select() on pipes and has no process groups, so the orchestrator peeks each pipe before reading and kills a hung test's tree with taskkill. Report names drop the drive colon, which Windows would read as an alternate data stream. --- tools/conftest.py | 74 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 14 deletions(-) diff --git a/tools/conftest.py b/tools/conftest.py index 19b7be193180..9f7845064ff9 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -174,6 +174,53 @@ def resolve_exit_code(num_failing: int, num_timeout: int, num_crashed: int, num_ return 0 +def _readable(fds, timeout): + """Return the pipes in ``fds`` that a read would not block on, waiting up to ``timeout`` seconds for one.""" + if os.name != "nt": + return select.select(fds, [], [], timeout)[0] + # Windows cannot select() on pipes, so poll each one instead. + deadline = time.monotonic() + timeout + while not (ready := [fd for fd in fds if _pipe_readable(fd)]) and time.monotonic() < deadline: + time.sleep(0.01) + return ready + + +def _pipe_readable(fd): + """Return whether the Windows pipe ``fd`` holds output or its writer has closed it.""" + import ctypes + import msvcrt + from ctypes import wintypes + + peek = ctypes.windll.kernel32.PeekNamedPipe + peek.argtypes = ( + wintypes.HANDLE, + wintypes.LPVOID, + wintypes.DWORD, + wintypes.LPDWORD, + wintypes.LPDWORD, + wintypes.LPDWORD, + ) + available = wintypes.DWORD() + if not peek(msvcrt.get_osfhandle(fd), None, 0, None, ctypes.byref(available), None): + return True # broken pipe: the read returns EOF at once + return available.value > 0 + + +def _kill_process_tree(process, pgid): + """Kill the test process and every process it started (Kit, Isaac Sim).""" + if pgid is None: + # Windows has no process groups; taskkill walks the tree from the test process instead. + result = subprocess.run(["taskkill", "/F", "/T", "/PID", str(process.pid)], capture_output=True, check=False) + if result.returncode != 0: + with contextlib.suppress(OSError): + process.kill() + return + try: + os.killpg(pgid, signal.SIGKILL) + except (ProcessLookupError, PermissionError, OSError): + process.kill() + + def _drain_ready_output(process, stdout_fd, stderr_fd, timeout=0.1, echo=True): """Read whatever is readable on the child's pipes, echoing it as it arrives. @@ -193,7 +240,7 @@ def _drain_ready_output(process, stdout_fd, stderr_fd, timeout=0.1, echo=True): stdout_chunk = b"" stderr_chunk = b"" try: - ready_fds, _, _ = select.select([stdout_fd, stderr_fd], [], [], timeout) + ready_fds = _readable([stdout_fd, stderr_fd], timeout) for fd in ready_fds: with contextlib.suppress(OSError): @@ -382,7 +429,7 @@ def capture_test_output_with_timeout(cmd, timeout, env, startup_deadline=0, repo universal_newlines=False, start_new_session=True, ) - pgid = os.getpgid(process.pid) + pgid = None if os.name == "nt" else os.getpgid(process.pid) stdout_fd = process.stdout.fileno() stderr_fd = process.stderr.fileno() @@ -439,11 +486,7 @@ def capture_test_output_with_timeout(cmd, timeout, env, startup_deadline=0, repo # Ahead of the system tables, which _get_diagnostics truncates off the end. pre_kill_diag = f"{hang_stacks}\n\n{pre_kill_diag}" - # Kill the entire process group (test + any Kit children). - try: - os.killpg(pgid, signal.SIGKILL) - except (ProcessLookupError, PermissionError, OSError): - process.kill() + _kill_process_tree(process, pgid) try: remaining_stdout, remaining_stderr = process.communicate(timeout=5) stdout_data += remaining_stdout @@ -465,12 +508,14 @@ def capture_test_output_with_timeout(cmd, timeout, env, startup_deadline=0, repo except Exception: pass - # Kill any orphaned child processes (Kit, Isaac Sim) left by the test. - try: - os.killpg(pgid, signal.SIGKILL) - time.sleep(1) - except (ProcessLookupError, PermissionError, OSError): - pass + # Kill any orphaned child processes (Kit, Isaac Sim) left by the test. On Windows the + # tree is unreachable once its root has exited. + if pgid is not None: + try: + os.killpg(pgid, signal.SIGKILL) + time.sleep(1) + except (ProcessLookupError, PermissionError, OSError): + pass wall_time = time.time() - start_time return process.returncode, stdout_data, stderr_data, "", wall_time, "" @@ -1050,7 +1095,8 @@ def _run_one_pass( # ``isaaclab_physx/.../test_articulation.py``) don't write to the same path # inside the shared ``/workspace/isaaclab`` mount and trigger false # shutdown_hang detections in sibling shards via the report-file existence check. - report_slug = str(ctx.test_file).replace("/", "__").replace("\\", "__") + # A Windows drive colon would name an alternate data stream instead of a file. + report_slug = str(ctx.test_file).replace("/", "__").replace("\\", "__").replace(":", "") report_file = f"tests/test-reports-{report_slug}{suffix}.xml" # Crash-durable progress log. pytest writes its JUnit XML only at session end, so this is # the only record of what passed, failed, and was in flight when a run dies before that. From bad4d1967f5d9e2b46fdcd01f2d53d3deb286d14 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Tue, 13 Oct 2026 02:26:30 -0700 Subject: [PATCH 02/21] Mark the Windows-safe isaaclab.utils tests for Windows CI --- source/isaaclab/changelog.d/jichuanh-windows-ci.skip | 0 source/isaaclab/test/utils/test_configclass.py | 2 +- source/isaaclab/test/utils/test_dict.py | 2 +- source/isaaclab/test/utils/test_episode_data.py | 2 +- .../test/utils/test_hdf5_dataset_file_handler.py | 10 ++++------ 5 files changed, 7 insertions(+), 9 deletions(-) create mode 100644 source/isaaclab/changelog.d/jichuanh-windows-ci.skip diff --git a/source/isaaclab/changelog.d/jichuanh-windows-ci.skip b/source/isaaclab/changelog.d/jichuanh-windows-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab/test/utils/test_configclass.py b/source/isaaclab/test/utils/test_configclass.py index 7d53871de683..e4e1ed1c5fa4 100644 --- a/source/isaaclab/test/utils/test_configclass.py +++ b/source/isaaclab/test/utils/test_configclass.py @@ -20,7 +20,7 @@ from isaaclab.utils.io import dump_yaml, load_yaml from isaaclab.utils.string import ResolvableString -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.windows_ci] """ Mock classes and functions. diff --git a/source/isaaclab/test/utils/test_dict.py b/source/isaaclab/test/utils/test_dict.py index b91720ab1a79..373718dacbd9 100644 --- a/source/isaaclab/test/utils/test_dict.py +++ b/source/isaaclab/test/utils/test_dict.py @@ -12,7 +12,7 @@ import isaaclab.utils.dict as dict_utils import isaaclab.utils.string as string_utils -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.windows_ci] def _test_function(x): diff --git a/source/isaaclab/test/utils/test_episode_data.py b/source/isaaclab/test/utils/test_episode_data.py index 1d9536dc05ac..7aa071a42a54 100644 --- a/source/isaaclab/test/utils/test_episode_data.py +++ b/source/isaaclab/test/utils/test_episode_data.py @@ -8,7 +8,7 @@ from isaaclab.test.utils import DeviceScope, test_devices from isaaclab.utils.datasets import EpisodeData -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.windows_ci] @pytest.mark.parametrize("device", test_devices(DeviceScope.CPU)) diff --git a/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py b/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py index 1fb78180ac5f..10c6c7238df4 100644 --- a/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py +++ b/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: BSD-3-Clause import json import os -import shutil import tempfile import uuid @@ -15,7 +14,7 @@ from isaaclab.test.utils import test_devices from isaaclab.utils.datasets import EpisodeData, HDF5DatasetFileHandler -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.windows_ci] def create_test_episode(device): @@ -41,10 +40,9 @@ def create_test_episode(device): @pytest.fixture def temp_dir(): """Create a temporary directory for test datasets.""" - temp_dir = tempfile.mkdtemp() - yield temp_dir - # cleanup after tests - shutil.rmtree(temp_dir) + # On Windows, libhdf5 releases its file handle shortly after close(), so the cleanup can race it. + with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as d: + yield d def test_create_dataset_file(temp_dir, monkeypatch): From efda6ebe0ad0d1b22c69a2f8fc2b3c1d43e60069 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Tue, 13 Oct 2026 02:26:31 -0700 Subject: [PATCH 03/21] Test ovrtx coexistence and Kit RTX camera rendering on Windows Moves the two checks the Windows workflow ran as inline scripts into windows_ci test files: ovrtx rendering alongside usd-exchange in either load order (nvbug 6817794), and the Cartpole camera task through Kit's RTX renderer. --- .../changelog.d/jichuanh-windows-ci.skip | 0 .../test_ovrtx_usd_exchange_coexistence.py | 78 +++++++++++++++++++ .../changelog.d/jichuanh-windows-ci.skip | 0 .../core/test_cartpole_camera_isaacsim_rtx.py | 45 +++++++++++ 4 files changed, 123 insertions(+) create mode 100644 source/isaaclab_ov/changelog.d/jichuanh-windows-ci.skip create mode 100644 source/isaaclab_ov/test/test_ovrtx_usd_exchange_coexistence.py create mode 100644 source/isaaclab_tasks/changelog.d/jichuanh-windows-ci.skip create mode 100644 source/isaaclab_tasks/test/core/test_cartpole_camera_isaacsim_rtx.py diff --git a/source/isaaclab_ov/changelog.d/jichuanh-windows-ci.skip b/source/isaaclab_ov/changelog.d/jichuanh-windows-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ov/test/test_ovrtx_usd_exchange_coexistence.py b/source/isaaclab_ov/test/test_ovrtx_usd_exchange_coexistence.py new file mode 100644 index 000000000000..5d53a95c9c19 --- /dev/null +++ b/source/isaaclab_ov/test/test_ovrtx_usd_exchange_coexistence.py @@ -0,0 +1,78 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""ovrtx renders in a process that has loaded usd-exchange, in either order. + +usd-exchange 3.0.0 and ovrtx 0.5.0 shipped differently built MaterialX DLLs under the same names, and on Windows +whichever loaded second failed (nvbug 6817794). ovrtx loads its runtime only when a renderer is created, so every +case creates one, each in a fresh interpreter so that the load order is the case's own. +""" + +import subprocess +import sys + +import pytest + +pytestmark = [ + pytest.mark.windows_ci, + pytest.mark.skipif(sys.platform != "win32", reason="the DLL name clash is specific to the Windows loader"), +] + + +def _load_usd(): + from pxr import Usd # noqa: F401 + + +def _load_materialx(): + import usdex.core # noqa: F401 + + from pxr import UsdMtlx # noqa: F401 + + +def _create_renderer(): + import ovrtx + + ovrtx.Renderer() + + +def _render_cartpole_camera(): + import torch + + from isaaclab_tasks.core.cartpole.cartpole_direct_camera_env import CartpoleCameraEnv + from isaaclab_tasks.utils import resolve_task_config + + env_cfg, _ = resolve_task_config( + "Isaac-Cartpole-Camera-Direct", "", overrides=("physics=newton_mjwarp", "renderer=ovrtx") + ) + env_cfg.scene.num_envs = 2 + env_cfg.frame_stack = 1 + env = CartpoleCameraEnv(cfg=env_cfg) + try: + obs, _ = env.reset() + image = obs["policy"] + assert torch.isfinite(image).all(), "Camera observation contains NaN or infinity" + assert image.amax() > image.amin(), "Camera observation is constant" + finally: + env.close() + + +_CASES = { + "ovrtx-alone": (_create_renderer,), + "usd-first": (_load_usd, _create_renderer), + "materialx-first": (_load_materialx, _create_renderer), + "ovrtx-first": (_create_renderer, _load_materialx), + "isaaclab-render": (_render_cartpole_camera,), +} + + +@pytest.mark.parametrize("case", _CASES) +def test_ovrtx_coexists_with_usd_exchange(case): + result = subprocess.run([sys.executable, __file__, case], capture_output=True, text=True, timeout=600, check=False) + assert result.returncode == 0, f"{case} exited with {result.returncode}:\n{result.stdout}\n{result.stderr}" + + +if __name__ == "__main__": + for step in _CASES[sys.argv[1]]: + step() diff --git a/source/isaaclab_tasks/changelog.d/jichuanh-windows-ci.skip b/source/isaaclab_tasks/changelog.d/jichuanh-windows-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_tasks/test/core/test_cartpole_camera_isaacsim_rtx.py b/source/isaaclab_tasks/test/core/test_cartpole_camera_isaacsim_rtx.py new file mode 100644 index 000000000000..9958e113e916 --- /dev/null +++ b/source/isaaclab_tasks/test/core/test_cartpole_camera_isaacsim_rtx.py @@ -0,0 +1,45 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""The Cartpole camera task renders through Kit's RTX renderer on Windows. + +Linux CI compares this scene against golden images in ``test_rendering_cartpole.py``. +""" + +import sys + +import pytest + +if sys.platform != "win32": + pytest.skip("Linux CI covers this scene with golden images", allow_module_level=True) + +from isaaclab.test.utils import launch_test_simulation + +launch_test_simulation(enable_cameras=True) + +import torch # noqa: E402 + +from isaaclab_tasks.core.cartpole.cartpole_direct_camera_env import CartpoleCameraEnv # noqa: E402 +from isaaclab_tasks.utils import resolve_task_config # noqa: E402 + +pytestmark = pytest.mark.windows_ci + + +def test_cartpole_camera_renders_and_steps(): + env_cfg, _ = resolve_task_config( + "Isaac-Cartpole-Camera-Direct", "", overrides=("physics=isaacsim_physx", "renderer=isaacsim_rtx") + ) + env_cfg.scene.num_envs = 2 + env_cfg.frame_stack = 1 + env = CartpoleCameraEnv(cfg=env_cfg) + try: + obs, _ = env.reset() + image = obs["policy"] + assert torch.isfinite(image).all(), "Camera observation contains NaN or infinity" + assert image.amax() > image.amin(), "Camera observation is constant" + _, reward, _, _, _ = env.step(torch.zeros(env.num_envs, 1, device=env.device)) + assert torch.isfinite(reward).all(), "Cartpole reward contains NaN or infinity" + finally: + env.close() From 9af768a97f227b2caf74e1cbe2e9dda94b463cfc Mon Sep 17 00:00:00 2001 From: jichuanh Date: Tue, 13 Oct 2026 02:26:32 -0700 Subject: [PATCH 04/21] Add Windows CI and Windows installation tests windows-ci.yaml mirrors arm-ci.yml: it syncs the locked environment and runs the windows_ci marker tests through tools/conftest.py, then the shared Cartpole smoke. install-ci.yml gains a Windows job that runs the installation suite natively. --- .github/workflows/install-ci.yml | 49 +++++++++ .github/workflows/windows-ci.yaml | 160 ++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 .github/workflows/windows-ci.yaml diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index 887af011be28..bcba7e8dc7e8 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -161,3 +161,52 @@ jobs: with: base-image: ${{ inputs.base_image || 'ubuntu:24.04' }} test-filter: ${{ inputs.test_filter }} + + # Windows has no Docker, so the suite runs natively. The composite actions above are + # bash and pin setup-uv v6, which crashes on exit on these runners + # (astral-sh/setup-uv#686). + install-tests-windows: + name: Installation Tests (Windows) + needs: [changes] + if: needs.changes.outputs.run_install_tests == 'true' + runs-on: [self-hosted, gpu-windows] + timeout-minutes: 90 + defaults: + run: + shell: powershell + env: + PYTHONUTF8: "1" + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.12.9" + + # By name, Windows resolves bash to WSL's launcher in System32 ahead of PATH. + - name: Build wheel + run: | + & "C:\Program Files\Git\bin\bash.exe" tools/wheel_builder/build.sh + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + - name: Run Tests + env: + TEST_FILTER: ${{ inputs.test_filter }} + run: | + $wheel = @(Get-ChildItem tools/wheel_builder/build/dist -Filter "isaaclab-*.whl") + if ($wheel.Count -ne 1) { throw "expected one wheel, found $($wheel.Count)" } + $pytestArgs = @("--run-gpu", "--tb=short", "-sv", "--junitxml=${{ github.workspace }}\results\results.xml") + if ($env:TEST_FILTER) { $pytestArgs += @("-k", $env:TEST_FILTER) } + uv run --no-project --python 3.12 --with pytest python tools/run_install_ci.py native --wheel $wheel[0].FullName -- @pytestArgs + exit $LASTEXITCODE + + - name: Upload JUnit XML report + if: always() + uses: actions/upload-artifact@v7 + with: + name: install-ci-junit-windows + path: ${{ github.workspace }}/results/results.xml + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/windows-ci.yaml b/.github/workflows/windows-ci.yaml new file mode 100644 index 000000000000..3824f728b886 --- /dev/null +++ b/.github/workflows/windows-ci.yaml @@ -0,0 +1,160 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Marker-gated tests on self-hosted Windows GPU runners, the counterpart of arm-ci.yml. +# Windows has no Docker, so the job syncs the locked uv environment on the runner and +# hands it to the same orchestrator (tools/conftest.py) the container jobs run. + +name: Windows CI + +on: + pull_request: + types: [opened, synchronize, reopened] + branches: + - main + - develop + - 'release/**' + push: + branches: + - main + - develop + - 'release/**' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + pull-requests: read + +env: + OMNI_KIT_ACCEPT_EULA: "yes" + # Decode subprocess output as UTF-8 instead of the cp1252 code page. + PYTHONUTF8: "1" + # A test the orchestrator kills on timeout would otherwise lose its buffered output. + PYTHONUNBUFFERED: "1" + +jobs: + changes: + name: Detect Changes + runs-on: ubuntu-latest + outputs: + should_run: ${{ steps.detect.outputs.should_run }} + steps: + - uses: actions/checkout@v6 + with: + sparse-checkout: .github/actions/detect-changes + sparse-checkout-cone-mode: false + - id: detect + uses: ./.github/actions/detect-changes + with: + triggered-jobs-push: windows-ci + triggered-jobs-pr: windows-ci + patterns: | + ^source/ :: Library source code + ^tools/ :: Build tooling + ^apps/ :: Standalone apps + ^scripts/ :: Standalone scripts + (^|/)pyproject\.toml$ :: Python project metadata + ^uv\.lock$ :: Locked dependency graph + ^VERSION$ :: Version file + ^\.github/workflows/windows-ci\.yaml$ :: This workflow file + + windows-ci: + name: windows-ci + needs: [changes] + if: needs.changes.outputs.should_run == 'true' + runs-on: [self-hosted, gpu-windows] + timeout-minutes: 90 + defaults: + run: + shell: powershell + steps: + - uses: actions/checkout@v6 + + # v6.8.0, which the Linux workflows pin, crashes on exit on these runners once uv + # is in the tool cache (astral-sh/setup-uv#686). + - name: Set up uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.12.9" + + - name: Sync environment + id: sync + timeout-minutes: 30 + run: uv sync --frozen --extra isaacsim --extra ovrtx --extra test + + # ovrtx and Kit compile their shaders into .venv, which checkout's clean deletes. + # The blobs are only valid for the GPU and driver that compiled them. + - name: Shader cache keys + id: shader-cache + run: | + $gpu = @(nvidia-smi --query-gpu=compute_cap,driver_version --format=csv,noheader) + if ($LASTEXITCODE -ne 0 -or $gpu.Count -eq 0) { throw "nvidia-smi failed: $gpu" } + $gpu = "sm$($gpu[0] -replace '\s', '' -replace ',', '-drv')" + foreach ($name in "ovrtx", "isaacsim") { + $pin = @(Select-String -LiteralPath uv.lock -Pattern "^name = `"$name`"$" -Context 0,1) + if ($pin.Count -ne 1) { throw "uv.lock pins $name $($pin.Count) times" } + $version = $pin[0].Context.PostContext[0] -replace '^version = "(.+)"$', '$1' + "$name=windows-$name-$version-$gpu" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + } + + # actions/cache archives with Git's tar, which finds gzip only on PATH (actions/cache#1297). + - name: Restore ovrtx shader cache + id: ovrtx-cache + uses: actions/cache/restore@v4 + env: + PATH: C:\Program Files\Git\usr\bin;C:\Windows\System32 + with: + path: .venv/Lib/site-packages/ovrtx/bin/cache + key: ${{ steps.shader-cache.outputs.ovrtx }} + + - name: Restore Kit shader cache + id: kit-cache + uses: actions/cache/restore@v4 + env: + PATH: C:\Program Files\Git\usr\bin;C:\Windows\System32 + with: + path: .venv/Lib/site-packages/isaacsim/kit/cache + key: ${{ steps.shader-cache.outputs.isaacsim }} + + - name: Run windows_ci marker tests + env: + CI_MARKER: windows_ci + TEST_RESULT_FILE: windows-ci-report.xml + run: uv run --frozen --no-sync python -m pytest tools + + # The orchestrator skips install_ci/, so the Cartpole state/camera probes run here. + - name: Run Cartpole smoke tests + if: ${{ !cancelled() && steps.sync.outcome == 'success' }} + run: uv run --frozen --no-sync python -m pytest -v source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py --junitxml=tests/windows-ci-smoke-report.xml + + - name: Save ovrtx shader cache + if: ${{ success() && steps.ovrtx-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + env: + PATH: C:\Program Files\Git\usr\bin;C:\Windows\System32 + with: + path: .venv/Lib/site-packages/ovrtx/bin/cache + key: ${{ steps.shader-cache.outputs.ovrtx }} + + - name: Save Kit shader cache + if: ${{ success() && steps.kit-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + env: + PATH: C:\Program Files\Git\usr\bin;C:\Windows\System32 + with: + path: .venv/Lib/site-packages/isaacsim/kit/cache + key: ${{ steps.shader-cache.outputs.isaacsim }} + + - name: Upload test reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: windows-ci-reports + path: tests/ + retention-days: 7 From 05cef35a5cebce6ac1fb0293ca7f50fb0c8b1fa6 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Tue, 13 Oct 2026 02:59:20 -0700 Subject: [PATCH 05/21] Keep the Windows installation venvs under a short path pytest's default base temp on the runner sits under the NetworkService profile, which pushes Kit's DLLs in the Isaac Sim venv past MAX_PATH. --- .github/workflows/install-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index bcba7e8dc7e8..b84404d6b84f 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -197,7 +197,9 @@ jobs: run: | $wheel = @(Get-ChildItem tools/wheel_builder/build/dist -Filter "isaaclab-*.whl") if ($wheel.Count -ne 1) { throw "expected one wheel, found $($wheel.Count)" } - $pytestArgs = @("--run-gpu", "--tb=short", "-sv", "--junitxml=${{ github.workspace }}\results\results.xml") + # pytest's default base sits under the NetworkService profile, deep enough that + # Kit's DLLs in a venv there pass the 260-character path limit. + $pytestArgs = @("--run-gpu", "--tb=short", "-sv", "--basetemp=$env:RUNNER_TEMP\i", "--junitxml=${{ github.workspace }}\results\results.xml") if ($env:TEST_FILTER) { $pytestArgs += @("-k", $env:TEST_FILTER) } uv run --no-project --python 3.12 --with pytest python tools/run_install_ci.py native --wheel $wheel[0].FullName -- @pytestArgs exit $LASTEXITCODE From 378000b2032dcb6527b057c785dd27e92c6a1731 Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Wed, 21 Oct 2026 20:04:53 -0700 Subject: [PATCH 06/21] Share the Cartpole camera check across renderers The Cartpole smoke now renders the camera through the Newton renderer and Kit's RTX renderer, each in a fresh interpreter, and the ovrtx coexistence test reuses that check instead of keeping its own copy. --- .../misc/cartpole_training_smoke.py | 33 ++++++++++---- .../test_ovrtx_usd_exchange_coexistence.py | 31 +++---------- .../changelog.d/jichuanh-windows-ci.skip | 0 .../core/test_cartpole_camera_isaacsim_rtx.py | 45 ------------------- 4 files changed, 32 insertions(+), 77 deletions(-) delete mode 100644 source/isaaclab_tasks/changelog.d/jichuanh-windows-ci.skip delete mode 100644 source/isaaclab_tasks/test/core/test_cartpole_camera_isaacsim_rtx.py diff --git a/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py b/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py index 08564d0afae3..c22bd4d8b53d 100644 --- a/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py +++ b/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py @@ -15,6 +15,8 @@ import sys from pathlib import Path +import pytest + _STATE_TRAIN_CMD = [ "train", "--rl_library", @@ -77,18 +79,23 @@ def test_train_cartpole_state_completes() -> None: _run_training(_STATE_TRAIN_CMD, timeout=600) -def test_render_cartpole_camera_produces_valid_observation_and_reward() -> None: - """Verify that camera Cartpole renders varying pixels and produces finite rewards.""" - import torch +def _render_cartpole_camera(physics: str, renderer: str) -> None: + """Render and step camera Cartpole, starting the runtime the physics and renderer need.""" + from isaaclab.test.utils import launch_test_simulation - from isaaclab_tasks.core.cartpole.cartpole_direct_camera_env import CartpoleCameraEnv from isaaclab_tasks.utils import resolve_task_config env_cfg, _ = resolve_task_config( - "Isaac-Cartpole-Camera-Direct", "", overrides=("physics=newton_mjwarp", "renderer=newton_renderer") + "Isaac-Cartpole-Camera-Direct", "", overrides=(f"physics={physics}", f"renderer={renderer}") ) env_cfg.scene.num_envs = 2 env_cfg.frame_stack = 1 + launch_test_simulation(env_cfg) + + import torch + + from isaaclab_tasks.core.cartpole.cartpole_direct_camera_env import CartpoleCameraEnv + env = None try: env = CartpoleCameraEnv(cfg=env_cfg) @@ -109,6 +116,18 @@ def test_render_cartpole_camera_produces_valid_observation_and_reward() -> None: env.close() +@pytest.mark.parametrize( + ("physics", "renderer"), [("newton_mjwarp", "newton_renderer"), ("isaacsim_physx", "isaacsim_rtx")] +) +def test_render_cartpole_camera_produces_valid_observation_and_reward(physics: str, renderer: str) -> None: + """Verify that camera Cartpole renders varying pixels and produces finite rewards.""" + # Kit stays up until its process exits, so each renderer runs in a fresh interpreter. + result = subprocess.run( + [sys.executable, __file__, physics, renderer], text=True, capture_output=True, timeout=1800, check=False + ) + assert result.returncode == 0, f"Rendering failed (rc={result.returncode}):\n{result.stdout}{result.stderr}" + + def test_train_cartpole_camera_completes() -> None: """Verify that camera-observation Cartpole completes short RSL-RL training.""" # A cold camera run compiles shaders before training begins. @@ -116,6 +135,4 @@ def test_train_cartpole_camera_completes() -> None: if __name__ == "__main__": - test_train_cartpole_state_completes() - test_render_cartpole_camera_produces_valid_observation_and_reward() - test_train_cartpole_camera_completes() + _render_cartpole_camera(*sys.argv[1:]) diff --git a/source/isaaclab_ov/test/test_ovrtx_usd_exchange_coexistence.py b/source/isaaclab_ov/test/test_ovrtx_usd_exchange_coexistence.py index 5d53a95c9c19..dcf0de05a93c 100644 --- a/source/isaaclab_ov/test/test_ovrtx_usd_exchange_coexistence.py +++ b/source/isaaclab_ov/test/test_ovrtx_usd_exchange_coexistence.py @@ -12,6 +12,7 @@ import subprocess import sys +from pathlib import Path import pytest @@ -37,39 +38,21 @@ def _create_renderer(): ovrtx.Renderer() -def _render_cartpole_camera(): - import torch - - from isaaclab_tasks.core.cartpole.cartpole_direct_camera_env import CartpoleCameraEnv - from isaaclab_tasks.utils import resolve_task_config - - env_cfg, _ = resolve_task_config( - "Isaac-Cartpole-Camera-Direct", "", overrides=("physics=newton_mjwarp", "renderer=ovrtx") - ) - env_cfg.scene.num_envs = 2 - env_cfg.frame_stack = 1 - env = CartpoleCameraEnv(cfg=env_cfg) - try: - obs, _ = env.reset() - image = obs["policy"] - assert torch.isfinite(image).all(), "Camera observation contains NaN or infinity" - assert image.amax() > image.amin(), "Camera observation is constant" - finally: - env.close() - - _CASES = { "ovrtx-alone": (_create_renderer,), "usd-first": (_load_usd, _create_renderer), "materialx-first": (_load_materialx, _create_renderer), "ovrtx-first": (_create_renderer, _load_materialx), - "isaaclab-render": (_render_cartpole_camera,), } +# Renders camera Cartpole through Isaac Lab, which loads both libraries. +_CARTPOLE_SMOKE = Path(__file__).resolve().parents[2] / "isaaclab/test/install_ci/misc/cartpole_training_smoke.py" + -@pytest.mark.parametrize("case", _CASES) +@pytest.mark.parametrize("case", [*_CASES, "isaaclab-render"]) def test_ovrtx_coexists_with_usd_exchange(case): - result = subprocess.run([sys.executable, __file__, case], capture_output=True, text=True, timeout=600, check=False) + script = [_CARTPOLE_SMOKE, "newton_mjwarp", "ovrtx"] if case == "isaaclab-render" else [__file__, case] + result = subprocess.run([sys.executable, *script], capture_output=True, text=True, timeout=600, check=False) assert result.returncode == 0, f"{case} exited with {result.returncode}:\n{result.stdout}\n{result.stderr}" diff --git a/source/isaaclab_tasks/changelog.d/jichuanh-windows-ci.skip b/source/isaaclab_tasks/changelog.d/jichuanh-windows-ci.skip deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/source/isaaclab_tasks/test/core/test_cartpole_camera_isaacsim_rtx.py b/source/isaaclab_tasks/test/core/test_cartpole_camera_isaacsim_rtx.py deleted file mode 100644 index 9958e113e916..000000000000 --- a/source/isaaclab_tasks/test/core/test_cartpole_camera_isaacsim_rtx.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""The Cartpole camera task renders through Kit's RTX renderer on Windows. - -Linux CI compares this scene against golden images in ``test_rendering_cartpole.py``. -""" - -import sys - -import pytest - -if sys.platform != "win32": - pytest.skip("Linux CI covers this scene with golden images", allow_module_level=True) - -from isaaclab.test.utils import launch_test_simulation - -launch_test_simulation(enable_cameras=True) - -import torch # noqa: E402 - -from isaaclab_tasks.core.cartpole.cartpole_direct_camera_env import CartpoleCameraEnv # noqa: E402 -from isaaclab_tasks.utils import resolve_task_config # noqa: E402 - -pytestmark = pytest.mark.windows_ci - - -def test_cartpole_camera_renders_and_steps(): - env_cfg, _ = resolve_task_config( - "Isaac-Cartpole-Camera-Direct", "", overrides=("physics=isaacsim_physx", "renderer=isaacsim_rtx") - ) - env_cfg.scene.num_envs = 2 - env_cfg.frame_stack = 1 - env = CartpoleCameraEnv(cfg=env_cfg) - try: - obs, _ = env.reset() - image = obs["policy"] - assert torch.isfinite(image).all(), "Camera observation contains NaN or infinity" - assert image.amax() > image.amin(), "Camera observation is constant" - _, reward, _, _, _ = env.step(torch.zeros(env.num_envs, 1, device=env.device)) - assert torch.isfinite(reward).all(), "Cartpole reward contains NaN or infinity" - finally: - env.close() From 73bf0ea17becf5158787b2d827eb022896c306e0 Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Thu, 22 Oct 2026 15:54:25 -0700 Subject: [PATCH 07/21] Drop the ovrtx coexistence test usd-exchange 3.0.1 fixed the DLL name clash the test guarded against (nvbug 6817794). The Cartpole smoke returns to the Newton probes it runs on main, so ARM CI runs what it runs there. --- .../misc/cartpole_training_smoke.py | 33 +++------- .../changelog.d/jichuanh-windows-ci.skip | 0 .../test_ovrtx_usd_exchange_coexistence.py | 61 ------------------- 3 files changed, 8 insertions(+), 86 deletions(-) delete mode 100644 source/isaaclab_ov/changelog.d/jichuanh-windows-ci.skip delete mode 100644 source/isaaclab_ov/test/test_ovrtx_usd_exchange_coexistence.py diff --git a/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py b/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py index c22bd4d8b53d..08564d0afae3 100644 --- a/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py +++ b/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py @@ -15,8 +15,6 @@ import sys from pathlib import Path -import pytest - _STATE_TRAIN_CMD = [ "train", "--rl_library", @@ -79,23 +77,18 @@ def test_train_cartpole_state_completes() -> None: _run_training(_STATE_TRAIN_CMD, timeout=600) -def _render_cartpole_camera(physics: str, renderer: str) -> None: - """Render and step camera Cartpole, starting the runtime the physics and renderer need.""" - from isaaclab.test.utils import launch_test_simulation +def test_render_cartpole_camera_produces_valid_observation_and_reward() -> None: + """Verify that camera Cartpole renders varying pixels and produces finite rewards.""" + import torch + from isaaclab_tasks.core.cartpole.cartpole_direct_camera_env import CartpoleCameraEnv from isaaclab_tasks.utils import resolve_task_config env_cfg, _ = resolve_task_config( - "Isaac-Cartpole-Camera-Direct", "", overrides=(f"physics={physics}", f"renderer={renderer}") + "Isaac-Cartpole-Camera-Direct", "", overrides=("physics=newton_mjwarp", "renderer=newton_renderer") ) env_cfg.scene.num_envs = 2 env_cfg.frame_stack = 1 - launch_test_simulation(env_cfg) - - import torch - - from isaaclab_tasks.core.cartpole.cartpole_direct_camera_env import CartpoleCameraEnv - env = None try: env = CartpoleCameraEnv(cfg=env_cfg) @@ -116,18 +109,6 @@ def _render_cartpole_camera(physics: str, renderer: str) -> None: env.close() -@pytest.mark.parametrize( - ("physics", "renderer"), [("newton_mjwarp", "newton_renderer"), ("isaacsim_physx", "isaacsim_rtx")] -) -def test_render_cartpole_camera_produces_valid_observation_and_reward(physics: str, renderer: str) -> None: - """Verify that camera Cartpole renders varying pixels and produces finite rewards.""" - # Kit stays up until its process exits, so each renderer runs in a fresh interpreter. - result = subprocess.run( - [sys.executable, __file__, physics, renderer], text=True, capture_output=True, timeout=1800, check=False - ) - assert result.returncode == 0, f"Rendering failed (rc={result.returncode}):\n{result.stdout}{result.stderr}" - - def test_train_cartpole_camera_completes() -> None: """Verify that camera-observation Cartpole completes short RSL-RL training.""" # A cold camera run compiles shaders before training begins. @@ -135,4 +116,6 @@ def test_train_cartpole_camera_completes() -> None: if __name__ == "__main__": - _render_cartpole_camera(*sys.argv[1:]) + test_train_cartpole_state_completes() + test_render_cartpole_camera_produces_valid_observation_and_reward() + test_train_cartpole_camera_completes() diff --git a/source/isaaclab_ov/changelog.d/jichuanh-windows-ci.skip b/source/isaaclab_ov/changelog.d/jichuanh-windows-ci.skip deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/source/isaaclab_ov/test/test_ovrtx_usd_exchange_coexistence.py b/source/isaaclab_ov/test/test_ovrtx_usd_exchange_coexistence.py deleted file mode 100644 index dcf0de05a93c..000000000000 --- a/source/isaaclab_ov/test/test_ovrtx_usd_exchange_coexistence.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""ovrtx renders in a process that has loaded usd-exchange, in either order. - -usd-exchange 3.0.0 and ovrtx 0.5.0 shipped differently built MaterialX DLLs under the same names, and on Windows -whichever loaded second failed (nvbug 6817794). ovrtx loads its runtime only when a renderer is created, so every -case creates one, each in a fresh interpreter so that the load order is the case's own. -""" - -import subprocess -import sys -from pathlib import Path - -import pytest - -pytestmark = [ - pytest.mark.windows_ci, - pytest.mark.skipif(sys.platform != "win32", reason="the DLL name clash is specific to the Windows loader"), -] - - -def _load_usd(): - from pxr import Usd # noqa: F401 - - -def _load_materialx(): - import usdex.core # noqa: F401 - - from pxr import UsdMtlx # noqa: F401 - - -def _create_renderer(): - import ovrtx - - ovrtx.Renderer() - - -_CASES = { - "ovrtx-alone": (_create_renderer,), - "usd-first": (_load_usd, _create_renderer), - "materialx-first": (_load_materialx, _create_renderer), - "ovrtx-first": (_create_renderer, _load_materialx), -} - -# Renders camera Cartpole through Isaac Lab, which loads both libraries. -_CARTPOLE_SMOKE = Path(__file__).resolve().parents[2] / "isaaclab/test/install_ci/misc/cartpole_training_smoke.py" - - -@pytest.mark.parametrize("case", [*_CASES, "isaaclab-render"]) -def test_ovrtx_coexists_with_usd_exchange(case): - script = [_CARTPOLE_SMOKE, "newton_mjwarp", "ovrtx"] if case == "isaaclab-render" else [__file__, case] - result = subprocess.run([sys.executable, *script], capture_output=True, text=True, timeout=600, check=False) - assert result.returncode == 0, f"{case} exited with {result.returncode}:\n{result.stdout}\n{result.stderr}" - - -if __name__ == "__main__": - for step in _CASES[sys.argv[1]]: - step() From ddbb985b538446446702a02cca438c022528079f Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Thu, 22 Oct 2026 15:54:52 -0700 Subject: [PATCH 08/21] Run Windows CI on the base install ARM CI's smoke runs Newton only, so Windows syncs the base environment with the test extra and no longer installs Isaac Sim or ovrtx, or keeps their shader caches. --- .github/workflows/windows-ci.yaml | 55 +------------------------------ 1 file changed, 1 insertion(+), 54 deletions(-) diff --git a/.github/workflows/windows-ci.yaml b/.github/workflows/windows-ci.yaml index 3824f728b886..d3af13a375c9 100644 --- a/.github/workflows/windows-ci.yaml +++ b/.github/workflows/windows-ci.yaml @@ -32,7 +32,6 @@ permissions: pull-requests: read env: - OMNI_KIT_ACCEPT_EULA: "yes" # Decode subprocess output as UTF-8 instead of the cp1252 code page. PYTHONUTF8: "1" # A test the orchestrator kills on timeout would otherwise lose its buffered output. @@ -86,41 +85,7 @@ jobs: - name: Sync environment id: sync timeout-minutes: 30 - run: uv sync --frozen --extra isaacsim --extra ovrtx --extra test - - # ovrtx and Kit compile their shaders into .venv, which checkout's clean deletes. - # The blobs are only valid for the GPU and driver that compiled them. - - name: Shader cache keys - id: shader-cache - run: | - $gpu = @(nvidia-smi --query-gpu=compute_cap,driver_version --format=csv,noheader) - if ($LASTEXITCODE -ne 0 -or $gpu.Count -eq 0) { throw "nvidia-smi failed: $gpu" } - $gpu = "sm$($gpu[0] -replace '\s', '' -replace ',', '-drv')" - foreach ($name in "ovrtx", "isaacsim") { - $pin = @(Select-String -LiteralPath uv.lock -Pattern "^name = `"$name`"$" -Context 0,1) - if ($pin.Count -ne 1) { throw "uv.lock pins $name $($pin.Count) times" } - $version = $pin[0].Context.PostContext[0] -replace '^version = "(.+)"$', '$1' - "$name=windows-$name-$version-$gpu" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - } - - # actions/cache archives with Git's tar, which finds gzip only on PATH (actions/cache#1297). - - name: Restore ovrtx shader cache - id: ovrtx-cache - uses: actions/cache/restore@v4 - env: - PATH: C:\Program Files\Git\usr\bin;C:\Windows\System32 - with: - path: .venv/Lib/site-packages/ovrtx/bin/cache - key: ${{ steps.shader-cache.outputs.ovrtx }} - - - name: Restore Kit shader cache - id: kit-cache - uses: actions/cache/restore@v4 - env: - PATH: C:\Program Files\Git\usr\bin;C:\Windows\System32 - with: - path: .venv/Lib/site-packages/isaacsim/kit/cache - key: ${{ steps.shader-cache.outputs.isaacsim }} + run: uv sync --frozen --extra test - name: Run windows_ci marker tests env: @@ -133,24 +98,6 @@ jobs: if: ${{ !cancelled() && steps.sync.outcome == 'success' }} run: uv run --frozen --no-sync python -m pytest -v source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py --junitxml=tests/windows-ci-smoke-report.xml - - name: Save ovrtx shader cache - if: ${{ success() && steps.ovrtx-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v4 - env: - PATH: C:\Program Files\Git\usr\bin;C:\Windows\System32 - with: - path: .venv/Lib/site-packages/ovrtx/bin/cache - key: ${{ steps.shader-cache.outputs.ovrtx }} - - - name: Save Kit shader cache - if: ${{ success() && steps.kit-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v4 - env: - PATH: C:\Program Files\Git\usr\bin;C:\Windows\System32 - with: - path: .venv/Lib/site-packages/isaacsim/kit/cache - key: ${{ steps.shader-cache.outputs.isaacsim }} - - name: Upload test reports if: always() uses: actions/upload-artifact@v7 From 64bf15295bfaadef400ccc498987139deaeec826 Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Thu, 22 Oct 2026 15:54:53 -0700 Subject: [PATCH 09/21] Keep the Windows CI environment on the runner Checkout cleans the workspace, so the environment moves to the runner's tool cache. A reused runner then re-syncs only what uv.lock changed and keeps the packages' compiled bytecode. --- .github/workflows/windows-ci.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows-ci.yaml b/.github/workflows/windows-ci.yaml index d3af13a375c9..f886c063fc14 100644 --- a/.github/workflows/windows-ci.yaml +++ b/.github/workflows/windows-ci.yaml @@ -82,10 +82,21 @@ jobs: with: version: "0.12.9" + # Checkout cleans the workspace, so the environment lives in the runner's tool cache, where a + # reused runner keeps the installed packages and their compiled bytecode between jobs. + - name: Locate environment + run: '"UV_PROJECT_ENVIRONMENT=$env:RUNNER_TOOL_CACHE\isaaclab-windows-ci" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append' + - name: Sync environment id: sync timeout-minutes: 30 - run: uv sync --frozen --extra test + run: | + uv sync --frozen --extra test + if ($LASTEXITCODE -ne 0) { + # A job killed mid-sync can leave the kept environment broken, so rebuild it once. + Remove-Item -Recurse -Force $env:UV_PROJECT_ENVIRONMENT -ErrorAction SilentlyContinue + uv sync --frozen --extra test + } - name: Run windows_ci marker tests env: From f870540c86633d14432b0ade661be91a73684aef Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Thu, 22 Oct 2026 15:55:13 -0700 Subject: [PATCH 10/21] TEMP: mask jobs with no signal while iterating Windows CI Skips ARM CI, the x86 and ARM installation tests, multi-GPU pytest, the license checks and the docs build. Must be reverted before final review. --- .github/workflows/arm-ci.yml | 3 ++- .github/workflows/docs.yaml | 4 ++-- .github/workflows/install-ci.yml | 6 ++++-- .github/workflows/license-check.yaml | 4 ++++ .github/workflows/test-multi-gpu-pytest.yaml | 5 +++-- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index e63c104de9a1..7e5f0a58b2db 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -69,7 +69,8 @@ jobs: contents: read pull-requests: read outputs: - should_run: ${{ steps.detect.outputs.should_run }} + # TEMP (revert before review): masked while iterating Windows CI on PR 5700. + should_run: 'false' steps: - uses: actions/checkout@v6 with: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 6b8dbe8354ed..133b7aef9208 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -55,8 +55,8 @@ jobs: name: Build Latest Docs runs-on: ubuntu-latest needs: [doc-build-type] - # run on non-deploy branches to build current version docs only - if: needs.doc-build-type.outputs.trigger-deploy != 'true' + # TEMP (revert before review): masked while iterating Windows CI on PR 5700. + if: false steps: - name: Checkout code diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index 96d039c04f40..85db3211df32 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -127,7 +127,8 @@ jobs: install-tests-x86: name: Installation Tests (x86) needs: [changes] - if: needs.changes.outputs.run_install_tests == 'true' + # TEMP (revert before review): masked while iterating Windows CI on PR 5700. + if: false runs-on: [self-hosted, gpu] timeout-minutes: 90 steps: @@ -146,7 +147,8 @@ jobs: install-tests-arm: name: Installation Tests (ARM) needs: [changes] - if: needs.changes.outputs.run_install_tests == 'true' + # TEMP (revert before review): masked while iterating Windows CI on PR 5700. + if: false runs-on: [self-hosted, aarch64] timeout-minutes: 90 steps: diff --git a/.github/workflows/license-check.yaml b/.github/workflows/license-check.yaml index 10c6420afd92..229e1b0cf953 100644 --- a/.github/workflows/license-check.yaml +++ b/.github/workflows/license-check.yaml @@ -15,6 +15,8 @@ concurrency: jobs: license-check: + # TEMP (revert before review): masked while iterating Windows CI on PR 5700. + if: false runs-on: ubuntu-24.04 steps: @@ -158,6 +160,8 @@ jobs: fi docker-license-check: name: Docker Dependency Licenses Check + # TEMP (revert before review): masked while iterating Windows CI on PR 5700. + if: false runs-on: [self-hosted, gpu] timeout-minutes: 90 diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index a7ec67803a0c..b28b1f268f4c 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -40,8 +40,9 @@ jobs: name: Detect Changes runs-on: ubuntu-latest outputs: - unit: ${{ steps.unit.outputs.should_run }} - smoke: ${{ steps.smoke.outputs.should_run }} + # TEMP (revert before review): masked while iterating Windows CI on PR 5700. + unit: 'false' + smoke: 'false' steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: From 61e8745b11e58b4f59a96ce710b5cc2a22085d20 Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Thu, 22 Oct 2026 15:55:29 -0700 Subject: [PATCH 11/21] TEMP: print the Windows runner's cache state Prints the runner, its boot time and the size of uv's cache, the kept environment and Warp's kernel cache before the sync and after the tests. Must be reverted before final review. --- .github/workflows/windows-ci.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/windows-ci.yaml b/.github/workflows/windows-ci.yaml index f886c063fc14..df5de5cbbf78 100644 --- a/.github/workflows/windows-ci.yaml +++ b/.github/workflows/windows-ci.yaml @@ -87,6 +87,17 @@ jobs: - name: Locate environment run: '"UV_PROJECT_ENVIRONMENT=$env:RUNNER_TOOL_CACHE\isaaclab-windows-ci" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append' + # TEMP (revert before review): runner cache state while iterating Windows CI on PR 5700. + - name: Runner cache state before (TEMP) + run: | + "runner=$env:RUNNER_NAME booted=$((Get-CimInstance Win32_OperatingSystem).LastBootUpTime)" + foreach ($p in @((uv cache dir), $env:UV_PROJECT_ENVIRONMENT, "$env:LOCALAPPDATA\NVIDIA\warp\Cache")) { + if (Test-Path $p) { + $m = Get-ChildItem -Recurse -File -Force $p -ErrorAction SilentlyContinue | Measure-Object Length -Sum + "$p : $($m.Count) files, $([int]($m.Sum / 1MB)) MB" + } else { "$p : missing" } + } + - name: Sync environment id: sync timeout-minutes: 30 @@ -109,6 +120,17 @@ jobs: if: ${{ !cancelled() && steps.sync.outcome == 'success' }} run: uv run --frozen --no-sync python -m pytest -v source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py --junitxml=tests/windows-ci-smoke-report.xml + - name: Runner cache state after (TEMP) + if: always() + run: | + "runner=$env:RUNNER_NAME booted=$((Get-CimInstance Win32_OperatingSystem).LastBootUpTime)" + foreach ($p in @((uv cache dir), $env:UV_PROJECT_ENVIRONMENT, "$env:LOCALAPPDATA\NVIDIA\warp\Cache")) { + if (Test-Path $p) { + $m = Get-ChildItem -Recurse -File -Force $p -ErrorAction SilentlyContinue | Measure-Object Length -Sum + "$p : $($m.Count) files, $([int]($m.Sum / 1MB)) MB" + } else { "$p : missing" } + } + - name: Upload test reports if: always() uses: actions/upload-artifact@v7 From 234820c281592b58d958a8d139586ebd495c9596 Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Thu, 22 Oct 2026 17:06:19 -0700 Subject: [PATCH 12/21] TEMP: time Cartpole camera training for every combination Adds a job that trains camera Cartpole for each physics and renderer pair in an environment kept on the runner, and masks the other Windows jobs meanwhile. Must be reverted before final review. --- .github/probe/cartpole_combos.ps1 | 21 ++++++++++++++++ .github/workflows/install-ci.yml | 3 ++- .github/workflows/windows-ci.yaml | 41 ++++++++++++++++++++++++++++++- 3 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 .github/probe/cartpole_combos.ps1 diff --git a/.github/probe/cartpole_combos.ps1 b/.github/probe/cartpole_combos.ps1 new file mode 100644 index 000000000000..9dfce0a20871 --- /dev/null +++ b/.github/probe/cartpole_combos.ps1 @@ -0,0 +1,21 @@ +# TEMP (revert before review): time Cartpole camera training for every physics x renderer pair. +$ErrorActionPreference = 'Continue' +$py = "$env:UV_PROJECT_ENVIRONMENT\Scripts\python.exe" +New-Item -ItemType Directory -Force tests | Out-Null +$rows = @() +foreach ($r in 'newton_renderer', 'ovrtx', 'isaacsim_rtx') { + foreach ($p in 'newton_mjwarp', 'newton_kamino', 'isaacsim_physx', 'ovphysx') { + $log = "tests\combo-$p-$r.log" + $argList = "-m isaaclab train --rl_library rsl_rl --task Isaac-Cartpole-Camera-Direct --num_envs 16 presets=$p,$r --max_iterations 2" + $sw = [Diagnostics.Stopwatch]::StartNew() + $proc = Start-Process -FilePath $py -ArgumentList $argList -NoNewWindow -PassThru -RedirectStandardOutput $log -RedirectStandardError "$log.err" + $null = $proc.Handle + if ($proc.WaitForExit(900000)) { $rc = $proc.ExitCode } else { taskkill /T /F /PID $proc.Id | Out-Null; $rc = 'timeout' } + $sw.Stop() + $trained = ($rc -eq 0) -and (Select-String -Quiet -Path $log -Pattern 'Training time:') + $rows += [pscustomobject]@{ physics = $p; renderer = $r; seconds = [int]$sw.Elapsed.TotalSeconds; rc = $rc; trained = $trained } + '{0,-15} {1,-16} {2,5} s rc={3} trained={4}' -f $p, $r, [int]$sw.Elapsed.TotalSeconds, $rc, $trained + } +} +($rows | Format-Table -AutoSize | Out-String).Trim() +"total: $(($rows | Measure-Object seconds -Sum).Sum) s, trained $(@($rows | Where-Object trained).Count)/$($rows.Count)" diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index 85db3211df32..ab1779e7e455 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -181,7 +181,8 @@ jobs: install-tests-windows: name: Installation Tests (Windows) needs: [changes] - if: needs.changes.outputs.run_install_tests == 'true' + # TEMP (revert before review): masked while timing the Cartpole combinations. + if: false runs-on: [self-hosted, gpu-windows] timeout-minutes: 90 defaults: diff --git a/.github/workflows/windows-ci.yaml b/.github/workflows/windows-ci.yaml index df5de5cbbf78..757710c0a9ef 100644 --- a/.github/workflows/windows-ci.yaml +++ b/.github/workflows/windows-ci.yaml @@ -66,7 +66,8 @@ jobs: windows-ci: name: windows-ci needs: [changes] - if: needs.changes.outputs.should_run == 'true' + # TEMP (revert before review): masked while timing the Cartpole combinations. + if: false runs-on: [self-hosted, gpu-windows] timeout-minutes: 90 defaults: @@ -138,3 +139,41 @@ jobs: name: windows-ci-reports path: tests/ retention-days: 7 + + # TEMP (revert before review): time Cartpole camera training for every physics x renderer combination. + windows-cartpole-combos: + name: windows-cartpole-combos (TEMP) + needs: [changes] + if: needs.changes.outputs.should_run == 'true' + runs-on: [self-hosted, gpu-windows] + timeout-minutes: 150 + defaults: + run: + shell: powershell + env: + OMNI_KIT_ACCEPT_EULA: "yes" + steps: + - uses: actions/checkout@v6 + + - name: Set up uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.12.9" + + - name: Locate environment + run: '"UV_PROJECT_ENVIRONMENT=$env:RUNNER_TOOL_CACHE\isaaclab-windows-combos" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append' + + - name: Sync environment + timeout-minutes: 40 + run: uv sync --frozen --extra test --extra isaacsim --extra ov + + - name: Train every combination + run: powershell -NoProfile -ExecutionPolicy Bypass -File .github/probe/cartpole_combos.ps1 + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: windows-cartpole-combos + path: tests/ + retention-days: 7 From 279866422685ce5851732fb76bca573f373f6088 Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Fri, 23 Oct 2026 11:06:47 -0700 Subject: [PATCH 13/21] Restore the Warp kernel cache in Windows CI Pull requests restore the newest snapshot of a Windows Warp cache collection and pushes to protected branches save what they compiled, the policy of the Linux test jobs. The key comes from the same collection_id.py; the shared actions run bash, which resolves to WSL on these runners. --- .github/workflows/windows-ci.yaml | 73 +++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/.github/workflows/windows-ci.yaml b/.github/workflows/windows-ci.yaml index 757710c0a9ef..7619b10fca87 100644 --- a/.github/workflows/windows-ci.yaml +++ b/.github/workflows/windows-ci.yaml @@ -62,6 +62,7 @@ jobs: ^uv\.lock$ :: Locked dependency graph ^VERSION$ :: Version file ^\.github/workflows/windows-ci\.yaml$ :: This workflow file + ^\.github/actions/(warp-cache-key/collection_id|run-package-tests/warp_cache_inventory)\.py$ :: Warp cache scripts windows-ci: name: windows-ci @@ -73,6 +74,10 @@ jobs: defaults: run: shell: powershell + env: + # Same policy as the Linux test jobs: pull requests only restore, so they never publish + # kernels built from unmerged code; pushes to protected branches also save. + WARP_CACHE_MODE: ${{ github.event_name == 'push' && 'save' || 'restore' }} steps: - uses: actions/checkout@v6 @@ -110,6 +115,48 @@ jobs: uv sync --frozen --extra test } + # The Warp kernel cache of the Linux test jobs (.github/actions/warp-cache-key), here in its own + # Windows collection. Those actions run bash, which on these runners resolves to WSL, so the + # key is rebuilt from the same collection_id.py. + - name: Resolve Warp cache key + id: warp-cache-key + run: | + $versions = uv run --frozen --no-sync python .github/actions/warp-cache-key/collection_id.py + if ($LASTEXITCODE -ne 0) { exit 1 } + $collection = "warp-v2-$env:RUNNER_OS-$env:RUNNER_ARCH-$versions" + $dir = "$env:RUNNER_TEMP\isaaclab-warp-cache" + @( + "collection=$collection" + "key=$collection-$env:GITHUB_SHA-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT" + "restore-keys=$collection-" + "host-dir=$dir" + ) | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + "WARP_CACHE_PATH=$dir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + - name: Restore Warp kernel cache + id: warp-cache-restore + uses: actions/cache/restore@v4 + with: + path: ${{ steps.warp-cache-key.outputs.host-dir }} + key: ${{ steps.warp-cache-key.outputs.key }} + restore-keys: ${{ steps.warp-cache-key.outputs.restore-keys }} + + - name: Report restored Warp cache + id: warp-cache-before + env: + MATCHED_KEY: ${{ steps.warp-cache-restore.outputs.cache-matched-key }} + COLLECTION: ${{ steps.warp-cache-key.outputs.collection }} + run: | + New-Item -ItemType Directory -Force $env:WARP_CACHE_PATH | Out-Null + if ($env:MATCHED_KEY) { "Warp cache hit: $env:MATCHED_KEY" } else { "::warning::Warp cache miss - no entry in collection $env:COLLECTION" } + uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $env:WARP_CACHE_PATH "Warp cache restored" + uv run --frozen --no-sync python tools/verify_warp_cache.py + if ($LASTEXITCODE -ne 0) { exit 1 } + if ($env:WARP_CACHE_MODE -eq 'save') { + $fingerprint = uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $env:WARP_CACHE_PATH --fingerprint + "fingerprint=$fingerprint" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + } + - name: Run windows_ci marker tests env: CI_MARKER: windows_ci @@ -121,6 +168,32 @@ jobs: if: ${{ !cancelled() && steps.sync.outcome == 'success' }} run: uv run --frozen --no-sync python -m pytest -v source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py --junitxml=tests/windows-ci-smoke-report.xml + # Not success(): compiled kernels stay valid when a test fails. Cancellation still skips, so a + # tree Warp was mid-write on is never published. A save run whose kernels all came from the + # restored snapshot publishes nothing, since every snapshot is a new immutable cache entry. + - name: Report Warp cache growth + id: warp-cache-growth + if: ${{ !cancelled() && steps.warp-cache-before.outcome == 'success' }} + env: + FINGERPRINT_BEFORE: ${{ steps.warp-cache-before.outputs.fingerprint }} + run: | + uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $env:WARP_CACHE_PATH "Warp cache final" + if ($env:WARP_CACHE_MODE -ne 'save') { exit 0 } + $changed = 'false' + if (Get-ChildItem -Recurse -File $env:WARP_CACHE_PATH | Select-Object -First 1) { + $fingerprint = uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $env:WARP_CACHE_PATH --fingerprint + if ($fingerprint -ne $env:FINGERPRINT_BEFORE) { $changed = 'true' } + } + "Warp cache changed: $changed" + "changed=$changed" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + + - name: Save Warp kernel cache + if: ${{ !cancelled() && steps.warp-cache-growth.outputs.changed == 'true' }} + uses: actions/cache/save@v4 + with: + path: ${{ steps.warp-cache-key.outputs.host-dir }} + key: ${{ steps.warp-cache-key.outputs.key }} + - name: Runner cache state after (TEMP) if: always() run: | From 31bd022b81c0a7a91899acf6df31327642b92195 Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Fri, 23 Oct 2026 11:09:35 -0700 Subject: [PATCH 14/21] TEMP: measure Windows CI fresh and warm with the caches restored Simulates a fresh instance (attempts 1-2 of windows-ci; the seed and fresh legs of the Cartpole combinations), lets this PR save the caches, and restores Warp, Kit and ovrtx shader caches in the combinations job. Must be reverted before final review. --- .github/probe/cache_state.ps1 | 27 ++++ .github/probe/simulate_fresh.ps1 | 15 +++ .github/workflows/windows-ci.yaml | 197 ++++++++++++++++++++++++++---- 3 files changed, 212 insertions(+), 27 deletions(-) create mode 100644 .github/probe/cache_state.ps1 create mode 100644 .github/probe/simulate_fresh.ps1 diff --git a/.github/probe/cache_state.ps1 b/.github/probe/cache_state.ps1 new file mode 100644 index 000000000000..3ce5471548df --- /dev/null +++ b/.github/probe/cache_state.ps1 @@ -0,0 +1,27 @@ +# TEMP (revert before review): print what this runner holds in each cache. +param([string]$Label) +"[$Label] runner=$env:RUNNER_NAME booted=$((Get-CimInstance Win32_OperatingSystem).LastBootUpTime)" +function Show-Size($path) { + if (Test-Path $path) { + $m = Get-ChildItem -Recurse -File -Force $path -ErrorAction SilentlyContinue | Measure-Object Length -Sum + '{0} : {1} files, {2} MB' -f $path, $m.Count, [int]($m.Sum / 1MB) + } else { "$path : missing" } +} +$site = "$env:UV_PROJECT_ENVIRONMENT\Lib\site-packages" +$paths = @((uv cache dir), $env:UV_PROJECT_ENVIRONMENT, "$env:LOCALAPPDATA\NVIDIA\warp\Cache", + "$env:RUNNER_TEMP\isaaclab-warp-cache", "$env:RUNNER_TEMP\isaaclab-ovrtx-shader-cache\kit", + "$env:RUNNER_TEMP\isaaclab-ovrtx-shader-cache\kitless") +foreach ($path in $paths) { Show-Size $path } +foreach ($dir in "$site\isaacsim\kit\cache", "$site\ovrtx\bin\cache") { + if (Test-Path $dir) { + Get-ChildItem -Force $dir | ForEach-Object { + $tag = if ($_.Attributes -band [IO.FileAttributes]::ReparsePoint) { ' (junction)' } else { '' } + (Show-Size $_.FullName) + $tag + } + } +} +if (Test-Path "$site\isaacsim\kit\logs") { + Get-ChildItem -Recurse "$site\isaacsim\kit\logs" -Filter *.log | + Select-String -Pattern 'driverShaderCachePath|driverAppShaderCachePath' | + Select-Object -First 4 | ForEach-Object { $_.Line } +} diff --git a/.github/probe/simulate_fresh.ps1 b/.github/probe/simulate_fresh.ps1 new file mode 100644 index 000000000000..16dc1e221157 --- /dev/null +++ b/.github/probe/simulate_fresh.ps1 @@ -0,0 +1,15 @@ +# TEMP (revert before review): return this runner to the state of a freshly launched instance, so a +# job measures the fresh-instance path whichever instance it lands on. +$sw = [Diagnostics.Stopwatch]::StartNew() +uv cache clean +foreach ($dir in @( + "$env:RUNNER_TOOL_CACHE\isaaclab-windows-ci", + "$env:RUNNER_TOOL_CACHE\isaaclab-windows-combos", + "$env:LOCALAPPDATA\NVIDIA\warp", + "$env:LOCALAPPDATA\NVIDIA\DXCache", + "$env:LOCALAPPDATA\NVIDIA\GLCache", + "$env:LOCALAPPDATA\ov")) { + # rmdir removes a junction without following it, which Remove-Item does not guarantee. + if (Test-Path $dir) { cmd /c rmdir /s /q "$dir"; "removed $dir" } +} +"simulated a fresh instance in $([int]$sw.Elapsed.TotalSeconds) s" diff --git a/.github/workflows/windows-ci.yaml b/.github/workflows/windows-ci.yaml index 7619b10fca87..ad11ac5f6b30 100644 --- a/.github/workflows/windows-ci.yaml +++ b/.github/workflows/windows-ci.yaml @@ -67,17 +67,15 @@ jobs: windows-ci: name: windows-ci needs: [changes] - # TEMP (revert before review): masked while timing the Cartpole combinations. - if: false + if: needs.changes.outputs.should_run == 'true' runs-on: [self-hosted, gpu-windows] timeout-minutes: 90 defaults: run: shell: powershell env: - # Same policy as the Linux test jobs: pull requests only restore, so they never publish - # kernels built from unmerged code; pushes to protected branches also save. - WARP_CACHE_MODE: ${{ github.event_name == 'push' && 'save' || 'restore' }} + # TEMP (revert before review): PR 5700 saves so the restore path can be measured. + WARP_CACHE_MODE: save steps: - uses: actions/checkout@v6 @@ -93,16 +91,14 @@ jobs: - name: Locate environment run: '"UV_PROJECT_ENVIRONMENT=$env:RUNNER_TOOL_CACHE\isaaclab-windows-ci" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append' - # TEMP (revert before review): runner cache state while iterating Windows CI on PR 5700. + # TEMP (revert before review): attempts 1 and 2 measure a fresh instance (cold, then + # restoring), attempt 3 a reused one. + - name: Simulate a fresh instance (TEMP) + if: github.run_attempt != '3' + run: powershell -NoProfile -ExecutionPolicy Bypass -File .github/probe/simulate_fresh.ps1 + - name: Runner cache state before (TEMP) - run: | - "runner=$env:RUNNER_NAME booted=$((Get-CimInstance Win32_OperatingSystem).LastBootUpTime)" - foreach ($p in @((uv cache dir), $env:UV_PROJECT_ENVIRONMENT, "$env:LOCALAPPDATA\NVIDIA\warp\Cache")) { - if (Test-Path $p) { - $m = Get-ChildItem -Recurse -File -Force $p -ErrorAction SilentlyContinue | Measure-Object Length -Sum - "$p : $($m.Count) files, $([int]($m.Sum / 1MB)) MB" - } else { "$p : missing" } - } + run: powershell -NoProfile -ExecutionPolicy Bypass -File .github/probe/cache_state.ps1 before - name: Sync environment id: sync @@ -196,14 +192,7 @@ jobs: - name: Runner cache state after (TEMP) if: always() - run: | - "runner=$env:RUNNER_NAME booted=$((Get-CimInstance Win32_OperatingSystem).LastBootUpTime)" - foreach ($p in @((uv cache dir), $env:UV_PROJECT_ENVIRONMENT, "$env:LOCALAPPDATA\NVIDIA\warp\Cache")) { - if (Test-Path $p) { - $m = Get-ChildItem -Recurse -File -Force $p -ErrorAction SilentlyContinue | Measure-Object Length -Sum - "$p : $($m.Count) files, $([int]($m.Sum / 1MB)) MB" - } else { "$p : missing" } - } + run: powershell -NoProfile -ExecutionPolicy Bypass -File .github/probe/cache_state.ps1 after - name: Upload test reports if: always() @@ -213,11 +202,18 @@ jobs: path: tests/ retention-days: 7 - # TEMP (revert before review): time Cartpole camera training for every physics x renderer combination. + # TEMP (revert before review): time Cartpole camera training for every physics x renderer combination, + # one leg at a time: seed (fresh instance, publishes the caches), fresh (fresh instance restoring + # them) and warm (the instance the fresh leg left behind). windows-cartpole-combos: - name: windows-cartpole-combos (TEMP) - needs: [changes] - if: needs.changes.outputs.should_run == 'true' + name: windows-cartpole-combos ${{ matrix.phase }} (TEMP) + needs: [changes, windows-ci] + if: ${{ !cancelled() && needs.changes.outputs.should_run == 'true' && github.run_attempt == '1' }} + strategy: + fail-fast: false + max-parallel: 1 + matrix: + phase: [seed, fresh, warm] runs-on: [self-hosted, gpu-windows] timeout-minutes: 150 defaults: @@ -225,6 +221,7 @@ jobs: shell: powershell env: OMNI_KIT_ACCEPT_EULA: "yes" + PHASE: ${{ matrix.phase }} steps: - uses: actions/checkout@v6 @@ -236,17 +233,163 @@ jobs: - name: Locate environment run: '"UV_PROJECT_ENVIRONMENT=$env:RUNNER_TOOL_CACHE\isaaclab-windows-combos" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append' + - name: Simulate a fresh instance + if: matrix.phase != 'warm' + run: powershell -NoProfile -ExecutionPolicy Bypass -File .github/probe/simulate_fresh.ps1 + + - name: Runner cache state before + run: powershell -NoProfile -ExecutionPolicy Bypass -File .github/probe/cache_state.ps1 before + - name: Sync environment timeout-minutes: 40 run: uv sync --frozen --extra test --extra isaacsim --extra ov + # Keys as in windows-ci and .github/actions/ovrtx-shader-cache/key.sh, with the installed Isaac Sim + # and ovrtx versions standing in for the image tag and the CI pin. kit/ replaces Kit's default + # driver shader cache directory with a junction, as the Linux jobs bind-mount over it. + - name: Resolve cache keys + id: keys + run: | + $suffix = "$env:GITHUB_SHA-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT-$env:PHASE" + $versions = uv run --frozen --no-sync python .github/actions/warp-cache-key/collection_id.py + if ($LASTEXITCODE -ne 0) { exit 1 } + $warp = "warp-v2-$env:RUNNER_OS-$env:RUNNER_ARCH-$versions" + + $driver = (nvidia-smi --query-gpu=driver_version --format=csv,noheader | Select-Object -First 1).Trim() + $arch = (nvidia-smi --query-gpu=compute_cap --format=csv,noheader | Select-Object -First 1).Trim().Replace('.', '') + if (-not $driver -or -not $arch) { "::error::nvidia-smi reported no driver version or compute capability"; exit 1 } + $isaacsim, $ovrtx = (uv run --frozen --no-sync python -c "from importlib.metadata import version; print(version('isaacsim'), version('ovrtx'))") -split ' ' + $base = "v1-$env:RUNNER_OS-$env:RUNNER_ARCH-sm$arch-drv$($driver -replace '[^A-Za-z0-9._-]', '_')" + $kit = "ovrtx-kit-$base-isaacsim$($isaacsim -replace '[^A-Za-z0-9._-]', '_')" + $kitless = "ovrtx-kitless-$base-ovrtx$($ovrtx -replace '[^A-Za-z0-9._-]', '_')" + + $warpDir = "$env:RUNNER_TEMP\isaaclab-warp-cache" + $kitDir = "$env:RUNNER_TEMP\isaaclab-ovrtx-shader-cache\kit" + $kitlessDir = "$env:RUNNER_TEMP\isaaclab-ovrtx-shader-cache\kitless" + New-Item -ItemType Directory -Force $warpDir, $kitDir, $kitlessDir | Out-Null + + $kitRoot = "$env:UV_PROJECT_ENVIRONMENT\Lib\site-packages\isaacsim\kit" + if (-not (Test-Path $kitRoot)) { "::error::Kit not found at $kitRoot"; exit 1 } + New-Item -ItemType Directory -Force "$kitRoot\cache" | Out-Null + $link = "$kitRoot\cache\nv_shadercache" + # cmd sees a dangling junction that Test-Path does not, and rmdir removes one without following it. + cmd /c "if exist $link rmdir /s /q $link" + New-Item -ItemType Junction -Path $link -Target $kitDir | Out-Null + + @( + "warp-collection=$warp", "warp-key=$warp-$suffix", "warp-restore-keys=$warp-", "warp-dir=$warpDir" + "kit-collection=$kit", "kit-key=$kit-$suffix", "kit-restore-keys=$kit-", "kit-dir=$kitDir" + "kitless-collection=$kitless", "kitless-key=$kitless-$suffix", "kitless-restore-keys=$kitless-", "kitless-dir=$kitlessDir" + ) | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + @( + "WARP_CACHE_PATH=$warpDir" + "OVRTX_SHADER_CACHE_PATH=$kitlessDir" + "OVRTX_KIT_SHADER_CACHE_PATH=$link" + ) | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + "warp: $warp"; "kit: $kit"; "kitless: $kitless" + + - name: Restore Warp kernel cache + id: restore-warp + uses: actions/cache/restore@v4 + with: + path: ${{ steps.keys.outputs.warp-dir }} + key: ${{ steps.keys.outputs.warp-key }} + restore-keys: ${{ steps.keys.outputs.warp-restore-keys }} + + - name: Restore OVRTX kit shader cache + id: restore-kit + uses: actions/cache/restore@v4 + with: + path: ${{ steps.keys.outputs.kit-dir }} + key: ${{ steps.keys.outputs.kit-key }} + restore-keys: ${{ steps.keys.outputs.kit-restore-keys }} + + - name: Restore OVRTX kitless shader cache + id: restore-kitless + uses: actions/cache/restore@v4 + with: + path: ${{ steps.keys.outputs.kitless-dir }} + key: ${{ steps.keys.outputs.kitless-key }} + restore-keys: ${{ steps.keys.outputs.kitless-restore-keys }} + + - name: Report restored caches + id: before + env: + WARP_HIT: ${{ steps.restore-warp.outputs.cache-matched-key }} + KIT_HIT: ${{ steps.restore-kit.outputs.cache-matched-key }} + KITLESS_HIT: ${{ steps.restore-kitless.outputs.cache-matched-key }} + WARP_DIR: ${{ steps.keys.outputs.warp-dir }} + KIT_DIR: ${{ steps.keys.outputs.kit-dir }} + KITLESS_DIR: ${{ steps.keys.outputs.kitless-dir }} + run: | + foreach ($tree in 'warp', 'kit', 'kitless') { + $hit = [Environment]::GetEnvironmentVariable("$($tree.ToUpper())_HIT") + $dir = [Environment]::GetEnvironmentVariable("$($tree.ToUpper())_DIR") + if ($hit) { "$tree cache hit: $hit" } else { "::warning::$tree cache miss" } + $fingerprint = uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $dir --fingerprint + "$tree-fingerprint=$fingerprint" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + } + uv run --frozen --no-sync python tools/verify_warp_cache.py + if ($LASTEXITCODE -ne 0) { exit 1 } + uv run --frozen --no-sync python tools/verify_ovrtx_shader_cache.py + if ($LASTEXITCODE -ne 0) { exit 1 } + - name: Train every combination run: powershell -NoProfile -ExecutionPolicy Bypass -File .github/probe/cartpole_combos.ps1 + - name: Report cache growth + id: growth + if: ${{ !cancelled() && steps.before.outcome == 'success' }} + env: + WARP_BEFORE: ${{ steps.before.outputs.warp-fingerprint }} + KIT_BEFORE: ${{ steps.before.outputs.kit-fingerprint }} + KITLESS_BEFORE: ${{ steps.before.outputs.kitless-fingerprint }} + WARP_DIR: ${{ steps.keys.outputs.warp-dir }} + KIT_DIR: ${{ steps.keys.outputs.kit-dir }} + KITLESS_DIR: ${{ steps.keys.outputs.kitless-dir }} + run: | + foreach ($tree in 'warp', 'kit', 'kitless') { + $before = [Environment]::GetEnvironmentVariable("$($tree.ToUpper())_BEFORE") + $dir = [Environment]::GetEnvironmentVariable("$($tree.ToUpper())_DIR") + $changed = 'false' + if (Get-ChildItem -Recurse -File $dir | Select-Object -First 1) { + $after = uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $dir --fingerprint + if ($after -ne $before) { $changed = 'true' } + } + $mb = [int]((Get-ChildItem -Recurse -File $dir | Measure-Object Length -Sum).Sum / 1MB) + "$tree cache: $mb MB, changed=$changed" + "$tree-changed=$changed" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + } + + - name: Save Warp kernel cache + if: ${{ !cancelled() && steps.growth.outputs.warp-changed == 'true' }} + uses: actions/cache/save@v4 + with: + path: ${{ steps.keys.outputs.warp-dir }} + key: ${{ steps.keys.outputs.warp-key }} + + - name: Save OVRTX kit shader cache + if: ${{ !cancelled() && steps.growth.outputs.kit-changed == 'true' }} + uses: actions/cache/save@v4 + with: + path: ${{ steps.keys.outputs.kit-dir }} + key: ${{ steps.keys.outputs.kit-key }} + + - name: Save OVRTX kitless shader cache + if: ${{ !cancelled() && steps.growth.outputs.kitless-changed == 'true' }} + uses: actions/cache/save@v4 + with: + path: ${{ steps.keys.outputs.kitless-dir }} + key: ${{ steps.keys.outputs.kitless-key }} + + - name: Runner cache state after + if: always() + run: powershell -NoProfile -ExecutionPolicy Bypass -File .github/probe/cache_state.ps1 after + - name: Upload logs if: always() uses: actions/upload-artifact@v7 with: - name: windows-cartpole-combos + name: windows-cartpole-combos-${{ matrix.phase }} path: tests/ retention-days: 7 From 0522fc86096b377422de01583cb4f065f3c6dc1e Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Fri, 23 Oct 2026 11:39:14 -0700 Subject: [PATCH 15/21] Revert the TEMP Windows CI measurement commits Reverts 31bd022b81c (measurement harness), 234820c2815 (Cartpole combinations job) and 61e8745b11e (runner cache-state print). --- .github/probe/cache_state.ps1 | 27 ---- .github/probe/cartpole_combos.ps1 | 21 --- .github/probe/simulate_fresh.ps1 | 15 --- .github/workflows/install-ci.yml | 3 +- .github/workflows/windows-ci.yaml | 210 +----------------------------- 5 files changed, 4 insertions(+), 272 deletions(-) delete mode 100644 .github/probe/cache_state.ps1 delete mode 100644 .github/probe/cartpole_combos.ps1 delete mode 100644 .github/probe/simulate_fresh.ps1 diff --git a/.github/probe/cache_state.ps1 b/.github/probe/cache_state.ps1 deleted file mode 100644 index 3ce5471548df..000000000000 --- a/.github/probe/cache_state.ps1 +++ /dev/null @@ -1,27 +0,0 @@ -# TEMP (revert before review): print what this runner holds in each cache. -param([string]$Label) -"[$Label] runner=$env:RUNNER_NAME booted=$((Get-CimInstance Win32_OperatingSystem).LastBootUpTime)" -function Show-Size($path) { - if (Test-Path $path) { - $m = Get-ChildItem -Recurse -File -Force $path -ErrorAction SilentlyContinue | Measure-Object Length -Sum - '{0} : {1} files, {2} MB' -f $path, $m.Count, [int]($m.Sum / 1MB) - } else { "$path : missing" } -} -$site = "$env:UV_PROJECT_ENVIRONMENT\Lib\site-packages" -$paths = @((uv cache dir), $env:UV_PROJECT_ENVIRONMENT, "$env:LOCALAPPDATA\NVIDIA\warp\Cache", - "$env:RUNNER_TEMP\isaaclab-warp-cache", "$env:RUNNER_TEMP\isaaclab-ovrtx-shader-cache\kit", - "$env:RUNNER_TEMP\isaaclab-ovrtx-shader-cache\kitless") -foreach ($path in $paths) { Show-Size $path } -foreach ($dir in "$site\isaacsim\kit\cache", "$site\ovrtx\bin\cache") { - if (Test-Path $dir) { - Get-ChildItem -Force $dir | ForEach-Object { - $tag = if ($_.Attributes -band [IO.FileAttributes]::ReparsePoint) { ' (junction)' } else { '' } - (Show-Size $_.FullName) + $tag - } - } -} -if (Test-Path "$site\isaacsim\kit\logs") { - Get-ChildItem -Recurse "$site\isaacsim\kit\logs" -Filter *.log | - Select-String -Pattern 'driverShaderCachePath|driverAppShaderCachePath' | - Select-Object -First 4 | ForEach-Object { $_.Line } -} diff --git a/.github/probe/cartpole_combos.ps1 b/.github/probe/cartpole_combos.ps1 deleted file mode 100644 index 9dfce0a20871..000000000000 --- a/.github/probe/cartpole_combos.ps1 +++ /dev/null @@ -1,21 +0,0 @@ -# TEMP (revert before review): time Cartpole camera training for every physics x renderer pair. -$ErrorActionPreference = 'Continue' -$py = "$env:UV_PROJECT_ENVIRONMENT\Scripts\python.exe" -New-Item -ItemType Directory -Force tests | Out-Null -$rows = @() -foreach ($r in 'newton_renderer', 'ovrtx', 'isaacsim_rtx') { - foreach ($p in 'newton_mjwarp', 'newton_kamino', 'isaacsim_physx', 'ovphysx') { - $log = "tests\combo-$p-$r.log" - $argList = "-m isaaclab train --rl_library rsl_rl --task Isaac-Cartpole-Camera-Direct --num_envs 16 presets=$p,$r --max_iterations 2" - $sw = [Diagnostics.Stopwatch]::StartNew() - $proc = Start-Process -FilePath $py -ArgumentList $argList -NoNewWindow -PassThru -RedirectStandardOutput $log -RedirectStandardError "$log.err" - $null = $proc.Handle - if ($proc.WaitForExit(900000)) { $rc = $proc.ExitCode } else { taskkill /T /F /PID $proc.Id | Out-Null; $rc = 'timeout' } - $sw.Stop() - $trained = ($rc -eq 0) -and (Select-String -Quiet -Path $log -Pattern 'Training time:') - $rows += [pscustomobject]@{ physics = $p; renderer = $r; seconds = [int]$sw.Elapsed.TotalSeconds; rc = $rc; trained = $trained } - '{0,-15} {1,-16} {2,5} s rc={3} trained={4}' -f $p, $r, [int]$sw.Elapsed.TotalSeconds, $rc, $trained - } -} -($rows | Format-Table -AutoSize | Out-String).Trim() -"total: $(($rows | Measure-Object seconds -Sum).Sum) s, trained $(@($rows | Where-Object trained).Count)/$($rows.Count)" diff --git a/.github/probe/simulate_fresh.ps1 b/.github/probe/simulate_fresh.ps1 deleted file mode 100644 index 16dc1e221157..000000000000 --- a/.github/probe/simulate_fresh.ps1 +++ /dev/null @@ -1,15 +0,0 @@ -# TEMP (revert before review): return this runner to the state of a freshly launched instance, so a -# job measures the fresh-instance path whichever instance it lands on. -$sw = [Diagnostics.Stopwatch]::StartNew() -uv cache clean -foreach ($dir in @( - "$env:RUNNER_TOOL_CACHE\isaaclab-windows-ci", - "$env:RUNNER_TOOL_CACHE\isaaclab-windows-combos", - "$env:LOCALAPPDATA\NVIDIA\warp", - "$env:LOCALAPPDATA\NVIDIA\DXCache", - "$env:LOCALAPPDATA\NVIDIA\GLCache", - "$env:LOCALAPPDATA\ov")) { - # rmdir removes a junction without following it, which Remove-Item does not guarantee. - if (Test-Path $dir) { cmd /c rmdir /s /q "$dir"; "removed $dir" } -} -"simulated a fresh instance in $([int]$sw.Elapsed.TotalSeconds) s" diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index ab1779e7e455..85db3211df32 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -181,8 +181,7 @@ jobs: install-tests-windows: name: Installation Tests (Windows) needs: [changes] - # TEMP (revert before review): masked while timing the Cartpole combinations. - if: false + if: needs.changes.outputs.run_install_tests == 'true' runs-on: [self-hosted, gpu-windows] timeout-minutes: 90 defaults: diff --git a/.github/workflows/windows-ci.yaml b/.github/workflows/windows-ci.yaml index ad11ac5f6b30..91536b228c15 100644 --- a/.github/workflows/windows-ci.yaml +++ b/.github/workflows/windows-ci.yaml @@ -74,8 +74,9 @@ jobs: run: shell: powershell env: - # TEMP (revert before review): PR 5700 saves so the restore path can be measured. - WARP_CACHE_MODE: save + # Same policy as the Linux test jobs: pull requests only restore, so they never publish + # kernels built from unmerged code; pushes to protected branches also save. + WARP_CACHE_MODE: ${{ github.event_name == 'push' && 'save' || 'restore' }} steps: - uses: actions/checkout@v6 @@ -91,15 +92,6 @@ jobs: - name: Locate environment run: '"UV_PROJECT_ENVIRONMENT=$env:RUNNER_TOOL_CACHE\isaaclab-windows-ci" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append' - # TEMP (revert before review): attempts 1 and 2 measure a fresh instance (cold, then - # restoring), attempt 3 a reused one. - - name: Simulate a fresh instance (TEMP) - if: github.run_attempt != '3' - run: powershell -NoProfile -ExecutionPolicy Bypass -File .github/probe/simulate_fresh.ps1 - - - name: Runner cache state before (TEMP) - run: powershell -NoProfile -ExecutionPolicy Bypass -File .github/probe/cache_state.ps1 before - - name: Sync environment id: sync timeout-minutes: 30 @@ -190,10 +182,6 @@ jobs: path: ${{ steps.warp-cache-key.outputs.host-dir }} key: ${{ steps.warp-cache-key.outputs.key }} - - name: Runner cache state after (TEMP) - if: always() - run: powershell -NoProfile -ExecutionPolicy Bypass -File .github/probe/cache_state.ps1 after - - name: Upload test reports if: always() uses: actions/upload-artifact@v7 @@ -201,195 +189,3 @@ jobs: name: windows-ci-reports path: tests/ retention-days: 7 - - # TEMP (revert before review): time Cartpole camera training for every physics x renderer combination, - # one leg at a time: seed (fresh instance, publishes the caches), fresh (fresh instance restoring - # them) and warm (the instance the fresh leg left behind). - windows-cartpole-combos: - name: windows-cartpole-combos ${{ matrix.phase }} (TEMP) - needs: [changes, windows-ci] - if: ${{ !cancelled() && needs.changes.outputs.should_run == 'true' && github.run_attempt == '1' }} - strategy: - fail-fast: false - max-parallel: 1 - matrix: - phase: [seed, fresh, warm] - runs-on: [self-hosted, gpu-windows] - timeout-minutes: 150 - defaults: - run: - shell: powershell - env: - OMNI_KIT_ACCEPT_EULA: "yes" - PHASE: ${{ matrix.phase }} - steps: - - uses: actions/checkout@v6 - - - name: Set up uv - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 - with: - version: "0.12.9" - - - name: Locate environment - run: '"UV_PROJECT_ENVIRONMENT=$env:RUNNER_TOOL_CACHE\isaaclab-windows-combos" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append' - - - name: Simulate a fresh instance - if: matrix.phase != 'warm' - run: powershell -NoProfile -ExecutionPolicy Bypass -File .github/probe/simulate_fresh.ps1 - - - name: Runner cache state before - run: powershell -NoProfile -ExecutionPolicy Bypass -File .github/probe/cache_state.ps1 before - - - name: Sync environment - timeout-minutes: 40 - run: uv sync --frozen --extra test --extra isaacsim --extra ov - - # Keys as in windows-ci and .github/actions/ovrtx-shader-cache/key.sh, with the installed Isaac Sim - # and ovrtx versions standing in for the image tag and the CI pin. kit/ replaces Kit's default - # driver shader cache directory with a junction, as the Linux jobs bind-mount over it. - - name: Resolve cache keys - id: keys - run: | - $suffix = "$env:GITHUB_SHA-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT-$env:PHASE" - $versions = uv run --frozen --no-sync python .github/actions/warp-cache-key/collection_id.py - if ($LASTEXITCODE -ne 0) { exit 1 } - $warp = "warp-v2-$env:RUNNER_OS-$env:RUNNER_ARCH-$versions" - - $driver = (nvidia-smi --query-gpu=driver_version --format=csv,noheader | Select-Object -First 1).Trim() - $arch = (nvidia-smi --query-gpu=compute_cap --format=csv,noheader | Select-Object -First 1).Trim().Replace('.', '') - if (-not $driver -or -not $arch) { "::error::nvidia-smi reported no driver version or compute capability"; exit 1 } - $isaacsim, $ovrtx = (uv run --frozen --no-sync python -c "from importlib.metadata import version; print(version('isaacsim'), version('ovrtx'))") -split ' ' - $base = "v1-$env:RUNNER_OS-$env:RUNNER_ARCH-sm$arch-drv$($driver -replace '[^A-Za-z0-9._-]', '_')" - $kit = "ovrtx-kit-$base-isaacsim$($isaacsim -replace '[^A-Za-z0-9._-]', '_')" - $kitless = "ovrtx-kitless-$base-ovrtx$($ovrtx -replace '[^A-Za-z0-9._-]', '_')" - - $warpDir = "$env:RUNNER_TEMP\isaaclab-warp-cache" - $kitDir = "$env:RUNNER_TEMP\isaaclab-ovrtx-shader-cache\kit" - $kitlessDir = "$env:RUNNER_TEMP\isaaclab-ovrtx-shader-cache\kitless" - New-Item -ItemType Directory -Force $warpDir, $kitDir, $kitlessDir | Out-Null - - $kitRoot = "$env:UV_PROJECT_ENVIRONMENT\Lib\site-packages\isaacsim\kit" - if (-not (Test-Path $kitRoot)) { "::error::Kit not found at $kitRoot"; exit 1 } - New-Item -ItemType Directory -Force "$kitRoot\cache" | Out-Null - $link = "$kitRoot\cache\nv_shadercache" - # cmd sees a dangling junction that Test-Path does not, and rmdir removes one without following it. - cmd /c "if exist $link rmdir /s /q $link" - New-Item -ItemType Junction -Path $link -Target $kitDir | Out-Null - - @( - "warp-collection=$warp", "warp-key=$warp-$suffix", "warp-restore-keys=$warp-", "warp-dir=$warpDir" - "kit-collection=$kit", "kit-key=$kit-$suffix", "kit-restore-keys=$kit-", "kit-dir=$kitDir" - "kitless-collection=$kitless", "kitless-key=$kitless-$suffix", "kitless-restore-keys=$kitless-", "kitless-dir=$kitlessDir" - ) | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - @( - "WARP_CACHE_PATH=$warpDir" - "OVRTX_SHADER_CACHE_PATH=$kitlessDir" - "OVRTX_KIT_SHADER_CACHE_PATH=$link" - ) | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - "warp: $warp"; "kit: $kit"; "kitless: $kitless" - - - name: Restore Warp kernel cache - id: restore-warp - uses: actions/cache/restore@v4 - with: - path: ${{ steps.keys.outputs.warp-dir }} - key: ${{ steps.keys.outputs.warp-key }} - restore-keys: ${{ steps.keys.outputs.warp-restore-keys }} - - - name: Restore OVRTX kit shader cache - id: restore-kit - uses: actions/cache/restore@v4 - with: - path: ${{ steps.keys.outputs.kit-dir }} - key: ${{ steps.keys.outputs.kit-key }} - restore-keys: ${{ steps.keys.outputs.kit-restore-keys }} - - - name: Restore OVRTX kitless shader cache - id: restore-kitless - uses: actions/cache/restore@v4 - with: - path: ${{ steps.keys.outputs.kitless-dir }} - key: ${{ steps.keys.outputs.kitless-key }} - restore-keys: ${{ steps.keys.outputs.kitless-restore-keys }} - - - name: Report restored caches - id: before - env: - WARP_HIT: ${{ steps.restore-warp.outputs.cache-matched-key }} - KIT_HIT: ${{ steps.restore-kit.outputs.cache-matched-key }} - KITLESS_HIT: ${{ steps.restore-kitless.outputs.cache-matched-key }} - WARP_DIR: ${{ steps.keys.outputs.warp-dir }} - KIT_DIR: ${{ steps.keys.outputs.kit-dir }} - KITLESS_DIR: ${{ steps.keys.outputs.kitless-dir }} - run: | - foreach ($tree in 'warp', 'kit', 'kitless') { - $hit = [Environment]::GetEnvironmentVariable("$($tree.ToUpper())_HIT") - $dir = [Environment]::GetEnvironmentVariable("$($tree.ToUpper())_DIR") - if ($hit) { "$tree cache hit: $hit" } else { "::warning::$tree cache miss" } - $fingerprint = uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $dir --fingerprint - "$tree-fingerprint=$fingerprint" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - } - uv run --frozen --no-sync python tools/verify_warp_cache.py - if ($LASTEXITCODE -ne 0) { exit 1 } - uv run --frozen --no-sync python tools/verify_ovrtx_shader_cache.py - if ($LASTEXITCODE -ne 0) { exit 1 } - - - name: Train every combination - run: powershell -NoProfile -ExecutionPolicy Bypass -File .github/probe/cartpole_combos.ps1 - - - name: Report cache growth - id: growth - if: ${{ !cancelled() && steps.before.outcome == 'success' }} - env: - WARP_BEFORE: ${{ steps.before.outputs.warp-fingerprint }} - KIT_BEFORE: ${{ steps.before.outputs.kit-fingerprint }} - KITLESS_BEFORE: ${{ steps.before.outputs.kitless-fingerprint }} - WARP_DIR: ${{ steps.keys.outputs.warp-dir }} - KIT_DIR: ${{ steps.keys.outputs.kit-dir }} - KITLESS_DIR: ${{ steps.keys.outputs.kitless-dir }} - run: | - foreach ($tree in 'warp', 'kit', 'kitless') { - $before = [Environment]::GetEnvironmentVariable("$($tree.ToUpper())_BEFORE") - $dir = [Environment]::GetEnvironmentVariable("$($tree.ToUpper())_DIR") - $changed = 'false' - if (Get-ChildItem -Recurse -File $dir | Select-Object -First 1) { - $after = uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $dir --fingerprint - if ($after -ne $before) { $changed = 'true' } - } - $mb = [int]((Get-ChildItem -Recurse -File $dir | Measure-Object Length -Sum).Sum / 1MB) - "$tree cache: $mb MB, changed=$changed" - "$tree-changed=$changed" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - } - - - name: Save Warp kernel cache - if: ${{ !cancelled() && steps.growth.outputs.warp-changed == 'true' }} - uses: actions/cache/save@v4 - with: - path: ${{ steps.keys.outputs.warp-dir }} - key: ${{ steps.keys.outputs.warp-key }} - - - name: Save OVRTX kit shader cache - if: ${{ !cancelled() && steps.growth.outputs.kit-changed == 'true' }} - uses: actions/cache/save@v4 - with: - path: ${{ steps.keys.outputs.kit-dir }} - key: ${{ steps.keys.outputs.kit-key }} - - - name: Save OVRTX kitless shader cache - if: ${{ !cancelled() && steps.growth.outputs.kitless-changed == 'true' }} - uses: actions/cache/save@v4 - with: - path: ${{ steps.keys.outputs.kitless-dir }} - key: ${{ steps.keys.outputs.kitless-key }} - - - name: Runner cache state after - if: always() - run: powershell -NoProfile -ExecutionPolicy Bypass -File .github/probe/cache_state.ps1 after - - - name: Upload logs - if: always() - uses: actions/upload-artifact@v7 - with: - name: windows-cartpole-combos-${{ matrix.phase }} - path: tests/ - retention-days: 7 From 599f5752b6f1dfac4370861f60c9f2f180eea47f Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Fri, 23 Oct 2026 11:39:39 -0700 Subject: [PATCH 16/21] Keep the Windows Warp cache beside the environment The runner empties RUNNER_TEMP at the start of every job and pull requests only restore, so a reused runner recompiled every kernel on each pull request. The cache now lives in the tool cache beside the environment; a save clears it first, so it never publishes kernels a pull request built. --- .github/workflows/windows-ci.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows-ci.yaml b/.github/workflows/windows-ci.yaml index 91536b228c15..61abc227fc91 100644 --- a/.github/workflows/windows-ci.yaml +++ b/.github/workflows/windows-ci.yaml @@ -112,7 +112,10 @@ jobs: $versions = uv run --frozen --no-sync python .github/actions/warp-cache-key/collection_id.py if ($LASTEXITCODE -ne 0) { exit 1 } $collection = "warp-v2-$env:RUNNER_OS-$env:RUNNER_ARCH-$versions" - $dir = "$env:RUNNER_TEMP\isaaclab-warp-cache" + # Beside the environment, so a reused runner keeps what earlier pull requests compiled. A save + # starts from the restored snapshot alone, so it never publishes kernels a pull request built. + $dir = "$env:RUNNER_TOOL_CACHE\isaaclab-warp-cache" + if ($env:WARP_CACHE_MODE -eq 'save') { cmd /c "if exist $dir rmdir /s /q $dir" } @( "collection=$collection" "key=$collection-$env:GITHUB_SHA-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT" From eb6c0247cb1ea5fe4a81bc80603f27f8b745f44e Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Fri, 23 Oct 2026 11:39:39 -0700 Subject: [PATCH 17/21] TEMP: mask Installation Tests (Windows) while iterating Windows CI It carries no signal for the Windows CI Warp cache change. Must be reverted before final review. --- .github/workflows/install-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index 85db3211df32..0dfea8709d85 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -181,7 +181,8 @@ jobs: install-tests-windows: name: Installation Tests (Windows) needs: [changes] - if: needs.changes.outputs.run_install_tests == 'true' + # TEMP (revert before review): no signal for the Windows CI Warp cache change. + if: false runs-on: [self-hosted, gpu-windows] timeout-minutes: 90 defaults: From e76ce8952206055d5964e44dddbb2daaab5571ee Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Fri, 23 Oct 2026 11:41:59 -0700 Subject: [PATCH 18/21] Check the Windows Warp cache path inline Warp reports its kernel cache directory with the \\?\ prefix on Windows, so tools/verify_warp_cache.py's os.path.commonpath raises. Its version check always passes on an environment synced from uv.lock, so Windows CI checks the path itself instead of changing the tool the Linux jobs share. --- .github/workflows/windows-ci.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows-ci.yaml b/.github/workflows/windows-ci.yaml index 61abc227fc91..b75dfec46709 100644 --- a/.github/workflows/windows-ci.yaml +++ b/.github/workflows/windows-ci.yaml @@ -141,8 +141,14 @@ jobs: New-Item -ItemType Directory -Force $env:WARP_CACHE_PATH | Out-Null if ($env:MATCHED_KEY) { "Warp cache hit: $env:MATCHED_KEY" } else { "::warning::Warp cache miss - no entry in collection $env:COLLECTION" } uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $env:WARP_CACHE_PATH "Warp cache restored" - uv run --frozen --no-sync python tools/verify_warp_cache.py - if ($LASTEXITCODE -ne 0) { exit 1 } + # tools/verify_warp_cache.py's path check, which on Windows fails on the \\?\ prefix Warp + # reports; its version check always passes on an environment synced from uv.lock. + $reported = uv run --frozen --no-sync python -c "import warp as wp; wp.config.log_level = wp.LOG_WARNING; wp.init(); print(wp.config.kernel_cache_dir)" | Select-Object -Last 1 + if (-not "$reported".Replace('\\?\', '').StartsWith($env:WARP_CACHE_PATH, [StringComparison]::OrdinalIgnoreCase)) { + "::error::Warp resolved its kernel cache to $reported, not $env:WARP_CACHE_PATH" + exit 1 + } + "Warp kernel cache: $reported" if ($env:WARP_CACHE_MODE -eq 'save') { $fingerprint = uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $env:WARP_CACHE_PATH --fingerprint "fingerprint=$fingerprint" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append From 9cb2a7e7c05868b54fd6f1235b4176a2b80d85b8 Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Fri, 23 Oct 2026 11:48:19 -0700 Subject: [PATCH 19/21] Train camera Cartpole on every backend pairing in Windows CI Windows CI installs the isaacsim and ov extras and trains camera Cartpole on each of the ten physics and renderer pairings that can share a process. The Kit and OVRTX driver shader caches join the Warp kernel cache under the Linux policy, so a fresh runner skips the shader compiles once main has saved them. The cache steps live in one script, since the shared actions run bash. --- .github/scripts/windows_ci_caches.ps1 | 117 +++++++++++++++ .github/workflows/windows-ci.yaml | 135 +++++++++--------- .../cartpole_backend_combinations_smoke.py | 48 +++++++ 3 files changed, 231 insertions(+), 69 deletions(-) create mode 100644 .github/scripts/windows_ci_caches.ps1 create mode 100644 source/isaaclab/test/install_ci/misc/cartpole_backend_combinations_smoke.py diff --git a/.github/scripts/windows_ci_caches.ps1 b/.github/scripts/windows_ci_caches.ps1 new file mode 100644 index 000000000000..83cbe4d140e7 --- /dev/null +++ b/.github/scripts/windows_ci_caches.ps1 @@ -0,0 +1,117 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# The Warp kernel and OVRTX shader caches of the Linux test jobs (.github/actions/warp-cache-key, +# .github/actions/ovrtx-shader-cache), each in its own Windows collection. Those actions run bash, +# which resolves to WSL on these runners. +# +# Usage: windows_ci_caches.ps1 keys|restored|growth +# +# keys resolves each cache's key and directory and exports the paths the tests read +# restored reports each restore and checks the paths took effect; a save also fingerprints them +# growth reports each cache's size; a save also reports whether the run changed it +# +# The directories survive on a reused runner, so a pull request keeps what earlier jobs compiled. +# CACHE_MODE=save clears them before the restore, so a save publishes only the restored snapshot +# and what that run compiled. + +param([Parameter(Mandatory)][ValidateSet('keys', 'restored', 'growth')][string]$Mode) + +$saving = $env:CACHE_MODE -eq 'save' +# warp: Warp kernels. kit: driver shaders Kit's RTX renderer compiles into its default directory. +# kitless: driver shaders the OVRTX renderer compiles, redirected by OVRTX_SHADER_CACHE_PATH. +$caches = [ordered]@{ + warp = "$env:RUNNER_TOOL_CACHE\isaaclab-warp-cache" + kit = "$env:UV_PROJECT_ENVIRONMENT\Lib\site-packages\isaacsim\kit\cache\nv_shadercache" + kitless = "$env:RUNNER_TOOL_CACHE\isaaclab-ovrtx-shader-cache" +} + +function Write-StepOutput([string[]]$Lines) { + $Lines | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append +} + +function Get-Fingerprint([string]$Dir) { + uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $Dir --fingerprint +} + +function Get-Megabytes([string]$Dir) { + [int]((Get-ChildItem -Recurse -File -Force $Dir | Measure-Object Length -Sum).Sum / 1MB) +} + +function ConvertTo-KeyPart([string]$Value) { + $Value -replace '[^A-Za-z0-9._-]', '_' +} + +switch ($Mode) { + 'keys' { + $warpVersions = uv run --frozen --no-sync python .github/actions/warp-cache-key/collection_id.py + if ($LASTEXITCODE -ne 0) { exit 1 } + $driver = (nvidia-smi --query-gpu=driver_version --format=csv,noheader | Select-Object -First 1).Trim() + $arch = (nvidia-smi --query-gpu=compute_cap --format=csv,noheader | Select-Object -First 1).Trim().Replace('.', '') + # The installed versions stand in for the image tag and the ovrtx pin the Linux keys use. + $isaacsim, $ovrtx = (uv run --frozen --no-sync python -c "from importlib.metadata import version; print(version('isaacsim'), version('ovrtx'))") -split ' ' + if (-not $driver -or -not $arch -or -not $ovrtx) { + "::error::Could not read the GPU driver, compute capability, or Isaac Sim and ovrtx versions" + exit 1 + } + $gpu = "v1-$env:RUNNER_OS-$env:RUNNER_ARCH-sm$arch-drv$(ConvertTo-KeyPart $driver)" + $collections = @{ + warp = "warp-v2-$env:RUNNER_OS-$env:RUNNER_ARCH-$warpVersions" + kit = "ovrtx-kit-$gpu-isaacsim$(ConvertTo-KeyPart $isaacsim)" + kitless = "ovrtx-kitless-$gpu-ovrtx$(ConvertTo-KeyPart $ovrtx)" + } + foreach ($name in $caches.Keys) { + $dir = $caches[$name] + if ($saving -and (Test-Path $dir)) { Remove-Item -Recurse -Force $dir } + New-Item -ItemType Directory -Force $dir | Out-Null + $collection = $collections[$name] + "${name}: $collection" + Write-StepOutput @( + "$name-collection=$collection" + "$name-key=$collection-$env:GITHUB_SHA-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT" + "$name-restore-keys=$collection-" + "$name-dir=$dir" + ) + } + @( + "WARP_CACHE_PATH=$($caches.warp)" + "OVRTX_SHADER_CACHE_PATH=$($caches.kitless)" + "OVRTX_KIT_SHADER_CACHE_PATH=$($caches.kit)" + ) | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + } + 'restored' { + foreach ($name in $caches.Keys) { + $hit = [Environment]::GetEnvironmentVariable("$($name.ToUpper())_HIT") + if ($hit) { "$name cache hit: $hit" } else { "::warning::$name cache miss" } + "$name cache: $(Get-Megabytes $caches[$name]) MB" + } + # tools/verify_warp_cache.py's path check, which fails on the \\?\ prefix Warp reports on + # Windows; its version check always passes on an environment synced from uv.lock. + $reported = uv run --frozen --no-sync python -c "import warp as wp; wp.config.log_level = wp.LOG_WARNING; wp.init(); print(wp.config.kernel_cache_dir)" | Select-Object -Last 1 + if (-not "$reported".Replace('\\?\', '').StartsWith($caches.warp, [StringComparison]::OrdinalIgnoreCase)) { + "::error::Warp resolved its kernel cache to $reported, not $($caches.warp)" + exit 1 + } + uv run --frozen --no-sync python tools/verify_ovrtx_shader_cache.py + if ($LASTEXITCODE -ne 0) { exit 1 } + if ($saving) { + foreach ($name in $caches.Keys) { Write-StepOutput "$name-fingerprint=$(Get-Fingerprint $caches[$name])" } + } + } + 'growth' { + foreach ($name in $caches.Keys) { + $dir = $caches[$name] + # Every snapshot is a new immutable cache entry, so a save that only read back what it + # restored, or found nothing, publishes nothing. + $changed = 'false' + if ($saving -and (Get-ChildItem -Recurse -File $dir | Select-Object -First 1)) { + $before = [Environment]::GetEnvironmentVariable("$($name.ToUpper())_BEFORE") + if ((Get-Fingerprint $dir) -ne $before) { $changed = 'true' } + } + "$name cache: $(Get-Megabytes $dir) MB, changed=$changed" + Write-StepOutput "$name-changed=$changed" + } + } +} diff --git a/.github/workflows/windows-ci.yaml b/.github/workflows/windows-ci.yaml index b75dfec46709..fffb615fcae5 100644 --- a/.github/workflows/windows-ci.yaml +++ b/.github/workflows/windows-ci.yaml @@ -62,7 +62,8 @@ jobs: ^uv\.lock$ :: Locked dependency graph ^VERSION$ :: Version file ^\.github/workflows/windows-ci\.yaml$ :: This workflow file - ^\.github/actions/(warp-cache-key/collection_id|run-package-tests/warp_cache_inventory)\.py$ :: Warp cache scripts + ^\.github/actions/(warp-cache-key/collection_id|run-package-tests/warp_cache_inventory)\.py$ :: Shared cache scripts + ^\.github/scripts/windows_ci_caches\.ps1$ :: Cache script windows-ci: name: windows-ci @@ -74,9 +75,10 @@ jobs: run: shell: powershell env: + OMNI_KIT_ACCEPT_EULA: "yes" # Same policy as the Linux test jobs: pull requests only restore, so they never publish - # kernels built from unmerged code; pushes to protected branches also save. - WARP_CACHE_MODE: ${{ github.event_name == 'push' && 'save' || 'restore' }} + # anything built from unmerged code; pushes to protected branches also save. + CACHE_MODE: ${{ github.event_name == 'push' && 'save' || 'restore' }} steps: - uses: actions/checkout@v6 @@ -96,63 +98,48 @@ jobs: id: sync timeout-minutes: 30 run: | - uv sync --frozen --extra test + uv sync --frozen --extra test --extra isaacsim --extra ov if ($LASTEXITCODE -ne 0) { # A job killed mid-sync can leave the kept environment broken, so rebuild it once. Remove-Item -Recurse -Force $env:UV_PROJECT_ENVIRONMENT -ErrorAction SilentlyContinue - uv sync --frozen --extra test + uv sync --frozen --extra test --extra isaacsim --extra ov } - # The Warp kernel cache of the Linux test jobs (.github/actions/warp-cache-key), here in its own - # Windows collection. Those actions run bash, which on these runners resolves to WSL, so the - # key is rebuilt from the same collection_id.py. - - name: Resolve Warp cache key - id: warp-cache-key - run: | - $versions = uv run --frozen --no-sync python .github/actions/warp-cache-key/collection_id.py - if ($LASTEXITCODE -ne 0) { exit 1 } - $collection = "warp-v2-$env:RUNNER_OS-$env:RUNNER_ARCH-$versions" - # Beside the environment, so a reused runner keeps what earlier pull requests compiled. A save - # starts from the restored snapshot alone, so it never publishes kernels a pull request built. - $dir = "$env:RUNNER_TOOL_CACHE\isaaclab-warp-cache" - if ($env:WARP_CACHE_MODE -eq 'save') { cmd /c "if exist $dir rmdir /s /q $dir" } - @( - "collection=$collection" - "key=$collection-$env:GITHUB_SHA-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT" - "restore-keys=$collection-" - "host-dir=$dir" - ) | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - "WARP_CACHE_PATH=$dir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - name: Resolve cache keys + id: cache-keys + run: .github/scripts/windows_ci_caches.ps1 keys - name: Restore Warp kernel cache - id: warp-cache-restore + id: restore-warp + uses: actions/cache/restore@v4 + with: + path: ${{ steps.cache-keys.outputs.warp-dir }} + key: ${{ steps.cache-keys.outputs.warp-key }} + restore-keys: ${{ steps.cache-keys.outputs.warp-restore-keys }} + + - name: Restore Kit shader cache + id: restore-kit + uses: actions/cache/restore@v4 + with: + path: ${{ steps.cache-keys.outputs.kit-dir }} + key: ${{ steps.cache-keys.outputs.kit-key }} + restore-keys: ${{ steps.cache-keys.outputs.kit-restore-keys }} + + - name: Restore OVRTX shader cache + id: restore-kitless uses: actions/cache/restore@v4 with: - path: ${{ steps.warp-cache-key.outputs.host-dir }} - key: ${{ steps.warp-cache-key.outputs.key }} - restore-keys: ${{ steps.warp-cache-key.outputs.restore-keys }} + path: ${{ steps.cache-keys.outputs.kitless-dir }} + key: ${{ steps.cache-keys.outputs.kitless-key }} + restore-keys: ${{ steps.cache-keys.outputs.kitless-restore-keys }} - - name: Report restored Warp cache - id: warp-cache-before + - name: Report restored caches + id: caches-restored env: - MATCHED_KEY: ${{ steps.warp-cache-restore.outputs.cache-matched-key }} - COLLECTION: ${{ steps.warp-cache-key.outputs.collection }} - run: | - New-Item -ItemType Directory -Force $env:WARP_CACHE_PATH | Out-Null - if ($env:MATCHED_KEY) { "Warp cache hit: $env:MATCHED_KEY" } else { "::warning::Warp cache miss - no entry in collection $env:COLLECTION" } - uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $env:WARP_CACHE_PATH "Warp cache restored" - # tools/verify_warp_cache.py's path check, which on Windows fails on the \\?\ prefix Warp - # reports; its version check always passes on an environment synced from uv.lock. - $reported = uv run --frozen --no-sync python -c "import warp as wp; wp.config.log_level = wp.LOG_WARNING; wp.init(); print(wp.config.kernel_cache_dir)" | Select-Object -Last 1 - if (-not "$reported".Replace('\\?\', '').StartsWith($env:WARP_CACHE_PATH, [StringComparison]::OrdinalIgnoreCase)) { - "::error::Warp resolved its kernel cache to $reported, not $env:WARP_CACHE_PATH" - exit 1 - } - "Warp kernel cache: $reported" - if ($env:WARP_CACHE_MODE -eq 'save') { - $fingerprint = uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $env:WARP_CACHE_PATH --fingerprint - "fingerprint=$fingerprint" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - } + WARP_HIT: ${{ steps.restore-warp.outputs.cache-matched-key }} + KIT_HIT: ${{ steps.restore-kit.outputs.cache-matched-key }} + KITLESS_HIT: ${{ steps.restore-kitless.outputs.cache-matched-key }} + run: .github/scripts/windows_ci_caches.ps1 restored - name: Run windows_ci marker tests env: @@ -165,31 +152,41 @@ jobs: if: ${{ !cancelled() && steps.sync.outcome == 'success' }} run: uv run --frozen --no-sync python -m pytest -v source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py --junitxml=tests/windows-ci-smoke-report.xml - # Not success(): compiled kernels stay valid when a test fails. Cancellation still skips, so a - # tree Warp was mid-write on is never published. A save run whose kernels all came from the - # restored snapshot publishes nothing, since every snapshot is a new immutable cache entry. - - name: Report Warp cache growth - id: warp-cache-growth - if: ${{ !cancelled() && steps.warp-cache-before.outcome == 'success' }} + - name: Run Cartpole backend combinations + if: ${{ !cancelled() && steps.sync.outcome == 'success' }} + run: uv run --frozen --no-sync python -m pytest -v source/isaaclab/test/install_ci/misc/cartpole_backend_combinations_smoke.py --junitxml=tests/windows-ci-combinations-report.xml + + # Not success(): compiled kernels and shaders stay valid when a test fails. Cancellation still + # skips, so a cache the run was mid-write on is never published. + - name: Report cache growth + id: caches-growth + if: ${{ !cancelled() && steps.caches-restored.outcome == 'success' }} env: - FINGERPRINT_BEFORE: ${{ steps.warp-cache-before.outputs.fingerprint }} - run: | - uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $env:WARP_CACHE_PATH "Warp cache final" - if ($env:WARP_CACHE_MODE -ne 'save') { exit 0 } - $changed = 'false' - if (Get-ChildItem -Recurse -File $env:WARP_CACHE_PATH | Select-Object -First 1) { - $fingerprint = uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $env:WARP_CACHE_PATH --fingerprint - if ($fingerprint -ne $env:FINGERPRINT_BEFORE) { $changed = 'true' } - } - "Warp cache changed: $changed" - "changed=$changed" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + WARP_BEFORE: ${{ steps.caches-restored.outputs.warp-fingerprint }} + KIT_BEFORE: ${{ steps.caches-restored.outputs.kit-fingerprint }} + KITLESS_BEFORE: ${{ steps.caches-restored.outputs.kitless-fingerprint }} + run: .github/scripts/windows_ci_caches.ps1 growth - name: Save Warp kernel cache - if: ${{ !cancelled() && steps.warp-cache-growth.outputs.changed == 'true' }} + if: ${{ !cancelled() && steps.caches-growth.outputs.warp-changed == 'true' }} + uses: actions/cache/save@v4 + with: + path: ${{ steps.cache-keys.outputs.warp-dir }} + key: ${{ steps.cache-keys.outputs.warp-key }} + + - name: Save Kit shader cache + if: ${{ !cancelled() && steps.caches-growth.outputs.kit-changed == 'true' }} + uses: actions/cache/save@v4 + with: + path: ${{ steps.cache-keys.outputs.kit-dir }} + key: ${{ steps.cache-keys.outputs.kit-key }} + + - name: Save OVRTX shader cache + if: ${{ !cancelled() && steps.caches-growth.outputs.kitless-changed == 'true' }} uses: actions/cache/save@v4 with: - path: ${{ steps.warp-cache-key.outputs.host-dir }} - key: ${{ steps.warp-cache-key.outputs.key }} + path: ${{ steps.cache-keys.outputs.kitless-dir }} + key: ${{ steps.cache-keys.outputs.kitless-key }} - name: Upload test reports if: always() diff --git a/source/isaaclab/test/install_ci/misc/cartpole_backend_combinations_smoke.py b/source/isaaclab/test/install_ci/misc/cartpole_backend_combinations_smoke.py new file mode 100644 index 000000000000..1e0cb24ada84 --- /dev/null +++ b/source/isaaclab/test/install_ci/misc/cartpole_backend_combinations_smoke.py @@ -0,0 +1,48 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Camera Cartpole training on every physics backend with every renderer. + +The caller owns environment preparation and needs the ``isaacsim`` and ``ov`` extras. Each pairing +trains in its own process. Isaac Sim / Kit cannot share a process with OvPhysX or OVRTX +(:func:`isaaclab.app.sim_launcher.validate_launch`), so those pairings are left out. +""" + +from __future__ import annotations + +import itertools +import subprocess +import sys +from pathlib import Path + +import pytest + +_PHYSICS = ("newton_mjwarp", "newton_kamino", "isaacsim_physx", "ovphysx") +_RENDERERS = ("newton_renderer", "ovrtx", "isaacsim_rtx") +_KIT = {"isaacsim_physx", "isaacsim_rtx"} +_KITLESS_ONLY = {"ovphysx", "ovrtx"} + +_COMBINATIONS = [ + pytest.param(physics, renderer, id=f"{physics}-{renderer}") + for physics, renderer in itertools.product(_PHYSICS, _RENDERERS) + if not ({physics, renderer} & _KIT and {physics, renderer} & _KITLESS_ONLY) +] + + +@pytest.mark.parametrize(("physics", "renderer"), _COMBINATIONS) +def test_train_cartpole_camera_completes(physics: str, renderer: str) -> None: + """Verify that camera Cartpole completes short RSL-RL training on one backend pairing.""" + isaaclab_root = next(parent for parent in Path(__file__).resolve().parents if (parent / "uv.lock").is_file()) + command = [ + *(sys.executable, "-m", "isaaclab", "train", "--rl_library", "rsl_rl"), + *("--task", "Isaac-Cartpole-Camera-Direct", "--num_envs", "16"), + *(f"presets={physics},{renderer}", "--max_iterations", "2"), + ] + # A cold run compiles kernels and shaders before training begins. + result = subprocess.run(command, cwd=isaaclab_root, text=True, capture_output=True, timeout=1800, check=False) + output = result.stdout + (result.stderr or "") + assert result.returncode == 0, f"Training failed (rc={result.returncode}):\n{output}" + assert "Traceback (most recent call last):" not in output, f"Training produced a traceback:\n{output}" + assert "Training time:" in output, f"Training did not report completion:\n{output}" From b050bfda70f4e332171672d84e8a91fbe451092f Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Fri, 23 Oct 2026 17:04:48 -0700 Subject: [PATCH 20/21] Ignore hdf5 test cleanup errors only on Windows --- source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py b/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py index 10c6c7238df4..304c9ece1b9a 100644 --- a/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py +++ b/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD-3-Clause import json import os +import sys import tempfile import uuid @@ -41,7 +42,7 @@ def create_test_episode(device): def temp_dir(): """Create a temporary directory for test datasets.""" # On Windows, libhdf5 releases its file handle shortly after close(), so the cleanup can race it. - with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as d: + with tempfile.TemporaryDirectory(ignore_cleanup_errors=sys.platform == "win32") as d: yield d From 23a67bea919470c88aa54ee45386e06d69dc7ca1 Mon Sep 17 00:00:00 2001 From: Jichuan Hu Date: Fri, 23 Oct 2026 17:04:49 -0700 Subject: [PATCH 21/21] Rebuild the Windows CI environment on runs that save caches --- .github/workflows/windows-ci.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/windows-ci.yaml b/.github/workflows/windows-ci.yaml index fffb615fcae5..ed6f6e8237b8 100644 --- a/.github/workflows/windows-ci.yaml +++ b/.github/workflows/windows-ci.yaml @@ -98,6 +98,12 @@ jobs: id: sync timeout-minutes: 30 run: | + if ($env:CACHE_MODE -eq 'save') { + # Pull requests run in the kept environment and uv cache, so a run that publishes caches + # installs from neither. + if (Test-Path $env:UV_PROJECT_ENVIRONMENT) { Remove-Item -Recurse -Force $env:UV_PROJECT_ENVIRONMENT } + $env:UV_CACHE_DIR = "$env:RUNNER_TEMP\uv-cache" + } uv sync --frozen --extra test --extra isaacsim --extra ov if ($LASTEXITCODE -ne 0) { # A job killed mid-sync can leave the kept environment broken, so rebuild it once.