diff --git a/.github/scripts/windows_ci_caches.ps1 b/.github/scripts/windows_ci_caches.ps1 new file mode 100644 index 000000000000..83cbe4d140e7 --- /dev/null +++ b/.github/scripts/windows_ci_caches.ps1 @@ -0,0 +1,117 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# The Warp kernel and OVRTX shader caches of the Linux test jobs (.github/actions/warp-cache-key, +# .github/actions/ovrtx-shader-cache), each in its own Windows collection. Those actions run bash, +# which resolves to WSL on these runners. +# +# Usage: windows_ci_caches.ps1 keys|restored|growth +# +# keys resolves each cache's key and directory and exports the paths the tests read +# restored reports each restore and checks the paths took effect; a save also fingerprints them +# growth reports each cache's size; a save also reports whether the run changed it +# +# The directories survive on a reused runner, so a pull request keeps what earlier jobs compiled. +# CACHE_MODE=save clears them before the restore, so a save publishes only the restored snapshot +# and what that run compiled. + +param([Parameter(Mandatory)][ValidateSet('keys', 'restored', 'growth')][string]$Mode) + +$saving = $env:CACHE_MODE -eq 'save' +# warp: Warp kernels. kit: driver shaders Kit's RTX renderer compiles into its default directory. +# kitless: driver shaders the OVRTX renderer compiles, redirected by OVRTX_SHADER_CACHE_PATH. +$caches = [ordered]@{ + warp = "$env:RUNNER_TOOL_CACHE\isaaclab-warp-cache" + kit = "$env:UV_PROJECT_ENVIRONMENT\Lib\site-packages\isaacsim\kit\cache\nv_shadercache" + kitless = "$env:RUNNER_TOOL_CACHE\isaaclab-ovrtx-shader-cache" +} + +function Write-StepOutput([string[]]$Lines) { + $Lines | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append +} + +function Get-Fingerprint([string]$Dir) { + uv run --frozen --no-sync python .github/actions/run-package-tests/warp_cache_inventory.py $Dir --fingerprint +} + +function Get-Megabytes([string]$Dir) { + [int]((Get-ChildItem -Recurse -File -Force $Dir | Measure-Object Length -Sum).Sum / 1MB) +} + +function ConvertTo-KeyPart([string]$Value) { + $Value -replace '[^A-Za-z0-9._-]', '_' +} + +switch ($Mode) { + 'keys' { + $warpVersions = uv run --frozen --no-sync python .github/actions/warp-cache-key/collection_id.py + if ($LASTEXITCODE -ne 0) { exit 1 } + $driver = (nvidia-smi --query-gpu=driver_version --format=csv,noheader | Select-Object -First 1).Trim() + $arch = (nvidia-smi --query-gpu=compute_cap --format=csv,noheader | Select-Object -First 1).Trim().Replace('.', '') + # The installed versions stand in for the image tag and the ovrtx pin the Linux keys use. + $isaacsim, $ovrtx = (uv run --frozen --no-sync python -c "from importlib.metadata import version; print(version('isaacsim'), version('ovrtx'))") -split ' ' + if (-not $driver -or -not $arch -or -not $ovrtx) { + "::error::Could not read the GPU driver, compute capability, or Isaac Sim and ovrtx versions" + exit 1 + } + $gpu = "v1-$env:RUNNER_OS-$env:RUNNER_ARCH-sm$arch-drv$(ConvertTo-KeyPart $driver)" + $collections = @{ + warp = "warp-v2-$env:RUNNER_OS-$env:RUNNER_ARCH-$warpVersions" + kit = "ovrtx-kit-$gpu-isaacsim$(ConvertTo-KeyPart $isaacsim)" + kitless = "ovrtx-kitless-$gpu-ovrtx$(ConvertTo-KeyPart $ovrtx)" + } + foreach ($name in $caches.Keys) { + $dir = $caches[$name] + if ($saving -and (Test-Path $dir)) { Remove-Item -Recurse -Force $dir } + New-Item -ItemType Directory -Force $dir | Out-Null + $collection = $collections[$name] + "${name}: $collection" + Write-StepOutput @( + "$name-collection=$collection" + "$name-key=$collection-$env:GITHUB_SHA-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT" + "$name-restore-keys=$collection-" + "$name-dir=$dir" + ) + } + @( + "WARP_CACHE_PATH=$($caches.warp)" + "OVRTX_SHADER_CACHE_PATH=$($caches.kitless)" + "OVRTX_KIT_SHADER_CACHE_PATH=$($caches.kit)" + ) | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + } + 'restored' { + foreach ($name in $caches.Keys) { + $hit = [Environment]::GetEnvironmentVariable("$($name.ToUpper())_HIT") + if ($hit) { "$name cache hit: $hit" } else { "::warning::$name cache miss" } + "$name cache: $(Get-Megabytes $caches[$name]) MB" + } + # tools/verify_warp_cache.py's path check, which fails on the \\?\ prefix Warp reports on + # Windows; its version check always passes on an environment synced from uv.lock. + $reported = uv run --frozen --no-sync python -c "import warp as wp; wp.config.log_level = wp.LOG_WARNING; wp.init(); print(wp.config.kernel_cache_dir)" | Select-Object -Last 1 + if (-not "$reported".Replace('\\?\', '').StartsWith($caches.warp, [StringComparison]::OrdinalIgnoreCase)) { + "::error::Warp resolved its kernel cache to $reported, not $($caches.warp)" + exit 1 + } + uv run --frozen --no-sync python tools/verify_ovrtx_shader_cache.py + if ($LASTEXITCODE -ne 0) { exit 1 } + if ($saving) { + foreach ($name in $caches.Keys) { Write-StepOutput "$name-fingerprint=$(Get-Fingerprint $caches[$name])" } + } + } + 'growth' { + foreach ($name in $caches.Keys) { + $dir = $caches[$name] + # Every snapshot is a new immutable cache entry, so a save that only read back what it + # restored, or found nothing, publishes nothing. + $changed = 'false' + if ($saving -and (Get-ChildItem -Recurse -File $dir | Select-Object -First 1)) { + $before = [Environment]::GetEnvironmentVariable("$($name.ToUpper())_BEFORE") + if ((Get-Fingerprint $dir) -ne $before) { $changed = 'true' } + } + "$name cache: $(Get-Megabytes $dir) MB, changed=$changed" + Write-StepOutput "$name-changed=$changed" + } + } +} diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index e63c104de9a1..7e5f0a58b2db 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -69,7 +69,8 @@ jobs: contents: read pull-requests: read outputs: - should_run: ${{ steps.detect.outputs.should_run }} + # TEMP (revert before review): masked while iterating Windows CI on PR 5700. + should_run: 'false' steps: - uses: actions/checkout@v6 with: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 6b8dbe8354ed..133b7aef9208 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -55,8 +55,8 @@ jobs: name: Build Latest Docs runs-on: ubuntu-latest needs: [doc-build-type] - # run on non-deploy branches to build current version docs only - if: needs.doc-build-type.outputs.trigger-deploy != 'true' + # TEMP (revert before review): masked while iterating Windows CI on PR 5700. + if: false steps: - name: Checkout code diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index 069a3193faa4..0dfea8709d85 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -127,7 +127,8 @@ jobs: install-tests-x86: name: Installation Tests (x86) needs: [changes] - if: needs.changes.outputs.run_install_tests == 'true' + # TEMP (revert before review): masked while iterating Windows CI on PR 5700. + if: false runs-on: [self-hosted, gpu] timeout-minutes: 90 steps: @@ -146,7 +147,8 @@ jobs: install-tests-arm: name: Installation Tests (ARM) needs: [changes] - if: needs.changes.outputs.run_install_tests == 'true' + # TEMP (revert before review): masked while iterating Windows CI on PR 5700. + if: false runs-on: [self-hosted, aarch64] timeout-minutes: 90 steps: @@ -172,3 +174,55 @@ jobs: continue-on-error: true run: | .github/scripts/runner_health.sh post-test + + # Windows has no Docker, so the suite runs natively. The composite actions above are + # bash and pin setup-uv v6, which crashes on exit on these runners + # (astral-sh/setup-uv#686). + install-tests-windows: + name: Installation Tests (Windows) + needs: [changes] + # TEMP (revert before review): no signal for the Windows CI Warp cache change. + if: false + runs-on: [self-hosted, gpu-windows] + timeout-minutes: 90 + defaults: + run: + shell: powershell + env: + PYTHONUTF8: "1" + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.12.9" + + # By name, Windows resolves bash to WSL's launcher in System32 ahead of PATH. + - name: Build wheel + run: | + & "C:\Program Files\Git\bin\bash.exe" tools/wheel_builder/build.sh + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + - name: Run Tests + env: + TEST_FILTER: ${{ inputs.test_filter }} + run: | + $wheel = @(Get-ChildItem tools/wheel_builder/build/dist -Filter "isaaclab-*.whl") + if ($wheel.Count -ne 1) { throw "expected one wheel, found $($wheel.Count)" } + # pytest's default base sits under the NetworkService profile, deep enough that + # Kit's DLLs in a venv there pass the 260-character path limit. + $pytestArgs = @("--run-gpu", "--tb=short", "-sv", "--basetemp=$env:RUNNER_TEMP\i", "--junitxml=${{ github.workspace }}\results\results.xml") + if ($env:TEST_FILTER) { $pytestArgs += @("-k", $env:TEST_FILTER) } + uv run --no-project --python 3.12 --with pytest python tools/run_install_ci.py native --wheel $wheel[0].FullName -- @pytestArgs + exit $LASTEXITCODE + + - name: Upload JUnit XML report + if: always() + uses: actions/upload-artifact@v7 + with: + name: install-ci-junit-windows + path: ${{ github.workspace }}/results/results.xml + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/license-check.yaml b/.github/workflows/license-check.yaml index 10c6420afd92..229e1b0cf953 100644 --- a/.github/workflows/license-check.yaml +++ b/.github/workflows/license-check.yaml @@ -15,6 +15,8 @@ concurrency: jobs: license-check: + # TEMP (revert before review): masked while iterating Windows CI on PR 5700. + if: false runs-on: ubuntu-24.04 steps: @@ -158,6 +160,8 @@ jobs: fi docker-license-check: name: Docker Dependency Licenses Check + # TEMP (revert before review): masked while iterating Windows CI on PR 5700. + if: false runs-on: [self-hosted, gpu] timeout-minutes: 90 diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index a7ec67803a0c..b28b1f268f4c 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -40,8 +40,9 @@ jobs: name: Detect Changes runs-on: ubuntu-latest outputs: - unit: ${{ steps.unit.outputs.should_run }} - smoke: ${{ steps.smoke.outputs.should_run }} + # TEMP (revert before review): masked while iterating Windows CI on PR 5700. + unit: 'false' + smoke: 'false' steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: diff --git a/.github/workflows/windows-ci.yaml b/.github/workflows/windows-ci.yaml new file mode 100644 index 000000000000..ed6f6e8237b8 --- /dev/null +++ b/.github/workflows/windows-ci.yaml @@ -0,0 +1,203 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Marker-gated tests on self-hosted Windows GPU runners, the counterpart of arm-ci.yml. +# Windows has no Docker, so the job syncs the locked uv environment on the runner and +# hands it to the same orchestrator (tools/conftest.py) the container jobs run. + +name: Windows CI + +on: + pull_request: + types: [opened, synchronize, reopened] + branches: + - main + - develop + - 'release/**' + push: + branches: + - main + - develop + - 'release/**' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + pull-requests: read + +env: + # Decode subprocess output as UTF-8 instead of the cp1252 code page. + PYTHONUTF8: "1" + # A test the orchestrator kills on timeout would otherwise lose its buffered output. + PYTHONUNBUFFERED: "1" + +jobs: + changes: + name: Detect Changes + runs-on: ubuntu-latest + outputs: + should_run: ${{ steps.detect.outputs.should_run }} + steps: + - uses: actions/checkout@v6 + with: + sparse-checkout: .github/actions/detect-changes + sparse-checkout-cone-mode: false + - id: detect + uses: ./.github/actions/detect-changes + with: + triggered-jobs-push: windows-ci + triggered-jobs-pr: windows-ci + patterns: | + ^source/ :: Library source code + ^tools/ :: Build tooling + ^apps/ :: Standalone apps + ^scripts/ :: Standalone scripts + (^|/)pyproject\.toml$ :: Python project metadata + ^uv\.lock$ :: Locked dependency graph + ^VERSION$ :: Version file + ^\.github/workflows/windows-ci\.yaml$ :: This workflow file + ^\.github/actions/(warp-cache-key/collection_id|run-package-tests/warp_cache_inventory)\.py$ :: Shared cache scripts + ^\.github/scripts/windows_ci_caches\.ps1$ :: Cache script + + windows-ci: + name: windows-ci + needs: [changes] + if: needs.changes.outputs.should_run == 'true' + runs-on: [self-hosted, gpu-windows] + timeout-minutes: 90 + defaults: + run: + shell: powershell + env: + OMNI_KIT_ACCEPT_EULA: "yes" + # Same policy as the Linux test jobs: pull requests only restore, so they never publish + # anything built from unmerged code; pushes to protected branches also save. + CACHE_MODE: ${{ github.event_name == 'push' && 'save' || 'restore' }} + steps: + - uses: actions/checkout@v6 + + # v6.8.0, which the Linux workflows pin, crashes on exit on these runners once uv + # is in the tool cache (astral-sh/setup-uv#686). + - name: Set up uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.12.9" + + # Checkout cleans the workspace, so the environment lives in the runner's tool cache, where a + # reused runner keeps the installed packages and their compiled bytecode between jobs. + - name: Locate environment + run: '"UV_PROJECT_ENVIRONMENT=$env:RUNNER_TOOL_CACHE\isaaclab-windows-ci" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append' + + - name: Sync environment + id: sync + timeout-minutes: 30 + run: | + if ($env:CACHE_MODE -eq 'save') { + # Pull requests run in the kept environment and uv cache, so a run that publishes caches + # installs from neither. + if (Test-Path $env:UV_PROJECT_ENVIRONMENT) { Remove-Item -Recurse -Force $env:UV_PROJECT_ENVIRONMENT } + $env:UV_CACHE_DIR = "$env:RUNNER_TEMP\uv-cache" + } + uv sync --frozen --extra test --extra isaacsim --extra ov + if ($LASTEXITCODE -ne 0) { + # A job killed mid-sync can leave the kept environment broken, so rebuild it once. + Remove-Item -Recurse -Force $env:UV_PROJECT_ENVIRONMENT -ErrorAction SilentlyContinue + uv sync --frozen --extra test --extra isaacsim --extra ov + } + + - name: Resolve cache keys + id: cache-keys + run: .github/scripts/windows_ci_caches.ps1 keys + + - name: Restore Warp kernel cache + id: restore-warp + uses: actions/cache/restore@v4 + with: + path: ${{ steps.cache-keys.outputs.warp-dir }} + key: ${{ steps.cache-keys.outputs.warp-key }} + restore-keys: ${{ steps.cache-keys.outputs.warp-restore-keys }} + + - name: Restore Kit shader cache + id: restore-kit + uses: actions/cache/restore@v4 + with: + path: ${{ steps.cache-keys.outputs.kit-dir }} + key: ${{ steps.cache-keys.outputs.kit-key }} + restore-keys: ${{ steps.cache-keys.outputs.kit-restore-keys }} + + - name: Restore OVRTX shader cache + id: restore-kitless + uses: actions/cache/restore@v4 + with: + path: ${{ steps.cache-keys.outputs.kitless-dir }} + key: ${{ steps.cache-keys.outputs.kitless-key }} + restore-keys: ${{ steps.cache-keys.outputs.kitless-restore-keys }} + + - name: Report restored caches + id: caches-restored + env: + WARP_HIT: ${{ steps.restore-warp.outputs.cache-matched-key }} + KIT_HIT: ${{ steps.restore-kit.outputs.cache-matched-key }} + KITLESS_HIT: ${{ steps.restore-kitless.outputs.cache-matched-key }} + run: .github/scripts/windows_ci_caches.ps1 restored + + - name: Run windows_ci marker tests + env: + CI_MARKER: windows_ci + TEST_RESULT_FILE: windows-ci-report.xml + run: uv run --frozen --no-sync python -m pytest tools + + # The orchestrator skips install_ci/, so the Cartpole state/camera probes run here. + - name: Run Cartpole smoke tests + if: ${{ !cancelled() && steps.sync.outcome == 'success' }} + run: uv run --frozen --no-sync python -m pytest -v source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py --junitxml=tests/windows-ci-smoke-report.xml + + - name: Run Cartpole backend combinations + if: ${{ !cancelled() && steps.sync.outcome == 'success' }} + run: uv run --frozen --no-sync python -m pytest -v source/isaaclab/test/install_ci/misc/cartpole_backend_combinations_smoke.py --junitxml=tests/windows-ci-combinations-report.xml + + # Not success(): compiled kernels and shaders stay valid when a test fails. Cancellation still + # skips, so a cache the run was mid-write on is never published. + - name: Report cache growth + id: caches-growth + if: ${{ !cancelled() && steps.caches-restored.outcome == 'success' }} + env: + WARP_BEFORE: ${{ steps.caches-restored.outputs.warp-fingerprint }} + KIT_BEFORE: ${{ steps.caches-restored.outputs.kit-fingerprint }} + KITLESS_BEFORE: ${{ steps.caches-restored.outputs.kitless-fingerprint }} + run: .github/scripts/windows_ci_caches.ps1 growth + + - name: Save Warp kernel cache + if: ${{ !cancelled() && steps.caches-growth.outputs.warp-changed == 'true' }} + uses: actions/cache/save@v4 + with: + path: ${{ steps.cache-keys.outputs.warp-dir }} + key: ${{ steps.cache-keys.outputs.warp-key }} + + - name: Save Kit shader cache + if: ${{ !cancelled() && steps.caches-growth.outputs.kit-changed == 'true' }} + uses: actions/cache/save@v4 + with: + path: ${{ steps.cache-keys.outputs.kit-dir }} + key: ${{ steps.cache-keys.outputs.kit-key }} + + - name: Save OVRTX shader cache + if: ${{ !cancelled() && steps.caches-growth.outputs.kitless-changed == 'true' }} + uses: actions/cache/save@v4 + with: + path: ${{ steps.cache-keys.outputs.kitless-dir }} + key: ${{ steps.cache-keys.outputs.kitless-key }} + + - name: Upload test reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: windows-ci-reports + path: tests/ + retention-days: 7 diff --git a/source/isaaclab/changelog.d/jichuanh-windows-ci.skip b/source/isaaclab/changelog.d/jichuanh-windows-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab/test/install_ci/misc/cartpole_backend_combinations_smoke.py b/source/isaaclab/test/install_ci/misc/cartpole_backend_combinations_smoke.py new file mode 100644 index 000000000000..1e0cb24ada84 --- /dev/null +++ b/source/isaaclab/test/install_ci/misc/cartpole_backend_combinations_smoke.py @@ -0,0 +1,48 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Camera Cartpole training on every physics backend with every renderer. + +The caller owns environment preparation and needs the ``isaacsim`` and ``ov`` extras. Each pairing +trains in its own process. Isaac Sim / Kit cannot share a process with OvPhysX or OVRTX +(:func:`isaaclab.app.sim_launcher.validate_launch`), so those pairings are left out. +""" + +from __future__ import annotations + +import itertools +import subprocess +import sys +from pathlib import Path + +import pytest + +_PHYSICS = ("newton_mjwarp", "newton_kamino", "isaacsim_physx", "ovphysx") +_RENDERERS = ("newton_renderer", "ovrtx", "isaacsim_rtx") +_KIT = {"isaacsim_physx", "isaacsim_rtx"} +_KITLESS_ONLY = {"ovphysx", "ovrtx"} + +_COMBINATIONS = [ + pytest.param(physics, renderer, id=f"{physics}-{renderer}") + for physics, renderer in itertools.product(_PHYSICS, _RENDERERS) + if not ({physics, renderer} & _KIT and {physics, renderer} & _KITLESS_ONLY) +] + + +@pytest.mark.parametrize(("physics", "renderer"), _COMBINATIONS) +def test_train_cartpole_camera_completes(physics: str, renderer: str) -> None: + """Verify that camera Cartpole completes short RSL-RL training on one backend pairing.""" + isaaclab_root = next(parent for parent in Path(__file__).resolve().parents if (parent / "uv.lock").is_file()) + command = [ + *(sys.executable, "-m", "isaaclab", "train", "--rl_library", "rsl_rl"), + *("--task", "Isaac-Cartpole-Camera-Direct", "--num_envs", "16"), + *(f"presets={physics},{renderer}", "--max_iterations", "2"), + ] + # A cold run compiles kernels and shaders before training begins. + result = subprocess.run(command, cwd=isaaclab_root, text=True, capture_output=True, timeout=1800, check=False) + output = result.stdout + (result.stderr or "") + assert result.returncode == 0, f"Training failed (rc={result.returncode}):\n{output}" + assert "Traceback (most recent call last):" not in output, f"Training produced a traceback:\n{output}" + assert "Training time:" in output, f"Training did not report completion:\n{output}" diff --git a/source/isaaclab/test/utils/test_configclass.py b/source/isaaclab/test/utils/test_configclass.py index 7d53871de683..e4e1ed1c5fa4 100644 --- a/source/isaaclab/test/utils/test_configclass.py +++ b/source/isaaclab/test/utils/test_configclass.py @@ -20,7 +20,7 @@ from isaaclab.utils.io import dump_yaml, load_yaml from isaaclab.utils.string import ResolvableString -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.windows_ci] """ Mock classes and functions. diff --git a/source/isaaclab/test/utils/test_dict.py b/source/isaaclab/test/utils/test_dict.py index b91720ab1a79..373718dacbd9 100644 --- a/source/isaaclab/test/utils/test_dict.py +++ b/source/isaaclab/test/utils/test_dict.py @@ -12,7 +12,7 @@ import isaaclab.utils.dict as dict_utils import isaaclab.utils.string as string_utils -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.windows_ci] def _test_function(x): diff --git a/source/isaaclab/test/utils/test_episode_data.py b/source/isaaclab/test/utils/test_episode_data.py index 1d9536dc05ac..7aa071a42a54 100644 --- a/source/isaaclab/test/utils/test_episode_data.py +++ b/source/isaaclab/test/utils/test_episode_data.py @@ -8,7 +8,7 @@ from isaaclab.test.utils import DeviceScope, test_devices from isaaclab.utils.datasets import EpisodeData -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.windows_ci] @pytest.mark.parametrize("device", test_devices(DeviceScope.CPU)) diff --git a/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py b/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py index 1fb78180ac5f..304c9ece1b9a 100644 --- a/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py +++ b/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: BSD-3-Clause import json import os -import shutil +import sys import tempfile import uuid @@ -15,7 +15,7 @@ from isaaclab.test.utils import test_devices from isaaclab.utils.datasets import EpisodeData, HDF5DatasetFileHandler -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.windows_ci] def create_test_episode(device): @@ -41,10 +41,9 @@ def create_test_episode(device): @pytest.fixture def temp_dir(): """Create a temporary directory for test datasets.""" - temp_dir = tempfile.mkdtemp() - yield temp_dir - # cleanup after tests - shutil.rmtree(temp_dir) + # On Windows, libhdf5 releases its file handle shortly after close(), so the cleanup can race it. + with tempfile.TemporaryDirectory(ignore_cleanup_errors=sys.platform == "win32") as d: + yield d def test_create_dataset_file(temp_dir, monkeypatch): diff --git a/tools/conftest.py b/tools/conftest.py index 8d9b943c7e33..4ee1d0f9a8e7 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -174,6 +174,53 @@ def resolve_exit_code(num_failing: int, num_timeout: int, num_crashed: int, num_ return 0 +def _readable(fds, timeout): + """Return the pipes in ``fds`` that a read would not block on, waiting up to ``timeout`` seconds for one.""" + if os.name != "nt": + return select.select(fds, [], [], timeout)[0] + # Windows cannot select() on pipes, so poll each one instead. + deadline = time.monotonic() + timeout + while not (ready := [fd for fd in fds if _pipe_readable(fd)]) and time.monotonic() < deadline: + time.sleep(0.01) + return ready + + +def _pipe_readable(fd): + """Return whether the Windows pipe ``fd`` holds output or its writer has closed it.""" + import ctypes + import msvcrt + from ctypes import wintypes + + peek = ctypes.windll.kernel32.PeekNamedPipe + peek.argtypes = ( + wintypes.HANDLE, + wintypes.LPVOID, + wintypes.DWORD, + wintypes.LPDWORD, + wintypes.LPDWORD, + wintypes.LPDWORD, + ) + available = wintypes.DWORD() + if not peek(msvcrt.get_osfhandle(fd), None, 0, None, ctypes.byref(available), None): + return True # broken pipe: the read returns EOF at once + return available.value > 0 + + +def _kill_process_tree(process, pgid): + """Kill the test process and every process it started (Kit, Isaac Sim).""" + if pgid is None: + # Windows has no process groups; taskkill walks the tree from the test process instead. + result = subprocess.run(["taskkill", "/F", "/T", "/PID", str(process.pid)], capture_output=True, check=False) + if result.returncode != 0: + with contextlib.suppress(OSError): + process.kill() + return + try: + os.killpg(pgid, signal.SIGKILL) + except (ProcessLookupError, PermissionError, OSError): + process.kill() + + def _drain_ready_output(process, stdout_fd, stderr_fd, timeout=0.1, echo=True): """Read whatever is readable on the child's pipes, echoing it as it arrives. @@ -193,7 +240,7 @@ def _drain_ready_output(process, stdout_fd, stderr_fd, timeout=0.1, echo=True): stdout_chunk = b"" stderr_chunk = b"" try: - ready_fds, _, _ = select.select([stdout_fd, stderr_fd], [], [], timeout) + ready_fds = _readable([stdout_fd, stderr_fd], timeout) for fd in ready_fds: with contextlib.suppress(OSError): @@ -382,7 +429,7 @@ def capture_test_output_with_timeout(cmd, timeout, env, startup_deadline=0, repo universal_newlines=False, start_new_session=True, ) - pgid = os.getpgid(process.pid) + pgid = None if os.name == "nt" else os.getpgid(process.pid) stdout_fd = process.stdout.fileno() stderr_fd = process.stderr.fileno() @@ -439,11 +486,7 @@ def capture_test_output_with_timeout(cmd, timeout, env, startup_deadline=0, repo # Ahead of the system tables, which _get_diagnostics truncates off the end. pre_kill_diag = f"{hang_stacks}\n\n{pre_kill_diag}" - # Kill the entire process group (test + any Kit children). - try: - os.killpg(pgid, signal.SIGKILL) - except (ProcessLookupError, PermissionError, OSError): - process.kill() + _kill_process_tree(process, pgid) try: remaining_stdout, remaining_stderr = process.communicate(timeout=5) stdout_data += remaining_stdout @@ -465,12 +508,14 @@ def capture_test_output_with_timeout(cmd, timeout, env, startup_deadline=0, repo except Exception: pass - # Kill any orphaned child processes (Kit, Isaac Sim) left by the test. - try: - os.killpg(pgid, signal.SIGKILL) - time.sleep(1) - except (ProcessLookupError, PermissionError, OSError): - pass + # Kill any orphaned child processes (Kit, Isaac Sim) left by the test. On Windows the + # tree is unreachable once its root has exited. + if pgid is not None: + try: + os.killpg(pgid, signal.SIGKILL) + time.sleep(1) + except (ProcessLookupError, PermissionError, OSError): + pass wall_time = time.time() - start_time return process.returncode, stdout_data, stderr_data, "", wall_time, "" @@ -1050,7 +1095,8 @@ def _run_one_pass( # ``isaaclab_physx/.../test_articulation.py``) don't write to the same path # inside the shared ``/workspace/isaaclab`` mount and trigger false # shutdown_hang detections in sibling shards via the report-file existence check. - report_slug = str(ctx.test_file).replace("/", "__").replace("\\", "__") + # A Windows drive colon would name an alternate data stream instead of a file. + report_slug = str(ctx.test_file).replace("/", "__").replace("\\", "__").replace(":", "") report_file = f"tests/test-reports-{report_slug}{suffix}.xml" # Crash-durable progress log. pytest writes its JUnit XML only at session end, so this is # the only record of what passed, failed, and was in flight when a run dies before that.