From 6456905a4071a52528f1179694878afb587d1059 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Wed, 30 Sep 2026 23:11:21 -0700 Subject: [PATCH 01/22] Run ARM CI tests through the x86 run-package-tests action ARM CI called run-tests directly in two containers split by OVPhysX, without the Warp and OVRTX shader caches x86 restores. Let run-package-tests use an image already built on the runner and pass a pytest marker through, then run the arm_ci tests in one step with the x86 kit-less settings: OVPhysX and OVRTX cases together, ovstage deselected, caches restored on pull requests and published on pushes. --- .github/actions/run-package-tests/action.yml | 34 +++++++++++++++--- .github/workflows/arm-ci.yml | 36 +++++++++----------- 2 files changed, 46 insertions(+), 24 deletions(-) diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index e04859354305..37ff5ac97ee8 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -5,7 +5,8 @@ name: 'Run Package Tests' description: > - Pulls the Docker image from ECR, runs pytest inside the container, uploads + Pulls the Docker image from ECR (or uses one already built on the runner), + runs pytest inside the container, uploads results as an artifact, and fails fork PRs on test failures. NOTE: The calling job must check out the code before using this action. @@ -27,6 +28,17 @@ inputs: description: 'ECR cache tag' default: 'cache-base' required: false + image-source: + description: >- + Where the test image comes from. 'ecr' pulls it from ECR, building and pushing it on + a miss. 'local' uses an image the calling job already built on this runner, for + runners ECR does not serve, such as the arm64 DGX Spark pool. + default: 'ecr' + required: false + ci-marker: + description: 'Optional pytest -m marker expression applied to every test file (e.g. arm_ci)' + default: '' + required: false filter-pattern: description: >- Pattern to filter test files (e.g., isaaclab_tasks); test files whose path @@ -111,14 +123,15 @@ inputs: newest compatible snapshot and never writes; every test job uses this, so a pull request can never publish kernels built from unmerged code. 'save' restores the same way and writes the result back as a new snapshot, making - post-merge a cumulative writeback; only warm-warp-cache uses it. + post-merge a cumulative writeback; only warm-warp-cache and ARM CI's + post-merge run use it. default: '' required: false ovrtx-shader-cache: description: >- OVRTX shader cache mode. Empty disables the cache. 'restore' reads the newest compatible snapshot and never writes. 'save' restores and writes - back a new snapshot; only warm-ovrtx-cache uses it. + back a new snapshot; only warm-ovrtx-cache and ARM CI's post-merge run use it. default: '' required: false ovrtx-shader-cache-trees: @@ -184,12 +197,24 @@ runs: "pre-test-cleanup" \ "true" + - name: Check image source + shell: bash + env: + IMAGE_SOURCE: ${{ inputs.image-source }} + run: | + case "$IMAGE_SOURCE" in + ecr|local) ;; + *) echo "::error::image-source must be 'ecr' or 'local', got '$IMAGE_SOURCE'"; exit 1 ;; + esac + - name: Record pull start time + if: inputs.image-source == 'ecr' id: pull-start shell: bash run: echo "time=$(date +%s)" >> "$GITHUB_OUTPUT" - name: Pull image from ECR + if: inputs.image-source == 'ecr' uses: ./.github/actions/ecr-build-push-pull with: image-tag: ${{ inputs.image-tag }} @@ -199,7 +224,7 @@ runs: cache-tag: ${{ inputs.cache-tag }} - name: Report pull duration - if: always() + if: always() && inputs.image-source == 'ecr' shell: bash run: | start_time="${{ steps.pull-start.outputs.time }}" @@ -348,6 +373,7 @@ runs: include-files: ${{ inputs.include-files }} test-node-ids-file: ${{ inputs.test-node-ids-file }} test-node-ids-key: ${{ inputs.test-node-ids-key }} + ci-marker: ${{ inputs.ci-marker }} volume-mount-source: ${{ github.workspace }} extra-pip-packages: ${{ inputs.extra-pip-packages }} extra-uv-packages: ${{ inputs.extra-uv-packages }} diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index a45a180a4815..adfe9fd557bf 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -175,31 +175,27 @@ jobs: uses: ./.github/actions/resolve-ov-pins id: ov_pins + # Same action and settings as x86's rendering-correctness-kitless (legacy) job, + # with the image built above instead of an ECR pull. Pushes to protected + # branches publish the arm64 Warp and OVRTX caches that pull requests restore, + # the role x86's cache-warm jobs play. - name: Run arm_ci marker tests - uses: ./.github/actions/run-tests + uses: ./.github/actions/run-package-tests with: - test-path: tools - result-file: arm-ci-report.xml - container-name: isaac-lab-arm-ci-${{ github.run_id }}-${{ github.run_attempt }} + image-source: local image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 - extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}" - # ovstage rendering cases are post-merge only on x86 as well. - test-k-expr: not ovphysx and not ovstage + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} ci-marker: arm_ci - volume-mount-source: ${{ github.workspace }} - - - name: Run arm_ci OVPhysX marker tests - uses: ./.github/actions/run-tests - with: - test-path: tools - result-file: arm-ci-ovphysx-report.xml - container-name: isaac-lab-arm-ci-ovphysx-${{ github.run_id }}-${{ github.run_attempt }} - image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 + # ovstage rendering cases are post-merge only on x86 as well. + test-k-expr: not ovstage extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}" - test-k-expr: ovphysx and not ovstage - ci-marker: arm_ci - volume-mount-source: ${{ github.workspace }} - + warp-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }} + ovrtx-shader-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }} + ovrtx-shader-cache-trees: kitless + result-file: arm-ci-report.xml + container-name: isaac-lab-arm-ci + omni-github-test-type: arm-ci - name: Run shared Cartpole smoke uses: ./.github/actions/run-tests From a018260c70c817c4f61114525a060b3e7ec205e0 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Wed, 30 Sep 2026 23:21:13 -0700 Subject: [PATCH 02/22] Drop ARM CI's duplicate Cartpole training smoke Installation Tests (ARM) already runs the same three Cartpole probes on the Spark pool, and x86's Docker CI has no such step, so ARM CI's separate smoke step only repeated them. --- .github/workflows/arm-ci.yml | 9 --------- .../isaaclab/changelog.d/jichuanh-arm-ci-align-x86.skip | 0 .../test/install_ci/misc/cartpole_training_smoke.py | 4 ++-- 3 files changed, 2 insertions(+), 11 deletions(-) create mode 100644 source/isaaclab/changelog.d/jichuanh-arm-ci-align-x86.skip diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index adfe9fd557bf..768454d9250f 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -197,15 +197,6 @@ jobs: container-name: isaac-lab-arm-ci omni-github-test-type: arm-ci - - name: Run shared Cartpole smoke - uses: ./.github/actions/run-tests - with: - test-path: source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py - result-file: arm-ci-cartpole-smoke-report.xml - container-name: isaac-lab-arm-ci-cartpole-${{ github.run_id }}-${{ github.run_attempt }} - image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 - volume-mount-source: ${{ github.workspace }} - - name: Upload test reports if: always() uses: actions/upload-artifact@v7 diff --git a/source/isaaclab/changelog.d/jichuanh-arm-ci-align-x86.skip b/source/isaaclab/changelog.d/jichuanh-arm-ci-align-x86.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py b/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py index 862c0467eb8a..f18007c3336f 100644 --- a/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py +++ b/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py @@ -6,8 +6,8 @@ """Shared state-training, camera-rendering, and camera-training Cartpole smoke probes. The caller owns environment preparation. Installation CI executes this file -inside the environment it just installed, while architecture CI executes the -same probes inside its prepared image. +inside the environment it just installed, while the kit-less image validation +runs the state-training probe inside that image. """ from __future__ import annotations From 44897179cc0d5280f58adca7086e4187f5c02c1a Mon Sep 17 00:00:00 2001 From: jichuanh Date: Wed, 30 Sep 2026 23:36:11 -0700 Subject: [PATCH 03/22] Re-enable the shadow-hand rendering and Isaac Sim PhysX Cartpole tests on ARM Both were disabled on ARM for real failures: the shadow-hand kit-less rendering file stalled mid-run on arm64 (#6581), and Isaac Sim PhysX training failed at environment creation on aarch64 with an invalidated simulation view. Restore the arm_ci marker and drop the aarch64 skip so DGX Spark runs both again. --- .../uv_run/test_uv_run_isaacsim_trains_cartpole.py | 8 -------- .../changelog.d/jichuanh-arm-ci-align-x86.skip | 0 .../test/core/test_rendering_shadow_hand_kitless.py | 4 ++-- 3 files changed, 2 insertions(+), 10 deletions(-) create mode 100644 source/isaaclab_tasks/changelog.d/jichuanh-arm-ci-align-x86.skip diff --git a/source/isaaclab/test/install_ci/uv_run/test_uv_run_isaacsim_trains_cartpole.py b/source/isaaclab/test/install_ci/uv_run/test_uv_run_isaacsim_trains_cartpole.py index 851cbba6b1f5..2b065072e99d 100644 --- a/source/isaaclab/test/install_ci/uv_run/test_uv_run_isaacsim_trains_cartpole.py +++ b/source/isaaclab/test/install_ci/uv_run/test_uv_run_isaacsim_trains_cartpole.py @@ -15,7 +15,6 @@ from __future__ import annotations -import platform import shutil import pytest @@ -52,13 +51,6 @@ class Test_Uv_Run_Isaacsim_Trains_Cartpole: def setup_class(cls): if not shutil.which("uv"): pytest.skip("uv is not available") - # PhysX training currently fails at env creation on aarch64 (DGX Spark): - # "[omni.physx.tensors.plugin] Simulation view object is invalidated and cannot - # be used again to call updateArticulationsKinematic". The environment itself - # installs fine from the lock; this matches the x86-only scope of the workflow - # this test replaces. Re-enable once PhysX training works on aarch64. - if platform.machine().lower() in ("aarch64", "arm64"): - pytest.skip("PhysX training is not functional on aarch64 yet") @pytest.mark.docker @pytest.mark.smoke diff --git a/source/isaaclab_tasks/changelog.d/jichuanh-arm-ci-align-x86.skip b/source/isaaclab_tasks/changelog.d/jichuanh-arm-ci-align-x86.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py index ea44307f02c2..b17e9c41a0c3 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py @@ -19,8 +19,8 @@ rendering_test_shadow_hand, ) -# no arm_ci marker: intermittently stalls the arm64 runner mid-file (renderer hang after a few -# passing cases, killed at the per-file timeout); x86 kitless coverage remains in place +pytestmark = pytest.mark.arm_ci + _RENDERING_PARAMS = group_rendering_params( make_kitless_rendering_params(MINIMAL_KITLESS_PHYSICS_RENDERER_AOV_COMBINATIONS) ) From adccfaedd47a484d077b293b16f573d7c432fc98 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Thu, 1 Oct 2026 01:38:19 -0700 Subject: [PATCH 04/22] Run the operational-space controller tests on ARM test_operational_space.py set pytestmark to arm_ci and then reassigned it to integration, so ARM CI's -m arm_ci deselected all of its tests. Declare both markers in one list at the top of the file. --- source/isaaclab/test/controllers/test_operational_space.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/isaaclab/test/controllers/test_operational_space.py b/source/isaaclab/test/controllers/test_operational_space.py index 2e7a3e4b09a9..1be680327664 100644 --- a/source/isaaclab/test/controllers/test_operational_space.py +++ b/source/isaaclab/test/controllers/test_operational_space.py @@ -13,7 +13,7 @@ import pytest import torch -pytestmark = pytest.mark.arm_ci +pytestmark = [pytest.mark.integration, pytest.mark.arm_ci] import isaaclab.envs.mdp as mdp import isaaclab.sim as sim_utils @@ -52,8 +52,6 @@ from isaaclab_assets import FRANKA_PANDA_LEGACY_CFG, G1_29DOF_CFG # isort:skip -pytestmark = pytest.mark.integration - @pytest.fixture def sim(): From 8938bf99c4f86bbc4ebc3f75d84ea0c0e27746f0 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Thu, 1 Oct 2026 00:13:11 -0700 Subject: [PATCH 05/22] Stop running the kuka visual-material kit-less rendering test on ARM No x86 job selects this file: the task package jobs exclude rendering tests and the kit-less rendering job lists its files explicitly without it. Drop its arm_ci marker so ARM CI runs the same rendering files as x86. --- .../test_rendering_kuka_visual_material_randomization_kitless.py | 1 - 1 file changed, 1 deletion(-) diff --git a/source/isaaclab_tasks/test/core/test_rendering_kuka_visual_material_randomization_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_kuka_visual_material_randomization_kitless.py index a7f2aacd7ced..fbeaf8a2894a 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_kuka_visual_material_randomization_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_kuka_visual_material_randomization_kitless.py @@ -18,7 +18,6 @@ ) pytestmark = [ - pytest.mark.arm_ci, pytest.mark.integration, pytest.mark.rendering, pytest.mark.kitless, From 3c2299461d5d87780d9a25b5c1ddd9167ebd316a Mon Sep 17 00:00:00 2001 From: jichuanh Date: Thu, 1 Oct 2026 03:12:19 -0700 Subject: [PATCH 06/22] Build Docker images for the runner's architecture by default docker-build defaulted to linux/amd64, so a fallback build on an arm64 runner (reached through ecr-build-push-pull, which passes no platform) would have produced an x86 image. Resolve an empty platform from RUNNER_ARCH; x86 runners still get linux/amd64 and the same cache keys. --- .github/actions/docker-build/action.yml | 28 ++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index 038deac54e98..b1e781a7e081 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -25,8 +25,10 @@ inputs: default: '.' required: false platform: - description: 'Target platform for `docker buildx build --platform`.' - default: 'linux/amd64' + description: >- + Target platform for `docker buildx build --platform`. Empty builds for the + runner's own architecture (X64 -> linux/amd64, ARM64 -> linux/arm64). + default: '' required: false cache-from: description: > @@ -68,6 +70,22 @@ runs: with: base-image-ref: ${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }} + - name: Resolve target platform + id: platform + shell: bash + env: + PLATFORM_INPUT: ${{ inputs.platform }} + run: | + platform="$PLATFORM_INPUT" + if [ -z "$platform" ]; then + case "$RUNNER_ARCH" in + X64) platform=linux/amd64 ;; + ARM64) platform=linux/arm64 ;; + *) echo "::error::No default platform for runner architecture '$RUNNER_ARCH'; pass platform"; exit 1 ;; + esac + fi + echo "platform=$platform" >> "$GITHUB_OUTPUT" + ##### 2: Host disk snapshot (pre) ##### - name: Host disk snapshot (pre) @@ -120,7 +138,7 @@ runs: dockerfile-path: ${{ inputs.dockerfile-path }} isaacsim-base-image: ${{ inputs.isaacsim-base-image }} isaacsim-version: ${{ inputs.isaacsim-version }} - platform: ${{ inputs.platform }} + platform: ${{ steps.platform.outputs.platform }} - name: Check deps-tag locally id: local-deps @@ -152,7 +170,7 @@ runs: run: | BUILD_ARGS=( --progress=plain - --platform "${{ inputs.platform }}" + --platform "${{ steps.platform.outputs.platform }}" -f "${{ inputs.dockerfile-path }}" --build-arg "ISAACSIM_BASE_IMAGE_ARG=${{ inputs.isaacsim-base-image }}" --build-arg "ISAACSIM_VERSION_ARG=${{ inputs.isaacsim-version }}" @@ -192,7 +210,7 @@ runs: # Dockerfile on stdin (no context to stage). Quoted heredoc so the # shell leaves ISAACLAB_PATH and IMAGE_USER for Docker to expand; the # image tag is an action input, substituted before bash ever runs. - docker build --platform "${{ inputs.platform }}" \ + docker build --platform "${{ steps.platform.outputs.platform }}" \ --build-arg "IMAGE_USER=${image_user:-root}" \ -t "${{ inputs.image-tag }}" - <<'DOCKERFILE' FROM ${{ inputs.image-tag }} From 7850851d20f24f82a36eaa380a57a2a1575886f6 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Thu, 1 Oct 2026 03:12:20 -0700 Subject: [PATCH 07/22] Drop run-package-tests' image-source input On a runner without ECR, ecr-build-push-pull already falls through to docker-build, which reuses an image with the same tag from the local store. ARM CI builds its image first, so run-package-tests picks it up without an ARM-specific input. --- .github/actions/run-package-tests/action.yml | 24 ++------------------ .github/workflows/arm-ci.yml | 9 ++++---- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index 37ff5ac97ee8..2d9b0e5ccaf9 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -5,8 +5,7 @@ name: 'Run Package Tests' description: > - Pulls the Docker image from ECR (or uses one already built on the runner), - runs pytest inside the container, uploads + Pulls the Docker image from ECR, runs pytest inside the container, uploads results as an artifact, and fails fork PRs on test failures. NOTE: The calling job must check out the code before using this action. @@ -28,13 +27,6 @@ inputs: description: 'ECR cache tag' default: 'cache-base' required: false - image-source: - description: >- - Where the test image comes from. 'ecr' pulls it from ECR, building and pushing it on - a miss. 'local' uses an image the calling job already built on this runner, for - runners ECR does not serve, such as the arm64 DGX Spark pool. - default: 'ecr' - required: false ci-marker: description: 'Optional pytest -m marker expression applied to every test file (e.g. arm_ci)' default: '' @@ -197,24 +189,12 @@ runs: "pre-test-cleanup" \ "true" - - name: Check image source - shell: bash - env: - IMAGE_SOURCE: ${{ inputs.image-source }} - run: | - case "$IMAGE_SOURCE" in - ecr|local) ;; - *) echo "::error::image-source must be 'ecr' or 'local', got '$IMAGE_SOURCE'"; exit 1 ;; - esac - - name: Record pull start time - if: inputs.image-source == 'ecr' id: pull-start shell: bash run: echo "time=$(date +%s)" >> "$GITHUB_OUTPUT" - name: Pull image from ECR - if: inputs.image-source == 'ecr' uses: ./.github/actions/ecr-build-push-pull with: image-tag: ${{ inputs.image-tag }} @@ -224,7 +204,7 @@ runs: cache-tag: ${{ inputs.cache-tag }} - name: Report pull duration - if: always() && inputs.image-source == 'ecr' + if: always() shell: bash run: | start_time="${{ steps.pull-start.outputs.time }}" diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index 768454d9250f..d332d8bdb589 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -175,14 +175,13 @@ jobs: uses: ./.github/actions/resolve-ov-pins id: ov_pins - # Same action and settings as x86's rendering-correctness-kitless (legacy) job, - # with the image built above instead of an ECR pull. Pushes to protected - # branches publish the arm64 Warp and OVRTX caches that pull requests restore, - # the role x86's cache-warm jobs play. + # Same action and settings as x86's rendering-correctness-kitless (legacy) job. + # With no ECR on the Spark, it picks up the image built above from the local + # store. Pushes to protected branches publish the arm64 Warp and OVRTX caches + # that pull requests restore, the role x86's cache-warm jobs play. - name: Run arm_ci marker tests uses: ./.github/actions/run-package-tests with: - image-source: local image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} From a7e156c78d914ae84663e731ea9aa43aefbb9797 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Thu, 1 Oct 2026 03:14:17 -0700 Subject: [PATCH 08/22] Read the default build platform from the Docker daemon Use the platform docker version reports instead of mapping RUNNER_ARCH by hand. --- .github/actions/docker-build/action.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index b1e781a7e081..9a1f5563caf4 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -27,7 +27,7 @@ inputs: platform: description: >- Target platform for `docker buildx build --platform`. Empty builds for the - runner's own architecture (X64 -> linux/amd64, ARM64 -> linux/arm64). + Docker daemon's own platform, as `docker version` reports it. default: '' required: false cache-from: @@ -76,14 +76,8 @@ runs: env: PLATFORM_INPUT: ${{ inputs.platform }} run: | - platform="$PLATFORM_INPUT" - if [ -z "$platform" ]; then - case "$RUNNER_ARCH" in - X64) platform=linux/amd64 ;; - ARM64) platform=linux/arm64 ;; - *) echo "::error::No default platform for runner architecture '$RUNNER_ARCH'; pass platform"; exit 1 ;; - esac - fi + platform="${PLATFORM_INPUT:-$(docker version --format '{{.Server.Os}}/{{.Server.Arch}}')}" + echo "Target platform: $platform" echo "platform=$platform" >> "$GITHUB_OUTPUT" ##### 2: Host disk snapshot (pre) ##### From bd5f11fc9535d7ee5a0d5ad43ecddcaf69e4882b Mon Sep 17 00:00:00 2001 From: jichuanh Date: Thu, 1 Oct 2026 03:16:47 -0700 Subject: [PATCH 09/22] Keep docker-build's platform default unchanged ARM CI passes platform to its own build step, and run-package-tests reuses that image from the local store before docker-build reads the platform, so no default is needed. --- .github/actions/docker-build/action.yml | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index 9a1f5563caf4..038deac54e98 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -25,10 +25,8 @@ inputs: default: '.' required: false platform: - description: >- - Target platform for `docker buildx build --platform`. Empty builds for the - Docker daemon's own platform, as `docker version` reports it. - default: '' + description: 'Target platform for `docker buildx build --platform`.' + default: 'linux/amd64' required: false cache-from: description: > @@ -70,16 +68,6 @@ runs: with: base-image-ref: ${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }} - - name: Resolve target platform - id: platform - shell: bash - env: - PLATFORM_INPUT: ${{ inputs.platform }} - run: | - platform="${PLATFORM_INPUT:-$(docker version --format '{{.Server.Os}}/{{.Server.Arch}}')}" - echo "Target platform: $platform" - echo "platform=$platform" >> "$GITHUB_OUTPUT" - ##### 2: Host disk snapshot (pre) ##### - name: Host disk snapshot (pre) @@ -132,7 +120,7 @@ runs: dockerfile-path: ${{ inputs.dockerfile-path }} isaacsim-base-image: ${{ inputs.isaacsim-base-image }} isaacsim-version: ${{ inputs.isaacsim-version }} - platform: ${{ steps.platform.outputs.platform }} + platform: ${{ inputs.platform }} - name: Check deps-tag locally id: local-deps @@ -164,7 +152,7 @@ runs: run: | BUILD_ARGS=( --progress=plain - --platform "${{ steps.platform.outputs.platform }}" + --platform "${{ inputs.platform }}" -f "${{ inputs.dockerfile-path }}" --build-arg "ISAACSIM_BASE_IMAGE_ARG=${{ inputs.isaacsim-base-image }}" --build-arg "ISAACSIM_VERSION_ARG=${{ inputs.isaacsim-version }}" @@ -204,7 +192,7 @@ runs: # Dockerfile on stdin (no context to stage). Quoted heredoc so the # shell leaves ISAACLAB_PATH and IMAGE_USER for Docker to expand; the # image tag is an action input, substituted before bash ever runs. - docker build --platform "${{ steps.platform.outputs.platform }}" \ + docker build --platform "${{ inputs.platform }}" \ --build-arg "IMAGE_USER=${image_user:-root}" \ -t "${{ inputs.image-tag }}" - <<'DOCKERFILE' FROM ${{ inputs.image-tag }} From de9960c22869463254a85a5da3ddc1c82d155ef0 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Thu, 1 Oct 2026 03:23:14 -0700 Subject: [PATCH 10/22] Pass the target platform through run-package-tests run-package-tests and ecr-build-push-pull now forward a platform (default linux/amd64) to the dependency hash and docker-build, so an image built inside a test job matches its runner. ARM CI passes linux/arm64; x86 callers keep the same default. --- .github/actions/ecr-build-push-pull/action.yml | 6 ++++++ .github/actions/run-package-tests/action.yml | 5 +++++ .github/workflows/arm-ci.yml | 1 + 3 files changed, 12 insertions(+) diff --git a/.github/actions/ecr-build-push-pull/action.yml b/.github/actions/ecr-build-push-pull/action.yml index 6c6b7eb6f069..7ff7a10ded17 100644 --- a/.github/actions/ecr-build-push-pull/action.yml +++ b/.github/actions/ecr-build-push-pull/action.yml @@ -23,6 +23,10 @@ inputs: description: 'Path to Dockerfile, relative to the repository root' default: 'docker/Dockerfile.base' required: false + platform: + description: 'Target platform for the build and its dependency-cache identity' + default: 'linux/amd64' + required: false ecr-url: description: > ECR repository URL (e.g. "123456789.dkr.ecr.us-west-2.amazonaws.com/my-repo"). @@ -208,6 +212,7 @@ runs: dockerfile-path: ${{ inputs.dockerfile-path }} isaacsim-base-image: ${{ inputs.isaacsim-base-image }} isaacsim-version: ${{ inputs.isaacsim-version }} + platform: ${{ inputs.platform }} - name: Check deps cache id: deps-cache @@ -250,6 +255,7 @@ runs: isaacsim-base-image: ${{ inputs.isaacsim-base-image }} isaacsim-version: ${{ inputs.isaacsim-version }} dockerfile-path: ${{ inputs.dockerfile-path }} + platform: ${{ inputs.platform }} cache-from: ${{ steps.resolve-ecr.outputs.available == 'true' && format('type=registry,ref={0}', env.CACHE_IMAGE) || '' }} cache-to: ${{ steps.resolve-ecr.outputs.available == 'true' && format('type=registry,ref={0},mode=max', env.CACHE_IMAGE) || '' }} deps-hash: ${{ steps.deps-hash.outputs.hash }} diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index 2d9b0e5ccaf9..393505230809 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -31,6 +31,10 @@ inputs: description: 'Optional pytest -m marker expression applied to every test file (e.g. arm_ci)' default: '' required: false + platform: + description: 'Target platform if the image has to be built' + default: 'linux/amd64' + required: false filter-pattern: description: >- Pattern to filter test files (e.g., isaaclab_tasks); test files whose path @@ -202,6 +206,7 @@ runs: isaacsim-version: ${{ inputs.isaacsim-version }} dockerfile-path: ${{ inputs.dockerfile-path }} cache-tag: ${{ inputs.cache-tag }} + platform: ${{ inputs.platform }} - name: Report pull duration if: always() diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index d332d8bdb589..00f650f81e67 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -183,6 +183,7 @@ jobs: uses: ./.github/actions/run-package-tests with: image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 + platform: linux/arm64 isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} ci-marker: arm_ci From f6b0e2d13264c6da550ebd352b5877b6d133ca07 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 5 Oct 2026 18:40:47 -0700 Subject: [PATCH 11/22] Preserve ARM camera smoke coverage with shared caches --- .github/actions/run-package-tests/action.yml | 17 +++++++++++++++++ .github/workflows/arm-ci.yml | 1 + .../install_ci/misc/cartpole_training_smoke.py | 5 ++--- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index eebc341a3713..c3189d172f33 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -152,6 +152,10 @@ inputs: description: 'Optional JUnit result filename; defaults to the workflow job identifier' default: '' required: false + smoke-test-path: + description: 'Optional pytest smoke-test file to run after the package tests, before cache publication' + default: '' + required: false container-name: description: 'Docker container name prefix (run-id is appended automatically)' required: true @@ -371,6 +375,19 @@ runs: standalone-script-visualizer: ${{ inputs.standalone-script-visualizer }} standalone-script-runtime-group: ${{ inputs.standalone-script-runtime-group }} + - name: Run smoke tests + if: inputs.smoke-test-path != '' + uses: ./.github/actions/run-tests + with: + test-path: ${{ inputs.smoke-test-path }} + result-file: ${{ github.job }}-smoke-report.xml + container-name: ${{ inputs.container-name }}-smoke-${{ github.run_id }}-${{ github.run_attempt }} + image-tag: ${{ inputs.image-tag }} + volume-mount-source: ${{ github.workspace }} + warp-cache-host-dir: ${{ steps.warp-cache-key.outputs.host-dir }} + ovrtx-shader-cache-host-dir: ${{ steps.ovrtx-cache.outputs.host-dir }} + omni-github-test-type: ${{ inputs.omni-github-test-type }} + - name: Check Test Results if: always() shell: bash diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index 53f6d391d7f2..647f69f62c95 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -192,6 +192,7 @@ jobs: warp-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }} ovrtx-shader-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }} ovrtx-shader-cache-trees: kitless + smoke-test-path: source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py result-file: arm-ci-report.xml container-name: isaac-lab-arm-ci omni-github-test-type: arm-ci diff --git a/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py b/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py index 92ee72a4df9e..08564d0afae3 100644 --- a/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py +++ b/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py @@ -5,9 +5,8 @@ """Shared state-training, camera-rendering, and camera-training Cartpole smoke probes. -The caller owns environment preparation. Installation CI executes this file -inside the environment it just installed, while the kit-less image validation -runs the state-training probe inside that image. +The caller owns environment preparation. Architecture CI executes all probes +inside its prepared image, while kit-less image validation runs the state-training probe. """ from __future__ import annotations From 8fa6f501ac597b3797e66f1e862ff8f56b9ca714 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 5 Oct 2026 18:40:57 -0700 Subject: [PATCH 12/22] Re-enable ARM PhysX training in the uv workspace tests --- source/isaaclab/test/install_ci/test_workspace.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/isaaclab/test/install_ci/test_workspace.py b/source/isaaclab/test/install_ci/test_workspace.py index e5be41d52590..939848cb223a 100644 --- a/source/isaaclab/test/install_ci/test_workspace.py +++ b/source/isaaclab/test/install_ci/test_workspace.py @@ -35,8 +35,6 @@ def test_workspace_install( def test_workspace_trains(workspace: tuple[str, Path, dict[str, str]], checkout: Path, run: Callable[..., str]) -> None: runtime, directory, env = workspace if runtime == "isaacsim": - if platform.machine().lower() in ("aarch64", "arm64"): - pytest.skip("Isaac Sim PhysX training is not functional on aarch64 yet") if platform.system() == "Linux": shim = directory / "venv/lib/python3.12/site-packages/isaacsim/kit/kernel/plugins/libcarb.env.shim.so" assert shim.is_file() From 7b73dafe1fe834c3ca088a210f1a635e704426a2 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 5 Oct 2026 18:50:01 -0700 Subject: [PATCH 13/22] Preserve the ARM system preload in installation probes --- source/isaaclab/test/install_ci/conftest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/isaaclab/test/install_ci/conftest.py b/source/isaaclab/test/install_ci/conftest.py index d08437e5ef54..db739e427284 100644 --- a/source/isaaclab/test/install_ci/conftest.py +++ b/source/isaaclab/test/install_ci/conftest.py @@ -58,10 +58,15 @@ def run() -> Callable[..., str]: clean_env["LD_PRELOAD"] = "/lib/aarch64-linux-gnu/libgomp.so.1" def execute(*args: str, cwd: Path, env: dict[str, str] | None = None, timeout: int = 900) -> str: + command_env = {**clean_env, **(env or {})} + # Keep platform-required libraries when a probe adds another preload. + preloads = [clean_env.get("LD_PRELOAD", ""), (env or {}).get("LD_PRELOAD", "")] + if any(preloads): + command_env["LD_PRELOAD"] = ":".join(preload for preload in preloads if preload) result = subprocess.run( args, cwd=cwd, - env={**clean_env, **(env or {})}, + env=command_env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, From a29615ac7f365efaf0b0b230bf06906cc23299fd Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 12 Oct 2026 12:07:46 -0700 Subject: [PATCH 14/22] Build ARM CI images in the runner daemon and run test files in parallel --- .github/actions/docker-build/action.yml | 18 +++++++++++++----- .github/workflows/arm-ci.yml | 1 + 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index e5d735205999..b053fa586df6 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -168,13 +168,21 @@ runs: BUILD_ARGS+=( --cache-to "${{ inputs.cache-to }}" ) fi - BUILDER_NAME="docker-build-${{ github.run_id }}-${{ github.job }}" - docker buildx create --use --driver docker-container --name "${BUILDER_NAME}" \ - || docker buildx use "${BUILDER_NAME}" - trap 'docker buildx rm "${BUILDER_NAME}" || true' EXIT + # Registry cache export needs a docker-container builder, whose image must + # then be exported and loaded into the daemon. Without one, build in the + # daemon itself so the image, layer cache and uv cache mount stay local. + if [ -n "${{ inputs.cache-from }}${{ inputs.cache-to }}" ]; then + BUILDER_NAME="docker-build-${{ github.run_id }}-${{ github.job }}" + docker buildx create --driver docker-container --name "${BUILDER_NAME}" \ + || true + trap 'docker buildx rm "${BUILDER_NAME}" || true' EXIT + BUILD_ARGS+=( --builder "${BUILDER_NAME}" --load ) + else + BUILD_ARGS+=( --builder default ) + fi echo "🔵 Building ${{ inputs.image-tag }}..." - docker buildx build --load "${BUILD_ARGS[@]}" "${{ inputs.context-path }}" + docker buildx build "${BUILD_ARGS[@]}" "${{ inputs.context-path }}" echo "was-built=true" >> "$GITHUB_OUTPUT" ##### 6: Layer the CI pytest harness onto the built image ##### diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index 647f69f62c95..12e48bbe6c40 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -188,6 +188,7 @@ jobs: ci-marker: arm_ci # ovstage rendering cases are post-merge only on x86 as well. test-k-expr: not ovstage + test-jobs: "4" extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}" warp-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }} ovrtx-shader-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }} From 3392b80f62293ef2a0addde90d56818c10f04961 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 12 Oct 2026 12:07:51 -0700 Subject: [PATCH 15/22] TEMP: Save ARM caches from PR runs for the warm benchmark --- .github/workflows/arm-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index 12e48bbe6c40..0063b5397bb1 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -190,8 +190,8 @@ jobs: test-k-expr: not ovstage test-jobs: "4" extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}" - warp-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }} - ovrtx-shader-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }} + warp-cache: save # TEMP(8233 warm benchmark): revert before merge + ovrtx-shader-cache: save # TEMP(8233 warm benchmark): revert before merge ovrtx-shader-cache-trees: kitless smoke-test-path: source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py result-file: arm-ci-report.xml From 3db7d91827a61a91bd3c2e2cbf749cafbf7a75c8 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 12 Oct 2026 12:15:57 -0700 Subject: [PATCH 16/22] Redirect the Newton RTX visualizer shader cache --- .../changelog.d/newton-rtx-shader-cache.fixed.rst | 2 ++ .../isaaclab_visualizers/newton/newton_visualizer.py | 6 ++++++ 2 files changed, 8 insertions(+) create mode 100644 source/isaaclab_visualizers/changelog.d/newton-rtx-shader-cache.fixed.rst diff --git a/source/isaaclab_visualizers/changelog.d/newton-rtx-shader-cache.fixed.rst b/source/isaaclab_visualizers/changelog.d/newton-rtx-shader-cache.fixed.rst new file mode 100644 index 000000000000..8fe3655977c8 --- /dev/null +++ b/source/isaaclab_visualizers/changelog.d/newton-rtx-shader-cache.fixed.rst @@ -0,0 +1,2 @@ +* Fixed the Newton RTX visualizer ignoring ``OVRTX_SHADER_CACHE_PATH``, so its + shaders recompiled on every launch instead of reusing the redirected cache. diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py index 9cf466e1884a..d35d6aa311c6 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py @@ -1138,6 +1138,8 @@ class NewtonRTXVisualizer(NewtonVisualizerBase): def initialize(self, sim: SimulationContext, *, cameras: list[PerspectiveCameraCfg | Camera]) -> None: """Attach to the populated scene and bind the model used to publish body poses.""" + import ovrtx + from isaaclab_ov.renderers.ovrtx_shader_cache import redirect_shader_cache from isaaclab_ov.stage import OvstageBackendCfg if self._is_initialized: @@ -1155,6 +1157,10 @@ def initialize(self, sim: SimulationContext, *, cameras: list[PerspectiveCameraC if cfg.background_color is not None: settings["omni:rtx:background:source:type"] = ("Token", "color") settings["omni:rtx:background:source:color"] = ("Color3f", cfg.background_color) + # Newton builds its own ovrtx.Renderer with this config; redirect the shader cache before it does. + renderer_config = ovrtx.RendererConfig() + renderer_config.log_level = "error" + redirect_shader_cache(renderer_config) self._viewer = NewtonViewerRTX( width=self.cfg.window.size[0], height=self.cfg.window.size[1], From dd17bc1f74420db91104a159236d67851fa02a06 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 12 Oct 2026 12:37:02 -0700 Subject: [PATCH 17/22] Serialize rendering test files that start OVRTX through rendering_test_utils --- tools/conftest.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/conftest.py b/tools/conftest.py index cbc4757ff5c1..19b7be193180 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -1402,8 +1402,15 @@ def _run_one_pass( def _starts_renderer(test_content: str) -> bool: - """Return whether a test file's source starts an RTX renderer: Kit's, by enabling cameras, or OVRTX.""" - return "enable_cameras=True" in test_content or "ovrtx" in test_content.lower() + """Return whether a test file's source starts an RTX renderer: Kit's, by enabling cameras, or OVRTX. + + Files built on ``rendering_test_utils`` pick their renderer through its helpers, so they name neither. + """ + return ( + "enable_cameras=True" in test_content + or "ovrtx" in test_content.lower() + or "rendering_test_utils" in test_content + ) def _test_file_job(test_file: str, max_slots: int) -> TestFileJob: From 998d85090fd50bcc3cc9b6d512fec66927c3613e Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 12 Oct 2026 12:37:03 -0700 Subject: [PATCH 18/22] Export daemon-built CI image layers with zstd --- .github/actions/docker-build/action.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index b053fa586df6..ebdd54d4d088 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -171,6 +171,8 @@ runs: # Registry cache export needs a docker-container builder, whose image must # then be exported and loaded into the daemon. Without one, build in the # daemon itself so the image, layer cache and uv cache mount stay local. + # A containerd image store still compresses new layers; zstd level 1 is + # several times faster than the default gzip for the multi-GB venv layer. if [ -n "${{ inputs.cache-from }}${{ inputs.cache-to }}" ]; then BUILDER_NAME="docker-build-${{ github.run_id }}-${{ github.job }}" docker buildx create --driver docker-container --name "${BUILDER_NAME}" \ @@ -178,7 +180,11 @@ runs: trap 'docker buildx rm "${BUILDER_NAME}" || true' EXIT BUILD_ARGS+=( --builder "${BUILDER_NAME}" --load ) else - BUILD_ARGS+=( --builder default ) + BUILD_ARGS+=( + --builder default + --provenance=false + --output "type=image,name=${{ inputs.image-tag }},compression=zstd,compression-level=1" + ) fi echo "🔵 Building ${{ inputs.image-tag }}..." From 295445746255ec7c1446c222a82a49dffd961d42 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 12 Oct 2026 13:15:44 -0700 Subject: [PATCH 19/22] Revert "TEMP: Save ARM caches from PR runs for the warm benchmark" This reverts commit 3392b80f62293ef2a0addde90d56818c10f04961. --- .github/workflows/arm-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index 0063b5397bb1..12e48bbe6c40 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -190,8 +190,8 @@ jobs: test-k-expr: not ovstage test-jobs: "4" extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}" - warp-cache: save # TEMP(8233 warm benchmark): revert before merge - ovrtx-shader-cache: save # TEMP(8233 warm benchmark): revert before merge + warp-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }} + ovrtx-shader-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }} ovrtx-shader-cache-trees: kitless smoke-test-path: source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py result-file: arm-ci-report.xml From 62226c711c678e4f1db13a5f6c84a25b60d48127 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 12 Oct 2026 14:37:32 -0700 Subject: [PATCH 20/22] Revert "Redirect the Newton RTX visualizer shader cache" This reverts commit 3db7d91827a61a91bd3c2e2cbf749cafbf7a75c8. --- .../changelog.d/newton-rtx-shader-cache.fixed.rst | 2 -- .../isaaclab_visualizers/newton/newton_visualizer.py | 6 ------ 2 files changed, 8 deletions(-) delete mode 100644 source/isaaclab_visualizers/changelog.d/newton-rtx-shader-cache.fixed.rst diff --git a/source/isaaclab_visualizers/changelog.d/newton-rtx-shader-cache.fixed.rst b/source/isaaclab_visualizers/changelog.d/newton-rtx-shader-cache.fixed.rst deleted file mode 100644 index 8fe3655977c8..000000000000 --- a/source/isaaclab_visualizers/changelog.d/newton-rtx-shader-cache.fixed.rst +++ /dev/null @@ -1,2 +0,0 @@ -* Fixed the Newton RTX visualizer ignoring ``OVRTX_SHADER_CACHE_PATH``, so its - shaders recompiled on every launch instead of reusing the redirected cache. diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py index d35d6aa311c6..9cf466e1884a 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py @@ -1138,8 +1138,6 @@ class NewtonRTXVisualizer(NewtonVisualizerBase): def initialize(self, sim: SimulationContext, *, cameras: list[PerspectiveCameraCfg | Camera]) -> None: """Attach to the populated scene and bind the model used to publish body poses.""" - import ovrtx - from isaaclab_ov.renderers.ovrtx_shader_cache import redirect_shader_cache from isaaclab_ov.stage import OvstageBackendCfg if self._is_initialized: @@ -1157,10 +1155,6 @@ def initialize(self, sim: SimulationContext, *, cameras: list[PerspectiveCameraC if cfg.background_color is not None: settings["omni:rtx:background:source:type"] = ("Token", "color") settings["omni:rtx:background:source:color"] = ("Color3f", cfg.background_color) - # Newton builds its own ovrtx.Renderer with this config; redirect the shader cache before it does. - renderer_config = ovrtx.RendererConfig() - renderer_config.log_level = "error" - redirect_shader_cache(renderer_config) self._viewer = NewtonViewerRTX( width=self.cfg.window.size[0], height=self.cfg.window.size[1], From 5a940d43a24e4de3f8f5350db0f6e5b600f71eae Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 12 Oct 2026 14:39:40 -0700 Subject: [PATCH 21/22] Run ARM Cartpole smoke tests outside the shared test action --- .github/actions/run-package-tests/action.yml | 30 +++++++------------- .github/workflows/arm-ci.yml | 30 ++++++++++++++++++-- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index c3189d172f33..3ec3a401042b 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -119,15 +119,15 @@ inputs: newest compatible snapshot and never writes; every test job uses this, so a pull request can never publish kernels built from unmerged code. 'save' restores the same way and writes the result back as a new snapshot, making - post-merge a cumulative writeback; only warm-warp-cache and ARM CI's - post-merge run use it. + post-merge a cumulative writeback; only trusted post-merge cache producers + use it. default: '' required: false ovrtx-shader-cache: description: >- OVRTX shader cache mode. Empty disables the cache. 'restore' reads the newest compatible snapshot and never writes. 'save' restores and writes - back a new snapshot; only warm-ovrtx-cache and ARM CI's post-merge run use it. + back a new snapshot; only trusted post-merge cache producers use it. default: '' required: false ovrtx-shader-cache-trees: @@ -152,13 +152,16 @@ inputs: description: 'Optional JUnit result filename; defaults to the workflow job identifier' default: '' required: false - smoke-test-path: - description: 'Optional pytest smoke-test file to run after the package tests, before cache publication' - default: '' - required: false container-name: description: 'Docker container name prefix (run-id is appended automatically)' required: true +outputs: + warp-cache-host-dir: + description: 'Host directory mounted as the Warp kernel cache; empty when the cache is disabled' + value: ${{ steps.warp-cache-key.outputs.host-dir }} + ovrtx-shader-cache-host-dir: + description: 'Host directory mounted as the OVRTX shader cache; empty when the cache is disabled' + value: ${{ steps.ovrtx-cache.outputs.host-dir }} runs: using: composite steps: @@ -375,19 +378,6 @@ runs: standalone-script-visualizer: ${{ inputs.standalone-script-visualizer }} standalone-script-runtime-group: ${{ inputs.standalone-script-runtime-group }} - - name: Run smoke tests - if: inputs.smoke-test-path != '' - uses: ./.github/actions/run-tests - with: - test-path: ${{ inputs.smoke-test-path }} - result-file: ${{ github.job }}-smoke-report.xml - container-name: ${{ inputs.container-name }}-smoke-${{ github.run_id }}-${{ github.run_attempt }} - image-tag: ${{ inputs.image-tag }} - volume-mount-source: ${{ github.workspace }} - warp-cache-host-dir: ${{ steps.warp-cache-key.outputs.host-dir }} - ovrtx-shader-cache-host-dir: ${{ steps.ovrtx-cache.outputs.host-dir }} - omni-github-test-type: ${{ inputs.omni-github-test-type }} - - name: Check Test Results if: always() shell: bash diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index 12e48bbe6c40..e63c104de9a1 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -174,11 +174,12 @@ jobs: uses: ./.github/actions/resolve-ov-pins id: ov_pins - # Same action and settings as x86's rendering-correctness-kitless (legacy) job. - # With no ECR on the Spark, it picks up the image built above from the local + # Same action as the x86 test jobs, with four file slots; rendering files still + # run one at a time. With no ECR on the Spark, it picks up the image built above from the local # store. Pushes to protected branches publish the arm64 Warp and OVRTX caches # that pull requests restore, the role x86's cache-warm jobs play. - name: Run arm_ci marker tests + id: tests uses: ./.github/actions/run-package-tests with: image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 @@ -193,11 +194,34 @@ jobs: warp-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }} ovrtx-shader-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }} ovrtx-shader-cache-trees: kitless - smoke-test-path: source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py result-file: arm-ci-report.xml container-name: isaac-lab-arm-ci omni-github-test-type: arm-ci + # The test orchestrator skips install_ci/, so the Cartpole state/camera probes + # run here in the same image, reusing the restored Warp and OVRTX caches. + - name: Run Cartpole smoke tests + uses: ./.github/actions/run-tests + with: + test-path: source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py + result-file: arm-ci-smoke-report.xml + container-name: isaac-lab-arm-ci-smoke-${{ github.run_id }}-${{ github.run_attempt }} + image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 + volume-mount-source: ${{ github.workspace }} + warp-cache-host-dir: ${{ steps.tests.outputs.warp-cache-host-dir }} + ovrtx-shader-cache-host-dir: ${{ steps.tests.outputs.ovrtx-shader-cache-host-dir }} + omni-github-test-type: arm-ci + + # run-tests records failures in the report rather than its exit code. + - name: Check Cartpole smoke results + shell: bash + run: | + report=reports/arm-ci-smoke-report.xml + if [ ! -f "$report" ] || grep -qE '(failures|errors)="[1-9][0-9]*"' "$report"; then + echo "::error::Cartpole smoke tests failed or produced no report: $report" + exit 1 + fi + - name: Upload test reports if: always() uses: actions/upload-artifact@v7 From d02aedda2f804617dcf9c146ec0b4700bc4a9829 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 12 Oct 2026 14:39:41 -0700 Subject: [PATCH 22/22] State which callers build in the runner daemon --- .github/actions/docker-build/action.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index ebdd54d4d088..16b985e21fb4 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -169,10 +169,11 @@ runs: fi # Registry cache export needs a docker-container builder, whose image must - # then be exported and loaded into the daemon. Without one, build in the - # daemon itself so the image, layer cache and uv cache mount stay local. - # A containerd image store still compresses new layers; zstd level 1 is - # several times faster than the default gzip for the multi-GB venv layer. + # then be exported and loaded into the daemon. Callers without a registry + # cache (ARM, which has no ECR) build in the daemon itself instead, so the + # image, layer cache and uv cache mount stay on the runner. A containerd + # image store still compresses new layers; zstd level 1 is several times + # faster than the default gzip for the multi-GB venv layer. if [ -n "${{ inputs.cache-from }}${{ inputs.cache-to }}" ]; then BUILDER_NAME="docker-build-${{ github.run_id }}-${{ github.job }}" docker buildx create --driver docker-container --name "${BUILDER_NAME}" \