[CI] Run ARM CI through the x86 test action and re-enable ARM tests

.github/actions/docker-build/action.yml

@@ -168,13 +168,28 @@ runs:
           BUILD_ARGS+=( --cache-to "${{ inputs.cache-to }}" )
         fi
 
-        BUILDER_NAME="docker-build-${{ github.run_id }}-${{ github.job }}"
-        docker buildx create --use --driver docker-container --name "${BUILDER_NAME}" \
-          || docker buildx use "${BUILDER_NAME}"
-        trap 'docker buildx rm "${BUILDER_NAME}" || true' EXIT
+        # Registry cache export needs a docker-container builder, whose image must
+        # then be exported and loaded into the daemon. Callers without a registry
+        # cache (ARM, which has no ECR) build in the daemon itself instead, so the
+        # image, layer cache and uv cache mount stay on the runner. A containerd
+        # image store still compresses new layers; zstd level 1 is several times
+        # faster than the default gzip for the multi-GB venv layer.
+        if [ -n "${{ inputs.cache-from }}${{ inputs.cache-to }}" ]; then
+          BUILDER_NAME="docker-build-${{ github.run_id }}-${{ github.job }}"
+          docker buildx create --driver docker-container --name "${BUILDER_NAME}" \
+            || true
+          trap 'docker buildx rm "${BUILDER_NAME}" || true' EXIT
+          BUILD_ARGS+=( --builder "${BUILDER_NAME}" --load )
+        else
+          BUILD_ARGS+=(
+            --builder default
+            --provenance=false
+            --output "type=image,name=${{ inputs.image-tag }},compression=zstd,compression-level=1"
+          )
+        fi
 
         echo "🔵 Building ${{ inputs.image-tag }}..."
-        docker buildx build --load "${BUILD_ARGS[@]}" "${{ inputs.context-path }}"
+        docker buildx build "${BUILD_ARGS[@]}" "${{ inputs.context-path }}"
         echo "was-built=true" >> "$GITHUB_OUTPUT"
 
     ##### 6: Layer the CI pytest harness onto the built image #####

.github/actions/ecr-build-push-pull/action.yml

@@ -23,6 +23,10 @@ inputs:
     description: 'Path to Dockerfile, relative to the repository root'
     default: 'docker/Dockerfile.base'
     required: false
+  platform:
+    description: 'Target platform for the build and its dependency-cache identity'
+    default: 'linux/amd64'
+    required: false
   ecr-url:
     description: >
       ECR repository URL (e.g. "123456789.dkr.ecr.us-west-2.amazonaws.com/my-repo").
@@ -208,6 +212,7 @@ runs:
         dockerfile-path: ${{ inputs.dockerfile-path }}
         isaacsim-base-image: ${{ inputs.isaacsim-base-image }}
         isaacsim-version: ${{ inputs.isaacsim-version }}
+        platform: ${{ inputs.platform }}
 
     - name: Check deps cache
       id: deps-cache
@@ -250,6 +255,7 @@ runs:
         isaacsim-base-image: ${{ inputs.isaacsim-base-image }}
         isaacsim-version: ${{ inputs.isaacsim-version }}
         dockerfile-path: ${{ inputs.dockerfile-path }}
+        platform: ${{ inputs.platform }}
         cache-from: ${{ steps.resolve-ecr.outputs.available == 'true' && format('type=registry,ref={0}', env.CACHE_IMAGE) || '' }}
         cache-to: ${{ steps.resolve-ecr.outputs.available == 'true' && format('type=registry,ref={0},mode=max', env.CACHE_IMAGE) || '' }}
         deps-hash: ${{ steps.deps-hash.outputs.hash }}

.github/actions/run-package-tests/action.yml

@@ -27,6 +27,14 @@ inputs:
     description: 'ECR cache tag'
     default: 'cache-base'
     required: false
+  ci-marker:
+    description: 'Optional pytest -m marker expression applied to every test file (e.g. arm_ci)'
+    default: ''
+    required: false
+  platform:
+    description: 'Target platform if the image has to be built'
+    default: 'linux/amd64'
+    required: false
   filter-pattern:
     description: >-
       Pattern to filter test files (e.g., isaaclab_tasks); test files whose path
@@ -111,14 +119,15 @@ inputs:
       newest compatible snapshot and never writes; every test job uses this, so
       a pull request can never publish kernels built from unmerged code. 'save'
       restores the same way and writes the result back as a new snapshot, making
-      post-merge a cumulative writeback; only warm-warp-cache uses it.
+      post-merge a cumulative writeback; only trusted post-merge cache producers
+      use it.
     default: ''
     required: false
   ovrtx-shader-cache:
     description: >-
       OVRTX shader cache mode. Empty disables the cache. 'restore' reads the
       newest compatible snapshot and never writes. 'save' restores and writes
-      back a new snapshot; only warm-ovrtx-cache uses it.
+      back a new snapshot; only trusted post-merge cache producers use it.
     default: ''
     required: false
   ovrtx-shader-cache-trees:
@@ -146,6 +155,13 @@ inputs:
   container-name:
     description: 'Docker container name prefix (run-id is appended automatically)'
     required: true
+outputs:
+  warp-cache-host-dir:
+    description: 'Host directory mounted as the Warp kernel cache; empty when the cache is disabled'
+    value: ${{ steps.warp-cache-key.outputs.host-dir }}
+  ovrtx-shader-cache-host-dir:
+    description: 'Host directory mounted as the OVRTX shader cache; empty when the cache is disabled'
+    value: ${{ steps.ovrtx-cache.outputs.host-dir }}
 runs:
   using: composite
   steps:
@@ -197,6 +213,7 @@ runs:
         isaacsim-version: ${{ inputs.isaacsim-version }}
         dockerfile-path: ${{ inputs.dockerfile-path }}
         cache-tag: ${{ inputs.cache-tag }}
+        platform: ${{ inputs.platform }}
 
     - name: Report pull duration
       if: always()
@@ -348,6 +365,7 @@ runs:
         include-files: ${{ inputs.include-files }}
         test-node-ids-file: ${{ inputs.test-node-ids-file }}
         test-node-ids-key: ${{ inputs.test-node-ids-key }}
+        ci-marker: ${{ inputs.ci-marker }}
         volume-mount-source: ${{ github.workspace }}
         extra-pip-packages: ${{ inputs.extra-pip-packages }}
         extra-uv-packages: ${{ inputs.extra-uv-packages }}

.github/workflows/arm-ci.yml

@@ -174,40 +174,53 @@ jobs:
       uses: ./.github/actions/resolve-ov-pins
       id: ov_pins
 
+    # Same action as the x86 test jobs, with four file slots; rendering files still
+    # run one at a time. With no ECR on the Spark, it picks up the image built above from the local
+    # store. Pushes to protected branches publish the arm64 Warp and OVRTX caches
+    # that pull requests restore, the role x86's cache-warm jobs play.
     - name: Run arm_ci marker tests
-      uses: ./.github/actions/run-tests
+      id: tests
+      uses: ./.github/actions/run-package-tests
       with:
-        test-path: tools
-        result-file: arm-ci-report.xml
-        container-name: isaac-lab-arm-ci-${{ github.run_id }}-${{ github.run_attempt }}
         image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64
-        extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}"
-        # ovstage rendering cases are post-merge only on x86 as well.
-        test-k-expr: not ovphysx and not ovstage
+        platform: linux/arm64
+        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
+        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
         ci-marker: arm_ci
-        volume-mount-source: ${{ github.workspace }}
-
-    - name: Run arm_ci OVPhysX marker tests
-      uses: ./.github/actions/run-tests
-      with:
-        test-path: tools
-        result-file: arm-ci-ovphysx-report.xml
-        container-name: isaac-lab-arm-ci-ovphysx-${{ github.run_id }}-${{ github.run_attempt }}
-        image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64
+        # ovstage rendering cases are post-merge only on x86 as well.
+        test-k-expr: not ovstage
+        test-jobs: "4"
         extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}"
-        test-k-expr: ovphysx and not ovstage
-        ci-marker: arm_ci
-        volume-mount-source: ${{ github.workspace }}
-
+        warp-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }}
+        ovrtx-shader-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }}
+        ovrtx-shader-cache-trees: kitless
+        result-file: arm-ci-report.xml
+        container-name: isaac-lab-arm-ci
+        omni-github-test-type: arm-ci
 
-    - name: Run shared Cartpole smoke
+    # The test orchestrator skips install_ci/, so the Cartpole state/camera probes
+    # run here in the same image, reusing the restored Warp and OVRTX caches.
+    - name: Run Cartpole smoke tests
       uses: ./.github/actions/run-tests
       with:
         test-path: source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py
-        result-file: arm-ci-cartpole-smoke-report.xml
-        container-name: isaac-lab-arm-ci-cartpole-${{ github.run_id }}-${{ github.run_attempt }}
+        result-file: arm-ci-smoke-report.xml
+        container-name: isaac-lab-arm-ci-smoke-${{ github.run_id }}-${{ github.run_attempt }}
         image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64
         volume-mount-source: ${{ github.workspace }}
+        warp-cache-host-dir: ${{ steps.tests.outputs.warp-cache-host-dir }}
+        ovrtx-shader-cache-host-dir: ${{ steps.tests.outputs.ovrtx-shader-cache-host-dir }}
+        omni-github-test-type: arm-ci
+
+    # run-tests records failures in the report rather than its exit code.
+    - name: Check Cartpole smoke results
+      shell: bash
+      run: |
+        report=reports/arm-ci-smoke-report.xml
+        if [ ! -f "$report" ] || grep -qE '(failures|errors)="[1-9][0-9]*"' "$report"; then
+          echo "::error::Cartpole smoke tests failed or produced no report: $report"
+          exit 1
+        fi
 
     - name: Upload test reports
       if: always()

source/isaaclab/test/controllers/test_operational_space.py

@@ -13,7 +13,7 @@
 import pytest
 import torch
 
-pytestmark = pytest.mark.arm_ci
+pytestmark = [pytest.mark.integration, pytest.mark.arm_ci]
 
 import isaaclab.envs.mdp as mdp
 import isaaclab.sim as sim_utils
@@ -52,8 +52,6 @@
 
 from isaaclab_assets import FRANKA_PANDA_LEGACY_CFG, G1_29DOF_CFG  # isort:skip
 
-pytestmark = pytest.mark.integration
-
 
 @pytest.fixture
 def sim():

source/isaaclab/test/install_ci/conftest.py

@@ -58,10 +58,15 @@ def run() -> Callable[..., str]:
         clean_env["LD_PRELOAD"] = "/lib/aarch64-linux-gnu/libgomp.so.1"
 
     def execute(*args: str, cwd: Path, env: dict[str, str] | None = None, timeout: int = 900) -> str:
+        command_env = {**clean_env, **(env or {})}
+        # Keep platform-required libraries when a probe adds another preload.
+        preloads = [clean_env.get("LD_PRELOAD", ""), (env or {}).get("LD_PRELOAD", "")]
+        if any(preloads):
+            command_env["LD_PRELOAD"] = ":".join(preload for preload in preloads if preload)
         result = subprocess.run(
             args,
             cwd=cwd,
-            env={**clean_env, **(env or {})},
+            env=command_env,
             text=True,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,

source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py

@@ -5,9 +5,8 @@
 
 """Shared state-training, camera-rendering, and camera-training Cartpole smoke probes.
 
-The caller owns environment preparation. Installation CI executes this file
-inside the environment it just installed, while architecture CI executes the
-same probes inside its prepared image.
+The caller owns environment preparation. Architecture CI executes all probes
+inside its prepared image, while kit-less image validation runs the state-training probe.
 """
 
 from __future__ import annotations

source/isaaclab/test/install_ci/test_workspace.py

@@ -35,8 +35,6 @@ def test_workspace_install(
 def test_workspace_trains(workspace: tuple[str, Path, dict[str, str]], checkout: Path, run: Callable[..., str]) -> None:
     runtime, directory, env = workspace
     if runtime == "isaacsim":
-        if platform.machine().lower() in ("aarch64", "arm64"):
-            pytest.skip("Isaac Sim PhysX training is not functional on aarch64 yet")
         if platform.system() == "Linux":
             shim = directory / "venv/lib/python3.12/site-packages/isaacsim/kit/kernel/plugins/libcarb.env.shim.so"
             assert shim.is_file()

source/isaaclab_tasks/test/core/test_rendering_kuka_visual_material_randomization_kitless.py

@@ -18,7 +18,6 @@
 )
 
 pytestmark = [
-    pytest.mark.arm_ci,
     pytest.mark.integration,
     pytest.mark.rendering,
     pytest.mark.kitless,

source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py

@@ -19,8 +19,8 @@
     rendering_test_shadow_hand,
 )
 
-# no arm_ci marker: intermittently stalls the arm64 runner mid-file (renderer hang after a few
-# passing cases, killed at the per-file timeout); x86 kitless coverage remains in place
+pytestmark = pytest.mark.arm_ci
+
 _RENDERING_PARAMS = group_rendering_params(
     make_kitless_rendering_params(MINIMAL_KITLESS_PHYSICS_RENDERER_AOV_COMBINATIONS)
 )

tools/conftest.py

@@ -1402,8 +1402,15 @@ def _run_one_pass(
 
 
 def _starts_renderer(test_content: str) -> bool:
-    """Return whether a test file's source starts an RTX renderer: Kit's, by enabling cameras, or OVRTX."""
-    return "enable_cameras=True" in test_content or "ovrtx" in test_content.lower()
+    """Return whether a test file's source starts an RTX renderer: Kit's, by enabling cameras, or OVRTX.
+
+    Files built on ``rendering_test_utils`` pick their renderer through its helpers, so they name neither.
+    """
+    return (
+        "enable_cameras=True" in test_content
+        or "ovrtx" in test_content.lower()
+        or "rendering_test_utils" in test_content
+    )
 
 
 def _test_file_job(test_file: str, max_slots: int) -> TestFileJob: