diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index 038deac54e98..182da3ef608a 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -168,13 +168,28 @@ runs: BUILD_ARGS+=( --cache-to "${{ inputs.cache-to }}" ) fi - BUILDER_NAME="docker-build-${{ github.run_id }}-${{ github.job }}" - docker buildx create --use --driver docker-container --name "${BUILDER_NAME}" \ - || docker buildx use "${BUILDER_NAME}" - trap 'docker buildx rm "${BUILDER_NAME}" || true' EXIT + # Registry cache export needs a docker-container builder, whose image must + # then be exported and loaded into the daemon. Callers without a registry + # cache (ARM, which has no ECR) build in the daemon itself instead, so the + # image, layer cache and uv cache mount stay on the runner. A containerd + # image store still compresses new layers; zstd level 1 is several times + # faster than the default gzip for the multi-GB venv layer. + if [ -n "${{ inputs.cache-from }}${{ inputs.cache-to }}" ]; then + BUILDER_NAME="docker-build-${{ github.run_id }}-${{ github.job }}" + docker buildx create --driver docker-container --name "${BUILDER_NAME}" \ + || true + trap 'docker buildx rm "${BUILDER_NAME}" || true' EXIT + BUILD_ARGS+=( --builder "${BUILDER_NAME}" --load ) + else + BUILD_ARGS+=( + --builder default + --provenance=false + --output "type=image,name=${{ inputs.image-tag }},compression=zstd,compression-level=1" + ) + fi echo "🔵 Building ${{ inputs.image-tag }}..." - docker buildx build --load "${BUILD_ARGS[@]}" "${{ inputs.context-path }}" + docker buildx build "${BUILD_ARGS[@]}" "${{ inputs.context-path }}" echo "was-built=true" >> "$GITHUB_OUTPUT" ##### 6: Layer the CI pytest harness onto the built image ##### diff --git a/.github/actions/ecr-build-push-pull/action.yml b/.github/actions/ecr-build-push-pull/action.yml index 6c6b7eb6f069..7ff7a10ded17 100644 --- a/.github/actions/ecr-build-push-pull/action.yml +++ b/.github/actions/ecr-build-push-pull/action.yml @@ -23,6 +23,10 @@ inputs: description: 'Path to Dockerfile, relative to the repository root' default: 'docker/Dockerfile.base' required: false + platform: + description: 'Target platform for the build and its dependency-cache identity' + default: 'linux/amd64' + required: false ecr-url: description: > ECR repository URL (e.g. "123456789.dkr.ecr.us-west-2.amazonaws.com/my-repo"). @@ -208,6 +212,7 @@ runs: dockerfile-path: ${{ inputs.dockerfile-path }} isaacsim-base-image: ${{ inputs.isaacsim-base-image }} isaacsim-version: ${{ inputs.isaacsim-version }} + platform: ${{ inputs.platform }} - name: Check deps cache id: deps-cache @@ -250,6 +255,7 @@ runs: isaacsim-base-image: ${{ inputs.isaacsim-base-image }} isaacsim-version: ${{ inputs.isaacsim-version }} dockerfile-path: ${{ inputs.dockerfile-path }} + platform: ${{ inputs.platform }} cache-from: ${{ steps.resolve-ecr.outputs.available == 'true' && format('type=registry,ref={0}', env.CACHE_IMAGE) || '' }} cache-to: ${{ steps.resolve-ecr.outputs.available == 'true' && format('type=registry,ref={0},mode=max', env.CACHE_IMAGE) || '' }} deps-hash: ${{ steps.deps-hash.outputs.hash }} diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index e04859354305..bf46296df977 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -27,6 +27,14 @@ inputs: description: 'ECR cache tag' default: 'cache-base' required: false + ci-marker: + description: 'Optional pytest -m marker expression applied to every test file (e.g. arm_ci)' + default: '' + required: false + platform: + description: 'Target platform if the image has to be built' + default: 'linux/amd64' + required: false filter-pattern: description: >- Pattern to filter test files (e.g., isaaclab_tasks); test files whose path @@ -111,14 +119,15 @@ inputs: newest compatible snapshot and never writes; every test job uses this, so a pull request can never publish kernels built from unmerged code. 'save' restores the same way and writes the result back as a new snapshot, making - post-merge a cumulative writeback; only warm-warp-cache uses it. + post-merge a cumulative writeback; only trusted post-merge cache producers + use it. default: '' required: false ovrtx-shader-cache: description: >- OVRTX shader cache mode. Empty disables the cache. 'restore' reads the newest compatible snapshot and never writes. 'save' restores and writes - back a new snapshot; only warm-ovrtx-cache uses it. + back a new snapshot; only trusted post-merge cache producers use it. default: '' required: false ovrtx-shader-cache-trees: @@ -146,6 +155,13 @@ inputs: container-name: description: 'Docker container name prefix (run-id is appended automatically)' required: true +outputs: + warp-cache-host-dir: + description: 'Host directory mounted as the Warp kernel cache; empty when the cache is disabled' + value: ${{ steps.warp-cache-key.outputs.host-dir }} + ovrtx-shader-cache-host-dir: + description: 'Host directory mounted as the OVRTX shader cache; empty when the cache is disabled' + value: ${{ steps.ovrtx-cache.outputs.host-dir }} runs: using: composite steps: @@ -197,6 +213,7 @@ runs: isaacsim-version: ${{ inputs.isaacsim-version }} dockerfile-path: ${{ inputs.dockerfile-path }} cache-tag: ${{ inputs.cache-tag }} + platform: ${{ inputs.platform }} - name: Report pull duration if: always() @@ -348,6 +365,7 @@ runs: include-files: ${{ inputs.include-files }} test-node-ids-file: ${{ inputs.test-node-ids-file }} test-node-ids-key: ${{ inputs.test-node-ids-key }} + ci-marker: ${{ inputs.ci-marker }} volume-mount-source: ${{ github.workspace }} extra-pip-packages: ${{ inputs.extra-pip-packages }} extra-uv-packages: ${{ inputs.extra-uv-packages }} diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index a45a180a4815..ddedccb0b560 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -175,40 +175,53 @@ jobs: uses: ./.github/actions/resolve-ov-pins id: ov_pins + # Same action as the x86 test jobs, with four file slots; rendering files still + # run one at a time. With no ECR on the Spark, it picks up the image built above from the local + # store. Pushes to protected branches publish the arm64 Warp and OVRTX caches + # that pull requests restore, the role x86's cache-warm jobs play. - name: Run arm_ci marker tests - uses: ./.github/actions/run-tests + id: tests + uses: ./.github/actions/run-package-tests with: - test-path: tools - result-file: arm-ci-report.xml - container-name: isaac-lab-arm-ci-${{ github.run_id }}-${{ github.run_attempt }} image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 - extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}" - # ovstage rendering cases are post-merge only on x86 as well. - test-k-expr: not ovphysx and not ovstage + platform: linux/arm64 + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} ci-marker: arm_ci - volume-mount-source: ${{ github.workspace }} - - - name: Run arm_ci OVPhysX marker tests - uses: ./.github/actions/run-tests - with: - test-path: tools - result-file: arm-ci-ovphysx-report.xml - container-name: isaac-lab-arm-ci-ovphysx-${{ github.run_id }}-${{ github.run_attempt }} - image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 + # ovstage rendering cases are post-merge only on x86 as well. + test-k-expr: not ovstage + test-jobs: "4" extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}" - test-k-expr: ovphysx and not ovstage - ci-marker: arm_ci - volume-mount-source: ${{ github.workspace }} - + warp-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }} + ovrtx-shader-cache: ${{ github.event_name == 'push' && 'save' || 'restore' }} + ovrtx-shader-cache-trees: kitless + result-file: arm-ci-report.xml + container-name: isaac-lab-arm-ci + omni-github-test-type: arm-ci - - name: Run shared Cartpole smoke + # The test orchestrator skips install_ci/, so the Cartpole state/camera probes + # run here in the same image, reusing the restored Warp and OVRTX caches. + - name: Run Cartpole smoke tests uses: ./.github/actions/run-tests with: test-path: source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py - result-file: arm-ci-cartpole-smoke-report.xml - container-name: isaac-lab-arm-ci-cartpole-${{ github.run_id }}-${{ github.run_attempt }} + result-file: arm-ci-smoke-report.xml + container-name: isaac-lab-arm-ci-smoke-${{ github.run_id }}-${{ github.run_attempt }} image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 volume-mount-source: ${{ github.workspace }} + warp-cache-host-dir: ${{ steps.tests.outputs.warp-cache-host-dir }} + ovrtx-shader-cache-host-dir: ${{ steps.tests.outputs.ovrtx-shader-cache-host-dir }} + omni-github-test-type: arm-ci + + # run-tests records failures in the report rather than its exit code. + - name: Check Cartpole smoke results + shell: bash + run: | + report=reports/arm-ci-smoke-report.xml + if [ ! -f "$report" ] || grep -qE '(failures|errors)="[1-9][0-9]*"' "$report"; then + echo "::error::Cartpole smoke tests failed or produced no report: $report" + exit 1 + fi - name: Upload test reports if: always() diff --git a/source/isaaclab/changelog.d/jichuanh-arm-ci-align-x86.skip b/source/isaaclab/changelog.d/jichuanh-arm-ci-align-x86.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab/test/controllers/test_operational_space.py b/source/isaaclab/test/controllers/test_operational_space.py index 76ed5a42165f..8a98daeed8f3 100644 --- a/source/isaaclab/test/controllers/test_operational_space.py +++ b/source/isaaclab/test/controllers/test_operational_space.py @@ -13,7 +13,7 @@ import pytest import torch -pytestmark = pytest.mark.arm_ci +pytestmark = [pytest.mark.integration, pytest.mark.arm_ci] import isaaclab.envs.mdp as mdp import isaaclab.sim as sim_utils @@ -52,8 +52,6 @@ from isaaclab_assets import FRANKA_PANDA_LEGACY_CFG, G1_29DOF_CFG # isort:skip -pytestmark = pytest.mark.integration - @pytest.fixture def sim(): diff --git a/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py b/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py index 862c0467eb8a..f4772a84ddde 100644 --- a/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py +++ b/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py @@ -6,8 +6,8 @@ """Shared state-training, camera-rendering, and camera-training Cartpole smoke probes. The caller owns environment preparation. Installation CI executes this file -inside the environment it just installed, while architecture CI executes the -same probes inside its prepared image. +inside the environment it just installed, architecture CI executes all probes +inside its prepared image, and kit-less image validation runs the state-training probe. """ from __future__ import annotations diff --git a/source/isaaclab/test/install_ci/uv_run/test_uv_run_isaacsim_trains_cartpole.py b/source/isaaclab/test/install_ci/uv_run/test_uv_run_isaacsim_trains_cartpole.py index 851cbba6b1f5..2b065072e99d 100644 --- a/source/isaaclab/test/install_ci/uv_run/test_uv_run_isaacsim_trains_cartpole.py +++ b/source/isaaclab/test/install_ci/uv_run/test_uv_run_isaacsim_trains_cartpole.py @@ -15,7 +15,6 @@ from __future__ import annotations -import platform import shutil import pytest @@ -52,13 +51,6 @@ class Test_Uv_Run_Isaacsim_Trains_Cartpole: def setup_class(cls): if not shutil.which("uv"): pytest.skip("uv is not available") - # PhysX training currently fails at env creation on aarch64 (DGX Spark): - # "[omni.physx.tensors.plugin] Simulation view object is invalidated and cannot - # be used again to call updateArticulationsKinematic". The environment itself - # installs fine from the lock; this matches the x86-only scope of the workflow - # this test replaces. Re-enable once PhysX training works on aarch64. - if platform.machine().lower() in ("aarch64", "arm64"): - pytest.skip("PhysX training is not functional on aarch64 yet") @pytest.mark.docker @pytest.mark.smoke diff --git a/source/isaaclab_tasks/changelog.d/jichuanh-arm-ci-align-x86.skip b/source/isaaclab_tasks/changelog.d/jichuanh-arm-ci-align-x86.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_tasks/test/core/test_rendering_kuka_visual_material_randomization_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_kuka_visual_material_randomization_kitless.py index a7f2aacd7ced..fbeaf8a2894a 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_kuka_visual_material_randomization_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_kuka_visual_material_randomization_kitless.py @@ -18,7 +18,6 @@ ) pytestmark = [ - pytest.mark.arm_ci, pytest.mark.integration, pytest.mark.rendering, pytest.mark.kitless, diff --git a/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py index ea44307f02c2..b17e9c41a0c3 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py @@ -19,8 +19,8 @@ rendering_test_shadow_hand, ) -# no arm_ci marker: intermittently stalls the arm64 runner mid-file (renderer hang after a few -# passing cases, killed at the per-file timeout); x86 kitless coverage remains in place +pytestmark = pytest.mark.arm_ci + _RENDERING_PARAMS = group_rendering_params( make_kitless_rendering_params(MINIMAL_KITLESS_PHYSICS_RENDERER_AOV_COMBINATIONS) ) diff --git a/tools/conftest.py b/tools/conftest.py index 3844e56f69fa..4980bd221253 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -1404,8 +1404,15 @@ def _run_one_pass( def _starts_renderer(test_content: str) -> bool: - """Return whether a test file's source starts an RTX renderer: Kit's, by enabling cameras, or OVRTX.""" - return "enable_cameras=True" in test_content or "ovrtx" in test_content.lower() + """Return whether a test file's source starts an RTX renderer: Kit's, by enabling cameras, or OVRTX. + + Files built on ``rendering_test_utils`` pick their renderer through its helpers, so they name neither. + """ + return ( + "enable_cameras=True" in test_content + or "ovrtx" in test_content.lower() + or "rendering_test_utils" in test_content + ) def _test_file_job(test_file: str, max_slots: int) -> TestFileJob: