[Backport release/3.0.0] [CI] Restore ARM CI on the expanded DGX Spark pool (#8229)

.github/workflows/arm-ci.yml

@@ -14,14 +14,14 @@
 # it could be retried. As its own workflow, arm-ci no longer gates the main
 # workflow's lifecycle, and it can be re-run independently.
 #
-# arm-ci is currently informational (continue-on-error), but it is wired to be
-# promotable to a REQUIRED status check: it triggers on every PR and gates the
-# expensive arm64 build behind the shared `changes` detector + `if:`. An
-# irrelevant PR SKIPS the job (which branch protection reads as green) instead
-# of never creating the check — a workflow-level `paths:` filter is deliberately
-# avoided because a not-triggered required check stays pending forever and
-# blocks the PR. This mirrors the `changes`+`if:` pattern build.yaml already
-# uses for its required test jobs.
+# arm-ci is not a required check, so a failure reports red without blocking
+# the merge. It is wired to be promotable to one: it triggers on every PR and
+# gates the expensive arm64 build behind the shared `changes` detector + `if:`.
+# An irrelevant PR SKIPS the job (which branch protection reads as green)
+# instead of never creating the check — a workflow-level `paths:` filter is
+# deliberately avoided because a not-triggered required check stays pending
+# forever and blocks the PR. This mirrors the `changes`+`if:` pattern
+# build.yaml already uses for its required test jobs.
 #
 # Trigger parity with the former build.yaml job:
 #   - push to protected branches always runs (post-merge integration);
@@ -153,7 +153,6 @@ jobs:
     needs: [changes, config]
     if: needs.changes.outputs.should_run == 'true'
     timeout-minutes: 60
-    continue-on-error: true
     steps:
     - uses: actions/checkout@v6
       with:
@@ -184,7 +183,8 @@ jobs:
         container-name: isaac-lab-arm-ci-${{ github.run_id }}-${{ github.run_attempt }}
         image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64
         extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}"
-        test-k-expr: not ovphysx
+        # ovstage rendering cases are post-merge only on x86 as well.
+        test-k-expr: not ovphysx and not ovstage
         ci-marker: arm_ci
         volume-mount-source: ${{ github.workspace }}
 
@@ -196,7 +196,7 @@ jobs:
         container-name: isaac-lab-arm-ci-ovphysx-${{ github.run_id }}-${{ github.run_attempt }}
         image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64
         extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}"
-        test-k-expr: ovphysx
+        test-k-expr: ovphysx and not ovstage
         ci-marker: arm_ci
         volume-mount-source: ${{ github.workspace }}
 

.github/workflows/config.yaml

@@ -9,7 +9,8 @@
 # (frozen at 6.1.0-alpha.2). Both images use Isaac Sim's NGC org, which is current and
 # which the CI credential can reach.
 isaacsim_image_name: nvcr.io/0947644777160149/internal/isaac-sim
-# Isaac Sim 6.2.0-alpha.12+develop.51655.36d2fc9a.gl with Kit 110.4.0-384466
-isaacsim_image_tag: latest-develop@sha256:a6c53e29d6f15ade5475e0d8b4bcff88c351a3815723ad36434e442728898c98
+# Isaac Sim 6.2.0-alpha.12+develop.51655.36d2fc9a.gl with Kit 110.4.0-384466.
+# Pin the multi-arch manifest list digest, not a per-architecture one: ARM CI pulls linux/arm64.
+isaacsim_image_tag: latest-develop@sha256:e78f716a1cae92d432b009e48b8d1184137515e244baefcdf8043ea48f5c4cc3
 isaaclab_image_name: nvcr.io/0947644777160149/internal/isaac-lab
 ovphysx_wheelhouse_image: ""

source/isaaclab/changelog.d/jichuanh-arm-ci-reenable.fixed.rst

@@ -0,0 +1,3 @@
+* Fixed CLI runs on Linux aarch64 from a virtual environment exiting at ``isaacsim`` import with
+  the ``LD_PRELOAD`` banner. The CLI now preloads the system ``libgomp.so.1`` by its full path,
+  which is the form Isaac Sim's check accepts.

source/isaaclab/isaaclab/cli/utils.py

@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
+import glob
 import os
 import platform
 import shutil
@@ -630,18 +631,19 @@ def _aarch64_libgomp_env(env: dict[str, str] | None) -> dict[str, str] | None:
     The torch wheel bundles its own libgomp, which loads first and conflicts with the
     library Isaac Sim expects, so isaacsim refuses to start unless the system libgomp is
     preloaded. The pip installation docs tell users to export LD_PRELOAD by hand; doing it
-    here makes first runs through the CLI work out of the box. The bare soname is used so
-    ``ld.so`` resolves the library through the ldconfig cache on any distro. Returns the
-    env unchanged on other platforms or when a libgomp is already preloaded.
+    here makes first runs through the CLI work out of the box. isaacsim only accepts the
+    system ``/lib/*/libgomp.so.1`` paths listed verbatim, so those full paths are prepended.
+    Returns the env unchanged on other platforms, without a system libgomp, or when every
+    such path is already preloaded.
     """
     if platform.system() != "Linux" or platform.machine().lower() not in ("aarch64", "arm64"):
         return env
-    libgomp = "libgomp.so.1"
     merged = dict(os.environ if env is None else env)
-    preload = merged.get("LD_PRELOAD", "")
-    if any(os.path.basename(entry) == libgomp for entry in preload.split(":") if entry):
+    preload = [entry for entry in merged.get("LD_PRELOAD", "").split(":") if entry]
+    missing = [path for path in sorted(glob.glob("/lib/*/libgomp.so.1")) if path not in preload]
+    if not missing:
         return env
-    merged["LD_PRELOAD"] = f"{libgomp}:{preload}" if preload else libgomp
+    merged["LD_PRELOAD"] = ":".join(missing + preload)
     return merged
 
 

source/isaaclab/test/cli/test_install.py

@@ -86,6 +86,22 @@ def test_run_python_command_uses_live_isaac_sim_with_active_python(tmp_path):
     assert run.call_args.kwargs["env"]["PYTHONEXE"] == active_python
 
 
+def test_run_python_command_preloads_system_libgomp_path_on_aarch64(tmp_path):
+    """isaacsim starts on aarch64 only when the system libgomp path is listed verbatim in LD_PRELOAD."""
+    with (
+        mock.patch("isaaclab.cli.utils.DEFAULT_ISAAC_SIM_PATH", tmp_path / "_isaac_sim"),
+        mock.patch("isaaclab.cli.utils.extract_python_exe", return_value=sys.executable),
+        mock.patch("isaaclab.cli.utils.platform.system", return_value="Linux"),
+        mock.patch("isaaclab.cli.utils.platform.machine", return_value="aarch64"),
+        mock.patch("isaaclab.cli.utils.glob.glob", return_value=["/lib/aarch64-linux-gnu/libgomp.so.1"]),
+        mock.patch("isaaclab.cli.utils.run_command") as run,
+        mock.patch.dict(os.environ, {"LD_PRELOAD": "libcarb.env.shim.so"}, clear=True),
+    ):
+        run_python_command("train.py", [])
+
+    assert run.call_args.kwargs["env"]["LD_PRELOAD"] == "/lib/aarch64-linux-gnu/libgomp.so.1:libcarb.env.shim.so"
+
+
 def test_run_python_command_accepts_virtual_environment_on_bundled_python(tmp_path):
     """A virtual environment created on a downloaded package's Python runs that interpreter."""
     local_sim = tmp_path / "_isaac_sim"