[Backport release/3.0.0] Complete develop backports

.agents/skills/isaaclab-auditing-an-issue

@@ -0,0 +1 @@
+../../skills/developer/issue-audit

.agents/skills/isaaclab-triaging-issue-backlog

@@ -0,0 +1 @@
+../../skills/developer/issue-backlog-triage

.dockerignore

@@ -21,6 +21,8 @@ docker/.container.cfg
 recordings/
 # ignore __pycache__
 **/__pycache__/
+**/.pytest_cache/
+**/.ruff_cache/
 **/*.egg-info/
 # ignore isaac sim symlink
 _isaac_sim

.github/actions/ecr-build-push-pull/README.md

@@ -16,6 +16,19 @@ ECR is also used as the BuildKit layer cache.
     ecr-url: (optional, complete url for ECR storage)
 ```
 
+## Verifying a freshly built image
+
+Pass `verify-test-path` to assert against the image before it is published:
+
+```yaml
+    verify-test-path: docker/test/test_image_invariants.py
+```
+
+The tests run only on a full build, with `IMAGE_TAG` set, so the caller's job needs `uv`
+(`astral-sh/setup-uv`). A failure fails the action with nothing pushed, so the next run
+rebuilds rather than serving the bad image from the deps cache. Exact-tag and deps-cache hits skip
+them: that image passed when it was built.
+
 ## ECR URL resolution order
 
 1. `ecr-url` input

.github/actions/ecr-build-push-pull/action.yml

@@ -37,6 +37,16 @@ inputs:
     description: Tag used for the ECR layer cache image (e.g. "cache-base", "cache-curobo").
     required: false
     default: 'cache'
+  verify-test-path:
+    description: >
+      Path to a test file or directory asserted against a freshly built image, before it is
+      tagged or pushed. Tests run with IMAGE_TAG set; a failure fails the action with nothing
+      published, so the next run rebuilds instead of inheriting the bad image from the cache.
+
+      Not run on an exact-tag or deps-cache hit: those serve an image that already passed when it
+      was built.
+    required: false
+    default: ''
   pull-on-deps-hit:
     description: >
       Pull the image locally after a deps-cache hit. Needed by jobs that run
@@ -244,6 +254,23 @@ runs:
         cache-to: ${{ steps.resolve-ecr.outputs.available == 'true' && format('type=registry,ref={0},mode=max', env.CACHE_IMAGE) || '' }}
         deps-hash: ${{ steps.deps-hash.outputs.hash }}
 
+    # Assert against the image while it is only local: the push steps below publish under both
+    # the commit tag and the deps tag, and a deps-cache hit later serves that image without
+    # rebuilding it, so anything published unverified stays unverified.
+    - name: Verify freshly built image
+      if: >
+        inputs.verify-test-path != '' &&
+        steps.pull-exact.outputs.hit != 'true' &&
+        steps.deps-cache.outputs.deps-cache-hit != 'true'
+      shell: bash
+      env:
+        IMAGE_TAG: ${{ inputs.image-tag }}
+        TEST_PATH: ${{ inputs.verify-test-path }}
+      run: |
+        set -euo pipefail
+        uv run --no-project --with pytest \
+          python -m pytest -q "${TEST_PATH}"
+
     - name: Tag built image with ECR-prefixed name
       if: >
         steps.resolve-ecr.outputs.available == 'true' &&

.github/actions/run-tests/run_tests.sh

@@ -413,13 +413,19 @@ run_tests() {
       fi
       if [ -n \"\${TEST_EXTRA_UV_PACKAGES:-}\" ]; then
         echo \"Installing extra packages with uv: \${TEST_EXTRA_UV_PACKAGES}\"
-        # isaaclab.sh prints an informational line before command output, and pip
-        # installs scripts into the user base when Isaac Sim site-packages is read-only.
-        isaac_python=\"\$(./isaaclab.sh -p -c 'import sys; print(sys.executable)' | tail -n 1)\"
-        isaac_user_site=\"\$(./isaaclab.sh -p -c 'import site; print(site.getusersitepackages())' | tail -n 1)\"
-        uv_executable=\"\$(./isaaclab.sh -p -c 'import pathlib, site; print(pathlib.Path(site.getuserbase()) / \"bin\" / \"uv\")' | tail -n 1)\"
-        if [ ! -x \"\${uv_executable}\" ]; then
-          bash /with-python-package-retries.sh ./isaaclab.sh -p -m pip install uv
+        # isaaclab.sh prints an [INFO] banner on stdout around the command output, so the
+        # banner is filtered rather than positionally skipped.
+        isaac_python=\"\$(./isaaclab.sh -p -c 'import sys; print(sys.executable)' | grep -v '^\[INFO\]' | tail -n 1)\"
+        isaac_user_site=\"\$(./isaaclab.sh -p -c 'import site; print(site.getusersitepackages())' | grep -v '^\[INFO\]' | tail -n 1)\"
+        # The image ships uv on PATH. Fall back to the user base only for images that do
+        # not: pip installs into the venv, not the user base, when the interpreter is a venv,
+        # so the user-base path is never created there.
+        uv_executable=\"\$(command -v uv || true)\"
+        if [ -z \"\${uv_executable}\" ]; then
+          uv_executable=\"\$(./isaaclab.sh -p -c 'import pathlib, site; print(pathlib.Path(site.getuserbase()) / \"bin\" / \"uv\")' | grep -v '^\[INFO\]' | tail -n 1)\"
+          if [ ! -x \"\${uv_executable}\" ]; then
+            bash /with-python-package-retries.sh ./isaaclab.sh -p -m pip install uv
+          fi
         fi
         bash /with-python-package-retries.sh \"\${uv_executable}\" pip install --python \"\${isaac_python}\" --target \"\${isaac_user_site}\" \${TEST_EXTRA_UV_PACKAGES}
         # Isaac Sim puts bundled packages ahead of the user site. Overlay only

.github/test-subsets/postmerge-rendering.toml

@@ -23,3 +23,17 @@ rendering-correctness-kitless-ovstage = [
   "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovstage-newton-ovrtx_renderer-static]",
   "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovstage-ovphysx-ovrtx_renderer-static]",
 ]
+
+# Nodes the shader-cache warmer runs: the RTX-renderer subset of the lists above,
+# which are the only ones that compile PSO blobs. isaacsim_rtx fills kit/ and ovrtx
+# fills kitless/; the newton_renderer nodes drive the Warp rasterizer and populate
+# neither tree. Exact node IDs rather than a -k expression so a parametrize-ID
+# rename fails collection instead of silently selecting nothing.
+ovrtx-cache-warm = [
+  "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[newton-isaacsim_rtx_renderer-static]",
+  "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[physx-isaacsim_rtx_renderer-static]",
+  "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[legacy-newton-ovrtx_renderer-static]",
+  "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[legacy-ovphysx-ovrtx_renderer-static]",
+  "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovstage-newton-ovrtx_renderer-static]",
+  "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovstage-ovphysx-ovrtx_renderer-static]",
+]

.github/workflows/build.yaml

@@ -46,7 +46,7 @@
 #      -e TEST_FILTER_PATTERN="isaaclab_physx" \
 #      -e TEST_INCLUDE_FILES="test_rigid_object_collection.py" \
 #      -v "$PWD":/workspace/isaaclab isaac-lab-base:latest \
-#      -c 'cd /workspace/isaaclab && /isaac-sim/python.sh -m pytest tools -v'
+#      -c 'cd /workspace/isaaclab && ./isaaclab.sh -p -m pytest tools -v'
 #
 # Remember to REVERT all temporary changes before merging!
 # =============================================================================
@@ -223,6 +223,7 @@ jobs:
         isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
         dockerfile-path: docker/Dockerfile.base
         cache-tag: cache-base
+        verify-test-path: docker/test/test_image_invariants.py
 
   build-curobo:
     name: Build cuRobo Docker Image
@@ -697,16 +698,6 @@ jobs:
     # Folded from the former standalone verify-base-non-root job: reuses the
     # base image already pulled by run-package-tests to keep the regression
     # check without burning a separate runner.
-    - name: Run Dockerfile non-root regression test
-      shell: bash
-      run: |
-        set -euo pipefail
-        docker run --rm \
-          -v "$PWD":/workspace/isaaclab \
-          --entrypoint bash \
-          "${{ needs.config.outputs.ci_image_tag }}" \
-          -lc 'cd /workspace/isaaclab && /isaac-sim/python.sh -m pytest docker/test/test_dockerfile_nonroot.py -q'
-
     - name: Verify Base runtime user is non-root
       shell: bash
       run: |
@@ -747,16 +738,6 @@ jobs:
     # Folded from the former standalone verify-curobo-non-root job: reuses
     # the curobo image already pulled by run-package-tests to keep the
     # regression check without burning a separate runner.
-    - name: Run Dockerfile non-root regression test
-      shell: bash
-      run: |
-        set -euo pipefail
-        docker run --rm \
-          -v "$PWD":/workspace/isaaclab \
-          --entrypoint bash \
-          "${{ needs.config.outputs.ci_image_tag }}-curobo" \
-          -lc 'cd /workspace/isaaclab && /isaac-sim/python.sh -m pytest docker/test/test_dockerfile_nonroot.py -q'
-
     - name: Verify cuRobo runtime user is non-root
       shell: bash
       run: |
@@ -1044,9 +1025,11 @@ jobs:
           test_rendering_cartpole_kitless.py
         # Only the two RTX paths compile PSO blobs: isaacsim_rtx fills kit/ and
         # ovrtx fills kitless/. newton_warp would exercise the Warp rasterizer,
-        # which populates neither tree.
-        test-k-expr: >-
-          (isaacsim_rtx or ovrtx) and (rgb or depth)
+        # which populates neither tree. The nodes are pinned by exact ID rather
+        # than a -k expression, so a parametrize-ID rename fails collection here
+        # instead of quietly leaving the warmer with nothing to run.
+        test-node-ids-file: .github/test-subsets/postmerge-rendering.toml
+        test-node-ids-key: ovrtx-cache-warm
         ovrtx-shader-cache: ${{ env.PUBLISHES_OVRTX_CACHE == 'true' && 'save' || 'restore' }}
         container-name: isaac-lab-ovrtx-cache-warm
         omni-github-test-type: ovrtx-cache-warm

.github/workflows/config.yaml

@@ -9,7 +9,7 @@
 # (frozen at 6.1.0-alpha.2). Both images use Isaac Sim's NGC org, which is current and
 # which the CI credential can reach.
 isaacsim_image_name: nvcr.io/0947644777160149/internal/isaac-sim
-# Test the release/3.0.0 branch against the latest Isaac Sim 6.1 release image.
-isaacsim_image_tag: latest-release-6-1
+# Isaac Sim 6.1.0-alpha.50 (b86cf6ce) includes Kit 110.3.0-360924's fix for NVBug 6566677.
+isaacsim_image_tag: latest-develop@sha256:223adbb0a6f1897ed78d40597dac2efc740843e8051779ffdcbb021a9edc1217
 isaaclab_image_name: nvcr.io/0947644777160149/internal/isaac-lab
 ovphysx_wheelhouse_image: ""

.github/workflows/docs.yaml

@@ -66,7 +66,7 @@ jobs:
 
     - name: Install docs dependencies
       run: |
-        bash "$GITHUB_WORKSPACE/.github/actions/_lib/with-python-package-retries.sh" uv sync --extra test
+        bash "$GITHUB_WORKSPACE/.github/actions/_lib/with-python-package-retries.sh" uv sync --extra dev
         echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
 
     - name: Build current version docs
@@ -98,7 +98,7 @@ jobs:
 
     - name: Install docs dependencies
       run: |
-        bash "$GITHUB_WORKSPACE/.github/actions/_lib/with-python-package-retries.sh" uv sync --extra test
+        bash "$GITHUB_WORKSPACE/.github/actions/_lib/with-python-package-retries.sh" uv sync --extra dev
         echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
 
     - name: Generate multi-version docs

.github/workflows/license-check.yaml

@@ -62,7 +62,7 @@ jobs:
         ISAACSIM_ACCEPT_EULA: YES
       run: |
         bash "$GITHUB_WORKSPACE/.github/actions/_lib/with-python-package-retries.sh" \
-          uv sync --extra all --extra isaacsim --extra test --extra rlinf --extra mimic
+          uv sync --extra all --extra isaacsim --extra dev --extra rlinf --extra mimic
         # ``[tool.uv.pip] prerelease = "allow"`` lets unpinned tools float onto
         # prereleases. pip-licenses 6.0.0a1 reports an empty License where 5.x reports
         # ``UNKNOWN``, which license-exceptions.json keys on, and joins multi-license

.github/workflows/nightly-changelog.yml

@@ -13,10 +13,10 @@
 # Scheduled workflow — must live on the default branch (``main``) for the
 # cron to register. See ``.github/workflows/README.md``.
 #
-# Adding a branch to the nightly set is a one-line edit to ``env.CRON_BRANCHES``
-# below. Each target branch uses its own ``tools/changelog/cli.py`` (the
-# same copy the PR gate already runs), so the nightly compile honours the
-# same rules that validated the fragments.
+# Scheduled runs combine ``env.CRON_BRANCHES`` with every remote
+# ``release/*`` branch that carries ``tools/changelog/cli.py``. This picks up
+# new compiler-enabled release branches automatically while leaving historical
+# branches that predate the fragment system untouched.
 #
 # The push uses a short-lived GitHub App installation token minted from
 # ``CHANGELOG_APP_ID`` + ``CHANGELOG_APP_PRIVATE_KEY`` (repo secrets). The
@@ -30,13 +30,11 @@
 
 name: Nightly Changelog Compilation
 
-# Branches the nightly cron compiles. Single source of truth — append a
-# ref here to extend the nightly set (the active release branch belongs
-# here). Each branch must carry ``tools/changelog/cli.py`` and the
-# isaaclab-bot App must be in its branch-ruleset bypass list.
-# Surrounding whitespace per entry is stripped by the resolver below.
+# Static branches the nightly cron compiles. The resolver also discovers every
+# remote ``release/*`` branch that carries ``tools/changelog/cli.py``. Every
+# resolved branch must allow the isaaclab-bot App to bypass its ruleset.
 env:
-  CRON_BRANCHES: develop,release/3.0.0-beta2
+  CRON_BRANCHES: develop
 
 on:
   schedule:
@@ -51,7 +49,7 @@ on:
         # develop) without needing to update the workflow. A branch
         # that lacks ``tools/changelog/cli.py`` fails the verify step
         # below with a clear error, which is the desired failure mode.
-        description: 'Branch to compile (e.g. develop, release/3.0.0-beta2). Must carry tools/changelog/cli.py.'
+        description: 'Branch to compile (e.g. develop, release/3.0.0). Must carry tools/changelog/cli.py.'
         required: true
         type: string
       dry_run:
@@ -67,10 +65,8 @@ permissions:
 
 jobs:
   resolve-branches:
-    # CSV → JSON array bridge. ``workflow_dispatch`` inputs can only be
-    # string / bool / choice, so the branch list arrives as a comma-
-    # separated string; the matrix below needs a JSON list to fan out.
-    # Mirrors the ``setup-versions`` job in ``daily-compatibility.yml``.
+    # Static/manual branches plus supported ``release/*`` refs → JSON array.
+    # ``workflow_dispatch`` remains exactly one explicitly requested branch.
     name: Resolve branch list
     runs-on: ubuntu-latest
     timeout-minutes: 1
@@ -79,18 +75,39 @@ jobs:
     steps:
       - id: b
         env:
-          # Schedule → the CRON_BRANCHES list. Manual → the single branch
-          # the maintainer entered. The two paths are intentionally
-          # asymmetric: cron is the configured set, manual is exactly one
-          # branch (required input).
+          # Schedule → static branches plus supported remote ``release/*``
+          # refs. Manual → exactly the branch the maintainer entered.
           BRANCHES: ${{ github.event_name == 'schedule' && env.CRON_BRANCHES || inputs.branch }}
           # ``EVENT_NAME`` mirrors ``github.event_name`` so the guard below
           # branches on it without re-interpolating into the shell.
           EVENT_NAME: ${{ github.event_name }}
+          REPOSITORY: ${{ github.repository }}
+          API_TOKEN: ${{ github.token }}
         run: |
-          # CSV → JSON array, trimming surrounding whitespace per entry.
-          # Manual produces a 1-element array; cron produces N elements.
-          arr=$(echo "$BRANCHES" | tr ',' '\n' | xargs -n1 | jq -R . | jq -s -c .)
+          branches=$(echo "$BRANCHES" | tr ',' '\n' | xargs -n1)
+          if [ "$EVENT_NAME" = "schedule" ]; then
+            release_branches=$(git ls-remote --heads "https://github.com/$REPOSITORY.git" 'refs/heads/release/*' \
+              | awk '{sub(/^refs\/heads\//, "", $2); print $2}')
+            while IFS= read -r branch; do
+              [ -z "$branch" ] && continue
+              encoded_branch=$(jq -rn --arg branch "$branch" '$branch | @uri')
+              http_status=$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
+                --header "Authorization: Bearer $API_TOKEN" \
+                --header 'Accept: application/vnd.github+json' \
+                --header 'X-GitHub-Api-Version: 2022-11-28' \
+                "https://api.github.com/repos/$REPOSITORY/contents/tools/changelog/cli.py?ref=$encoded_branch")
+              case "$http_status" in
+                200) branches=$(printf '%s\n%s\n' "$branches" "$branch") ;;
+                404) echo "::notice::Skipping $branch because it predates the changelog compiler." ;;
+                *)
+                  echo "::error::GitHub Contents API returned HTTP $http_status for $branch."
+                  exit 1
+                  ;;
+              esac
+            done <<< "$release_branches"
+          fi
+          # Trim, de-duplicate, and convert the resolved branches to a JSON array.
+          arr=$(printf '%s\n' "$branches" | sed '/^$/d' | sort -u | jq -R . | jq -s -c .)
           # Manual trigger contract: exactly one branch. A maintainer who
           # pastes a comma-separated list into the dispatch form should
           # see a clear error, not a silent multi-branch fan-out.
@@ -109,9 +126,8 @@ jobs:
     strategy:
       # Independent branches: one failing shouldn't cancel the others.
       # Each matrix entry shows up as a separate job tile in the Actions
-      # UI, so ``release/3.0.0-beta2`` failing doesn't hide ``develop``'s
-      # success (and ``gh run rerun --failed`` re-runs only the failed
-      # entry). Mirrors ``daily-compatibility.yml``'s matrix style.
+      # UI, so one release ref failing doesn't hide another ref's success
+      # (and ``gh run rerun --failed`` re-runs only the failed entry).
       fail-fast: false
       matrix:
         branch: ${{ fromJson(needs.resolve-branches.outputs.branches) }}