From 92dda6677c5ade52d4c3a8f8cfad14dd1d466c6e Mon Sep 17 00:00:00 2001 From: Mustafa Haiderbhai Date: Fri, 23 Oct 2026 16:22:19 -0700 Subject: [PATCH 1/3] Run contributed environment tests on the base CI image No contributed environment imports cuRobo, and the base image installs a superset of the cuRobo image's Python extras. Running the job on the base image removes its dependency on the cuRobo image build, which takes over five minutes in about a quarter of PR runs and up to 46 minutes. Give the contrib files their own skip list so CUROBO_TESTS again lists only tests that need cuRobo. --- .github/workflows/build.yaml | 10 +++++----- tools/test_settings.py | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9d692f1943af..fd808637d5c9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -742,15 +742,17 @@ jobs: exit 1 fi + # No contributed environment needs cuRobo, so this job uses the base image and + # does not wait for the cuRobo image build. test-contrib-environments: name: test-contrib-environments runs-on: [self-hosted, gpu] timeout-minutes: 120 continue-on-error: true - needs: [build-curobo, config] + needs: [build, config] if: >- github.event_name != 'push' && - needs.build-curobo.result == 'success' + needs.build.result == 'success' steps: - uses: actions/checkout@v6 with: @@ -758,11 +760,9 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: - image-tag: ${{ needs.config.outputs.ci_image_tag }}-curobo + image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - dockerfile-path: docker/Dockerfile.curobo - cache-tag: cache-curobo include-files: >- test_contrib_environments_kitless.py, test_contrib_environments_kit.py, diff --git a/tools/test_settings.py b/tools/test_settings.py index c7c616c5dd04..2785a77b7f6e 100644 --- a/tools/test_settings.py +++ b/tools/test_settings.py @@ -120,15 +120,22 @@ CUROBO_TESTS = [ *CUROBO_PLANNER_TESTS, "test_generate_dataset_skillgen.py", +] +"""A list of tests that require cuRobo installation. + +These tests are skipped in the base image CI jobs and run separately in the +dedicated ``test-curobo`` CI job which uses the cuRobo Docker image. +""" + +CONTRIB_ENVIRONMENT_TESTS = [ "test_contrib_environments_kit.py", "test_contrib_environments_kit_cameras.py", "test_contrib_environments_kitless.py", ] -"""A list of tests that require cuRobo installation. +"""Contributed-environment smoke tests. -These tests are skipped in the base image CI jobs and run separately in the -dedicated ``test-curobo`` and ``test-contrib-environments`` CI jobs which use the cuRobo -Docker image. +These tests are skipped in the package CI jobs and run in the dedicated +``test-contrib-environments`` CI job. """ QUARANTINED_TESTS: list[str] = [ @@ -154,6 +161,8 @@ "test_record_video.py", # Failing # curobo / skillgen - require cuRobo installation; run via test-curobo and test-skillgen CI jobs *CUROBO_TESTS, + # contributed environments - run via the test-contrib-environments CI job + *CONTRIB_ENVIRONMENT_TESTS, # quarantined tests - run in dedicated CI job that does not block PR merges *QUARANTINED_TESTS, "test_environments_training.py", # Long-running RL training test; runs in dedicated CI job From 7d5ddca747e98a84071ae571263eb3928bf26fa9 Mon Sep 17 00:00:00 2001 From: Mustafa Haiderbhai Date: Fri, 23 Oct 2026 16:22:24 -0700 Subject: [PATCH 2/3] Warm Warp kernels for the determinism and contrib suites PR test jobs only restore the Warp kernel cache, and the post-merge warmer filled it from the rigid Newton core environments alone. Newton's deterministic modes and the contributed environments compile kernels outside that set, so every PR recompiled them: the determinism file is the long pole of isaaclab_tasks [2/3] and the contrib job compiles more than the whole restored cache. Add both files to the warmer. The old -k prefix matched every test in the existing files, so dropping it keeps their selection unchanged. --- .github/workflows/build.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index fd808637d5c9..8e92ee95ab45 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1236,12 +1236,17 @@ jobs: # Warm every supported rigid core environment on Newton. MJWarp # specializes kernels per model config, so a small representative task # set leaves most of the cache to be compiled again by each PR test - # shard. The multi-agent suite supplies Handover. + # shard. The multi-agent suite supplies Handover. Newton deterministic + # modes generate different kernels, which only the determinism suite + # compiles, and the contributed environments are not core tasks; uncached, + # both are the long poles of their PR jobs. include-files: >- test_environments_newton.py, - test_multi_agent_environments.py + test_multi_agent_environments.py, + test_environment_determinism.py, + test_contrib_environments_kitless.py # Keep this warmer focused on rigid environments. - test-k-expr: "test_environments and not (Soft or Cloth or Cable)" + test-k-expr: "not (Soft or Cloth or Cable)" warp-cache: ${{ env.PUBLISHES_WARP_CACHE == 'true' && 'save' || 'restore' }} container-name: isaac-lab-warp-cache-warm omni-github-test-type: warp-cache-warm From cb7ea3d691863516f318495d10044c25027f4c3b Mon Sep 17 00:00:00 2001 From: Mustafa Haiderbhai Date: Fri, 23 Oct 2026 16:22:28 -0700 Subject: [PATCH 3/3] Run four standalone demo launches at a time Both standalone demo jobs are among the slowest PR jobs, and each is a single file whose launch time is spread across only two xdist workers. Other package jobs already run four simulator processes at once on the same runners. On 16 pinned cores with warm caches, four workers cut the Kit group from 195 s to 105 s and the non-Kit group from 195 s to 123 s, with the same results. --- .github/workflows/build.yaml | 4 ++-- tools/test_settings.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8e92ee95ab45..f76960e5cb8f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -445,7 +445,7 @@ jobs: test-k-expr: "not (multi_mesh_raycaster or newton_raycast or raycaster_sensor)" warp-cache: restore ovrtx-shader-cache: restore - test-jobs: "2" + test-jobs: "4" result-file: "test-standalone-demos-kit-report.xml" container-name: isaac-lab-standalone-demos-kit-test omni-github-test-type: standalone-demo @@ -479,7 +479,7 @@ jobs: test-k-expr: "not (multi_mesh_raycaster or newton_raycast or raycaster_sensor)" warp-cache: restore ovrtx-shader-cache: restore - test-jobs: "2" + test-jobs: "4" result-file: "test-standalone-demos-non-kit-report.xml" container-name: isaac-lab-standalone-demos-non-kit-test omni-github-test-type: standalone-demo diff --git a/tools/test_settings.py b/tools/test_settings.py index 2785a77b7f6e..3bd2aa796d3e 100644 --- a/tools/test_settings.py +++ b/tools/test_settings.py @@ -82,8 +82,8 @@ """Per-test startup timeouts for cold external asset downloads.""" PYTEST_WORKERS = { - # Standalone launches already use independent subprocesses; overlap two simulator startups. - "test_standalone_scripts.py": 2, + # Standalone launches already use independent subprocesses; overlap four simulator startups. + "test_standalone_scripts.py": 4, # Contributed-environment smoke tests: environment runs of several seconds to 2 min each. The camera file # stays whole: its workers would each start the RTX renderer, and one environment dominates it. "test_contrib_environments_kit.py": 2,