diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 33b8a7fa3e63..621c2e999b29 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,6 +67,13 @@ on: - develop - 'release/**' workflow_dispatch: + inputs: + marker: + description: >- + Run only the tests carrying this registered pytest marker, e.g. kitless. A job with no + marked file passes with an empty report. Empty runs every test. + type: string + default: '' # Concurrency control to prevent parallel runs on the same PR concurrency: @@ -130,11 +137,20 @@ jobs: isaaclab_image_name: ${{ steps.load.outputs.isaaclab_image_name }} ovphysx_wheelhouse_image: ${{ steps.load.outputs.ovphysx_wheelhouse_image }} ci_image_tag: ${{ steps.image_tag.outputs.ci_image_tag }} + marker_jobs: ${{ steps.marker.outputs.marker_jobs }} steps: - uses: actions/checkout@v6 with: fetch-depth: 1 - sparse-checkout: .github/workflows/config.yaml + # The marker check below reads the workflow, the registered markers and the test files. + sparse-checkout: | + .github/workflows/config.yaml + .github/workflows/build.yaml + .github/test-subsets/ + pyproject.toml + scripts/ + source/ + tools/ sparse-checkout-cone-mode: false - id: load run: | @@ -168,6 +184,23 @@ jobs: echo "ci_image_tag=isaac-lab-ci:${sanitized_ref}-${SHA}" >> "$GITHUB_OUTPUT" echo "CI image tag: isaac-lab-ci:${sanitized_ref}-${SHA}" + # A marker run skips jobs with no marked test file here, before they wait for a GPU runner + # and pull the CI image. + - id: marker + env: + MARKER: ${{ inputs.marker }} + EVENT_NAME: ${{ github.event_name }} + run: | + set -euo pipefail + if [ -z "${MARKER}" ]; then + echo 'marker_jobs={}' >> "$GITHUB_OUTPUT" + exit 0 + fi + workflow_json="${RUNNER_TEMP}/build.json" + yq -o=json .github/workflows/build.yaml > "$workflow_json" + python3 tools/ci_marker_jobs.py --workflow "$workflow_json" --marker "$MARKER" --event-name "$EVENT_NAME" \ + >> "$GITHUB_OUTPUT" + test-dockerfile-contract: name: dockerfile-contract # Static analysis of the Dockerfile text: no image, no GPU, no Isaac Sim. @@ -258,7 +291,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-tasks']) steps: - uses: actions/checkout@v6 with: @@ -266,6 +300,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -286,7 +321,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-tasks-2']) steps: - uses: actions/checkout@v6 with: @@ -294,6 +330,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -314,7 +351,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-tasks-3']) steps: - uses: actions/checkout@v6 with: @@ -322,6 +360,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -341,7 +380,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-core']) steps: - uses: actions/checkout@v6 with: @@ -349,6 +389,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -368,7 +409,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-core-2']) steps: - uses: actions/checkout@v6 with: @@ -376,6 +418,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -395,7 +438,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-core-3']) steps: - uses: actions/checkout@v6 with: @@ -403,6 +447,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -426,7 +471,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-standalone-demos-kit']) steps: - uses: actions/checkout@v6 with: @@ -434,6 +480,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -457,7 +504,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-standalone-demos-non-kit']) steps: - uses: actions/checkout@v6 with: @@ -468,6 +516,7 @@ jobs: id: ov_pins - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -491,7 +540,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-rl']) steps: - uses: actions/checkout@v6 with: @@ -499,6 +549,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -515,7 +566,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-mimic']) steps: - uses: actions/checkout@v6 with: @@ -523,6 +575,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -538,7 +591,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-contrib']) steps: - uses: actions/checkout@v6 with: @@ -546,6 +600,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -562,7 +617,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-teleop']) steps: - uses: actions/checkout@v6 with: @@ -570,6 +626,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -586,7 +643,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-visualizers']) steps: - uses: actions/checkout@v6 with: @@ -594,6 +652,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -610,7 +669,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-assets']) steps: - uses: actions/checkout@v6 with: @@ -618,6 +678,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -633,7 +694,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-experimental']) steps: - uses: actions/checkout@v6 with: @@ -641,6 +703,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -656,7 +719,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-newton']) steps: - uses: actions/checkout@v6 with: @@ -664,6 +728,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -680,7 +745,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-physx']) steps: - uses: actions/checkout@v6 with: @@ -688,6 +754,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -704,7 +771,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-isaaclab-ov']) env: USE_OVPHYSX_WHEELHOUSE: ${{ needs.config.outputs.ovphysx_wheelhouse_image != '' && ((github.event_name == 'pull_request' && github.base_ref == 'develop' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name != 'pull_request' && github.ref_name == 'develop')) }} steps: @@ -717,6 +785,7 @@ jobs: id: ov_pins - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -753,7 +822,8 @@ jobs: needs: [build-curobo, config] if: >- github.event_name != 'push' && - needs.build-curobo.result == 'success' + needs.build-curobo.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-curobo']) steps: - uses: actions/checkout@v6 with: @@ -761,6 +831,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }}-curobo isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -794,7 +865,8 @@ jobs: needs: [build-curobo, config] if: >- github.event_name != 'push' && - needs.build-curobo.result == 'success' + needs.build-curobo.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-contrib-environments']) steps: - uses: actions/checkout@v6 with: @@ -802,6 +874,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }}-curobo isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -826,7 +899,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-record-video']) steps: - uses: actions/checkout@v6 with: @@ -834,6 +908,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -849,7 +924,9 @@ jobs: timeout-minutes: 120 continue-on-error: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }} needs: [build, config] - if: needs.build.result == 'success' + if: >- + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-rendering-correctness']) steps: - uses: actions/checkout@v6 with: @@ -857,6 +934,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -888,7 +966,9 @@ jobs: timeout-minutes: 120 continue-on-error: ${{ github.event_name == 'workflow_dispatch' }} needs: [build, config] - if: needs.build.result == 'success' + if: >- + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-rendering-correctness-kitless-legacy']) env: USE_OVPHYSX_WHEELHOUSE: ${{ needs.config.outputs.ovphysx_wheelhouse_image != '' && ((github.event_name == 'pull_request' && github.base_ref == 'develop' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name != 'pull_request' && github.ref_name == 'develop')) }} steps: @@ -901,6 +981,7 @@ jobs: id: ov_pins - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -935,7 +1016,8 @@ jobs: needs: [build, config] if: >- github.event_name != 'pull_request' && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['test-rendering-correctness-kitless-ovstage']) env: USE_OVPHYSX_WHEELHOUSE: ${{ needs.config.outputs.ovphysx_wheelhouse_image != '' && ((github.event_name == 'pull_request' && github.base_ref == 'develop' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name != 'pull_request' && github.ref_name == 'develop')) }} steps: @@ -948,6 +1030,7 @@ jobs: id: ov_pins - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -1259,7 +1342,8 @@ jobs: cancel-in-progress: false if: >- (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['warm-warp-cache']) env: # Caches are branch-scoped: publishing only helps from a PR base branch # (the on.push.branches list). A dispatch from elsewhere runs as a dry run @@ -1273,6 +1357,7 @@ jobs: lfs: true - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} @@ -1304,7 +1389,8 @@ jobs: cancel-in-progress: false if: >- (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && - needs.build.result == 'success' + needs.build.result == 'success' && + (!inputs.marker || fromJSON(needs.config.outputs.marker_jobs)['warm-ovrtx-cache']) env: PUBLISHES_OVRTX_CACHE: ${{ github.ref_name == 'main' || github.ref_name == 'develop' || startsWith(github.ref_name, 'release/') }} steps: @@ -1317,6 +1403,7 @@ jobs: id: ov_pins - uses: ./.github/actions/run-package-tests with: + ci-marker: ${{ inputs.marker }} image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} diff --git a/.github/workflows/tools-tests.yml b/.github/workflows/tools-tests.yml index 3062204d6dd9..1d8b0fb4d2d8 100644 --- a/.github/workflows/tools-tests.yml +++ b/.github/workflows/tools-tests.yml @@ -19,6 +19,8 @@ on: # Shared CI action helpers and their co-located tests. - ".github/actions/_lib/**" - ".github/workflows/tools-tests.yml" + # test_ci_marker_jobs.py checks the marker gates in this workflow. + - ".github/workflows/build.yaml" workflow_dispatch: concurrency: @@ -68,7 +70,7 @@ jobs: # --noconftest keeps tools/conftest.py out of the session. That file is the CI test # orchestrator - its pytest_sessionstart scans source/ and scripts/ and runs the whole # suite, so loading it here would ignore the files named below. - run: python3 -m pytest tools/test_crash_journal.py tools/test_file_scheduler.py + run: python3 -m pytest tools/test_crash_journal.py tools/test_file_scheduler.py tools/test_ci_marker_jobs.py .github/actions/_lib/test_registry_credential_fallback.py -v --noconftest - uses: astral-sh/setup-uv@v6 diff --git a/source/isaaclab/changelog.d/jichuanh-ci-marker-selection.skip b/source/isaaclab/changelog.d/jichuanh-ci-marker-selection.skip new file mode 100644 index 000000000000..9e1cbe23414d --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-ci-marker-selection.skip @@ -0,0 +1 @@ +Rejected unregistered CI markers in the test orchestrator. diff --git a/source/isaaclab/test/cli/test_test_orchestrator_result_handling.py b/source/isaaclab/test/cli/test_test_orchestrator_result_handling.py index 07d712f4ba0b..9ee0cd0289d1 100644 --- a/source/isaaclab/test/cli/test_test_orchestrator_result_handling.py +++ b/source/isaaclab/test/cli/test_test_orchestrator_result_handling.py @@ -14,7 +14,7 @@ import sys import xml.etree.ElementTree as ElementTree from pathlib import Path -from types import ModuleType +from types import ModuleType, SimpleNamespace import pytest @@ -718,3 +718,50 @@ def test_external_git_asset_tests_receive_extended_startup_deadline(): assert startup_deadline == orchestrator.test_settings.GIT_ASSET_STARTUP_TIMEOUT assert orchestrator._resolve_startup_deadline("test_sample.py", timeout=10000, is_cold_cache_test=False) == 120 + + +def _start_marker_session(orchestrator, monkeypatch, ci_marker: str): + """Run the orchestrator's session start for ``ci_marker`` and return the exit code it requests.""" + for name in ( + "TEST_FILTER_PATTERN", + "TEST_EXCLUDE_PATTERN", + "TEST_INCLUDE_FILES", + "TEST_QUARANTINED_ONLY", + "TEST_CUROBO_ONLY", + "ISAACSIM_CI_SHORT", + "TEST_NODE_IDS", + "TEST_NODE_IDS_FILE", + "TEST_NODE_IDS_KEY", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("CI_MARKER", ci_marker) + config = SimpleNamespace(getini=lambda name: ["kitless: runs without Kit", "arm_ci: runs on arm"], option=None) + with pytest.raises(pytest.exit.Exception) as exit_info: + orchestrator.pytest_sessionstart(SimpleNamespace(config=config)) + return exit_info.value.returncode + + +@pytest.mark.parametrize("ci_marker", ["kitles", "kitless and not arm_ci"]) +def test_unregistered_ci_marker_fails_before_selection(ci_marker, monkeypatch): + """A marker run matches files by name, so an expression or a misspelling must fail instead of passing empty.""" + orchestrator = _load_orchestrator_module() + + def unexpected_selection(*args, **kwargs): + raise AssertionError("test files were selected for an unregistered marker") + + monkeypatch.setattr(orchestrator, "_collect_test_files", unexpected_selection) + + assert _start_marker_session(orchestrator, monkeypatch, ci_marker) == 1 + + +def test_registered_ci_marker_without_marked_files_passes_empty(monkeypatch, tmp_path): + """A job whose files carry no marker passes with an empty report instead of failing.""" + orchestrator = _load_orchestrator_module() + unmarked = tmp_path / "test_unmarked.py" + unmarked.write_text("def test_unmarked():\n pass\n") + monkeypatch.setattr(orchestrator, "_collect_test_files", lambda *args, **kwargs: [str(unmarked)]) + monkeypatch.setenv("TEST_RESULT_FILE", "marker_report.xml") + monkeypatch.chdir(tmp_path) + + assert _start_marker_session(orchestrator, monkeypatch, "kitless") == 0 + assert (tmp_path / "tests" / "marker_report.xml").is_file() diff --git a/source/isaaclab_newton/changelog.d/jichuanh-ci-marker-selection.skip b/source/isaaclab_newton/changelog.d/jichuanh-ci-marker-selection.skip new file mode 100644 index 000000000000..51bcbf53b393 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/jichuanh-ci-marker-selection.skip @@ -0,0 +1 @@ +Marked the test files that run without Kit as kitless so a marker run can select them. diff --git a/source/isaaclab_newton/test/assets/test_cable_object.py b/source/isaaclab_newton/test/assets/test_cable_object.py index 3b4398a23389..e4f470184c2c 100644 --- a/source/isaaclab_newton/test/assets/test_cable_object.py +++ b/source/isaaclab_newton/test/assets/test_cable_object.py @@ -29,6 +29,8 @@ from isaaclab_contrib.coupling import CouplerEntryCfg, CouplerProxyCfg, CouplerProxyMappingCfg +pytestmark = pytest.mark.kitless + @configclass class _CableSceneCfg(InteractiveSceneCfg): diff --git a/source/isaaclab_newton/test/assets/test_joint_coordinates.py b/source/isaaclab_newton/test/assets/test_joint_coordinates.py index b4e53d699c96..6efb0a35ff04 100644 --- a/source/isaaclab_newton/test/assets/test_joint_coordinates.py +++ b/source/isaaclab_newton/test/assets/test_joint_coordinates.py @@ -23,6 +23,8 @@ from newton.selection import ArticulationView from scipy.spatial.transform import Rotation +pytestmark = pytest.mark.kitless + # One revolute, one ball, one revolute -- the layout that hides an off-by-one when the tables are # built by walking the model instead of the view's own per-joint counts. COORD_COUNTS = [1, 4, 1] diff --git a/source/isaaclab_newton/test/assets/test_mpm_object.py b/source/isaaclab_newton/test/assets/test_mpm_object.py index 3dacc6141880..cbf9db9b85a8 100644 --- a/source/isaaclab_newton/test/assets/test_mpm_object.py +++ b/source/isaaclab_newton/test/assets/test_mpm_object.py @@ -11,6 +11,8 @@ import pytest import torch +pytestmark = pytest.mark.kitless + newton = pytest.importorskip("newton") from isaaclab_newton.assets.mpm_object import MPMObject, MPMObjectCfg diff --git a/source/isaaclab_newton/test/cloner/test_clone_xform_math.py b/source/isaaclab_newton/test/cloner/test_clone_xform_math.py index 78507b19e4a7..3de464eeabed 100644 --- a/source/isaaclab_newton/test/cloner/test_clone_xform_math.py +++ b/source/isaaclab_newton/test/cloner/test_clone_xform_math.py @@ -10,6 +10,8 @@ import warp as wp from isaaclab_newton.cloner.newton_clone_utils import _quat_rotate +pytestmark = pytest.mark.kitless + _TOL = 1e-5 # Hand-picked quaternions (xyzw) that cover identity and the 180-degree cases where a sign diff --git a/source/isaaclab_newton/test/cloner/test_collision_approximation.py b/source/isaaclab_newton/test/cloner/test_collision_approximation.py index fee301834066..a946745358b4 100644 --- a/source/isaaclab_newton/test/cloner/test_collision_approximation.py +++ b/source/isaaclab_newton/test/cloner/test_collision_approximation.py @@ -12,6 +12,8 @@ from pxr import Sdf, Usd, UsdGeom, UsdPhysics, UsdShade +pytestmark = pytest.mark.kitless + _SOURCE = "/World/Asset" diff --git a/source/isaaclab_newton/test/cloner/test_newton_builder_world_hook.py b/source/isaaclab_newton/test/cloner/test_newton_builder_world_hook.py index 73103102995b..ac6b9ec4f5a3 100644 --- a/source/isaaclab_newton/test/cloner/test_newton_builder_world_hook.py +++ b/source/isaaclab_newton/test/cloner/test_newton_builder_world_hook.py @@ -23,6 +23,8 @@ from isaaclab.cloner import make_clone_plan from isaaclab.sim import SimulationCfg, SimulationContext, SpawnerCfg, build_simulation_context +pytestmark = pytest.mark.kitless + replicate_module = importlib.import_module("isaaclab_newton.cloner.replicate") diff --git a/source/isaaclab_newton/test/cloner/test_rename_builder_labels.py b/source/isaaclab_newton/test/cloner/test_rename_builder_labels.py index 07bc6e32ef48..4fee8e291911 100644 --- a/source/isaaclab_newton/test/cloner/test_rename_builder_labels.py +++ b/source/isaaclab_newton/test/cloner/test_rename_builder_labels.py @@ -11,6 +11,7 @@ import newton import numpy as np +import pytest import warp as wp from isaaclab_newton.cloner import NewtonReplicateContext from isaaclab_newton.cloner import newton_clone_utils as newton_clone_utils_module @@ -27,6 +28,8 @@ from isaaclab.sim import SimulationContext, SpawnerCfg from isaaclab.sim.schemas import define_deformable_curve_properties +pytestmark = pytest.mark.kitless + class TestReplicateBuilderMapping(unittest.TestCase): def test_world_prototypes_preserve_repeated_instances_and_default_poses(self): diff --git a/source/isaaclab_newton/test/cloner/test_visual_shape_import.py b/source/isaaclab_newton/test/cloner/test_visual_shape_import.py index 6cd49c001fba..9ff3a49c13f7 100644 --- a/source/isaaclab_newton/test/cloner/test_visual_shape_import.py +++ b/source/isaaclab_newton/test/cloner/test_visual_shape_import.py @@ -12,6 +12,8 @@ from pxr import Usd, UsdGeom, UsdPhysics +pytestmark = pytest.mark.kitless + @pytest.mark.parametrize("load_visual_shapes", [True, False]) @pytest.mark.parametrize("dynamic", [True, False]) diff --git a/source/isaaclab_newton/test/controllers/test_newton_controllers.py b/source/isaaclab_newton/test/controllers/test_newton_controllers.py index 61b3fa32af33..48b62dec6957 100644 --- a/source/isaaclab_newton/test/controllers/test_newton_controllers.py +++ b/source/isaaclab_newton/test/controllers/test_newton_controllers.py @@ -24,6 +24,8 @@ from isaaclab.utils import math as math_utils +pytestmark = pytest.mark.kitless + _NUM_ENVS = 3 _NUM_JOINTS = 7 _DEVICE = "cpu" diff --git a/source/isaaclab_newton/test/controllers/test_newton_ik_solver.py b/source/isaaclab_newton/test/controllers/test_newton_ik_solver.py index 7ea6e8733cce..d338747111c1 100644 --- a/source/isaaclab_newton/test/controllers/test_newton_ik_solver.py +++ b/source/isaaclab_newton/test/controllers/test_newton_ik_solver.py @@ -7,6 +7,7 @@ import isaaclab_newton.controllers.ik.newton_ik_objectives as objectives_module import isaaclab_newton.controllers.ik.newton_ik_solver as ik_solver_module +import pytest import torch import warp as wp from isaaclab_newton.controllers.ik.newton_ik_objectives import ( @@ -24,6 +25,8 @@ from isaaclab.utils import configclass +pytestmark = pytest.mark.kitless + # Maps the stub body names used across these tests to Newton link indices. _LINKS = {"ee": 0, "torso": 1, "custom": 0} diff --git a/source/isaaclab_newton/test/envs/test_events.py b/source/isaaclab_newton/test/envs/test_events.py index ee3c4877e1cd..e1604e7a0f24 100644 --- a/source/isaaclab_newton/test/envs/test_events.py +++ b/source/isaaclab_newton/test/envs/test_events.py @@ -36,6 +36,8 @@ from isaaclab.utils import configclass from isaaclab.visualizers import VisualizerCfg +pytestmark = pytest.mark.kitless + @configclass class ActionsCfg: diff --git a/source/isaaclab_newton/test/physics/test_mjwarp_tendon_control.py b/source/isaaclab_newton/test/physics/test_mjwarp_tendon_control.py index 93c6413fb639..da9490ca595b 100644 --- a/source/isaaclab_newton/test/physics/test_mjwarp_tendon_control.py +++ b/source/isaaclab_newton/test/physics/test_mjwarp_tendon_control.py @@ -15,6 +15,8 @@ ) from newton.solvers import SolverMuJoCo +pytestmark = pytest.mark.kitless + _TENDON = int(SolverMuJoCo.TrnType.TENDON) _JOINT = 0 diff --git a/source/isaaclab_newton/test/physics/test_mpm_reset_mask_contract.py b/source/isaaclab_newton/test/physics/test_mpm_reset_mask_contract.py index 3f534458494d..38f1738d2ab4 100644 --- a/source/isaaclab_newton/test/physics/test_mpm_reset_mask_contract.py +++ b/source/isaaclab_newton/test/physics/test_mpm_reset_mask_contract.py @@ -13,6 +13,8 @@ from isaaclab_newton.physics import MPMSolverCfg, NewtonMPMManager from newton.solvers import SolverImplicitMPM +pytestmark = pytest.mark.kitless + @pytest.fixture(scope="module") def cpu_mpm_solver_and_state(): diff --git a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py index 2dfc71d0cf0d..5a8178bb1ca0 100644 --- a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py +++ b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py @@ -69,6 +69,8 @@ from isaaclab.sim import SimulationCfg, SimulationContext, build_simulation_context from isaaclab.test.utils import test_devices +pytestmark = pytest.mark.kitless + # --------------------------------------------------------------------------- # Lightweight (no sim) parametrisation # --------------------------------------------------------------------------- diff --git a/source/isaaclab_newton/test/physics/test_vbd_core.py b/source/isaaclab_newton/test/physics/test_vbd_core.py index 9ec4548bc458..3e3e162e42bc 100644 --- a/source/isaaclab_newton/test/physics/test_vbd_core.py +++ b/source/isaaclab_newton/test/physics/test_vbd_core.py @@ -20,6 +20,8 @@ from isaaclab.sim import BackendCfg, SimulationContext from isaaclab.utils import replace +pytestmark = pytest.mark.kitless + # The soft-contact and simulation axes are independent, so each value is covered once. @pytest.mark.parametrize( diff --git a/source/isaaclab_newton/test/renderers/test_segmentation.py b/source/isaaclab_newton/test/renderers/test_segmentation.py index e38194b21735..8a708f45ccd0 100644 --- a/source/isaaclab_newton/test/renderers/test_segmentation.py +++ b/source/isaaclab_newton/test/renderers/test_segmentation.py @@ -27,6 +27,8 @@ from isaaclab.renderers.segmentation_colors import BACKGROUND_ID, UNLABELLED_ID, pack_rgba, random_color_from_id from isaaclab.sim.utils.semantics import add_labels +pytestmark = pytest.mark.kitless + def _cfg(**overrides): """Minimal renderer-cfg stand-in exposing only the fields the mapper reads.""" diff --git a/source/isaaclab_newton/test/renderers/test_visual_material.py b/source/isaaclab_newton/test/renderers/test_visual_material.py index c873f109bb87..2663f8498646 100644 --- a/source/isaaclab_newton/test/renderers/test_visual_material.py +++ b/source/isaaclab_newton/test/renderers/test_visual_material.py @@ -5,6 +5,7 @@ """Tests for Newton's compiled visual-material writers.""" +import pytest import torch import warp as wp from isaaclab_newton.physics import NewtonBackendCfg, NewtonBuilderCfg @@ -21,6 +22,8 @@ from isaaclab.renderers.base_renderer import VisualMaterialBatch from isaaclab.sim import SimulationContext +pytestmark = pytest.mark.kitless + def _srgb(colors: torch.Tensor) -> torch.Tensor: return torch.where(colors <= 0.0031308, 12.92 * colors, 1.055 * colors.pow(1.0 / 2.4) - 0.055) diff --git a/source/isaaclab_newton/test/sensors/test_camera_opencv_distortion_newton.py b/source/isaaclab_newton/test/sensors/test_camera_opencv_distortion_newton.py index d5fb86f9fc33..a3cdce732fa3 100644 --- a/source/isaaclab_newton/test/sensors/test_camera_opencv_distortion_newton.py +++ b/source/isaaclab_newton/test/sensors/test_camera_opencv_distortion_newton.py @@ -28,7 +28,7 @@ from isaaclab.utils.configclass import configclass from isaaclab.utils.math import create_rotation_matrix_from_view, quat_from_matrix -pytestmark = [pytest.mark.integration, pytest.mark.rendering] +pytestmark = [pytest.mark.integration, pytest.mark.rendering, pytest.mark.kitless] SIM_DT = 1.0 / 60.0 WIDTH, HEIGHT = 640, 480 diff --git a/source/isaaclab_newton/test/sensors/test_contact_sensor.py b/source/isaaclab_newton/test/sensors/test_contact_sensor.py index 7babb6d24639..dde406fd214e 100644 --- a/source/isaaclab_newton/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_newton/test/sensors/test_contact_sensor.py @@ -57,6 +57,8 @@ from isaaclab_assets.robots.allegro import ALLEGRO_HAND_CFG +pytestmark = pytest.mark.kitless + ## # Scene Configuration ## diff --git a/source/isaaclab_newton/test/sensors/test_contact_sensor_history.py b/source/isaaclab_newton/test/sensors/test_contact_sensor_history.py index 934758cdb923..f2bcbac81ce8 100644 --- a/source/isaaclab_newton/test/sensors/test_contact_sensor_history.py +++ b/source/isaaclab_newton/test/sensors/test_contact_sensor_history.py @@ -9,6 +9,7 @@ import warnings +import pytest import torch import warp as wp from isaaclab_newton.sensors.contact_sensor.contact_sensor_data import ContactSensorData @@ -17,6 +18,8 @@ update_contact_sensor_kernel, ) +pytestmark = pytest.mark.kitless + def test_force_matrix_history_rolls_newest_first_and_honors_mask(): """Test newest-first ordering of every history buffer without advancing masked environments. diff --git a/source/isaaclab_newton/test/sensors/test_frame_transformer.py b/source/isaaclab_newton/test/sensors/test_frame_transformer.py index ad1389bc23b2..94d52bf9db7d 100644 --- a/source/isaaclab_newton/test/sensors/test_frame_transformer.py +++ b/source/isaaclab_newton/test/sensors/test_frame_transformer.py @@ -32,6 +32,8 @@ ## from isaaclab_assets.robots.anymal import ANYMAL_C_CFG # isort:skip +pytestmark = pytest.mark.kitless + def quat_from_euler_rpy(roll, pitch, yaw, degrees=False): """Converts Euler XYZ to Quaternion (x, y, z, w).""" diff --git a/source/isaaclab_newton/test/sensors/test_imu.py b/source/isaaclab_newton/test/sensors/test_imu.py index fbb77ecbe00a..6e50a335f142 100644 --- a/source/isaaclab_newton/test/sensors/test_imu.py +++ b/source/isaaclab_newton/test/sensors/test_imu.py @@ -22,6 +22,8 @@ from isaaclab.terrains import TerrainImporterCfg from isaaclab.utils import configclass +pytestmark = pytest.mark.kitless + @configclass class ImuTestSceneCfg(InteractiveSceneCfg): diff --git a/source/isaaclab_newton/test/sensors/test_joint_wrench_sensor.py b/source/isaaclab_newton/test/sensors/test_joint_wrench_sensor.py index 578338477b42..cdc7c971c266 100644 --- a/source/isaaclab_newton/test/sensors/test_joint_wrench_sensor.py +++ b/source/isaaclab_newton/test/sensors/test_joint_wrench_sensor.py @@ -36,6 +36,8 @@ from isaaclab_assets.robots.ant import ANT_CFG +pytestmark = pytest.mark.kitless + def _make_single_joint_articulation_cfg() -> ArticulationCfg: """Single-joint revolute test articulation (root ``CenterPivot`` + arm ``Arm``).""" diff --git a/source/isaaclab_newton/test/sensors/test_newton_raycast_sensor.py b/source/isaaclab_newton/test/sensors/test_newton_raycast_sensor.py index 0415133dae15..7d089fbc5cba 100644 --- a/source/isaaclab_newton/test/sensors/test_newton_raycast_sensor.py +++ b/source/isaaclab_newton/test/sensors/test_newton_raycast_sensor.py @@ -40,6 +40,8 @@ from isaaclab.terrains import TerrainImporterCfg from isaaclab.utils import configclass +pytestmark = pytest.mark.kitless + SENSOR_HEIGHT = 2.0 RAY_OFFSET = 0.2 RAY_START_HEIGHT = SENSOR_HEIGHT - RAY_OFFSET diff --git a/source/isaaclab_newton/test/sensors/test_pva.py b/source/isaaclab_newton/test/sensors/test_pva.py index f7bdb49e9137..34ae63de71dc 100644 --- a/source/isaaclab_newton/test/sensors/test_pva.py +++ b/source/isaaclab_newton/test/sensors/test_pva.py @@ -22,6 +22,8 @@ from isaaclab.terrains import TerrainImporterCfg from isaaclab.utils import configclass +pytestmark = pytest.mark.kitless + @configclass class PvaTestSceneCfg(InteractiveSceneCfg): diff --git a/source/isaaclab_newton/test/sensors/test_site_injection.py b/source/isaaclab_newton/test/sensors/test_site_injection.py index 2e1ac96f8744..bea23e01f037 100644 --- a/source/isaaclab_newton/test/sensors/test_site_injection.py +++ b/source/isaaclab_newton/test/sensors/test_site_injection.py @@ -13,6 +13,8 @@ from isaaclab.utils.warp.math_ops import transform_to_vec_quat +pytestmark = pytest.mark.kitless + # --------------------------------------------------------------------------- # transform_to_vec_quat # --------------------------------------------------------------------------- diff --git a/source/isaaclab_newton/test/sim/test_cable_usd_import.py b/source/isaaclab_newton/test/sim/test_cable_usd_import.py index 7bcfae13e9de..cce29078fb39 100644 --- a/source/isaaclab_newton/test/sim/test_cable_usd_import.py +++ b/source/isaaclab_newton/test/sim/test_cable_usd_import.py @@ -10,6 +10,8 @@ from isaaclab.sim.spawners.materials import CableMaterialCfg from isaaclab.sim.spawners.shapes import CableCfg +pytestmark = pytest.mark.kitless + # Cable joint degrees of freedom, in the order Newton lays them out. _SHEAR_X, _SHEAR_Y, _STRETCH, _BEND_X, _BEND_Y, _TWIST = range(6) diff --git a/source/isaaclab_newton/test/sim/test_mpm_spawners.py b/source/isaaclab_newton/test/sim/test_mpm_spawners.py index 234a03ffcda6..bc4a087601b9 100644 --- a/source/isaaclab_newton/test/sim/test_mpm_spawners.py +++ b/source/isaaclab_newton/test/sim/test_mpm_spawners.py @@ -20,7 +20,7 @@ import isaaclab.sim as sim_utils -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.kitless] @pytest.fixture diff --git a/source/isaaclab_newton/test/sim/test_mpm_visualization.py b/source/isaaclab_newton/test/sim/test_mpm_visualization.py index 3b7e283e0988..82739938bf01 100644 --- a/source/isaaclab_newton/test/sim/test_mpm_visualization.py +++ b/source/isaaclab_newton/test/sim/test_mpm_visualization.py @@ -15,6 +15,8 @@ import isaaclab.sim as sim_utils +pytestmark = pytest.mark.kitless + _PRIM_PATHS = ["/World/envs/env_0/Sand/Particles", "/World/envs/env_1/Sand/Particles"] """Environment-namespaced points prims, matching the ``{ENV_REGEX_NS}//Particles`` paths used by MPMObject.""" diff --git a/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py b/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py index b435f251c456..f7bbcce8309a 100644 --- a/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py +++ b/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py @@ -36,6 +36,8 @@ from isaaclab.sim import SimulationCfg, SimulationContext, build_simulation_context from isaaclab.utils import configclass +pytestmark = pytest.mark.kitless + NEWTON_SIM_CFG = SimulationCfg(physics=NewtonCfg(solver_cfg=MJWarpSolverCfg())) WORLD_MARKER_POS = (5.0, 3.0, 1.0) SITE_PATH = "/World/Robot/SiteFrame" diff --git a/source/isaaclab_ov/changelog.d/jichuanh-ci-marker-selection.skip b/source/isaaclab_ov/changelog.d/jichuanh-ci-marker-selection.skip new file mode 100644 index 000000000000..51bcbf53b393 --- /dev/null +++ b/source/isaaclab_ov/changelog.d/jichuanh-ci-marker-selection.skip @@ -0,0 +1 @@ +Marked the test files that run without Kit as kitless so a marker run can select them. diff --git a/source/isaaclab_ov/test/assets/test_articulation.py b/source/isaaclab_ov/test/assets/test_articulation.py index e9b758f99eff..989905ba9ad0 100644 --- a/source/isaaclab_ov/test/assets/test_articulation.py +++ b/source/isaaclab_ov/test/assets/test_articulation.py @@ -72,7 +72,7 @@ from isaaclab_assets import FRANKA_PANDA_CFG # noqa: E402 -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.integration, pytest.mark.kitless] _FIXTURE = Path(__file__).parent / "data" / "articulation_ordering_branching.usda" _STIFFNESS, _DAMPING, _MAX_FORCE = 5.0, 0.5, 100.0 diff --git a/source/isaaclab_ov/test/assets/test_deformable_object.py b/source/isaaclab_ov/test/assets/test_deformable_object.py index bc9fc6a2e04b..94a3e4eb1364 100644 --- a/source/isaaclab_ov/test/assets/test_deformable_object.py +++ b/source/isaaclab_ov/test/assets/test_deformable_object.py @@ -44,6 +44,8 @@ pretriangulated_surface_deformable_spawn_cfg, ) +pytestmark = pytest.mark.kitless + wp.init() diff --git a/source/isaaclab_ov/test/assets/test_rigid_object.py b/source/isaaclab_ov/test/assets/test_rigid_object.py index 76f573229da5..8fcef55a979b 100644 --- a/source/isaaclab_ov/test/assets/test_rigid_object.py +++ b/source/isaaclab_ov/test/assets/test_rigid_object.py @@ -46,7 +46,7 @@ from isaaclab.utils import configclass # noqa: E402 from isaaclab.utils.math import quat_apply_inverse, quat_mul # noqa: E402 -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.integration, pytest.mark.kitless] _NUM_CUBES = 2 _NUM_ENVS, _NUM_BODIES = 2, 3 diff --git a/source/isaaclab_ov/test/envs/test_events.py b/source/isaaclab_ov/test/envs/test_events.py index b84fab5973a6..7348f79d112b 100644 --- a/source/isaaclab_ov/test/envs/test_events.py +++ b/source/isaaclab_ov/test/envs/test_events.py @@ -32,6 +32,8 @@ from isaaclab.sim import SimulationCfg, build_simulation_context from isaaclab.test.utils import DeviceScope, test_devices +pytestmark = pytest.mark.kitless + @pytest.fixture(scope="module", params=test_devices(DeviceScope.CUDA)) def event_env(request, tmp_path_factory): diff --git a/source/isaaclab_ov/test/physics/test_ovphysx_compat.py b/source/isaaclab_ov/test/physics/test_ovphysx_compat.py index ea9cb1f471e9..7951836a07f3 100644 --- a/source/isaaclab_ov/test/physics/test_ovphysx_compat.py +++ b/source/isaaclab_ov/test/physics/test_ovphysx_compat.py @@ -22,6 +22,7 @@ bool(_MISSING_MODULES), reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}", ), + pytest.mark.kitless, ] if not _MISSING_MODULES: diff --git a/source/isaaclab_ov/test/physics/test_ovphysx_gravity.py b/source/isaaclab_ov/test/physics/test_ovphysx_gravity.py index 8661417e0bb9..beb117425a48 100644 --- a/source/isaaclab_ov/test/physics/test_ovphysx_gravity.py +++ b/source/isaaclab_ov/test/physics/test_ovphysx_gravity.py @@ -23,6 +23,8 @@ from isaaclab.sim import SimulationCfg, build_simulation_context from isaaclab.test.utils import DeviceScope, test_devices +pytestmark = pytest.mark.kitless + @pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) def test_gravity_event_changes_rigid_body_motion(device): diff --git a/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py b/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py index 854318865b7c..922f41080bce 100644 --- a/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py +++ b/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py @@ -20,6 +20,8 @@ from isaaclab.test.utils import DeviceScope, test_devices +pytestmark = pytest.mark.kitless + pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") _CPU_DEVICES = test_devices(DeviceScope.CPU) diff --git a/source/isaaclab_ov/test/physics/test_ovphysx_scene_data_backend.py b/source/isaaclab_ov/test/physics/test_ovphysx_scene_data_backend.py index e32f26dccf04..bd5f49f8f798 100644 --- a/source/isaaclab_ov/test/physics/test_ovphysx_scene_data_backend.py +++ b/source/isaaclab_ov/test/physics/test_ovphysx_scene_data_backend.py @@ -12,6 +12,8 @@ import pytest +pytestmark = pytest.mark.kitless + # The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; # CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") diff --git a/source/isaaclab_ov/test/sensors/test_contact_sensor.py b/source/isaaclab_ov/test/sensors/test_contact_sensor.py index af5143faac0a..c23b12bb8ad1 100644 --- a/source/isaaclab_ov/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_ov/test/sensors/test_contact_sensor.py @@ -52,6 +52,8 @@ from isaaclab.test.utils import test_devices # noqa: E402 from isaaclab.utils import configclass, replace # noqa: E402 +pytestmark = pytest.mark.kitless + wp.init() diff --git a/source/isaaclab_ov/test/sensors/test_contact_sensor_kernels.py b/source/isaaclab_ov/test/sensors/test_contact_sensor_kernels.py index 9d6b0f276587..0ab13aaa3b6f 100644 --- a/source/isaaclab_ov/test/sensors/test_contact_sensor_kernels.py +++ b/source/isaaclab_ov/test/sensors/test_contact_sensor_kernels.py @@ -12,6 +12,8 @@ from isaaclab.test.utils import DeviceScope, test_devices +pytestmark = pytest.mark.kitless + @pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("use_mask, capacity", [(False, None), (True, 5)]) diff --git a/source/isaaclab_ov/test/sensors/test_frame_transformer.py b/source/isaaclab_ov/test/sensors/test_frame_transformer.py index 6ac9e360f630..a4acb7a881cc 100644 --- a/source/isaaclab_ov/test/sensors/test_frame_transformer.py +++ b/source/isaaclab_ov/test/sensors/test_frame_transformer.py @@ -24,6 +24,8 @@ import torch import warp as wp +pytestmark = pytest.mark.kitless + # OVRTX-only CI jobs collect the consolidated isaaclab_ov test suite without # the optional ovphysx wheel. Skip the OVPhysX tests gracefully in that case. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") diff --git a/source/isaaclab_ov/test/sensors/test_imu.py b/source/isaaclab_ov/test/sensors/test_imu.py index 8ea15cae95c7..9a36e76181df 100644 --- a/source/isaaclab_ov/test/sensors/test_imu.py +++ b/source/isaaclab_ov/test/sensors/test_imu.py @@ -32,6 +32,8 @@ # --------------------------------------------------------------------------- import pytest +pytestmark = pytest.mark.kitless + pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") _TT_module = pytest.importorskip( "isaaclab_ov.tensor_types", diff --git a/source/isaaclab_ov/test/sensors/test_joint_wrench_sensor.py b/source/isaaclab_ov/test/sensors/test_joint_wrench_sensor.py index b35efb30eb6d..5d57caa46435 100644 --- a/source/isaaclab_ov/test/sensors/test_joint_wrench_sensor.py +++ b/source/isaaclab_ov/test/sensors/test_joint_wrench_sensor.py @@ -45,6 +45,8 @@ from isaaclab_assets.robots.ant import ANT_CFG # noqa: E402 +pytestmark = pytest.mark.kitless + wp.init() diff --git a/source/isaaclab_ov/test/sensors/test_pva.py b/source/isaaclab_ov/test/sensors/test_pva.py index 39dff9597416..b10bbf60f8f1 100644 --- a/source/isaaclab_ov/test/sensors/test_pva.py +++ b/source/isaaclab_ov/test/sensors/test_pva.py @@ -27,6 +27,8 @@ # --------------------------------------------------------------------------- import pytest +pytestmark = pytest.mark.kitless + pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") _TT_module = pytest.importorskip( "isaaclab_ov.tensor_types", diff --git a/source/isaaclab_ov/test/sensors/test_ray_caster.py b/source/isaaclab_ov/test/sensors/test_ray_caster.py index eaede51018d9..66912d77acb1 100644 --- a/source/isaaclab_ov/test/sensors/test_ray_caster.py +++ b/source/isaaclab_ov/test/sensors/test_ray_caster.py @@ -9,11 +9,14 @@ from types import SimpleNamespace +import pytest import torch import warp as wp from isaaclab_ov import tensor_types as TT from isaaclab_ov.sensors.ray_caster import ray_caster as ray_caster_module +pytestmark = pytest.mark.kitless + class _FakeBinding: shape = (3, 7) diff --git a/source/isaaclab_ov/test/sim/test_ovphysx_view.py b/source/isaaclab_ov/test/sim/test_ovphysx_view.py index 8bca85fad3cd..9d50cf817045 100644 --- a/source/isaaclab_ov/test/sim/test_ovphysx_view.py +++ b/source/isaaclab_ov/test/sim/test_ovphysx_view.py @@ -41,6 +41,8 @@ ) from ovphysx.types import TensorType # noqa: E402 +pytestmark = pytest.mark.kitless + wp.init() wp.set_device("cpu") _HAS_CUDA = wp.get_cuda_device_count() > 0 diff --git a/source/isaaclab_ov/test/sim/test_views_xform_prim_ovphysx.py b/source/isaaclab_ov/test/sim/test_views_xform_prim_ovphysx.py index 956177675018..c5b6991f315b 100644 --- a/source/isaaclab_ov/test/sim/test_views_xform_prim_ovphysx.py +++ b/source/isaaclab_ov/test/sim/test_views_xform_prim_ovphysx.py @@ -25,6 +25,8 @@ from isaaclab.sim import SimulationCfg, build_simulation_context # noqa: E402 from isaaclab.sim.views import FrameView # noqa: E402 +pytestmark = pytest.mark.kitless + OVPHYSX_SIM_CFG = SimulationCfg(physics=OvPhysxCfg()) diff --git a/source/isaaclab_ov/test/test_cloner.py b/source/isaaclab_ov/test/test_cloner.py index 83df0541380f..e334d3ef58a3 100644 --- a/source/isaaclab_ov/test/test_cloner.py +++ b/source/isaaclab_ov/test/test_cloner.py @@ -20,6 +20,8 @@ from isaaclab.cloner import make_clone_plan from isaaclab.sim import SimulationContext +pytestmark = pytest.mark.kitless + def _pose_matrix(position: tuple[float, float, float], quaternion: tuple[float, float, float, float]) -> Gf.Matrix4d: """Build a USD pose matrix from an xyzw quaternion.""" diff --git a/source/isaaclab_ov/test/test_ovrtx_annotator_utils.py b/source/isaaclab_ov/test/test_ovrtx_annotator_utils.py index 7cf343b56485..3ff13f345ae2 100644 --- a/source/isaaclab_ov/test/test_ovrtx_annotator_utils.py +++ b/source/isaaclab_ov/test/test_ovrtx_annotator_utils.py @@ -18,6 +18,7 @@ bool(_MISSING_MODULES), reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}", ), + pytest.mark.kitless, ] if not _MISSING_MODULES: diff --git a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py index edb38ad2b0ca..a8e0c5395248 100644 --- a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py +++ b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py @@ -32,6 +32,7 @@ bool(_MISSING_MODULES), reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}", ), + pytest.mark.kitless, ] if not _MISSING_MODULES: diff --git a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py index 529ab0a469b5..aba6736e0aa3 100644 --- a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py +++ b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py @@ -21,7 +21,10 @@ _REQUIRED_MODULES = ("isaaclab_ov", "ovrtx", "pxr") _MISSING_MODULES = [module for module in _REQUIRED_MODULES if importlib.util.find_spec(module) is None] -pytestmark = pytest.mark.skipif(bool(_MISSING_MODULES), reason=f"requires optional modules: {_MISSING_MODULES}") +pytestmark = [ + pytest.mark.skipif(bool(_MISSING_MODULES), reason=f"requires optional modules: {_MISSING_MODULES}"), + pytest.mark.kitless, +] if not _MISSING_MODULES: import isaaclab_ov.renderers.ovrtx_renderer as ovrtx_renderer_module diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index 9831b5931465..11b01b7ad1f5 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -34,6 +34,7 @@ bool(_MISSING_MODULES), reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}", ), + pytest.mark.kitless, ] if not _MISSING_MODULES: diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_kernels.py b/source/isaaclab_ov/test/test_ovrtx_renderer_kernels.py index 09d5dcb5e52c..0d5e0831c3fa 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_kernels.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_kernels.py @@ -15,6 +15,8 @@ from isaaclab.renderers.segmentation_colors import pack_rgba, random_color_from_id +pytestmark = pytest.mark.kitless + DEVICE = "cuda:0" diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_rigid_object_rendering.py b/source/isaaclab_ov/test/test_ovrtx_renderer_rigid_object_rendering.py index 3235e0af19a9..79ade794310c 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_rigid_object_rendering.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_rigid_object_rendering.py @@ -32,6 +32,7 @@ pytest.mark.rendering, pytest.mark.isaacsim_ci, pytest.mark.skipif(bool(_MISSING_MODULES), reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}"), + pytest.mark.kitless, ] if not _MISSING_MODULES: diff --git a/source/isaaclab_ov/test/test_ovrtx_shader_cache_redirect.py b/source/isaaclab_ov/test/test_ovrtx_shader_cache_redirect.py index dda1a37cba0a..2d9708753913 100644 --- a/source/isaaclab_ov/test/test_ovrtx_shader_cache_redirect.py +++ b/source/isaaclab_ov/test/test_ovrtx_shader_cache_redirect.py @@ -19,6 +19,8 @@ redirect_shader_cache, ) +pytestmark = pytest.mark.kitless + _CACHE_PATH = "/tmp/isaaclab-ovrtx-kitless-cache" # Only ever forwarded to the patched applier factory, so its contents never matter. diff --git a/source/isaaclab_ov/test/test_ovrtx_usd.py b/source/isaaclab_ov/test/test_ovrtx_usd.py index 698d13612af1..db0a0afc28b8 100644 --- a/source/isaaclab_ov/test/test_ovrtx_usd.py +++ b/source/isaaclab_ov/test/test_ovrtx_usd.py @@ -20,6 +20,7 @@ bool(_MISSING_MODULES), reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}", ), + pytest.mark.kitless, ] if not _MISSING_MODULES: diff --git a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py index 69d97d575fae..697cb8617d8e 100644 --- a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py +++ b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py @@ -16,9 +16,10 @@ _REQUIRED_MODULES = ("isaaclab_ov", "ovrtx") _MISSING_MODULES = [module for module in _REQUIRED_MODULES if importlib.util.find_spec(module) is None] -pytestmark = pytest.mark.skipif( - bool(_MISSING_MODULES), reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}" -) +pytestmark = [ + pytest.mark.skipif(bool(_MISSING_MODULES), reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}"), + pytest.mark.kitless, +] if not _MISSING_MODULES: from isaaclab_ov.renderers.ovrtx_renderer import OVRTXRenderer diff --git a/source/isaaclab_ov/test/test_randomize_rigid_body_collider_offsets_mdp.py b/source/isaaclab_ov/test/test_randomize_rigid_body_collider_offsets_mdp.py index f704f0e1fe35..992f0577dab9 100644 --- a/source/isaaclab_ov/test/test_randomize_rigid_body_collider_offsets_mdp.py +++ b/source/isaaclab_ov/test/test_randomize_rigid_body_collider_offsets_mdp.py @@ -38,6 +38,8 @@ from isaaclab_assets import CARTPOLE_CFG # isort:skip # noqa: E402 +pytestmark = pytest.mark.kitless + wp.init() diff --git a/source/isaaclab_ov/test/test_randomize_rigid_body_material_mdp.py b/source/isaaclab_ov/test/test_randomize_rigid_body_material_mdp.py index baca4bd4b497..39dc7486467d 100644 --- a/source/isaaclab_ov/test/test_randomize_rigid_body_material_mdp.py +++ b/source/isaaclab_ov/test/test_randomize_rigid_body_material_mdp.py @@ -33,6 +33,8 @@ from isaaclab.test.utils import DeviceScope, test_devices # noqa: E402 from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR # noqa: E402 +pytestmark = pytest.mark.kitless + wp.init() diff --git a/source/isaaclab_physx/changelog.d/jichuanh-ci-marker-selection.fixed.rst b/source/isaaclab_physx/changelog.d/jichuanh-ci-marker-selection.fixed.rst new file mode 100644 index 000000000000..117be864a69e --- /dev/null +++ b/source/isaaclab_physx/changelog.d/jichuanh-ci-marker-selection.fixed.rst @@ -0,0 +1,2 @@ +* Fixed a test module that starts Kit crashing when pytest runs with a ``-m`` marker expression naming a marker + other than ``isaacsim_ci``, ``windows_ci`` or ``arm_ci``, such as ``-m kitless``. diff --git a/source/isaaclab_physx/isaaclab_physx/app/kit_launcher.py b/source/isaaclab_physx/isaaclab_physx/app/kit_launcher.py index 2c57c67e4f8a..4dea26534023 100644 --- a/source/isaaclab_physx/isaaclab_physx/app/kit_launcher.py +++ b/source/isaaclab_physx/isaaclab_physx/app/kit_launcher.py @@ -67,9 +67,8 @@ def _sanitize_sys_argv_for_kit(argv: list[str]) -> list[str]: indexes_to_remove: set[int] = set() for index, argument in enumerate(argv): if argument == "-m" and index + 1 < len(argv): - marker_expression = argv[index + 1] - if any(marker in marker_expression for marker in ("pytest", "isaacsim_ci", "windows_ci", "arm_ci")): - indexes_to_remove.update((index, index + 1)) + # Under pytest every ``-m`` is a marker expression, and Kit crashes on one it does not know. + indexes_to_remove.update((index, index + 1)) elif ( (argument.startswith("--config-file=") and "pyproject.toml" in argument) or argument == "--capture=no" diff --git a/source/isaaclab_physx/test/app/test_kit_launcher.py b/source/isaaclab_physx/test/app/test_kit_launcher.py index 0fd29d744e35..29e513a2bdd5 100644 --- a/source/isaaclab_physx/test/app/test_kit_launcher.py +++ b/source/isaaclab_physx/test/app/test_kit_launcher.py @@ -57,11 +57,12 @@ def test_sanitize_sys_argv_preserves_user_verbosity_outside_pytest(monkeypatch): assert result is argv -def test_sanitize_sys_argv_removes_pytest_marker_pair(monkeypatch): - """Remove a pytest marker option together with its expression.""" +@pytest.mark.parametrize("marker_expression", ["not isaacsim_ci", "kitless"]) +def test_sanitize_sys_argv_removes_pytest_marker_pair(marker_expression, monkeypatch): + """Remove a pytest marker option together with its expression, whichever markers it names.""" monkeypatch.setitem(sys.modules, "pytest", object()) - result = _sanitize_sys_argv_for_kit(["test_script.py", "-m", "not isaacsim_ci", "--keep"]) + result = _sanitize_sys_argv_for_kit(["test_script.py", "-m", marker_expression, "--keep"]) assert result == ["test_script.py", "--keep"] diff --git a/source/isaaclab_tasks/changelog.d/jichuanh-ci-marker-selection.skip b/source/isaaclab_tasks/changelog.d/jichuanh-ci-marker-selection.skip new file mode 100644 index 000000000000..51bcbf53b393 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/jichuanh-ci-marker-selection.skip @@ -0,0 +1 @@ +Marked the test files that run without Kit as kitless so a marker run can select them. diff --git a/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py index ff2367b550db..bbf003a09255 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py @@ -19,7 +19,7 @@ rendering_test_cartpole, ) -pytestmark = pytest.mark.arm_ci +pytestmark = [pytest.mark.arm_ci, pytest.mark.kitless] _RENDERING_PARAMS = [ pytest.param(*param.values, False, marks=param.marks, id=param.id) diff --git a/source/isaaclab_tasks/test/core/test_rendering_franka_cable_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_franka_cable_kitless.py index 4170237a2664..0c1d55af56bf 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_franka_cable_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_franka_cable_kitless.py @@ -18,6 +18,8 @@ rendering_test_franka_cable, ) +pytestmark = pytest.mark.kitless + _RENDERING_PARAMS = group_rendering_params(make_kitless_rendering_params_franka()) _COMPARISON_SCORES: list[dict] = [] diff --git a/source/isaaclab_tasks/test/core/test_rendering_franka_cloth_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_franka_cloth_kitless.py index 77ba8da567b5..0ed86fd86a53 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_franka_cloth_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_franka_cloth_kitless.py @@ -18,6 +18,8 @@ rendering_test_franka_cloth, ) +pytestmark = pytest.mark.kitless + _RENDERING_PARAMS = group_rendering_params(make_kitless_rendering_params_franka()) _COMPARISON_SCORES: list[dict] = [] diff --git a/source/isaaclab_tasks/test/core/test_rendering_franka_soft_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_franka_soft_kitless.py index e34b3657bcdb..fc6448f4ace9 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_franka_soft_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_franka_soft_kitless.py @@ -18,6 +18,8 @@ rendering_test_franka_soft, ) +pytestmark = pytest.mark.kitless + _RENDERING_PARAMS = group_rendering_params(make_kitless_rendering_params_franka()) _COMPARISON_SCORES: list[dict] = [] diff --git a/source/isaaclab_tasks/test/core/test_rendering_lift_kuka_hetero_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_lift_kuka_hetero_kitless.py index 92aa1df826be..a0936bffee79 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_lift_kuka_hetero_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_lift_kuka_hetero_kitless.py @@ -18,6 +18,8 @@ rendering_test_lift_kuka, ) +pytestmark = pytest.mark.kitless + # OVPhysX heterogeneous rendering goldens are not available yet. _RENDERING_PARAMS = group_rendering_params( [param for param in make_kitless_rendering_params_lift() if param.values[1] != "ovphysx"] diff --git a/source/isaaclab_tasks/test/core/test_rendering_lift_kuka_homo_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_lift_kuka_homo_kitless.py index bda9c02b288c..a4c95a8fb1a7 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_lift_kuka_homo_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_lift_kuka_homo_kitless.py @@ -18,7 +18,7 @@ rendering_test_lift_kuka, ) -pytestmark = pytest.mark.arm_ci +pytestmark = [pytest.mark.arm_ci, pytest.mark.kitless] _RENDERING_PARAMS = group_rendering_params(make_kitless_rendering_params_lift()) _COMPARISON_SCORES: list[dict] = [] diff --git a/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py index b17e9c41a0c3..dd98588d1a2c 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py @@ -19,7 +19,7 @@ rendering_test_shadow_hand, ) -pytestmark = pytest.mark.arm_ci +pytestmark = [pytest.mark.arm_ci, pytest.mark.kitless] _RENDERING_PARAMS = group_rendering_params( make_kitless_rendering_params(MINIMAL_KITLESS_PHYSICS_RENDERER_AOV_COMBINATIONS) diff --git a/tools/ci_marker_jobs.py b/tools/ci_marker_jobs.py new file mode 100644 index 000000000000..7dde4c203ee8 --- /dev/null +++ b/tools/ci_marker_jobs.py @@ -0,0 +1,135 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Report which workflow test jobs select at least one test file carrying a pytest marker. + +``build.yaml`` runs this in its config job so that a marker run skips jobs with nothing to run +before they wait for a GPU runner and pull the CI image. It applies the same file selection as +``tools/conftest.py`` to each job's ``run-package-tests`` inputs. + +Prints ``marker_jobs=`` for ``$GITHUB_OUTPUT``. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys + +import tomllib + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from test_selection import collect_test_files, files_with_marker # isort: skip + +WORKSPACE_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +RUN_PACKAGE_TESTS = "./.github/actions/run-package-tests" +# The only expression the selection inputs use: a value that applies to push runs alone. +EVENT_EXPRESSION = re.compile(r"^\$\{\{\s*github\.event_name == '(\w+)' && '([^']*)' \|\| ''\s*\}\}$") + +# run-package-tests inputs that decide which test files a job runs. +SELECTION_INPUTS = ( + "filter-pattern", + "exclude-pattern", + "include-files", + "quarantined-only", + "curobo-only", + "shard-index", + "shard-count", + "test-node-ids-file", + "test-node-ids-key", +) + + +def registered_markers() -> set[str]: + with open(os.path.join(WORKSPACE_ROOT, "pyproject.toml"), "rb") as f: + markers = tomllib.load(f)["tool"]["pytest"]["ini_options"]["markers"] + return {entry.split(":", 1)[0].strip() for entry in markers} + + +def resolve(value, event_name: str) -> str | None: + """Return an input's value for this event, or None for an expression this script cannot evaluate.""" + value = str(value) + if "${{" not in value: + return value + match = EVENT_EXPRESSION.match(value.strip()) + if match is None: + return None + return match[2] if match[1] == event_name else "" + + +def node_id_files(node_ids_file: str, node_ids_key: str) -> set[str]: + with open(os.path.join(WORKSPACE_ROOT, node_ids_file), "rb") as f: + node_ids = tomllib.load(f)[node_ids_key] + return {os.path.normpath(os.path.join(WORKSPACE_ROOT, node_id.split("::", 1)[0])) for node_id in node_ids} + + +def marked_files(inputs: dict[str, str], marker: str) -> list[str]: + """Select a job's test files the way ``run_tests.sh`` and ``tools/conftest.py`` do, then keep marked ones.""" + filter_pattern = inputs.get("filter-pattern", "") + exclude_pattern = inputs.get("exclude-pattern", "") + # run_tests.sh turns the legacy "not " filter into an exclude entry. + if filter_pattern.startswith("not "): + exclude_pattern = ",".join(p for p in (exclude_pattern, filter_pattern.removeprefix("not ")) if p) + filter_pattern = "" + include_files = {f.strip() for f in inputs.get("include-files", "").split(",") if f.strip()} + configured_files = set() + if inputs.get("test-node-ids-file") and inputs.get("test-node-ids-key"): + configured_files = node_id_files(inputs["test-node-ids-file"], inputs["test-node-ids-key"]) + include_files.update(os.path.basename(path) for path in configured_files) + + test_files = collect_test_files( + [os.path.join(WORKSPACE_ROOT, "scripts"), os.path.join(WORKSPACE_ROOT, "source")], + filter_pattern, + exclude_pattern, + include_files, + inputs.get("quarantined-only", "false") == "true", + inputs.get("curobo-only", "false") == "true", + shard_index=inputs.get("shard-index", ""), + shard_count=inputs.get("shard-count", ""), + ) + if configured_files: + test_files = [f for f in test_files if os.path.normpath(f) in configured_files] + return files_with_marker(test_files, marker) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--workflow", required=True, help="Workflow file converted to JSON (yq -o=json).") + parser.add_argument("--marker", required=True, help="Single registered pytest marker.") + parser.add_argument("--event-name", required=True, help="github.event_name of the run.") + args = parser.parse_args() + + if args.marker not in registered_markers(): + print(f"::error::Marker {args.marker!r} is not a single marker registered in pyproject.toml") + return 1 + + with open(args.workflow) as f: + jobs = json.load(f)["jobs"] + result = {} + for job_id, job in jobs.items(): + for step in job.get("steps", []): + inputs = step.get("with") or {} + # Jobs that ignore the marker (e.g. performance-smoke) do not forward it and always run. + if step.get("uses") != RUN_PACKAGE_TESTS or "inputs.marker" not in str(inputs.get("ci-marker", "")): + continue + resolved = {key: resolve(inputs[key], args.event_name) for key in SELECTION_INPUTS if key in inputs} + unresolved = sorted(key for key, value in resolved.items() if value is None) + if unresolved: + # Running a job that turns out to be empty is cheaper than skipping one that is not. + result[job_id] = True + print(f"{job_id}: runs, cannot evaluate {', '.join(unresolved)}", file=sys.stderr) + continue + files = marked_files(resolved, args.marker) + result[job_id] = bool(files) + print(f"{job_id}: {len(files)} file(s) marked {args.marker}", file=sys.stderr) + + print(f"marker_jobs={json.dumps(result, separators=(',', ':'))}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/conftest.py b/tools/conftest.py index 19b7be193180..b5dba6017015 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -28,6 +28,7 @@ import test_settings as test_settings # isort: skip from crash_journal import JOURNAL_ENV_VAR, create_crash_report # isort: skip from _file_scheduler import JobContext, TestFileJob, run_test_files # isort: skip +from test_selection import collect_test_files, files_with_marker # isort: skip logging.basicConfig(level=logging.INFO, format="%(message)s") logger = logging.getLogger(__name__) @@ -1580,67 +1581,20 @@ def _collect_test_files( curobo_only, ): """Collect test files from source directories, applying all active filters.""" - test_files = [] - for source_dir in source_dirs: - if not os.path.exists(source_dir): - logger.error(f"Error: source directory not found at {source_dir}") - pytest.exit("Source directory not found", returncode=1) - - for root, _, files in os.walk(source_dir): - # source/isaaclab/test/install_ci/ has its own pytest config and conftest. - # It is run via .github/actions/install-ci-run, never via this collector, - # so skip the whole subtree to keep install_ci tests out of build.yaml jobs. - if "install_ci" in root.replace("\\", "/").split("/"): - continue - - for file in files: - if not (file.startswith("test_") and file.endswith(".py")): - continue - - # Mode-exclusive filters (each bypasses TESTS_TO_SKIP) - if quarantined_only: - if file not in test_settings.QUARANTINED_TESTS: - continue - elif curobo_only: - if file not in test_settings.CUROBO_TESTS: - continue - else: - # An explicit include_files entry overrides TESTS_TO_SKIP, allowing - # dedicated jobs (e.g. test-environments-training) to run tests that - # are otherwise excluded from general CI runs. - if file in test_settings.TESTS_TO_SKIP and file not in include_files: - logger.debug(f"Skipping {file} as it's in the skip list") - continue - - full_path = os.path.join(root, file) - - if filter_pattern and filter_pattern not in full_path: - logger.debug(f"Skipping {full_path} (does not match include pattern: {filter_pattern})") - continue - if exclude_pattern and any(p.strip() in full_path for p in exclude_pattern.split(",")): - logger.debug(f"Skipping {full_path} (matches exclude pattern: {exclude_pattern})") - continue - if include_files and file not in include_files: - logger.debug(f"Skipping {full_path} (not in include files list)") - continue - - test_files.append(full_path) - - # Sort test files deterministically to ensure consistent test ordering. - test_files.sort() - - # Apply file-level sharding: select every Nth file from the deterministic order. - # Skip when include_files is set — in that case the test's own conftest handles - # sharding at the test-item level (e.g. parametrized test cases). - shard_index = os.environ.get("TEST_SHARD_INDEX", "") - shard_count = os.environ.get("TEST_SHARD_COUNT", "") - if shard_index and shard_count and not include_files: - shard_index = int(shard_index) - shard_count = int(shard_count) - test_files = [f for i, f in enumerate(test_files) if i % shard_count == shard_index] - logger.info(f"Shard {shard_index}/{shard_count}: selected {len(test_files)} test files") - - return test_files + try: + return collect_test_files( + source_dirs, + filter_pattern, + exclude_pattern, + include_files, + quarantined_only, + curobo_only, + shard_index=os.environ.get("TEST_SHARD_INDEX", ""), + shard_count=os.environ.get("TEST_SHARD_COUNT", ""), + ) + except FileNotFoundError as exc: + logger.error(f"Error: {exc}") + pytest.exit("Source directory not found", returncode=1) def _load_test_node_ids_from_toml(workspace_root: str) -> list[str]: @@ -1758,6 +1712,11 @@ def pytest_sessionstart(session): # filter is owned by Isaac Sim's external CI pipeline; the CI_MARKER path # leaves that contract untouched. ci_marker = os.environ.get("CI_MARKER", "") + # The file scan below matches a single marker name, and a job with no marked file passes empty, + # so an expression or a misspelt name would pass every job without running anything. + registered_markers = {entry.split(":", 1)[0].strip() for entry in session.config.getini("markers")} + if ci_marker and ci_marker not in registered_markers: + pytest.exit(f"CI_MARKER {ci_marker!r} is not a single marker registered in pyproject.toml", returncode=1) test_node_ids_by_file = _collect_test_node_ids_by_file(workspace_root) # Parse include files list (comma-separated paths) @@ -1813,22 +1772,7 @@ def pytest_sessionstart(session): test_files = new_test_files if ci_marker: - # Match both `@pytest.mark.` (per-function) and - # `pytestmark = pytest.mark.` / `pytestmark = [..., pytest.mark., ...]` - # (module-level) by looking for the common `pytest.mark.` substring. - marker_token = f"pytest.mark.{ci_marker}" - new_test_files = [] - for test_file in test_files: - try: - with open(test_file) as f: - if marker_token in f.read(): - new_test_files.append(test_file) - except OSError as exc: - raise RuntimeError( - f"ci_marker post-scan could not read {test_file}; refusing to" - f" silently drop a potentially marker-tagged file" - ) from exc - test_files = new_test_files + test_files = files_with_marker(test_files, ci_marker) if test_node_ids_by_file: configured_files = set(test_node_ids_by_file) @@ -1842,6 +1786,10 @@ def pytest_sessionstart(session): logger.info("No quarantined tests configured — nothing to run.") _write_empty_report() pytest.exit("No quarantined tests configured", returncode=0) + if ci_marker: + logger.info(f"No selected test file carries marker '{ci_marker}' — nothing to run.") + _write_empty_report() + pytest.exit("No test files found for marker", returncode=0) if filter_pattern: logger.info(f"No test files found matching filter pattern '{filter_pattern}' — nothing to run.") _write_empty_report() diff --git a/tools/test_ci_marker_jobs.py b/tools/test_ci_marker_jobs.py new file mode 100644 index 000000000000..cb68f75f30b5 --- /dev/null +++ b/tools/test_ci_marker_jobs.py @@ -0,0 +1,103 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for ``tools/ci_marker_jobs.py``.""" + +from __future__ import annotations + +import json +import os + +import pytest +import yaml +from ci_marker_jobs import RUN_PACKAGE_TESTS, WORKSPACE_ROOT, main, resolve + +BUILD_WORKFLOW = os.path.join(WORKSPACE_ROOT, ".github", "workflows", "build.yaml") + + +def _marker_jobs(tmp_path, monkeypatch, capsys, workflow: dict, marker: str = "kitless") -> dict[str, bool]: + workflow_json = tmp_path / "build.json" + workflow_json.write_text(json.dumps(workflow)) + monkeypatch.setattr( + "sys.argv", + [ + "ci_marker_jobs.py", + "--workflow", + str(workflow_json), + "--marker", + marker, + "--event-name", + "workflow_dispatch", + ], + ) + assert main() == 0 + output = capsys.readouterr().out.strip() + assert output.startswith("marker_jobs=") + return json.loads(output.removeprefix("marker_jobs=")) + + +def _job(**inputs) -> dict: + return {"steps": [{"uses": RUN_PACKAGE_TESTS, "with": {"ci-marker": "${{ inputs.marker }}", **inputs}}]} + + +@pytest.mark.parametrize( + ("value", "event_name", "expected"), + [ + ("isaaclab_ov", "workflow_dispatch", "isaaclab_ov"), + ("${{ github.event_name == 'push' && 'rendering' || '' }}", "push", "rendering"), + ("${{ github.event_name == 'push' && 'rendering' || '' }}", "workflow_dispatch", ""), + ("${{ needs.config.outputs.subset }}", "workflow_dispatch", None), + ], +) +def test_resolve_evaluates_only_event_conditional_values(value, event_name, expected): + assert resolve(value, event_name) == expected + + +def test_jobs_without_marked_files_are_skipped(tmp_path, monkeypatch, capsys): + jobs = _marker_jobs( + tmp_path, + monkeypatch, + capsys, + { + "jobs": { + "ov": _job(**{"filter-pattern": "isaaclab_ov"}), + "no-files": _job(**{"filter-pattern": "no_such_package"}), + # The legacy "not " filter excludes instead of including; core has kitless files. + "core": _job(**{"filter-pattern": "not isaaclab_"}), + "unknown-input": _job(**{"filter-pattern": "${{ needs.config.outputs.subset }}"}), + "ignores-marker": {"steps": [{"uses": RUN_PACKAGE_TESTS, "with": {"filter-pattern": "isaaclab_ov"}}]}, + } + }, + ) + + assert jobs == {"ov": True, "no-files": False, "core": True, "unknown-input": True} + + +def test_unregistered_marker_fails(tmp_path, monkeypatch, capsys): + workflow_json = tmp_path / "build.json" + workflow_json.write_text(json.dumps({"jobs": {}})) + monkeypatch.setattr( + "sys.argv", + ["ci_marker_jobs.py", "--workflow", str(workflow_json), "--marker", "kitles", "--event-name", "push"], + ) + + assert main() == 1 + assert "kitles" in capsys.readouterr().out + + +def test_every_job_forwarding_the_marker_is_gated_on_its_result(): + with open(BUILD_WORKFLOW) as f: + jobs = yaml.safe_load(f)["jobs"] + + forwarding = { + job_id + for job_id, job in jobs.items() + for step in job.get("steps", []) + if "inputs.marker" in str((step.get("with") or {}).get("ci-marker", "")) + } + gated = {job_id for job_id, job in jobs.items() if f"marker_jobs)['{job_id}']" in str(job.get("if", ""))} + + assert forwarding + assert forwarding == gated diff --git a/tools/test_selection.py b/tools/test_selection.py new file mode 100644 index 000000000000..6c9c8ce3d9e0 --- /dev/null +++ b/tools/test_selection.py @@ -0,0 +1,119 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Test file selection shared by the CI test orchestrator and the CI marker job check. + +Only the standard library and :mod:`test_settings` are imported, so the selection also runs on a +plain runner before any Isaac Lab image is pulled. +""" + +from __future__ import annotations + +import logging +import os + +import test_settings # isort: skip + +logger = logging.getLogger(__name__) + + +def collect_test_files( + source_dirs, + filter_pattern, + exclude_pattern, + include_files, + quarantined_only, + curobo_only, + shard_index="", + shard_count="", +): + """Collect test files from source directories, applying all active filters. + + Raises: + FileNotFoundError: If a source directory does not exist. + """ + test_files = [] + for source_dir in source_dirs: + if not os.path.exists(source_dir): + raise FileNotFoundError(f"Source directory not found at {source_dir}") + + for root, _, files in os.walk(source_dir): + # source/isaaclab/test/install_ci/ has its own pytest config and conftest. + # It is run via .github/actions/install-ci-run, never via this collector, + # so skip the whole subtree to keep install_ci tests out of build.yaml jobs. + if "install_ci" in root.replace("\\", "/").split("/"): + continue + + for file in files: + if not (file.startswith("test_") and file.endswith(".py")): + continue + + # Mode-exclusive filters (each bypasses TESTS_TO_SKIP) + if quarantined_only: + if file not in test_settings.QUARANTINED_TESTS: + continue + elif curobo_only: + if file not in test_settings.CUROBO_TESTS: + continue + else: + # An explicit include_files entry overrides TESTS_TO_SKIP, allowing + # dedicated jobs (e.g. test-environments-training) to run tests that + # are otherwise excluded from general CI runs. + if file in test_settings.TESTS_TO_SKIP and file not in include_files: + logger.debug(f"Skipping {file} as it's in the skip list") + continue + + full_path = os.path.join(root, file) + + if filter_pattern and filter_pattern not in full_path: + logger.debug(f"Skipping {full_path} (does not match include pattern: {filter_pattern})") + continue + if exclude_pattern and any(p.strip() in full_path for p in exclude_pattern.split(",")): + logger.debug(f"Skipping {full_path} (matches exclude pattern: {exclude_pattern})") + continue + if include_files and file not in include_files: + logger.debug(f"Skipping {full_path} (not in include files list)") + continue + + test_files.append(full_path) + + # Sort test files deterministically to ensure consistent test ordering. + test_files.sort() + + # Apply file-level sharding: select every Nth file from the deterministic order. + # Skip when include_files is set — in that case the test's own conftest handles + # sharding at the test-item level (e.g. parametrized test cases). + if shard_index and shard_count and not include_files: + shard_index = int(shard_index) + shard_count = int(shard_count) + test_files = [f for i, f in enumerate(test_files) if i % shard_count == shard_index] + logger.info(f"Shard {shard_index}/{shard_count}: selected {len(test_files)} test files") + + return test_files + + +def files_with_marker(test_files, marker): + """Keep the test files that carry ``marker``. + + Matches both ``@pytest.mark.`` (per-function) and ``pytestmark = pytest.mark.`` / + ``pytestmark = [..., pytest.mark., ...]`` (module-level) through their common + ``pytest.mark.`` substring. + + Raises: + RuntimeError: If a test file cannot be read, rather than silently dropping a marked file. + """ + marker_token = f"pytest.mark.{marker}" + marked_files = [] + for test_file in test_files: + try: + with open(test_file) as f: + if marker_token in f.read(): + marked_files.append(test_file) + except OSError as exc: + raise RuntimeError( + f"ci_marker post-scan could not read {test_file}; refusing to silently drop a potentially" + " marker-tagged file" + ) from exc + return marked_files