diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 01df1aec1962..a9ea008f14de 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -341,16 +341,67 @@ jobs: #endregion + #region compute affected packages + compute-affected-packages: + name: Compute Affected CI Jobs + runs-on: ubuntu-latest + needs: [changes] + if: needs.changes.outputs.run_docker_tests == 'true' + outputs: + affected_jobs: ${{ steps.compute.outputs.affected_jobs }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install pyyaml + run: pip install pyyaml --quiet + - name: Compute affected jobs from file-level import graph + id: compute + env: + BASE_REF: ${{ github.event.pull_request.base.ref || github.event.before }} + EVENT_NAME: ${{ github.event_name }} + run: | + set -euo pipefail + + if [ "$EVENT_NAME" = "pull_request" ]; then + # Find the merge base so rebased PRs compare correctly + git fetch origin "$BASE_REF" --depth=100 2>/dev/null || true + MERGE_BASE=$(git merge-base HEAD "origin/${BASE_REF}" 2>/dev/null || echo "") + REF="${MERGE_BASE:-origin/${BASE_REF}}" + + AFFECTED=$(python3 tools/package_impact_graph.py --since "$REF" --json 2>/dev/null \ + | python3 -c "import json,sys; m=json.load(sys.stdin); print(json.dumps(sorted(m['jobs'].keys())))") + else + # Non-PR push: run all jobs (fail-open). Derive the full list from + # the graph so it stays in sync when new jobs are added to build.yaml. + AFFECTED=$(python3 tools/package_impact_graph.py 2>/dev/null \ + | python3 -c "import json,sys; g=json.load(sys.stdin); print(json.dumps(g['all_ci_jobs']))") + fi + + echo "affected_jobs=${AFFECTED}" >> "$GITHUB_OUTPUT" + echo "### Affected CI jobs" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo '```json' >> "$GITHUB_STEP_SUMMARY" + echo "${AFFECTED}" | python3 -m json.tool >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + #endregion + #region test jobs test-isaaclab-tasks: name: isaaclab_tasks [1/3] runs-on: [self-hosted, gpu] timeout-minutes: 180 continue-on-error: true - needs: [build, config] + needs: [build, config, compute-affected-packages] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (needs.compute-affected-packages.result != 'success' || + contains(fromJSON(needs.compute-affected-packages.outputs.affected_jobs), 'test-isaaclab-tasks')) steps: - uses: actions/checkout@v6 with: @@ -372,10 +423,12 @@ jobs: runs-on: [self-hosted, gpu] timeout-minutes: 180 continue-on-error: true - needs: [build, config] + needs: [build, config, compute-affected-packages] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (needs.compute-affected-packages.result != 'success' || + contains(fromJSON(needs.compute-affected-packages.outputs.affected_jobs), 'test-isaaclab-tasks')) steps: - uses: actions/checkout@v6 with: @@ -397,10 +450,12 @@ jobs: runs-on: [self-hosted, gpu] timeout-minutes: 180 continue-on-error: true - needs: [build, config] + needs: [build, config, compute-affected-packages] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (needs.compute-affected-packages.result != 'success' || + contains(fromJSON(needs.compute-affected-packages.outputs.affected_jobs), 'test-isaaclab-tasks')) steps: - uses: actions/checkout@v6 with: @@ -421,10 +476,12 @@ jobs: name: isaaclab (core) [1/3] runs-on: [self-hosted, gpu] timeout-minutes: 180 - needs: [build, config] + needs: [build, config, compute-affected-packages] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (needs.compute-affected-packages.result != 'success' || + contains(fromJSON(needs.compute-affected-packages.outputs.affected_jobs), 'test-isaaclab-core')) steps: - uses: actions/checkout@v6 with: @@ -444,10 +501,12 @@ jobs: name: isaaclab (core) [2/3] runs-on: [self-hosted, gpu] timeout-minutes: 180 - needs: [build, config] + needs: [build, config, compute-affected-packages] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (needs.compute-affected-packages.result != 'success' || + contains(fromJSON(needs.compute-affected-packages.outputs.affected_jobs), 'test-isaaclab-core')) steps: - uses: actions/checkout@v6 with: @@ -467,10 +526,12 @@ jobs: name: isaaclab (core) [3/3] runs-on: [self-hosted, gpu] timeout-minutes: 180 - needs: [build, config] + needs: [build, config, compute-affected-packages] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (needs.compute-affected-packages.result != 'success' || + contains(fromJSON(needs.compute-affected-packages.outputs.affected_jobs), 'test-isaaclab-core')) steps: - uses: actions/checkout@v6 with: @@ -490,10 +551,12 @@ jobs: name: isaaclab_rl runs-on: [self-hosted, gpu] timeout-minutes: 180 - needs: [build, config] + needs: [build, config, compute-affected-packages] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (needs.compute-affected-packages.result != 'success' || + contains(fromJSON(needs.compute-affected-packages.outputs.affected_jobs), 'test-isaaclab-rl')) steps: - uses: actions/checkout@v6 with: @@ -512,10 +575,12 @@ jobs: name: isaaclab_mimic runs-on: [self-hosted, gpu] timeout-minutes: 180 - needs: [build, config] + needs: [build, config, compute-affected-packages] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (needs.compute-affected-packages.result != 'success' || + contains(fromJSON(needs.compute-affected-packages.outputs.affected_jobs), 'test-isaaclab-mimic')) steps: - uses: actions/checkout@v6 with: @@ -533,10 +598,12 @@ jobs: name: isaaclab_contrib runs-on: [self-hosted, gpu] timeout-minutes: 180 - needs: [build, config] + needs: [build, config, compute-affected-packages] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (needs.compute-affected-packages.result != 'success' || + contains(fromJSON(needs.compute-affected-packages.outputs.affected_jobs), 'test-isaaclab-contrib')) steps: - uses: actions/checkout@v6 with: @@ -554,10 +621,12 @@ jobs: name: isaaclab_teleop runs-on: [self-hosted, gpu] timeout-minutes: 180 - needs: [build, config] + needs: [build, config, compute-affected-packages] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (needs.compute-affected-packages.result != 'success' || + contains(fromJSON(needs.compute-affected-packages.outputs.affected_jobs), 'test-isaaclab-teleop')) steps: - uses: actions/checkout@v6 with: @@ -575,10 +644,12 @@ jobs: name: isaaclab_visualizers runs-on: [self-hosted, gpu] timeout-minutes: 180 - needs: [build, config] + needs: [build, config, compute-affected-packages] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (needs.compute-affected-packages.result != 'success' || + contains(fromJSON(needs.compute-affected-packages.outputs.affected_jobs), 'test-isaaclab-visualizers')) steps: - uses: actions/checkout@v6 with: @@ -596,10 +667,12 @@ jobs: name: isaaclab_assets runs-on: [self-hosted, gpu] timeout-minutes: 180 - needs: [build, config] + needs: [build, config, compute-affected-packages] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (needs.compute-affected-packages.result != 'success' || + contains(fromJSON(needs.compute-affected-packages.outputs.affected_jobs), 'test-isaaclab-assets')) steps: - uses: actions/checkout@v6 with: @@ -617,10 +690,12 @@ jobs: name: isaaclab_newton runs-on: [self-hosted, gpu] timeout-minutes: 180 - needs: [build, config] + needs: [build, config, compute-affected-packages] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (needs.compute-affected-packages.result != 'success' || + contains(fromJSON(needs.compute-affected-packages.outputs.affected_jobs), 'test-isaaclab-newton')) steps: - uses: actions/checkout@v6 with: @@ -638,10 +713,12 @@ jobs: name: isaaclab_physx runs-on: [self-hosted, gpu] timeout-minutes: 180 - needs: [build, config] + needs: [build, config, compute-affected-packages] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (needs.compute-affected-packages.result != 'success' || + contains(fromJSON(needs.compute-affected-packages.outputs.affected_jobs), 'test-isaaclab-physx')) steps: - uses: actions/checkout@v6 with: @@ -659,10 +736,12 @@ jobs: name: isaaclab_ov runs-on: [self-hosted, gpu] timeout-minutes: 180 - needs: [build, config] + needs: [build, config, compute-affected-packages] if: >- github.event_name != 'push' && - needs.build.result == 'success' + needs.build.result == 'success' && + (needs.compute-affected-packages.result != 'success' || + contains(fromJSON(needs.compute-affected-packages.outputs.affected_jobs), 'test-isaaclab-ov')) env: USE_OVPHYSX_WHEELHOUSE: ${{ needs.config.outputs.ovphysx_wheelhouse_resource != '' && ((github.event_name == 'pull_request' && github.base_ref == 'develop' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name != 'pull_request' && github.ref_name == 'develop')) }} steps: diff --git a/tools/package_impact_graph.py b/tools/package_impact_graph.py new file mode 100644 index 000000000000..5a33d279c438 --- /dev/null +++ b/tools/package_impact_graph.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Build a file-level interpackage import graph and compute CI job impact. + +Usage +----- +# Print the full impact map (JSON): +python tools/package_impact_graph.py + +# Print CI jobs triggered by specific changed files: +python tools/package_impact_graph.py --changed source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py + +# Print CI jobs triggered by all files changed vs a git ref: +python tools/package_impact_graph.py --since HEAD~1 + +The tool walks every .py file under source/, parses its import statements +(skipping TYPE_CHECKING guards), and records which isaaclab* packages each file +imports. From that file-level graph it derives a package-level reverse-dep map: +for each package P, which other packages Q have at least one file that imports +from P. When P changes, the tests for P plus every Q that (transitively) imports +P must run. + +The output is a JSON object: + + { + "package_to_ci_jobs": { + "isaaclab_newton": ["test-isaaclab-newton", "test-isaaclab-physx", ...], + ... + }, + "file_imports": { + "isaaclab_newton/isaaclab_newton/foo.py": ["isaaclab", "isaaclab_physx"], + ... + }, + "reverse_deps": { + "isaaclab_newton": ["isaaclab_physx", "isaaclab_contrib", ...], + ... + } + } +""" + +from __future__ import annotations + +import argparse +import ast +import json +import re +import subprocess +import sys +from collections import defaultdict +from pathlib import Path + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_PREFIX = "isaaclab" +_DEP_NAME_RE = re.compile(r"^([A-Za-z0-9_-]+)") +_BUILD_YAML = Path(__file__).resolve().parent.parent / ".github" / "workflows" / "build.yaml" + + +def _derive_pkg_to_job_map(packages: list[str]) -> dict[str, list[str]]: + """Derive the package→CI-jobs mapping by parsing .github/workflows/build.yaml. + + For each job that uses ``.github/actions/run-package-tests``, extracts the + ``filter-pattern`` value. ``TEST_FILTER_PATTERN`` is applied as a path + substring filter by ``tools/conftest.py``, so a package is covered by a job + when the pattern is a substring of the package name. + + A package can match multiple patterns — for example ``isaaclab_tasks_experimental`` + matches both ``"isaaclab_tasks"`` and any other overlapping pattern. All + matching canonical labels are returned so every affected job runs. + + Sharded jobs (e.g. ``test-isaaclab-tasks``, ``test-isaaclab-tasks-2``) share + the same filter-pattern; the shortest job ID becomes the canonical label so + all shards check against the same string in ``contains(fromJSON(...))``. + + Returns an empty map if ``build.yaml`` is missing or ``pyyaml`` is unavailable. + """ + if not _BUILD_YAML.is_file(): + return {} + try: + import yaml # pyyaml + except ImportError: + print( + "Warning: pyyaml not installed; run `pip install pyyaml` for dynamic job mapping.", + file=sys.stderr, + ) + return {} + + try: + with _BUILD_YAML.open() as f: + workflow = yaml.safe_load(f) + except Exception as exc: + print(f"Warning: could not parse build.yaml: {exc}", file=sys.stderr) + return {} + + # Collect (job_id, filter_pattern) for every test job that uses run-package-tests. + pattern_to_job_ids: dict[str, list[str]] = defaultdict(list) + for job_id, job_def in workflow.get("jobs", {}).items(): + if not isinstance(job_def, dict): + continue + for step in job_def.get("steps", []): + if not isinstance(step, dict): + continue + uses = str(step.get("uses", "")) + if ".github/actions/run-package-tests" not in uses: + continue + with_block = step.get("with") or {} + pattern = str(with_block.get("filter-pattern", "")).strip() + # Skip missing patterns and GitHub Actions expressions + if pattern and not pattern.startswith("${{"): + pattern_to_job_ids[pattern].append(job_id) + break # only one run-package-tests step per job + + # For sharded jobs with identical patterns, use the shortest ID as the + # canonical label (e.g. "test-isaaclab-tasks" for all three shards). + pattern_to_canonical: dict[str, str] = { + pattern: sorted(ids, key=len)[0] for pattern, ids in pattern_to_job_ids.items() + } + + # Map each source package to ALL canonical labels whose filter-pattern matches. + # A positive pattern matches when it is a substring of the package name. + # A "not X" pattern matches when X is NOT a substring of the package name. + pkg_to_jobs: dict[str, list[str]] = {} + for pkg in packages: + labels: set[str] = set() + for pattern, label in pattern_to_canonical.items(): + if pattern.startswith("not "): + excluded = pattern[4:].strip() + matched = excluded not in pkg + else: + matched = pattern in pkg + if matched: + labels.add(label) + if labels: + pkg_to_jobs[pkg] = sorted(labels) + + return pkg_to_jobs + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _repo_root() -> Path: + for parent in Path(__file__).resolve().parents: + if (parent / "pyproject.toml").is_file() and (parent / "source").is_dir(): + return parent + raise RuntimeError("Cannot find repo root.") + + +def _all_packages(source: Path) -> list[str]: + """Return the name of every subpackage directory that has a pyproject.toml.""" + return sorted(p.name for p in source.iterdir() if p.is_dir() and (p / "pyproject.toml").is_file()) + + +def _owner_package(py_file: Path, source: Path) -> str | None: + """Return the isaaclab* package that owns *py_file*, or None if not under source/.""" + try: + rel = py_file.relative_to(source) + except ValueError: + return None + return rel.parts[0] if rel.parts else None + + +def _runtime_imports(py_file: Path) -> set[str]: + """Parse *py_file* and return the isaaclab* packages it imports at runtime. + + Imports inside ``if TYPE_CHECKING:`` guards are excluded because they + create no runtime dependency. + """ + try: + tree = ast.parse(py_file.read_text(encoding="utf-8", errors="replace")) + except SyntaxError: + return set() + + tc_ids: set[int] = set() + for node in ast.walk(tree): + if isinstance(node, ast.If): + t = node.test + if (isinstance(t, ast.Name) and t.id == "TYPE_CHECKING") or ( + isinstance(t, ast.Attribute) and t.attr == "TYPE_CHECKING" + ): + for child in ast.walk(node): + tc_ids.add(id(child)) + + names: set[str] = set() + for node in ast.walk(tree): + if id(node) in tc_ids: + continue + if isinstance(node, ast.Import): + for alias in node.names: + top = alias.name.split(".")[0] + if top.startswith(_PREFIX): + names.add(top) + elif isinstance(node, ast.ImportFrom) and node.module: + top = node.module.split(".")[0] + if top.startswith(_PREFIX): + names.add(top) + return names + + +# --------------------------------------------------------------------------- +# Graph construction +# --------------------------------------------------------------------------- + + +def build_graph(source: Path) -> dict: + """Walk source/ and return the full dependency graph. + + Returns a dict with keys: + - ``file_imports``: {rel_path: [pkg, ...]} — runtime imports per file + - ``reverse_deps``: {pkg: [pkg, ...]} — packages that import each pkg + - ``package_to_ci_jobs``: {pkg: [job, ...]} + """ + packages = set(_all_packages(source)) + pkg_to_job = _derive_pkg_to_job_map(sorted(packages)) + + # file_imports[rel_path] = sorted list of isaaclab* packages imported + file_imports: dict[str, list[str]] = {} + + # For each package P, which packages Q have files that import P? + reverse_deps: dict[str, set[str]] = defaultdict(set) + + for pkg in packages: + pkg_dir = source / pkg + for py_file in sorted(pkg_dir.rglob("*.py")): + imports = _runtime_imports(py_file) + # Exclude self-imports + pkg_imports = sorted(imports - {pkg}) + if pkg_imports: + rel = str(py_file.relative_to(source)) + file_imports[rel] = pkg_imports + for imported_pkg in pkg_imports: + reverse_deps[imported_pkg].add(pkg) + + # BFS that also records the shortest import chain from start to each affected pkg. + # Returns {affected_pkg: [start, intermediate..., affected_pkg]} + def _transitive_affected_with_paths(start: str) -> dict[str, list[str]]: + paths: dict[str, list[str]] = {start: [start]} + queue = [start] + while queue: + pkg = queue.pop(0) + for importer in reverse_deps.get(pkg, set()): + if importer not in paths: + paths[importer] = paths[pkg] + [importer] + queue.append(importer) + return paths + + package_to_ci_jobs: dict[str, list[str]] = {} + # Also store the shortest chain that explains each (pkg, job) pair + package_job_chains: dict[str, dict[str, list[str]]] = {} + + for pkg in packages: + affected_paths = _transitive_affected_with_paths(pkg) + jobs: dict[str, list[str]] = {} # job -> shortest chain that explains it + for affected_pkg, chain in affected_paths.items(): + for job in pkg_to_job.get(affected_pkg, []): + if job not in jobs or len(chain) < len(jobs[job]): + jobs[job] = chain + package_to_ci_jobs[pkg] = sorted(jobs) + package_job_chains[pkg] = {j: c for j, c in jobs.items()} + + all_ci_jobs = sorted({j for labels in pkg_to_job.values() for j in labels}) + return { + "file_imports": dict(sorted(file_imports.items())), + "reverse_deps": {k: sorted(v) for k, v in sorted(reverse_deps.items())}, + "package_to_ci_jobs": dict(sorted(package_to_ci_jobs.items())), + "package_job_chains": package_job_chains, + "all_ci_jobs": all_ci_jobs, + } + + +# --------------------------------------------------------------------------- +# CI job lookup for changed files +# --------------------------------------------------------------------------- + + +def build_manifest(changed: list[str], source: Path, graph: dict) -> dict: + """Return a manifest explaining which jobs run and why. + + The manifest has shape:: + + { + "jobs": { + "test-isaaclab-newton": [ + { + "file": "source/isaaclab_newton/isaaclab_newton/foo.py", + "owner": "isaaclab_newton", + "chain": ["isaaclab_newton"], + } + ], + "test-isaaclab-tasks": [ + { + "file": "source/isaaclab_newton/isaaclab_newton/foo.py", + "owner": "isaaclab_newton", + "chain": ["isaaclab_newton", "isaaclab_tasks"], + } + ], + }, + "changed_files": [...], + "non_python_files": [...], + } + + Each entry under a job shows the shortest reverse-import chain from the + changed file's owning package to the package whose tests that job covers. + """ + # job -> list of {file, owner, chain} entries + job_reasons: dict[str, list[dict]] = defaultdict(list) + non_python: list[str] = [] + + for path_str in changed: + py_file = Path(path_str) + # Try to resolve; fall back to repo-relative if it doesn't exist yet + try: + resolved = py_file.resolve() + except Exception: + resolved = py_file + + if py_file.suffix != ".py": + non_python.append(path_str) + for job in graph.get("all_ci_jobs", []): + job_reasons[job].append({"file": path_str, "owner": None, "chain": ["(non-python)"]}) + continue + + owner = _owner_package(resolved, source) + if owner is None: + for job in graph.get("all_ci_jobs", []): + job_reasons[job].append({"file": path_str, "owner": None, "chain": ["(unknown)"]}) + continue + + chains = graph.get("package_job_chains", {}).get(owner, {}) + for job, chain in chains.items(): + job_reasons[job].append({"file": path_str, "owner": owner, "chain": chain}) + + return { + "jobs": dict(sorted(job_reasons.items())), + "changed_files": list(changed), + "non_python_files": non_python, + } + + +def _changed_files_since(ref: str) -> list[str]: + result = subprocess.run( + ["git", "diff", "--name-only", ref], + capture_output=True, + text=True, + check=True, + ) + return [line for line in result.stdout.splitlines() if line.strip()] + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "--changed", + nargs="+", + metavar="FILE", + help="Print CI jobs triggered by these changed files.", + ) + group.add_argument( + "--since", + metavar="GIT_REF", + help="Print CI jobs triggered by files changed since GIT_REF (e.g. HEAD~1, main).", + ) + parser.add_argument( + "--json", + action="store_true", + help="Emit full graph as JSON (default when neither --changed nor --since given).", + ) + args = parser.parse_args() + + root = _repo_root() + source = root / "source" + + print("Building file-level import graph...", file=sys.stderr) + graph = build_graph(source) + + if args.changed or args.since: + changed = args.changed or _changed_files_since(args.since) + manifest = build_manifest(changed, source, graph) + + if args.json: + print(json.dumps(manifest, indent=2)) + else: + jobs = manifest["jobs"] + if not jobs: + print("(no CI jobs affected)") + return + + # Human-readable manifest + print(f"Changed files ({len(manifest['changed_files'])}):") + for f in manifest["changed_files"]: + print(f" {f}") + print() + print(f"CI jobs to run ({len(jobs)}):") + for job, reasons in sorted(jobs.items()): + print(f"\n {job}") + # Deduplicate reasons by chain string + seen_chains: set[str] = set() + for r in reasons: + chain_str = " → ".join(r["chain"]) + key = f"{r['file']}|{chain_str}" + if key in seen_chains: + continue + seen_chains.add(key) + print(f" ← {r['file']}") + print(f" chain: {chain_str}") + else: + print(json.dumps(graph, indent=2)) + + +if __name__ == "__main__": + main()