diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 590952695..d8f938d8f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -117,6 +117,7 @@ jobs: e2e-bots-any: ${{ steps.e2e.outputs.bots-any }} go-packages: ${{ steps.units.outputs.go-packages }} web-tests: ${{ steps.units.outputs.web-tests }} + ci-jobs: ${{ steps.jobs.outputs.ci-jobs }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -150,6 +151,15 @@ jobs: HEAD_SHA: ${{ github.sha }} CI_FULL_RUN: "${{ contains(github.event.pull_request.labels.*.name, 'ci: full-run') }}" run: python3 hack/ci_scope.py units --base "$BASE_SHA" --head "$HEAD_SHA" + - name: map edited ci.yaml lines to their jobs + id: jobs + # A ci.yaml edit runs only the jobs it sits in, plus the jobs they need; + # an edit outside any job or in changes/report sets `all` (spec 023 US3, OD-3). + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + HEAD_SHA: ${{ github.sha }} + CI_FULL_RUN: "${{ contains(github.event.pull_request.labels.*.name, 'ci: full-run') }}" + run: python3 hack/ci_scope.py jobs --base "$BASE_SHA" --head "$HEAD_SHA" - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 id: filter with: @@ -254,7 +264,7 @@ jobs: - 'charts/gameplane/crd-manifests/**' - 'hack/check-codegen-drift_test.sh' ci: - - '.github/workflows/ci.yaml' + # ci.yaml and .golangci.yml map to their own jobs (step `jobs`, OD-3). - '.github/actions/**' - 'hack/ci_scope.py' - 'hack/test_ci_scope.py' @@ -264,12 +274,7 @@ jobs: - 'go.work' - 'go.work.sum' - 'test/e2e/buckets.sh' - # golangci-lint walks up from `working-directory` to find this - # config, so every `lint (…)` leg picks up an edit here even - # though none of them declare it as an input. Force the lint - # job on so a config change is tested against real code - # instead of silently taking effect on the next unrelated PR. - - '.golangci.yml' + - 'test/e2e/suite-components.json' github: - '.github/workflows/**' - '.github/actions/**' @@ -366,7 +371,7 @@ jobs: E2ETREE: ${{ steps.filter.outputs.e2etree }} GAMECOVERAGE: ${{ steps.filter.outputs.gamecoverage }} UNIVERSAL: ${{ steps.e2e.outputs.universal }} - CI: ${{ steps.filter.outputs.ci == 'true' || steps.scope.outputs.all == 'true' || steps.e2e.outputs.all == 'true' }} + CI: ${{ steps.filter.outputs.ci == 'true' || steps.scope.outputs.all == 'true' || steps.e2e.outputs.all == 'true' || steps.jobs.outputs.all == 'true' }} CAPTURE: ${{ steps.filter.outputs.capture_overhead }} SUBMODULES: ${{ steps.filter.outputs.submodules }} CODEGEN: ${{ steps.filter.outputs.codegen }} @@ -422,6 +427,7 @@ jobs: BOT_MATRIX: ${{ steps.bots.outputs.matrix }} E2E_OUTPUTS: ${{ toJSON(steps.e2e.outputs) }} UNITS_OUTPUTS: ${{ toJSON(steps.units.outputs) }} + CI_JOBS: ${{ steps.jobs.outputs.ci-jobs }} run: python3 hack/ci_scope.py summary --base "$BASE_SHA" --head "$HEAD_SHA" # Build the operator/api/agent (+ gameprobe) images ONCE and publish them as @@ -431,7 +437,7 @@ jobs: build-images: name: build e2e images needs: [changes] - if: needs.changes.outputs.images == 'true' + if: (needs.changes.outputs.images == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'build-images') runs-on: ubuntu-latest timeout-minutes: 15 permissions: @@ -445,7 +451,7 @@ jobs: build-images-arm64: name: build e2e images (arm64) needs: [changes] - if: needs.changes.outputs.images == 'true' + if: (needs.changes.outputs.images == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'build-images-arm64') runs-on: ubuntu-24.04-arm timeout-minutes: 20 permissions: @@ -487,7 +493,7 @@ jobs: capture-sidecar-setcap-proof: name: capture-sidecar CAP_NET_RAW survival needs: [changes] - if: needs.changes.outputs.go == 'true' + if: (needs.changes.outputs.go == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'capture-sidecar-setcap-proof') runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -595,7 +601,7 @@ jobs: vuln-images: name: vuln (images) needs: [changes, build-images] - if: needs.changes.outputs.images == 'true' + if: (needs.changes.outputs.images == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'vuln-images') runs-on: ubuntu-latest timeout-minutes: 20 permissions: @@ -627,7 +633,7 @@ jobs: docs: name: docs and repository checks needs: [changes] - if: needs.changes.outputs.lint == 'true' || needs.changes.outputs.specs == 'true' || needs.changes.outputs.docs == 'true' + if: (needs.changes.outputs.lint == 'true' || needs.changes.outputs.specs == 'true' || needs.changes.outputs.docs == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'docs') runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -665,7 +671,7 @@ jobs: codegen-drift: name: codegen drift needs: [changes] - if: needs.changes.outputs.codegen == 'true' || github.event_name == 'push' + if: (needs.changes.outputs.codegen == 'true' || github.event_name == 'push') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'codegen-drift') runs-on: ubuntu-latest timeout-minutes: 20 permissions: @@ -691,7 +697,7 @@ jobs: submodule-freshness: name: submodule freshness needs: [changes] - if: needs.changes.outputs.submodules == 'true' || github.event_name == 'push' + if: (needs.changes.outputs.submodules == 'true' || github.event_name == 'push') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'submodule-freshness') runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -734,7 +740,7 @@ jobs: vuln-go: name: vuln (go) needs: [changes] - if: needs.changes.outputs.go == 'true' || github.event_name == 'push' + if: (needs.changes.outputs.go == 'true' || github.event_name == 'push') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'vuln-go') runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -754,7 +760,7 @@ jobs: lint: name: lint (${{ matrix.module }}) needs: [changes] - if: needs.changes.outputs.lint == 'true' + if: (needs.changes.outputs.lint == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'lint') runs-on: ubuntu-latest timeout-minutes: 15 permissions: @@ -830,7 +836,7 @@ jobs: go: name: go (${{ matrix.module }} / ${{ matrix.arch }}) needs: [changes] - if: needs.changes.outputs.go == 'true' + if: (needs.changes.outputs.go == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'go') # amd64 on the standard hosted runner; arm64 on the free public-repo # arm runner. Unit + envtest run on both arches every PR — this is where # real arch bugs (build tags, sqlite codepaths, struct alignment) surface. @@ -974,7 +980,7 @@ jobs: web: name: web needs: [changes] - if: needs.changes.outputs.web == 'true' + if: (needs.changes.outputs.web == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'web') runs-on: ubuntu-latest timeout-minutes: 15 permissions: @@ -1121,7 +1127,7 @@ jobs: web-e2e-mock: name: web e2e (mock) needs: [changes] - if: needs.changes.outputs.web == 'true' + if: (needs.changes.outputs.web == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'web-e2e-mock') runs-on: ubuntu-latest permissions: contents: read @@ -1154,7 +1160,7 @@ jobs: helm: name: helm lint needs: [changes] - if: needs.changes.outputs.charts == 'true' + if: (needs.changes.outputs.charts == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'helm') runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -1167,7 +1173,7 @@ jobs: standalone-compose: name: standalone Compose smoke needs: [changes] - if: needs.changes.outputs.standalone == 'true' + if: (needs.changes.outputs.standalone == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'standalone-compose') runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -1213,7 +1219,7 @@ jobs: chart-template: name: chart render needs: [changes] - if: needs.changes.outputs.charts == 'true' + if: (needs.changes.outputs.charts == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'chart-template') runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -1576,7 +1582,7 @@ jobs: workflow-lint: name: workflow lint needs: [changes] - if: needs.changes.outputs.github == 'true' + if: (needs.changes.outputs.github == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'workflow-lint') runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -1646,7 +1652,7 @@ jobs: go-e2e-unit: name: go (test/e2e) needs: [changes] - if: needs.changes.outputs.e2e-unit == 'true' + if: (needs.changes.outputs.e2e-unit == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'go-e2e-unit') runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -1675,7 +1681,7 @@ jobs: api-postgres: name: api (postgres) needs: [changes] - if: contains(fromJSON(needs.changes.outputs.modules), 'api') + if: (contains(fromJSON(needs.changes.outputs.modules), 'api')) || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'api-postgres') runs-on: ubuntu-latest timeout-minutes: 15 permissions: @@ -1735,7 +1741,7 @@ jobs: e2e-buckets: name: e2e bucket coverage needs: [changes] - if: needs.changes.outputs.e2e == 'true' + if: (needs.changes.outputs.e2e == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-buckets') runs-on: ubuntu-latest timeout-minutes: 5 permissions: @@ -1773,7 +1779,7 @@ jobs: e2e-go: name: e2e ${{ matrix.bucket }} / ${{ matrix.arch }} (kind) needs: [changes, build-images] - if: needs.changes.outputs.e2e == 'true' + if: (needs.changes.outputs.e2e == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-go') runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: ${{ matrix.job_timeout }} permissions: @@ -1880,7 +1886,7 @@ jobs: e2e-go-arm64: name: e2e ${{ matrix.bucket }} / ${{ matrix.arch }} (kind) needs: [changes, build-images-arm64] - if: needs.changes.outputs.e2e == 'true' + if: (needs.changes.outputs.e2e == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-go-arm64') runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: ${{ matrix.job_timeout }} permissions: @@ -1946,7 +1952,7 @@ jobs: e2e-multicluster: name: e2e multicluster / ${{ matrix.arch }} (kind x2) needs: [changes, build-images] - if: needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-multicluster == 'true' + if: (needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-multicluster == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-multicluster') runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 60 # Same, across two clusters. permissions: @@ -2011,7 +2017,7 @@ jobs: e2e-multicluster-arm64: name: e2e multicluster / ${{ matrix.arch }} (kind x2) needs: [changes, build-images-arm64] - if: needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-multicluster == 'true' + if: (needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-multicluster == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-multicluster-arm64') runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 60 # Same, across two clusters. permissions: @@ -2038,7 +2044,7 @@ jobs: e2e-upgrade: name: e2e upgrade / ${{ matrix.arch }} (kind) needs: [changes, build-images] - if: needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-upgrade == 'true' + if: (needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-upgrade == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-upgrade') runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 60 # Same, plus a chart upgrade across versions. permissions: @@ -2088,7 +2094,7 @@ jobs: e2e-upgrade-arm64: name: e2e upgrade / ${{ matrix.arch }} (kind) needs: [changes, build-images-arm64] - if: needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-upgrade == 'true' + if: (needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-upgrade == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-upgrade-arm64') runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 60 # Same, plus a chart upgrade across versions. permissions: @@ -2108,7 +2114,7 @@ jobs: e2e-web-live: name: e2e web live / ${{ matrix.arch }} (kind) needs: [changes, build-images] - if: needs.changes.outputs.weblive == 'true' && needs.changes.outputs.e2e-web-live == 'true' + if: (needs.changes.outputs.weblive == 'true' && needs.changes.outputs.e2e-web-live == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-web-live') runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 60 # Same, plus a live dashboard drive. permissions: @@ -2188,7 +2194,7 @@ jobs: e2e-web-live-arm64: name: e2e web live / ${{ matrix.arch }} (kind) needs: [changes, build-images-arm64] - if: needs.changes.outputs.weblive == 'true' && needs.changes.outputs.e2e-web-live == 'true' + if: (needs.changes.outputs.weblive == 'true' && needs.changes.outputs.e2e-web-live == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-web-live-arm64') runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 60 # Same, plus a live dashboard drive. permissions: @@ -2205,7 +2211,7 @@ jobs: e2e-game-bot: name: e2e game bot / ${{ matrix.test }} (kind) needs: [changes, build-images] - if: needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-bots-any == 'true' + if: (needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-bots-any == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-game-bot') # amd64 only: this boots the external itzg/minecraft-server (multi-arch) AND # passivelemon/terraria-docker, whose arm64 support is unverified upstream — # so an arm64 leg would fail on the Terraria pull. Add an arch axis here once @@ -2273,7 +2279,8 @@ jobs: if: >- (github.event_name == 'push' && github.ref == 'refs/heads/master' && needs.changes.outputs.e2e == 'true') || (github.event_name == 'pull_request' && needs.changes.outputs.capture_overhead == 'true' && needs.changes.outputs.e2e-capture-overhead == 'true') || - (github.event_name == 'workflow_dispatch' && needs.changes.outputs.e2e == 'true') + (github.event_name == 'workflow_dispatch' && needs.changes.outputs.e2e == 'true') || + contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-capture-overhead') runs-on: ubuntu-latest timeout-minutes: 45 continue-on-error: true @@ -2334,7 +2341,7 @@ jobs: e2e-postgres: name: e2e postgres / amd64 (kind) needs: [changes, build-images] - if: needs.changes.outputs.e2e == 'true' + if: (needs.changes.outputs.e2e == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-postgres') runs-on: ubuntu-latest timeout-minutes: 75 # Boots a kind cluster with PostgreSQL, installs the chart, runs three buckets. permissions: @@ -2419,7 +2426,7 @@ jobs: e2e-kind-ingress-smoke: name: ingress-nginx smoke (kind) needs: [changes] - if: needs.changes.outputs.kindsmoke == 'true' + if: (needs.changes.outputs.kindsmoke == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-kind-ingress-smoke') runs-on: ubuntu-latest timeout-minutes: 15 permissions: diff --git a/docs/agent-architecture.md b/docs/agent-architecture.md index 02e2a47e5..3617b9cf6 100644 --- a/docs/agent-architecture.md +++ b/docs/agent-architecture.md @@ -47,6 +47,7 @@ This document is the **first stop** for AI agents modifying or extending Gamepla | Task | Read | | --- | --- | | Translations, language packs, localized server or module messages (i18n) | `docs/translating.md`, `specs/019-localization-language-packs/contracts/` (`pack-format.md`, `server-messages.md`, `i18n-api.md`, `module-locales.md`, `check-tool.md`), and the Internationalization section of `web/specs.md` | +| CI test selection: which jobs, packages and tests a PR runs (`hack/ci_scope.py`, `hack/check_changed_coverage.py`, `test/e2e/suite-components.json`, the `changes` job in `.github/workflows/ci.yaml`) | `docs/contributing.md` ("How CI picks tests"), `specs/023-change-based-test-selection/contracts/` (`ci-scope-cli.md`, `suite-components.md`, `changed-coverage.md`) | ## How to use this index 1. Identify the domain of your task (e.g., "Add an auth provider"). diff --git a/docs/contributing.md b/docs/contributing.md index affec7301..a73c7e67c 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -72,6 +72,45 @@ These jobs block merging. Each gate with a fixture has a `hack/*_test.sh` self-t - **`codegen drift`** runs `make generate manifests` and `make tidy`, then fails if `git diff` is not empty. Run the same commands locally and commit what they change (CLAUDE.md rule 7). - **`submodule freshness`** runs `hack/check-submodule-freshness.sh`. It fails when the `modules` or `website` pointer names a commit that is not on that repo's `main`, when a gitlink is missing, or when the script cannot fetch that repo's `main`. For a pointer that is not on `main`, merge the submodule PR first, then bump the pointer to the merged commit. Otherwise restore the gitlink, or fix the fetch error, and re-run. +### How CI picks tests + +On a pull request, CI runs only the work your change can affect (spec 023). The +`detect changes` job decides, and its job summary has a **Test selection** table that +lists every suite as `run` or `skip`, with the changed files that chose it or the reason +everything ran. + +What a partial run narrows: + +- **Go unit tests** run only the packages that own a changed file, plus every package + that imports them, across modules. A module's own `go.mod`, `go.sum` or + `.testcoverage.yml` reruns that whole module. The module coverage gate then becomes a + changed-lines gate: the lines you add must meet the module's own minimum + (`hack/check_changed_coverage.py`). +- **Web unit tests** run `vitest related` on the changed `web/src` files, with the same + changed-lines gate at the Vitest `lines` threshold. Lint and build always run whole. + Shared inputs (`package.json`, the lockfile, the Vite/Vitest/TS/ESLint configs, + `src/test/**`, `i18n/locales/**`) run the full suite. +- **Go e2e tests** run only the tests whose components changed, per + `test/e2e/suite-components.json`. Jobs with no selected test are skipped. +- **`ci.yaml` edits** run only the jobs whose lines changed, plus the jobs they need. + `.golangci.yml` runs only the lint legs. Other workflow files run only `workflow lint`. + +Everything still runs on every push to `master`, and when a selection input changes +(`hack/ci_scope.py`, `hack/check_changed_coverage.py`, their tests, `Makefile`, +`go.work`, `go.work.sum`, `.github/actions/**`, `test/e2e/buckets.sh`, +`test/e2e/suite-components.json`). It also runs everything for a `ci.yaml` edit outside +any job or in `changes`/`report`, a path no rule maps, and any selection error. + +To force a full run on your PR, add the label `ci: full-run` (read on the next push), or +start the `ci` workflow by hand on your branch (`workflow_dispatch`). + +When you add an e2e test or a component, keep `test/e2e/suite-components.json` in step: +add the test name pattern under `suites` with the components and paths it exercises, or +list the component under `unexercised` with a reason. The `e2e bucket coverage` job +(`ci_scope.py verify-suites`) fails when a registered test matches no entry or a +component is missing. The full rules are in +`specs/023-change-based-test-selection/contracts/suite-components.md`. + ## Testing Run the whole suite: diff --git a/hack/ci_scope.py b/hack/ci_scope.py index e8dbeaae7..3cb7ac20c 100644 --- a/hack/ci_scope.py +++ b/hack/ci_scope.py @@ -22,8 +22,8 @@ ] # Mirrors the paths-filter `ci` filter in .github/workflows/ci.yaml; keep both in sync. FORCE_ALL_FILES = { - ".github/workflows/ci.yaml", "hack/ci_scope.py", "hack/test_ci_scope.py", "Makefile", - "go.work", "go.work.sum", "test/e2e/buckets.sh", ".golangci.yml", + "hack/ci_scope.py", "hack/test_ci_scope.py", "Makefile", "go.work", "go.work.sum", + "test/e2e/buckets.sh", "test/e2e/suite-components.json", "hack/check_changed_coverage.py", "hack/test_check_changed_coverage.py", } FORCE_ALL_PREFIXES = (".github/actions/",) @@ -33,7 +33,7 @@ SUITES_FILE = "test/e2e/suite-components.json" # Paths no Go e2e test reads. Their own jobs keep their paths-filter gates. NO_E2E_FILES = { - "design.pen", "LICENSE", ".claudeignore", ".coderabbit.yaml", ".editorconfig", + "design.pen", "LICENSE", ".claudeignore", ".coderabbit.yaml", ".editorconfig", ".golangci.yml", ".gitattributes", ".gitignore", ".gitmodules", ".hadolint.yaml", "deploy/kind/up.sh", "deploy/kind/down.sh", "test/e2e/joincoverage.sh", "test/e2e/joincoverage_test.sh", @@ -63,6 +63,49 @@ # Web files that change every Vitest run (research R6). WEB_ALL_FILES = {"web/package.json", "web/package-lock.json", "web/vitest.config.ts", "web/vite.config.ts", "web/eslint.config.js"} +CI_FILE = ".github/workflows/ci.yaml" +# ci.yaml jobs that compute or report the selection; an edit there forces full scope (R8). +CI_SELECTION_JOBS = {"changes", "report"} +CI_JOB_KEY = re.compile(r"^ ([A-Za-z0-9_-]+):\s*(?:#.*)?$") +CI_TOP_KEY = re.compile(r"^[A-Za-z_][\w-]*:") +# e2e jobs whose own switch output a ci.yaml edit inside them turns on (mode e2e). +E2E_JOB_SWITCHES = { + "e2e-multicluster": "e2e-multicluster", "e2e-multicluster-arm64": "e2e-multicluster", + "e2e-upgrade": "e2e-upgrade", "e2e-upgrade-arm64": "e2e-upgrade", + "e2e-web-live": "e2e-web-live", "e2e-web-live-arm64": "e2e-web-live", + "e2e-capture-overhead": "e2e-capture-overhead", +} +# Summary row prefixes and the ci.yaml jobs behind them; the first match wins. +ROW_JOBS = ( + ("lint (", ("lint",)), + ("go (test/e2e)", ("go-e2e-unit",)), + ("go (", ("go",)), + ("api (postgres)", ("api-postgres",)), + ("web e2e (mock)", ("web-e2e-mock",)), + ("web", ("web",)), + ("helm lint", ("helm",)), + ("chart render", ("chart-template",)), + ("standalone Compose smoke", ("standalone-compose",)), + ("workflow lint", ("workflow-lint",)), + ("docs and repository checks", ("docs",)), + ("codegen drift", ("codegen-drift",)), + ("submodule freshness", ("submodule-freshness",)), + ("vuln (go)", ("vuln-go",)), + ("capture-sidecar CAP_NET_RAW survival", ("capture-sidecar-setcap-proof",)), + ("dependency review", ("dependency-review",)), + ("telemetry default gate", ("telemetry-default-gate",)), + ("build e2e images", ("build-images", "build-images-arm64")), + ("vuln (images)", ("vuln-images",)), + ("e2e bucket coverage", ("e2e-buckets",)), + ("e2e multicluster", ("e2e-multicluster", "e2e-multicluster-arm64")), + ("e2e upgrade", ("e2e-upgrade", "e2e-upgrade-arm64")), + ("e2e postgres", ("e2e-postgres",)), + ("e2e game bot", ("e2e-game-bot",)), + ("e2e web live", ("e2e-web-live", "e2e-web-live-arm64")), + ("e2e capture overhead", ("e2e-capture-overhead",)), + ("ingress-nginx smoke", ("e2e-kind-ingress-smoke",)), + ("e2e ", ("e2e-go", "e2e-go-arm64")), +) def is_unmapped(path, modules): @@ -113,9 +156,10 @@ def changed_paths(root, base, head, *, history=False): def affected_modules(root, paths): """Select changed modules and the reverse dependency closure, including tests.""" modules = workspace_modules(root) - shared = {"Makefile", "go.work", "go.work.sum", ".golangci.yml", "test/e2e/buckets.sh"} + # ci.yaml and .golangci.yml map to their own jobs through ci_jobs() (OD-3). + shared = {"Makefile", "go.work", "go.work.sum", "test/e2e/buckets.sh"} if paths is None or any( - p in shared or p.startswith((".github/", "hack/ci_scope", "hack/test_ci_scope", + p in shared or p.startswith((".github/actions/", "hack/ci_scope", "hack/test_ci_scope", "hack/check_changed_coverage", "hack/test_check_changed_coverage")) or Path(p).name in {"go.mod", "go.sum"} @@ -607,6 +651,120 @@ def unit_outputs(root, paths, reason): } +def ci_blocks(text): + """Return ({line number: job id or None}, {job id: needed job ids}) for a workflow.""" + owner, needs = {}, {} + job, in_jobs, collecting = None, False, None + for number, line in enumerate(text.splitlines(), 1): + if CI_TOP_KEY.match(line): + in_jobs, job, collecting = line.split(":", 1)[0] == "jobs", None, None + elif in_jobs and CI_JOB_KEY.match(line): + job, collecting = CI_JOB_KEY.match(line)[1], None + needs.setdefault(job, set()) + elif job and line.startswith(" needs:"): + collecting = job + line = line[len(" needs:"):] + elif re.match(r"^ \S", line): + collecting = None + owner[number] = job + if collecting: + needs[collecting].update(re.findall(r"[A-Za-z0-9_][A-Za-z0-9_-]*", + line.split("#", 1)[0])) + return owner, needs + + +def changed_ci_lines(root, base, head): + """Return ([(old line, text)], [(new line, text)]) for ci.yaml lines changed base..head.""" + diff = subprocess.run(["git", "diff", "-U0", "--no-color", base, head, "--", CI_FILE], + cwd=root, check=True, capture_output=True, text=True).stdout + removed, added, old, new, started = [], [], 0, 0, False + for line in diff.splitlines(): + hunk = re.match(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@", line) + if hunk: + old, new, started = int(hunk[1]), int(hunk[2]), True + elif started and line.startswith("-"): + removed.append((old, line[1:])) + old += 1 + elif started and line.startswith("+"): + added.append((new, line[1:])) + new += 1 + return removed, added + + +def git_text(root, revision, path): + """Return a file's text at revision, or "" when it does not exist there.""" + result = subprocess.run(["git", "show", f"{revision}:{path}"], cwd=root, + capture_output=True, text=True, check=False) + return result.stdout if result.returncode == 0 else "" + + +def ci_jobs(root, base, head, paths): + """Map shared CI file changes to ci.yaml job ids (research R8, OD-3). + + Returns (jobs, forced). jobs holds every job whose ci.yaml lines changed, `lint` for + .golangci.yml, `workflow-lint` for any .github/ or .golangci.yml change, and every job + they need. forced is True when a changed ci.yaml line that is not blank or a comment + sits outside any job or inside the changes or report job. + """ + jobs, forced, needs = set(), False, {} + if any(p.startswith(".github/") or p == ".golangci.yml" for p in paths): + jobs.add("workflow-lint") + if ".golangci.yml" in paths: + jobs.add("lint") + if CI_FILE in paths: + removed, added = changed_ci_lines(root, base, head) + for revision, changes in ((base, removed), (head, added)): + owner, found = ci_blocks(git_text(root, revision, CI_FILE)) + needs.update(found) + for number, text in changes: + if not text.strip() or text.strip().startswith("#"): + continue + job = owner.get(number) + if job is None or job in CI_SELECTION_JOBS: + forced = True + else: + jobs.add(job) + queue = list(jobs) + while queue: + for name in needs.get(queue.pop(), ()): + if name not in jobs: + jobs.add(name) + queue.append(name) + return sorted(jobs - CI_SELECTION_JOBS), forced + + +def scoped(env, base, head): + """Return (paths, reason, ci.yaml jobs): the diff, why everything runs, edited jobs.""" + event_reason = full_scope_reason(env, []) + paths = None if event_reason else changed_paths(ROOT, base, head) + reason = full_scope_reason(env, paths) + if reason: + return paths, reason, [] + jobs, forced = ci_jobs(ROOT, base, head, paths) + return paths, ("selection-rule change" if forced else None), jobs + + +def widen_e2e(values, full, jobs): + """Run every test of each e2e job whose ci.yaml block changed (research R8).""" + if {"e2e-go", "e2e-go-arm64"} & set(jobs): + values["e2e-run"] = full["e2e-run"] + for job, key in E2E_JOB_SWITCHES.items(): + if job in jobs: + values[key] = True + if "e2e-game-bot" in jobs: + values["bot-matrix"], values["bots-any"] = full["bot-matrix"], full["bots-any"] + return values + + +def widen_units(values, jobs): + """Run every package or web test of a go or web job whose ci.yaml block changed.""" + if "go" in jobs: + values["go-packages"] = {m: ["./..."] for m in values["go-packages"]} + if "web" in jobs: + values["web-tests"] = "all" + return values + + def verify_suites(suites, buckets, modules): """Return the violations of contracts/suite-components.md rules 1, 2 and 5.""" errors = [] @@ -650,6 +808,7 @@ def summarize(env, paths, reason): bots = load_json(env, "BOT_MATRIX") e2e = load_json(env, "E2E_OUTPUTS") units = load_json(env, "UNITS_OUTPUTS") + edited_jobs = set(jlist(env, "CI_JOBS")) go_selected = decoded(units, "go-packages", {}) web_selected = decoded(units, "web-tests", "all") # Without the e2e step's outputs every e2e suite follows the coarse e2e gate. @@ -706,6 +865,9 @@ def go_note(module): def add(suite, run, feed=(), run_reason="", skip_reason="not affected", note=""): """Append one selection row with a run or skip decision and its reason.""" + jobs = next((found for prefix, found in ROW_JOBS if suite.startswith(prefix)), ()) + if not run and edited_jobs & set(jobs): + run, run_reason = True, "ci.yaml: lines in this job changed" if run and reason: why = f"full: {reason}" elif run and run_reason: @@ -805,7 +967,7 @@ def bot_matrix(names): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("mode", choices=("modules", "images", "dockerfiles", "bots", "summary", - "e2e", "verify-suites", "units")) + "e2e", "verify-suites", "units", "jobs")) parser.add_argument("--base", default="") parser.add_argument("--head", default="HEAD") args = parser.parse_args() @@ -853,25 +1015,32 @@ def main(): if args.mode == "e2e": env = os.environ try: - event_reason = full_scope_reason(env, []) - paths = None if event_reason else changed_paths(ROOT, args.base, args.head) - values = e2e_outputs(ROOT, paths, full_scope_reason(env, paths)) + paths, reason, jobs = scoped(env, args.base, args.head) + values = e2e_outputs(ROOT, paths, reason) + if jobs: + values = widen_e2e(values, e2e_outputs(ROOT, None, "ci.yaml job edit"), jobs) except Exception as exc: # any doubt selects every e2e test (FR-002) print(f"::warning::e2e selection failed, running every e2e test: {exc}") values = e2e_outputs(ROOT, None, f"error: {exc}") emit(values) return + if args.mode == "jobs": + try: + _, reason, jobs = scoped(os.environ, args.base, args.head) + except Exception as exc: # any doubt selects the full suite (FR-002) + print(f"::warning::ci.yaml job mapping failed, running everything: {exc}") + reason, jobs = f"error: {exc}", [] + emit({"all": bool(reason), "reason": reason or "", "ci-jobs": jobs}) + return if args.mode == "units": env = os.environ try: - event_reason = full_scope_reason(env, []) - paths = None if event_reason else changed_paths(ROOT, args.base, args.head) - reason = full_scope_reason(env, paths) + paths, reason, jobs = scoped(env, args.base, args.head) modules = workspace_modules(ROOT) unmapped = [p for p in paths or [] if is_unmapped(p, modules)] if reason is None and unmapped: reason = f"unmapped path {unmapped[0]}" - values = unit_outputs(ROOT, paths, reason) + values = widen_units(unit_outputs(ROOT, paths, reason), jobs) except Exception as exc: # any doubt runs every unit test (FR-002) print(f"::warning::unit selection failed, running every unit test: {exc}") values = unit_outputs(ROOT, None, f"error: {exc}") @@ -881,14 +1050,15 @@ def main(): # label is the full-suite backstop. env = os.environ try: - event_reason = full_scope_reason(env, []) - paths = None if event_reason else changed_paths(ROOT, args.base, args.head) + paths, reason, jobs = scoped(env, args.base, args.head) + if paths is not None and reason and full_scope_reason(env, paths) is None: + paths = None # a ci.yaml edit outside any job or in changes/report (R8) if paths is not None and any(is_unmapped(p, workspace_modules(ROOT)) for p in paths): paths = None # mode e2e reports the path; every module runs (FR-002) selected = affected_modules(ROOT, paths) except Exception as exc: # any doubt selects the full suite (FR-002) print(f"::warning::test selection failed, running everything: {exc}") - paths = None + paths, jobs = None, [] selected = sorted(workspace_modules(ROOT)) modules = sorted(workspace_modules(ROOT)) emit({ @@ -897,8 +1067,11 @@ def main(): "lint": bool(selected), "go": bool(set(selected) - {"test/e2e"}), "e2e-unit": "test/e2e" in selected, - "lint-exclude": [{"module": m} for m in modules if m not in selected], - "go-exclude": [{"module": m} for m in modules if m not in selected and m != "test/e2e"], + # An edit inside the lint or go job of ci.yaml runs that job on every module. + "lint-exclude": [{"module": m} for m in modules + if m not in selected and "lint" not in jobs], + "go-exclude": [{"module": m} for m in modules + if m not in selected and m != "test/e2e" and "go" not in jobs], }) diff --git a/hack/test_ci_scope.py b/hack/test_ci_scope.py index 8e946f979..753dcd2d2 100644 --- a/hack/test_ci_scope.py +++ b/hack/test_ci_scope.py @@ -11,7 +11,8 @@ from ci_scope import (E2E_BUCKETS, FORCE_ALL_FILES, ROOT, affected_modules, bot_matrix, bucket_lists, changed_paths, e2e_outputs, full_scope_reason, image_inputs, is_unmapped, load_suites, main, publish_paths, select_images, summarize, - verify_suites, go_packages, web_tests, + verify_suites, go_packages, web_tests, CI_FILE, ci_blocks, ci_jobs, + widen_e2e, widen_units, workspace_modules) @@ -65,7 +66,7 @@ def test_unrelated_module_stays_isolated(self): def test_shared_inputs_and_dependency_changes_select_every_module(self): for path in ("go.work", "go.work.sum", "api/go.mod", "api/go.sum", - "Makefile", ".golangci.yml", ".github/workflows/ci.yaml", + "Makefile", ".github/actions/go-cache/action.yml", "hack/ci_scope.py", "test/e2e/buckets.sh", "web/Dockerfile"): with self.subTest(path=path): @@ -678,5 +679,186 @@ def test_whole_runs_add_no_note(self): self.assertTrue(rows["web"].startswith("lint and build only; ")) +GIT = ["git", "-c", "user.name=test", "-c", "user.email=test@example.com", + "-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null"] +CI_FIXTURE = """name: ci +on: + push: +permissions: + contents: read +concurrency: + group: ci +env: + A: "1" +jobs: + changes: + runs-on: ubuntu-latest + steps: + - run: echo changes + web: + needs: [changes] + steps: + - run: echo web + build-images: + needs: + - changes + steps: + - run: echo build + e2e-go: + needs: [changes, build-images] + steps: + - run: echo e2e + report: + needs: + [ + changes, + web, + ] + steps: + - run: echo report +""" + + +class CiJobsTests(unittest.TestCase): + """ci.yaml lines map to the jobs they sit in (spec 023 T042, research R8, OD-3).""" + + def setUp(self): + """Create a git repository holding the fixture workflow.""" + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.git("init", "-q") + write_file(self.root, CI_FILE, CI_FIXTURE) + self.base = self.commit() + + def git(self, *args): + """Run git in the fixture repository and return its trimmed output.""" + return subprocess.run([*GIT, *args], cwd=self.root, check=True, capture_output=True, + text=True).stdout.strip() + + def commit(self): + """Commit every change and return the new commit id.""" + self.git("add", "-A") + self.git("commit", "-q", "--allow-empty", "-m", "change") + return self.git("rev-parse", "HEAD") + + def edit(self, old, new): + """Apply one edit to the base workflow and return ci_jobs for it.""" + self.git("reset", "-q", "--hard", self.base) + text = (self.root / CI_FILE).read_text(encoding="utf-8") + self.assertIn(old, text) + write_file(self.root, CI_FILE, text.replace(old, new, 1)) + return ci_jobs(self.root, self.base, self.commit(), [CI_FILE]) + + def test_needs_are_parsed_in_every_form(self): + self.assertEqual(ci_blocks(CI_FIXTURE)[1], { + "changes": set(), "web": {"changes"}, "build-images": {"changes"}, + "e2e-go": {"changes", "build-images"}, "report": {"changes", "web"}}) + + def test_comment_or_blank_line_selects_only_workflow_lint(self): + for new in (" # note\n - run: echo web\n", "\n - run: echo web\n"): + with self.subTest(new=new): + self.assertEqual(self.edit(" - run: echo web\n", new), + (["workflow-lint"], False)) + + def test_step_change_selects_its_job(self): + self.assertEqual(self.edit("echo web\n", "echo web2\n"), (["web", "workflow-lint"], False)) + + def test_selected_job_brings_the_jobs_it_needs(self): + self.assertEqual(self.edit("echo e2e\n", "echo e2e2\n"), + (["build-images", "e2e-go", "workflow-lint"], False)) + + def test_removed_line_maps_against_the_base_file(self): + self.assertEqual(self.edit(" - run: echo build\n", ""), + (["build-images", "workflow-lint"], False)) + + def test_top_level_and_selection_job_edits_force_full_scope(self): + for old, new in ((" push:\n", " push:\n workflow_dispatch:\n"), + ("contents: read", "contents: write"), ("group: ci", "group: ci2"), + ('A: "1"', 'A: "2"'), ("echo changes", "echo changes2"), + ("echo report", "echo report2")): + with self.subTest(old=old): + self.assertTrue(self.edit(old, new)[1]) + + def test_golangci_selects_lint_legs_only(self): + self.assertEqual(ci_jobs(self.root, self.base, self.base, [".golangci.yml"]), + (["lint", "workflow-lint"], False)) + self.assertIsNone(full_scope_reason({}, [".golangci.yml"])) + self.assertIsNone(full_scope_reason({}, [CI_FILE])) + + def test_other_workflow_files_select_workflow_lint_only(self): + for path in (".github/workflows/publish-edge.yaml", ".github/zizmor.yml", + ".github/dependabot.yml"): + with self.subTest(path=path): + self.assertEqual(ci_jobs(self.root, self.base, self.base, [path]), + (["workflow-lint"], False)) + + def test_selection_inputs_still_force_full_scope(self): + for path in ("hack/ci_scope.py", "hack/test_ci_scope.py", "Makefile", "go.work", + "go.work.sum", ".github/actions/go-cache/action.yml", + "test/e2e/buckets.sh", "test/e2e/suite-components.json", + "hack/check_changed_coverage.py"): + with self.subTest(path=path): + self.assertEqual(full_scope_reason({}, [path]), "selection-rule change") + + +class CiJobsModuleTests(unittest.TestCase): + """Shared CI config no longer selects every Go module (spec 023 T044).""" + + def setUp(self): + """Create the shared Go workspace fixture for each test.""" + make_go_workspace(self) + + def test_ci_yaml_and_golangci_select_no_module(self): + for path in (CI_FILE, ".golangci.yml", ".github/workflows/publish-edge.yaml"): + with self.subTest(path=path): + self.assertEqual(affected_modules(self.root, [path]), []) + + +class WidenTests(unittest.TestCase): + """An edited job widens its own selection (spec 023 T045).""" + + def test_widen_e2e(self): + values = {"e2e-run": {"operator": "^(TestA)$"}, "e2e-upgrade": False, + "e2e-web-live": False, "bot-matrix": {"test": []}, "bots-any": False} + full = {"e2e-run": {"operator": "^(TestA|TestB)$", "api-auth": "^(TestC)$"}, + "bot-matrix": {"test": ["TestBot"]}, "bots-any": True} + widened = widen_e2e(dict(values), full, ["e2e-go-arm64", "e2e-upgrade", "e2e-game-bot"]) + self.assertEqual(widened["e2e-run"], full["e2e-run"]) + self.assertTrue(widened["e2e-upgrade"]) + self.assertFalse(widened["e2e-web-live"]) + self.assertEqual((widened["bot-matrix"], widened["bots-any"]), ({"test": ["TestBot"]}, True)) + self.assertEqual(widen_e2e(dict(values), full, ["web"]), values) + + def test_widen_units(self): + values = {"go-packages": {"api": ["./internal/a"], "lib": []}, "web-tests": []} + self.assertEqual(widen_units(dict(values), ["go", "web"]), + {"go-packages": {"api": ["./..."], "lib": ["./..."]}, "web-tests": "all"}) + self.assertEqual(widen_units(dict(values), ["helm"]), values) + + +class EditedJobSummaryTests(unittest.TestCase): + """The summary marks rows run by an edited ci.yaml job (spec 023 T045).""" + + def setUp(self): + """Create the shared Go workspace fixture for each test.""" + make_go_workspace(self) + + def test_edited_job_row_runs(self): + env = { + "GITHUB_EVENT_NAME": "pull_request", + "SCOPE_OUTPUTS": json.dumps({}), "FILTER_OUTPUTS": json.dumps({}), + "GATE_OUTPUTS": json.dumps({}), "BOT_MATRIX": json.dumps({"test": []}), + "CI_JOBS": json.dumps(["web", "workflow-lint"]), + } + with patch("ci_scope.ROOT", self.root): + rows, _ = summarize(env, [CI_FILE], None) + rows = {row["suite"]: row for row in rows} + self.assertEqual(rows["web"]["decision"], "run") + self.assertEqual(rows["web"]["reason"], "ci.yaml: lines in this job changed") + self.assertEqual(rows["workflow lint"]["decision"], "run") + self.assertEqual(rows["web e2e (mock)"]["decision"], "skip") + + if __name__ == "__main__": unittest.main() diff --git a/specs/023-change-based-test-selection/briefs/ci-line-mapping.md b/specs/023-change-based-test-selection/briefs/ci-line-mapping.md new file mode 100644 index 000000000..1e851b188 --- /dev/null +++ b/specs/023-change-based-test-selection/briefs/ci-line-mapping.md @@ -0,0 +1,728 @@ +# Brief: PR 4, shared CI configuration mapped per job (T042–T046) + +Branch `claude/project-thread-63yei6-pr4`, stacked on PR 3. Facts below were checked +once by the scout; fix agents apply them blind. Every edit is exact: find the quoted +"old" text and replace it with "new". Do not touch any other line. Never edit an +existing test, except section 4b, which needs valgul's sign-off first. + +Scout facts: + +- In `ci.yaml`, job keys are the two-space keys under `jobs:`. `needs:` appears as + ` needs: [a, b]`, as a `- a` list, or as a multi-line `[ … ]` flow list (job + `report`). +- `ci.yaml` and `.golangci.yml` force everything today in three places: + `FORCE_ALL_FILES`; the `shared` set plus the `.github/` prefix in `affected_modules`; + and the paths-filter `ci` list. +- The `lint` job reads `lint-exclude`, `go` reads `go-exclude` and `GO_PACKAGES`, + `web` reads `web-tests`, the `e2e-go*` jobs read `e2e-run`, and `e2e-game-bot` reads + `e2e-bot-matrix`. A ci.yaml edit inside one of those jobs must therefore widen that + job's own selection. A job-level `if:` is not enough on its own. + +## 1. `hack/ci_scope.py` + +### 1a. `FORCE_ALL_FILES` + +old: +```python +FORCE_ALL_FILES = { + ".github/workflows/ci.yaml", "hack/ci_scope.py", "hack/test_ci_scope.py", "Makefile", + "go.work", "go.work.sum", "test/e2e/buckets.sh", ".golangci.yml", + "hack/check_changed_coverage.py", "hack/test_check_changed_coverage.py", +} +``` +new: +```python +FORCE_ALL_FILES = { + "hack/ci_scope.py", "hack/test_ci_scope.py", "Makefile", "go.work", "go.work.sum", + "test/e2e/buckets.sh", "test/e2e/suite-components.json", + "hack/check_changed_coverage.py", "hack/test_check_changed_coverage.py", +} +``` + +### 1b. `NO_E2E_FILES`: add `.golangci.yml` (otherwise it is unmapped) + +old: +```python + "design.pen", "LICENSE", ".claudeignore", ".coderabbit.yaml", ".editorconfig", +``` +new: +```python + "design.pen", "LICENSE", ".claudeignore", ".coderabbit.yaml", ".editorconfig", ".golangci.yml", +``` + +### 1c. Constants: insert after the `WEB_ALL_FILES = {...}` statement (two lines) + +```python +CI_FILE = ".github/workflows/ci.yaml" +# ci.yaml jobs that compute or report the selection; an edit there forces full scope (R8). +CI_SELECTION_JOBS = {"changes", "report"} +CI_JOB_KEY = re.compile(r"^ ([A-Za-z0-9_-]+):\s*(?:#.*)?$") +CI_TOP_KEY = re.compile(r"^[A-Za-z_][\w-]*:") +# e2e jobs whose own switch output a ci.yaml edit inside them turns on (mode e2e). +E2E_JOB_SWITCHES = { + "e2e-multicluster": "e2e-multicluster", "e2e-multicluster-arm64": "e2e-multicluster", + "e2e-upgrade": "e2e-upgrade", "e2e-upgrade-arm64": "e2e-upgrade", + "e2e-web-live": "e2e-web-live", "e2e-web-live-arm64": "e2e-web-live", + "e2e-capture-overhead": "e2e-capture-overhead", +} +# Summary row prefixes and the ci.yaml jobs behind them; the first match wins. +ROW_JOBS = ( + ("lint (", ("lint",)), + ("go (test/e2e)", ("go-e2e-unit",)), + ("go (", ("go",)), + ("api (postgres)", ("api-postgres",)), + ("web e2e (mock)", ("web-e2e-mock",)), + ("web", ("web",)), + ("helm lint", ("helm",)), + ("chart render", ("chart-template",)), + ("standalone Compose smoke", ("standalone-compose",)), + ("workflow lint", ("workflow-lint",)), + ("docs and repository checks", ("docs",)), + ("codegen drift", ("codegen-drift",)), + ("submodule freshness", ("submodule-freshness",)), + ("vuln (go)", ("vuln-go",)), + ("capture-sidecar CAP_NET_RAW survival", ("capture-sidecar-setcap-proof",)), + ("dependency review", ("dependency-review",)), + ("telemetry default gate", ("telemetry-default-gate",)), + ("build e2e images", ("build-images", "build-images-arm64")), + ("vuln (images)", ("vuln-images",)), + ("e2e bucket coverage", ("e2e-buckets",)), + ("e2e multicluster", ("e2e-multicluster", "e2e-multicluster-arm64")), + ("e2e upgrade", ("e2e-upgrade", "e2e-upgrade-arm64")), + ("e2e postgres", ("e2e-postgres",)), + ("e2e game bot", ("e2e-game-bot",)), + ("e2e web live", ("e2e-web-live", "e2e-web-live-arm64")), + ("e2e capture overhead", ("e2e-capture-overhead",)), + ("ingress-nginx smoke", ("e2e-kind-ingress-smoke",)), + ("e2e ", ("e2e-go", "e2e-go-arm64")), +) +``` + +### 1d. `affected_modules` + +old: +```python + shared = {"Makefile", "go.work", "go.work.sum", ".golangci.yml", "test/e2e/buckets.sh"} + if paths is None or any( + p in shared or p.startswith((".github/", "hack/ci_scope", "hack/test_ci_scope", +``` +new: +```python + # ci.yaml and .golangci.yml map to their own jobs through ci_jobs() (OD-3). + shared = {"Makefile", "go.work", "go.work.sum", "test/e2e/buckets.sh"} + if paths is None or any( + p in shared or p.startswith((".github/actions/", "hack/ci_scope", "hack/test_ci_scope", +``` + +### 1e. New functions: insert right before `def verify_suites(suites, buckets, modules):` + +```python +def ci_blocks(text): + """Return ({line number: job id or None}, {job id: needed job ids}) for a workflow.""" + owner, needs = {}, {} + job, in_jobs, collecting = None, False, None + for number, line in enumerate(text.splitlines(), 1): + if CI_TOP_KEY.match(line): + in_jobs, job, collecting = line.split(":", 1)[0] == "jobs", None, None + elif in_jobs and CI_JOB_KEY.match(line): + job, collecting = CI_JOB_KEY.match(line)[1], None + needs.setdefault(job, set()) + elif job and line.startswith(" needs:"): + collecting = job + line = line[len(" needs:"):] + elif re.match(r"^ \S", line): + collecting = None + owner[number] = job + if collecting: + needs[collecting].update(re.findall(r"[A-Za-z0-9_-]+", line.split("#", 1)[0])) + return owner, needs + + +def changed_ci_lines(root, base, head): + """Return ([(old line, text)], [(new line, text)]) for ci.yaml lines changed base..head.""" + diff = subprocess.run(["git", "diff", "-U0", "--no-color", base, head, "--", CI_FILE], + cwd=root, check=True, capture_output=True, text=True).stdout + removed, added, old, new, started = [], [], 0, 0, False + for line in diff.splitlines(): + hunk = re.match(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@", line) + if hunk: + old, new, started = int(hunk[1]), int(hunk[2]), True + elif started and line.startswith("-"): + removed.append((old, line[1:])) + old += 1 + elif started and line.startswith("+"): + added.append((new, line[1:])) + new += 1 + return removed, added + + +def git_text(root, revision, path): + """Return a file's text at revision, or "" when it does not exist there.""" + result = subprocess.run(["git", "show", f"{revision}:{path}"], cwd=root, + capture_output=True, text=True, check=False) + return result.stdout if result.returncode == 0 else "" + + +def ci_jobs(root, base, head, paths): + """Map shared CI file changes to ci.yaml job ids (research R8, OD-3). + + Returns (jobs, forced). jobs holds every job whose ci.yaml lines changed, `lint` for + .golangci.yml, `workflow-lint` for any .github/ or .golangci.yml change, and every job + they need. forced is True when a changed ci.yaml line that is not blank or a comment + sits outside any job or inside the changes or report job. + """ + jobs, forced, needs = set(), False, {} + if any(p.startswith(".github/") or p == ".golangci.yml" for p in paths): + jobs.add("workflow-lint") + if ".golangci.yml" in paths: + jobs.add("lint") + if CI_FILE in paths: + removed, added = changed_ci_lines(root, base, head) + for revision, changes in ((base, removed), (head, added)): + owner, found = ci_blocks(git_text(root, revision, CI_FILE)) + needs.update(found) + for number, text in changes: + if not text.strip() or text.strip().startswith("#"): + continue + job = owner.get(number) + if job is None or job in CI_SELECTION_JOBS: + forced = True + else: + jobs.add(job) + queue = list(jobs) + while queue: + for name in needs.get(queue.pop(), ()): + if name not in jobs: + jobs.add(name) + queue.append(name) + return sorted(jobs - CI_SELECTION_JOBS), forced + + +def scoped(env, base, head): + """Return (paths, reason, ci.yaml jobs): the diff, why everything runs, edited jobs.""" + event_reason = full_scope_reason(env, []) + paths = None if event_reason else changed_paths(ROOT, base, head) + reason = full_scope_reason(env, paths) + if reason: + return paths, reason, [] + jobs, forced = ci_jobs(ROOT, base, head, paths) + return paths, ("selection-rule change" if forced else None), jobs + + +def widen_e2e(values, full, jobs): + """Run every test of each e2e job whose ci.yaml block changed (research R8).""" + if {"e2e-go", "e2e-go-arm64"} & set(jobs): + values["e2e-run"] = full["e2e-run"] + for job, key in E2E_JOB_SWITCHES.items(): + if job in jobs: + values[key] = True + if "e2e-game-bot" in jobs: + values["bot-matrix"], values["bots-any"] = full["bot-matrix"], full["bots-any"] + return values + + +def widen_units(values, jobs): + """Run every package or web test of a go or web job whose ci.yaml block changed.""" + if "go" in jobs: + values["go-packages"] = {m: ["./..."] for m in values["go-packages"]} + if "web" in jobs: + values["web-tests"] = "all" + return values +``` + +### 1f. `summarize`: rows turned on by an edited ci.yaml job + +old: +```python + units = load_json(env, "UNITS_OUTPUTS") +``` +new: +```python + units = load_json(env, "UNITS_OUTPUTS") + edited_jobs = set(jlist(env, "CI_JOBS")) +``` + +old: +```python + def add(suite, run, feed=(), run_reason="", skip_reason="not affected", note=""): + """Append one selection row with a run or skip decision and its reason.""" +``` +new: +```python + def add(suite, run, feed=(), run_reason="", skip_reason="not affected", note=""): + """Append one selection row with a run or skip decision and its reason.""" + jobs = next((found for prefix, found in ROW_JOBS if suite.startswith(prefix)), ()) + if not run and edited_jobs & set(jobs): + run, run_reason = True, "ci.yaml: lines in this job changed" +``` + +### 1g. `main` + +Mode choices. old: +```python + "e2e", "verify-suites", "units")) +``` +new: +```python + "e2e", "verify-suites", "units", "jobs")) +``` + +Mode e2e. old: +```python + event_reason = full_scope_reason(env, []) + paths = None if event_reason else changed_paths(ROOT, args.base, args.head) + values = e2e_outputs(ROOT, paths, full_scope_reason(env, paths)) +``` +new: +```python + paths, reason, jobs = scoped(env, args.base, args.head) + values = e2e_outputs(ROOT, paths, reason) + if jobs: + values = widen_e2e(values, e2e_outputs(ROOT, None, "ci.yaml job edit"), jobs) +``` + +Mode units. old: +```python + event_reason = full_scope_reason(env, []) + paths = None if event_reason else changed_paths(ROOT, args.base, args.head) + reason = full_scope_reason(env, paths) + modules = workspace_modules(ROOT) + unmapped = [p for p in paths or [] if is_unmapped(p, modules)] + if reason is None and unmapped: + reason = f"unmapped path {unmapped[0]}" + values = unit_outputs(ROOT, paths, reason) +``` +new: +```python + paths, reason, jobs = scoped(env, args.base, args.head) + modules = workspace_modules(ROOT) + unmapped = [p for p in paths or [] if is_unmapped(p, modules)] + if reason is None and unmapped: + reason = f"unmapped path {unmapped[0]}" + values = widen_units(unit_outputs(ROOT, paths, reason), jobs) +``` + +New mode `jobs`: insert right before the line ` if args.mode == "units":` +```python + if args.mode == "jobs": + try: + _, reason, jobs = scoped(os.environ, args.base, args.head) + except Exception as exc: # any doubt selects the full suite (FR-002) + print(f"::warning::ci.yaml job mapping failed, running everything: {exc}") + reason, jobs = f"error: {exc}", [] + emit({"all": bool(reason), "reason": reason or "", "ci-jobs": jobs}) + return +``` + +Mode modules. old: +```python + event_reason = full_scope_reason(env, []) + paths = None if event_reason else changed_paths(ROOT, args.base, args.head) + if paths is not None and any(is_unmapped(p, workspace_modules(ROOT)) for p in paths): + paths = None # mode e2e reports the path; every module runs (FR-002) + selected = affected_modules(ROOT, paths) + except Exception as exc: # any doubt selects the full suite (FR-002) + print(f"::warning::test selection failed, running everything: {exc}") + paths = None + selected = sorted(workspace_modules(ROOT)) +``` +new: +```python + paths, reason, jobs = scoped(env, args.base, args.head) + if paths is not None and reason and full_scope_reason(env, paths) is None: + paths = None # a ci.yaml edit outside any job or in changes/report (R8) + if paths is not None and any(is_unmapped(p, workspace_modules(ROOT)) for p in paths): + paths = None # mode e2e reports the path; every module runs (FR-002) + selected = affected_modules(ROOT, paths) + except Exception as exc: # any doubt selects the full suite (FR-002) + print(f"::warning::test selection failed, running everything: {exc}") + paths, jobs = None, [] + selected = sorted(workspace_modules(ROOT)) +``` +old: +```python + "lint-exclude": [{"module": m} for m in modules if m not in selected], + "go-exclude": [{"module": m} for m in modules if m not in selected and m != "test/e2e"], +``` +new: +```python + # An edit inside the lint or go job of ci.yaml runs that job on every module. + "lint-exclude": [{"module": m} for m in modules + if m not in selected and "lint" not in jobs], + "go-exclude": [{"module": m} for m in modules + if m not in selected and m != "test/e2e" and "go" not in jobs], +``` + +## 2. `.github/workflows/ci.yaml` + +2a. `changes` job outputs. old: +```yaml + web-tests: ${{ steps.units.outputs.web-tests }} +``` +new: +```yaml + web-tests: ${{ steps.units.outputs.web-tests }} + ci-jobs: ${{ steps.jobs.outputs.ci-jobs }} +``` + +2b. New step. Insert before ` - uses: dorny/paths-filter@` (right after the `units` step): +```yaml + - name: map edited ci.yaml lines to their jobs + id: jobs + # A ci.yaml edit runs only the jobs it sits in, plus the jobs they need; + # an edit outside any job or in changes/report sets `all` (spec 023 US3, OD-3). + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + HEAD_SHA: ${{ github.sha }} + CI_FULL_RUN: "${{ contains(github.event.pull_request.labels.*.name, 'ci: full-run') }}" + run: python3 hack/ci_scope.py jobs --base "$BASE_SHA" --head "$HEAD_SHA" +``` + +2c. paths-filter `ci` list. old: +```yaml + ci: + - '.github/workflows/ci.yaml' + - '.github/actions/**' +``` +new: +```yaml + ci: + # ci.yaml and .golangci.yml map to their own jobs (step `jobs`, OD-3). + - '.github/actions/**' +``` +old: +```yaml + - 'test/e2e/buckets.sh' + # golangci-lint walks up from `working-directory` to find this + # config, so every `lint (…)` leg picks up an edit here even + # though none of them declare it as an input. Force the lint + # job on so a config change is tested against real code + # instead of silently taking effect on the next unrelated PR. + - '.golangci.yml' + github: +``` +new: +```yaml + - 'test/e2e/buckets.sh' + - 'test/e2e/suite-components.json' + github: +``` + +2d. combine. old: +```yaml + CI: ${{ steps.filter.outputs.ci == 'true' || steps.scope.outputs.all == 'true' || steps.e2e.outputs.all == 'true' }} +``` +new: +```yaml + CI: ${{ steps.filter.outputs.ci == 'true' || steps.scope.outputs.all == 'true' || steps.e2e.outputs.all == 'true' || steps.jobs.outputs.all == 'true' }} +``` + +2e. Summary env. old: +```yaml + UNITS_OUTPUTS: ${{ toJSON(steps.units.outputs) }} +``` +new: +```yaml + UNITS_OUTPUTS: ${{ toJSON(steps.units.outputs) }} + CI_JOBS: ${{ steps.jobs.outputs.ci-jobs }} +``` + +2f. Job gates (T045): run this script once from the repo root. It is a +deterministic rewrite: one `if:` line per listed job. +```sh +python3 - <<'EOF' +import re +from pathlib import Path +path = Path(".github/workflows/ci.yaml") +lines = path.read_text(encoding="utf-8").split("\n") +jobs = ["build-images", "build-images-arm64", "capture-sidecar-setcap-proof", "vuln-images", + "docs", "codegen-drift", "submodule-freshness", "vuln-go", "lint", "go", "web", + "web-e2e-mock", "helm", "standalone-compose", "chart-template", "workflow-lint", + "go-e2e-unit", "api-postgres", "e2e-buckets", "e2e-go", "e2e-go-arm64", + "e2e-multicluster", "e2e-multicluster-arm64", "e2e-upgrade", "e2e-upgrade-arm64", + "e2e-web-live", "e2e-web-live-arm64", "e2e-game-bot", "e2e-postgres", + "e2e-kind-ingress-smoke"] +done, job, in_jobs = set(), None, False +for i, line in enumerate(lines): + if re.match(r"^[A-Za-z_][\w-]*:", line): + in_jobs, job = line.startswith("jobs:"), None + elif in_jobs and re.match(r"^ ([A-Za-z0-9_-]+):\s*$", line): + job = line.strip()[:-1] + elif job in jobs and job not in done and line.startswith(" if: "): + expr = line[len(" if: "):] + assert not expr.startswith(">"), job + lines[i] = (f" if: ({expr}) || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), " + f"'{job}')") + done.add(job) +missing = set(jobs) - done +assert not missing, missing +path.write_text("\n".join(lines), encoding="utf-8") +print("gated", len(done)) +EOF +``` +Expected output: `gated 30`. + +2g. `e2e-capture-overhead` (folded `if: >-`, edited by hand). old: +```yaml + (github.event_name == 'workflow_dispatch' && needs.changes.outputs.e2e == 'true') + runs-on: ubuntu-latest + timeout-minutes: 45 +``` +new: +```yaml + (github.event_name == 'workflow_dispatch' && needs.changes.outputs.e2e == 'true') || + contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-capture-overhead') + runs-on: ubuntu-latest + timeout-minutes: 45 +``` + +## 3. Specs + +3a. `specs/023-change-based-test-selection/contracts/ci-scope-cli.md`. old: +```markdown +| `ci-jobs` | JSON list of job ids selected by the `ci.yaml` line mapper | Folded into each job's gate | +``` +new: +```markdown +| `ci-jobs` | JSON list of job ids selected by the `ci.yaml` line mapper | Emitted by mode `jobs`. Folded into each job's gate | +``` +Insert before the line `## Mode \`bots\` — changed`: +```markdown +## Mode `jobs` (new) + +Maps shared CI file changes to `ci.yaml` job ids (research R8, settled OD-3). The `jobs` +step of the `changes` job runs right after `units`. Outputs: `all` (boolean), +`reason` (`selection-rule change` when a changed line sits outside any job or inside +`changes` or `report`, else empty), and `ci-jobs`. + +`ci-jobs` lists every job whose block holds a changed line that is neither blank nor a +comment, plus every job those jobs need. `.golangci.yml` adds `lint`, and any `.github/` +or `.golangci.yml` change adds `workflow-lint`. Lines are read from `git diff -U0`: +removed lines map against the base file and added lines against the head file. Each +listed job's gate is ORed with `contains(fromJSON(needs.changes.outputs.ci-jobs), '')`. +Modes `modules`, `units` and `e2e` widen the job's own selection, so the job runs all of +its legs, packages or tests. Still forcing everything (FR-014): `hack/ci_scope.py`, +`hack/test_ci_scope.py`, `hack/check_changed_coverage.py` and its test, `Makefile`, +`go.work`, `go.work.sum`, `.github/actions/**`, `test/e2e/buckets.sh`, +`test/e2e/suite-components.json`. + +``` + +3b. `specs/023-change-based-test-selection/tasks.md`: change `- [ ] T042` … `- [ ] T046` +(five lines) to `- [X]`. + +## 4. Tests: `hack/test_ci_scope.py` + +### 4a. New tests (no existing test changes) + +Import: old +```python + verify_suites, go_packages, web_tests, +``` +new +```python + verify_suites, go_packages, web_tests, CI_FILE, ci_blocks, ci_jobs, + widen_e2e, widen_units, +``` + +Insert right before `if __name__ == "__main__":`: + +```python +GIT = ["git", "-c", "user.name=test", "-c", "user.email=test@example.com", + "-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null"] +CI_FIXTURE = """name: ci +on: + push: +permissions: + contents: read +concurrency: + group: ci +env: + A: "1" +jobs: + changes: + runs-on: ubuntu-latest + steps: + - run: echo changes + web: + needs: [changes] + steps: + - run: echo web + build-images: + needs: + - changes + steps: + - run: echo build + e2e-go: + needs: [changes, build-images] + steps: + - run: echo e2e + report: + needs: + [ + changes, + web, + ] + steps: + - run: echo report +""" + + +class CiJobsTests(unittest.TestCase): + """ci.yaml lines map to the jobs they sit in (spec 023 T042, research R8, OD-3).""" + + def setUp(self): + """Create a git repository holding the fixture workflow.""" + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.git("init", "-q") + write_file(self.root, CI_FILE, CI_FIXTURE) + self.base = self.commit() + + def git(self, *args): + """Run git in the fixture repository and return its trimmed output.""" + return subprocess.run([*GIT, *args], cwd=self.root, check=True, capture_output=True, + text=True).stdout.strip() + + def commit(self): + """Commit every change and return the new commit id.""" + self.git("add", "-A") + self.git("commit", "-q", "--allow-empty", "-m", "change") + return self.git("rev-parse", "HEAD") + + def edit(self, old, new): + """Apply one edit to the base workflow and return ci_jobs for it.""" + self.git("reset", "-q", "--hard", self.base) + text = (self.root / CI_FILE).read_text(encoding="utf-8") + self.assertIn(old, text) + write_file(self.root, CI_FILE, text.replace(old, new, 1)) + return ci_jobs(self.root, self.base, self.commit(), [CI_FILE]) + + def test_needs_are_parsed_in_every_form(self): + self.assertEqual(ci_blocks(CI_FIXTURE)[1], { + "changes": set(), "web": {"changes"}, "build-images": {"changes"}, + "e2e-go": {"changes", "build-images"}, "report": {"changes", "web"}}) + + def test_comment_or_blank_line_selects_only_workflow_lint(self): + for new in (" # note\n - run: echo web\n", "\n - run: echo web\n"): + with self.subTest(new=new): + self.assertEqual(self.edit(" - run: echo web\n", new), + (["workflow-lint"], False)) + + def test_step_change_selects_its_job(self): + self.assertEqual(self.edit("echo web\n", "echo web2\n"), (["web", "workflow-lint"], False)) + + def test_selected_job_brings_the_jobs_it_needs(self): + self.assertEqual(self.edit("echo e2e\n", "echo e2e2\n"), + (["build-images", "e2e-go", "workflow-lint"], False)) + + def test_removed_line_maps_against_the_base_file(self): + self.assertEqual(self.edit(" - run: echo build\n", ""), + (["build-images", "workflow-lint"], False)) + + def test_top_level_and_selection_job_edits_force_full_scope(self): + for old, new in ((" push:\n", " push:\n workflow_dispatch:\n"), + ("contents: read", "contents: write"), ("group: ci", "group: ci2"), + ('A: "1"', 'A: "2"'), ("echo changes", "echo changes2"), + ("echo report", "echo report2")): + with self.subTest(old=old): + self.assertTrue(self.edit(old, new)[1]) + + def test_golangci_selects_lint_legs_only(self): + self.assertEqual(ci_jobs(self.root, self.base, self.base, [".golangci.yml"]), + (["lint", "workflow-lint"], False)) + self.assertIsNone(full_scope_reason({}, [".golangci.yml"])) + self.assertIsNone(full_scope_reason({}, [CI_FILE])) + + def test_other_workflow_files_select_workflow_lint_only(self): + for path in (".github/workflows/publish-edge.yaml", ".github/zizmor.yml", + ".github/dependabot.yml"): + with self.subTest(path=path): + self.assertEqual(ci_jobs(self.root, self.base, self.base, [path]), + (["workflow-lint"], False)) + + def test_selection_inputs_still_force_full_scope(self): + for path in ("hack/ci_scope.py", "hack/test_ci_scope.py", "Makefile", "go.work", + "go.work.sum", ".github/actions/go-cache/action.yml", + "test/e2e/buckets.sh", "test/e2e/suite-components.json", + "hack/check_changed_coverage.py"): + with self.subTest(path=path): + self.assertEqual(full_scope_reason({}, [path]), "selection-rule change") + + +class CiJobsModuleTests(unittest.TestCase): + """Shared CI config no longer selects every Go module (spec 023 T044).""" + + def setUp(self): + """Create the shared Go workspace fixture for each test.""" + make_go_workspace(self) + + def test_ci_yaml_and_golangci_select_no_module(self): + for path in (CI_FILE, ".golangci.yml", ".github/workflows/publish-edge.yaml"): + with self.subTest(path=path): + self.assertEqual(affected_modules(self.root, [path]), []) + + +class WidenTests(unittest.TestCase): + """An edited job widens its own selection (spec 023 T045).""" + + def test_widen_e2e(self): + values = {"e2e-run": {"operator": "^(TestA)$"}, "e2e-upgrade": False, + "e2e-web-live": False, "bot-matrix": {"test": []}, "bots-any": False} + full = {"e2e-run": {"operator": "^(TestA|TestB)$", "api-auth": "^(TestC)$"}, + "bot-matrix": {"test": ["TestBot"]}, "bots-any": True} + widened = widen_e2e(dict(values), full, ["e2e-go-arm64", "e2e-upgrade", "e2e-game-bot"]) + self.assertEqual(widened["e2e-run"], full["e2e-run"]) + self.assertTrue(widened["e2e-upgrade"]) + self.assertFalse(widened["e2e-web-live"]) + self.assertEqual((widened["bot-matrix"], widened["bots-any"]), ({"test": ["TestBot"]}, True)) + self.assertEqual(widen_e2e(dict(values), full, ["web"]), values) + + def test_widen_units(self): + values = {"go-packages": {"api": ["./internal/a"], "lib": []}, "web-tests": []} + self.assertEqual(widen_units(dict(values), ["go", "web"]), + {"go-packages": {"api": ["./..."], "lib": ["./..."]}, "web-tests": "all"}) + self.assertEqual(widen_units(dict(values), ["helm"]), values) + + +class EditedJobSummaryTests(unittest.TestCase): + """The summary marks rows run by an edited ci.yaml job (spec 023 T045).""" + + def setUp(self): + """Create the shared Go workspace fixture for each test.""" + make_go_workspace(self) + + def test_edited_job_row_runs(self): + env = { + "GITHUB_EVENT_NAME": "pull_request", + "SCOPE_OUTPUTS": json.dumps({}), "FILTER_OUTPUTS": json.dumps({}), + "GATE_OUTPUTS": json.dumps({}), "BOT_MATRIX": json.dumps({"test": []}), + "CI_JOBS": json.dumps(["web", "workflow-lint"]), + } + with patch("ci_scope.ROOT", self.root): + rows, _ = summarize(env, [CI_FILE], None) + rows = {row["suite"]: row for row in rows} + self.assertEqual(rows["web"]["decision"], "run") + self.assertEqual(rows["web"]["reason"], "ci.yaml: lines in this job changed") + self.assertEqual(rows["workflow lint"]["decision"], "run") + self.assertEqual(rows["web e2e (mock)"]["decision"], "skip") +``` + +### 4b. Existing assertion change: apply ONLY after valgul's sign-off (card, 2026-10-22) + +In `test_shared_inputs_and_dependency_changes_select_every_module`, old: +```python + "Makefile", ".golangci.yml", ".github/workflows/ci.yaml", +``` +new: +```python + "Makefile", +``` + +## 5. Verification (compile only, rule 8) + +```sh +python3 -c "import ast; [ast.parse(open(f).read()) for f in ('hack/ci_scope.py', 'hack/test_ci_scope.py')]" +python3 -c "import yaml; yaml.safe_load(open('.github/workflows/ci.yaml'))" +``` diff --git a/specs/023-change-based-test-selection/briefs/docs.md b/specs/023-change-based-test-selection/briefs/docs.md new file mode 100644 index 000000000..b1eeaafa0 --- /dev/null +++ b/specs/023-change-based-test-selection/briefs/docs.md @@ -0,0 +1,68 @@ +# Brief: PR 5, docs (T048–T050) + +Branch `claude/project-thread-63yei6-pr5`, stacked on PR 4. Apply exactly; touch nothing else. + +## 1. `docs/contributing.md`: new subsection + +Insert right before the line `## Testing` (it follows the `### Security and drift gates` +list), keeping one blank line before and after: + +```markdown +### How CI picks tests + +On a pull request, CI runs only the work your change can affect (spec 023). The +`detect changes` job decides, and its job summary has a **Test selection** table that +lists every suite as `run` or `skip`, with the changed files that chose it or the reason +everything ran. + +What a partial run narrows: + +- **Go unit tests** run only the packages that own a changed file, plus every package + that imports them, across modules. A module's own `go.mod`, `go.sum` or + `.testcoverage.yml` reruns that whole module. The module coverage gate then becomes a + changed-lines gate: the lines you add must meet the module's own minimum + (`hack/check_changed_coverage.py`). +- **Web unit tests** run `vitest related` on the changed `web/src` files, with the same + changed-lines gate at the Vitest `lines` threshold. Lint and build always run whole. + Shared inputs (`package.json`, the lockfile, the Vite/Vitest/TS/ESLint configs, + `src/test/**`, `i18n/locales/**`) run the full suite. +- **Go e2e tests** run only the tests whose components changed, per + `test/e2e/suite-components.json`. Jobs with no selected test are skipped. +- **`ci.yaml` edits** run only the jobs whose lines changed, plus the jobs they need. + `.golangci.yml` runs only the lint legs. Other workflow files run only `workflow lint`. + +Everything still runs on every push to `master`, and when a selection input changes +(`hack/ci_scope.py`, `hack/check_changed_coverage.py`, their tests, `Makefile`, +`go.work`, `go.work.sum`, `.github/actions/**`, `test/e2e/buckets.sh`, +`test/e2e/suite-components.json`). It also runs everything for a `ci.yaml` edit outside +any job or in `changes`/`report`, a path no rule maps, and any selection error. + +To force a full run on your PR, add the label `ci: full-run` (read on the next push), or +start the `ci` workflow by hand on your branch (`workflow_dispatch`). + +When you add an e2e test or a component, keep `test/e2e/suite-components.json` in step: +add the test name pattern under `suites` with the components and paths it exercises, or +list the component under `unexercised` with a reason. The `e2e bucket coverage` job +(`ci_scope.py verify-suites`) fails when a registered test matches no entry or a +component is missing. The full rules are in +`specs/023-change-based-test-selection/contracts/suite-components.md`. +``` + +## 2. `docs/agent-architecture.md`: task map row (T049) + +old: +```markdown +| Translations, language packs, localized server or module messages (i18n) | +``` +Keep that whole row unchanged and add this row directly below it: +```markdown +| CI test selection: which jobs, packages and tests a PR runs (`hack/ci_scope.py`, `hack/check_changed_coverage.py`, `test/e2e/suite-components.json`, the `changes` job in `.github/workflows/ci.yaml`) | `docs/contributing.md` ("How CI picks tests"), `specs/023-change-based-test-selection/contracts/` (`ci-scope-cli.md`, `suite-components.md`, `changed-coverage.md`) | +``` + +## 3. `specs/023-change-based-test-selection/tasks.md` + +Change `- [ ] T048`, `- [ ] T049` and `- [ ] T050` to `- [X]`. + +## 4. Checks + +Only check that the links and paths named above exist (`ls`). No link checker, no lint. diff --git a/specs/023-change-based-test-selection/contracts/ci-scope-cli.md b/specs/023-change-based-test-selection/contracts/ci-scope-cli.md index 213621206..5c4ed8766 100644 --- a/specs/023-change-based-test-selection/contracts/ci-scope-cli.md +++ b/specs/023-change-based-test-selection/contracts/ci-scope-cli.md @@ -26,7 +26,7 @@ path forces everything (research R8). | `e2e-exclude` | Not emitted (decision 2). Unselected legs skip through `RUN_REGEX` instead | Would have been read by `e2e-go` and `e2e-go-arm64` | | `e2e-run` | JSON object: bucket → anchored `-run` regex of the selected tests | Emitted by mode `e2e`. Replaces `buckets.sh regex ` in the e2e jobs (OD-6); equals it on full scope | | `e2e-multicluster`, `e2e-upgrade`, `e2e-web-live`, `e2e-capture-overhead` | `"true"`/`"false"` | Emitted by mode `e2e`. Folded into the existing `combine` step booleans | -| `ci-jobs` | JSON list of job ids selected by the `ci.yaml` line mapper | Folded into each job's gate | +| `ci-jobs` | JSON list of job ids selected by the `ci.yaml` line mapper | Emitted by mode `jobs`. Folded into each job's gate | | `unmapped` | JSON list of paths | Emitted by mode `e2e`. Non-empty forces full scope and fails `selection-rules` | | `selection` | JSON list of SelectionResult | Emitted by mode `summary`; read by the summary table and `report` | @@ -92,6 +92,24 @@ gives `["./..."]`. For web, `package.json`, `package-lock.json`, `vitest.config. `vite.config.ts`, `tsconfig*.json`, `eslint.config.js`, `src/test/**`, `i18n/locales/**`, a deleted `web/src` file and any other `web/` file outside `src` and `e2e` give `"all"`. +## Mode `jobs` (new) + +Maps shared CI file changes to `ci.yaml` job ids (research R8, settled OD-3). The `jobs` +step of the `changes` job runs right after `units`. Outputs: `all` (boolean), +`reason` (`selection-rule change` when a changed line sits outside any job or inside +`changes` or `report`, else empty), and `ci-jobs`. + +`ci-jobs` lists every job whose block holds a changed line that is neither blank nor a +comment, plus every job those jobs need. `.golangci.yml` adds `lint`, and any `.github/` +or `.golangci.yml` change adds `workflow-lint`. Lines are read from `git diff -U0`: +removed lines map against the base file and added lines against the head file. Each +listed job's gate is ORed with `contains(fromJSON(needs.changes.outputs.ci-jobs), '')`. +Modes `modules`, `units` and `e2e` widen the job's own selection, so the job runs all of +its legs, packages or tests. Still forcing everything (FR-014): `hack/ci_scope.py`, +`hack/test_ci_scope.py`, `hack/check_changed_coverage.py` and its test, `Makefile`, +`go.work`, `go.work.sum`, `.github/actions/**`, `test/e2e/buckets.sh`, +`test/e2e/suite-components.json`. + ## Mode `bots` — changed Emits only the selected `bot-fast` tests. When none is selected it emits an empty diff --git a/specs/023-change-based-test-selection/tasks.md b/specs/023-change-based-test-selection/tasks.md index 66b409742..f741b15f8 100644 --- a/specs/023-change-based-test-selection/tasks.md +++ b/specs/023-change-based-test-selection/tasks.md @@ -141,14 +141,14 @@ Repository root of `GameplanePanel/Gameplane`. Main files: `hack/ci_scope.py`, ` ### Tests for User Story 3 -- [ ] T042 [P] [US3] Add `hack/test_ci_scope.py` cases: comment-only `ci.yaml` change selects only `workflow-lint`; a step change inside job `web` selects `web` and `workflow-lint`; a change in `on:`, `env:`, `concurrency:`, `permissions:`, or inside the `changes` or `report` job forces full scope with reason `selection-rule change`; `.golangci.yml` selects only `lint` legs; `hack/ci_scope.py`, `Makefile`, `go.work`, `go.work.sum`, `.github/actions/**` still force full scope +- [X] T042 [P] [US3] Add `hack/test_ci_scope.py` cases: comment-only `ci.yaml` change selects only `workflow-lint`; a step change inside job `web` selects `web` and `workflow-lint`; a change in `on:`, `env:`, `concurrency:`, `permissions:`, or inside the `changes` or `report` job forces full scope with reason `selection-rule change`; `.golangci.yml` selects only `lint` legs; `hack/ci_scope.py`, `Makefile`, `go.work`, `go.work.sum`, `.github/actions/**` still force full scope ### Implementation for User Story 3 -- [ ] T043 [US3] In `hack/ci_scope.py`, add the `ci.yaml` line mapper from research R8 (job blocks by two-space keys under `jobs:`, blank/comment lines ignored) using `git diff -U0` line numbers, and emit `ci-jobs` -- [ ] T044 [US3] In `hack/ci_scope.py`, narrow the shared-file list per OD-3: remove `.github/` and `.golangci.yml` from the force-all set, map them per research R8, keep `hack/ci_scope*`, `hack/check_changed_coverage.py`, `Makefile`, `go.work*`, `.github/actions/**`, `test/e2e/buckets.sh`, `test/e2e/suite-components.json` as force-all -- [ ] T045 [US3] In `.github/workflows/ci.yaml`, OR each job's existing gate with `contains(fromJSON(needs.changes.outputs.ci-jobs), '')`, keeping `workflow-lint` selected for any `.github/` change -- [ ] T046 [US3] Compile checks only; open PR 4 with labels `type: ci`, `area: shared` +- [X] T043 [US3] In `hack/ci_scope.py`, add the `ci.yaml` line mapper from research R8 (job blocks by two-space keys under `jobs:`, blank/comment lines ignored) using `git diff -U0` line numbers, and emit `ci-jobs` +- [X] T044 [US3] In `hack/ci_scope.py`, narrow the shared-file list per OD-3: remove `.github/` and `.golangci.yml` from the force-all set, map them per research R8, keep `hack/ci_scope*`, `hack/check_changed_coverage.py`, `Makefile`, `go.work*`, `.github/actions/**`, `test/e2e/buckets.sh`, `test/e2e/suite-components.json` as force-all +- [X] T045 [US3] In `.github/workflows/ci.yaml`, OR each job's existing gate with `contains(fromJSON(needs.changes.outputs.ci-jobs), '')`, keeping `workflow-lint` selected for any `.github/` change +- [X] T046 [US3] Compile checks only; open PR 4 with labels `type: ci`, `area: shared` - [ ] T047 [US3] Run quickstart scenarios 9–11 as draft PRs on PR 4's branch, record runner-minutes for scenario 9 (SC-003) in `validation.md`, close them and delete their branches **Checkpoint**: all four stories work independently. @@ -157,9 +157,9 @@ Repository root of `GameplanePanel/Gameplane`. Main files: `hack/ci_scope.py`, ` ## Phase 7: Polish & Docs — PR 5 -- [ ] T048 [P] Add a "How CI picks tests" section to `docs/contributing.md`: what a partial run skips, the `ci: full-run` label and manual run, how to add a `suite-components.json` entry, where to read the run summary -- [ ] T049 [P] Add the new selection files to the task map in `docs/agent-architecture.md` -- [ ] T050 Compile and link checks only; open PR 5 with labels `type: docs`, `area: shared` +- [X] T048 [P] Add a "How CI picks tests" section to `docs/contributing.md`: what a partial run skips, the `ci: full-run` label and manual run, how to add a `suite-components.json` entry, where to read the run summary +- [X] T049 [P] Add the new selection files to the task map in `docs/agent-architecture.md` +- [X] T050 Compile and link checks only; open PR 5 with labels `type: docs`, `area: shared` - [ ] T051 After the first `master` push following PR 5's merge, confirm every job ran (FR-006, SC-006) and record it in `validation.md` - [ ] T052 After 30 merged PRs, measure median runner-minutes against the 30 before PR 2 via the Actions API and record SC-002 in `validation.md`; for 60 days review each red `master` run for a selection miss (SC-004)