diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 68ae1c2e4..a98c83fe1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -107,6 +107,14 @@ jobs: scope: ${{ steps.summary.outputs.scope }} scope-reason: ${{ steps.summary.outputs.reason }} selection: ${{ steps.summary.outputs.selection }} + e2e-run: ${{ steps.e2e.outputs.e2e-run }} + e2e-unmapped: ${{ steps.e2e.outputs.unmapped }} + e2e-multicluster: ${{ steps.e2e.outputs.e2e-multicluster }} + e2e-upgrade: ${{ steps.e2e.outputs.e2e-upgrade }} + e2e-web-live: ${{ steps.e2e.outputs.e2e-web-live }} + e2e-capture-overhead: ${{ steps.e2e.outputs.e2e-capture-overhead }} + e2e-bot-matrix: ${{ steps.e2e.outputs.bot-matrix }} + e2e-bots-any: ${{ steps.e2e.outputs.bots-any }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -122,6 +130,15 @@ jobs: - name: partition every bot-fast test onto its own runner id: bots run: python3 hack/ci_scope.py bots + - name: select the e2e tests the change affects + id: e2e + # Per-test e2e selection from test/e2e/suite-components.json (spec 023 US1). + # An unmapped path sets `all`, which `combine` folds into CI below. + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + HEAD_SHA: ${{ github.sha }} + CI_FULL_RUN: "${{ contains(github.event.pull_request.labels.*.name, 'ci: full-run') }}" + run: python3 hack/ci_scope.py e2e --base "$BASE_SHA" --head "$HEAD_SHA" - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 id: filter with: @@ -333,7 +350,8 @@ jobs: STANDALONE: ${{ steps.filter.outputs.standalone }} E2ETREE: ${{ steps.filter.outputs.e2etree }} GAMECOVERAGE: ${{ steps.filter.outputs.gamecoverage }} - CI: ${{ steps.filter.outputs.ci == 'true' || steps.scope.outputs.all == 'true' }} + UNIVERSAL: ${{ steps.e2e.outputs.universal }} + CI: ${{ steps.filter.outputs.ci == 'true' || steps.scope.outputs.all == 'true' || steps.e2e.outputs.all == 'true' }} CAPTURE: ${{ steps.filter.outputs.capture_overhead }} SUBMODULES: ${{ steps.filter.outputs.submodules }} CODEGEN: ${{ steps.filter.outputs.codegen }} @@ -346,7 +364,7 @@ jobs: webv=false; { [ "$WEB" = true ] || [ "$CI" = true ]; } && webv=true chv=false; { [ "$CHARTS" = true ] || [ "$CI" = true ]; } && chv=true standalonev=false; { [ "$STANDALONE" = true ] || [ "$CI" = true ]; } && standalonev=true - e2ev=false; { [ "$GO" = true ] || [ "$CHARTS" = true ] || [ "$E2ETREE" = true ] || [ "$GAMECOVERAGE" = true ] || [ "$CI" = true ]; } && e2ev=true + e2ev=false; { [ "$GO" = true ] || [ "$CHARTS" = true ] || [ "$E2ETREE" = true ] || [ "$GAMECOVERAGE" = true ] || [ "$UNIVERSAL" = true ] || [ "$CI" = true ]; } && e2ev=true wlv=false; { [ "$e2ev" = true ] || [ "$WEB" = true ] || [ "$CI" = true ]; } && wlv=true ghv=false; { [ "$GITHUB" = true ] || [ "$CI" = true ]; } && ghv=true specsv=false; { [ "$SPECS" = true ] || [ "$CI" = true ]; } && specsv=true @@ -387,6 +405,7 @@ jobs: FILTER_OUTPUTS: ${{ toJSON(steps.filter.outputs) }} GATE_OUTPUTS: ${{ toJSON(steps.combine.outputs) }} BOT_MATRIX: ${{ steps.bots.outputs.matrix }} + E2E_OUTPUTS: ${{ toJSON(steps.e2e.outputs) }} run: python3 hack/ci_scope.py summary --base "$BASE_SHA" --head "$HEAD_SHA" # Build the operator/api/agent (+ gameprobe) images ONCE and publish them as @@ -1661,6 +1680,18 @@ jobs: submodules: true - name: verify buckets are disjoint + exhaustive run: ./test/e2e/buckets.sh verify + - name: verify every e2e test declares what it exercises + run: python3 hack/ci_scope.py verify-suites + - name: selection rules map every changed path + # An unmapped path already forced the full suite; this makes the gap visible + # so a rule gets added to hack/ci_scope.py (spec 023 quickstart scenario 12). + env: + UNMAPPED: ${{ needs.changes.outputs.e2e-unmapped }} + run: | + if [ -n "$UNMAPPED" ] && [ "$UNMAPPED" != "[]" ]; then + echo "::error::no selection rule maps these paths: $UNMAPPED" + exit 1 + fi - name: verify game protocol coverage run: ./test/e2e/joincoverage.sh verify # The fixtures are what prove the gate above can actually fail. Without @@ -1729,25 +1760,34 @@ jobs: env: CLUSTER: gameplane-e2e CTX: kind-gameplane-e2e + # Only this bucket's selected tests (spec 023 OD-6); empty skips the leg. + RUN_REGEX: ${{ fromJSON(needs.changes.outputs.e2e-run)[matrix.bucket] || '' }} + TAIL_REGEX: ${{ matrix.tail == 'ratelimit' && fromJSON(needs.changes.outputs.e2e-run).ratelimit || '' }} steps: &e2e-go-steps - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' - uses: ./.github/actions/go-cache + if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' with: key-suffix: e2e - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' - name: install kind + if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' uses: helm/kind-action@1544676c07bb3570fcb70166beaa07c4528fd4aa # v1.15.1 with: install_only: true - name: load images + if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' uses: ./.github/actions/e2e-images with: arch: ${{ matrix.arch }} - name: bring up cluster + helm install + if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e - name: run go e2e (${{ matrix.bucket }}) + if: env.RUN_REGEX != '' run: | - RUN_REGEX="$(./test/e2e/buckets.sh regex '${{ matrix.bucket }}')" cd test/e2e && \ GAMEPLANE_E2E_REUSE_CLUSTER=1 \ GAMEPLANE_E2E_CLUSTER="$CLUSTER" \ @@ -1757,13 +1797,12 @@ jobs: # LAST on purpose: this test deliberately drains the shared per-IP # and per-username login buckets; anything running after it queues # behind the refill. - if: matrix.tail == 'ratelimit' + if: env.TAIL_REGEX != '' run: | - RUN_REGEX="$(./test/e2e/buckets.sh regex ratelimit)" cd test/e2e && \ GAMEPLANE_E2E_REUSE_CLUSTER=1 \ GAMEPLANE_E2E_CLUSTER="$CLUSTER" \ - go test -tags=e2e -timeout 5m -v -run "$RUN_REGEX" ./... + go test -tags=e2e -timeout 5m -v -run "$TAIL_REGEX" ./... - name: dump cluster state on failure (e2e-go) if: failure() uses: ./.github/actions/dump-cluster-state @@ -1828,6 +1867,9 @@ jobs: env: CLUSTER: gameplane-e2e CTX: kind-gameplane-e2e + # Same selection as e2e-go, so both arches always run the same tests (FR-010). + RUN_REGEX: ${{ fromJSON(needs.changes.outputs.e2e-run)[matrix.bucket] || '' }} + TAIL_REGEX: ${{ matrix.tail == 'ratelimit' && fromJSON(needs.changes.outputs.e2e-run).ratelimit || '' }} steps: *e2e-go-steps # Dual-cluster ?cluster= dispatch + scoped RBAC (test/e2e/multicluster_e2e_test.go). @@ -1838,7 +1880,7 @@ jobs: e2e-multicluster: name: e2e multicluster / ${{ matrix.arch }} (kind x2) needs: [changes, build-images] - if: needs.changes.outputs.e2e == 'true' + if: needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-multicluster == 'true' runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 60 # Same, across two clusters. permissions: @@ -1903,7 +1945,7 @@ jobs: e2e-multicluster-arm64: name: e2e multicluster / ${{ matrix.arch }} (kind x2) needs: [changes, build-images-arm64] - if: needs.changes.outputs.e2e == 'true' + if: needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-multicluster == 'true' runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 60 # Same, across two clusters. permissions: @@ -1930,7 +1972,7 @@ jobs: e2e-upgrade: name: e2e upgrade / ${{ matrix.arch }} (kind) needs: [changes, build-images] - if: needs.changes.outputs.e2e == 'true' + if: needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-upgrade == 'true' runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 60 # Same, plus a chart upgrade across versions. permissions: @@ -1980,7 +2022,7 @@ jobs: e2e-upgrade-arm64: name: e2e upgrade / ${{ matrix.arch }} (kind) needs: [changes, build-images-arm64] - if: needs.changes.outputs.e2e == 'true' + if: needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-upgrade == 'true' runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 60 # Same, plus a chart upgrade across versions. permissions: @@ -2000,7 +2042,7 @@ jobs: e2e-web-live: name: e2e web live / ${{ matrix.arch }} (kind) needs: [changes, build-images] - if: needs.changes.outputs.weblive == 'true' + if: needs.changes.outputs.weblive == 'true' && needs.changes.outputs.e2e-web-live == 'true' runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 60 # Same, plus a live dashboard drive. permissions: @@ -2080,7 +2122,7 @@ jobs: e2e-web-live-arm64: name: e2e web live / ${{ matrix.arch }} (kind) needs: [changes, build-images-arm64] - if: needs.changes.outputs.weblive == 'true' + if: needs.changes.outputs.weblive == 'true' && needs.changes.outputs.e2e-web-live == 'true' runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 60 # Same, plus a live dashboard drive. permissions: @@ -2097,7 +2139,7 @@ jobs: e2e-game-bot: name: e2e game bot / ${{ matrix.test }} (kind) needs: [changes, build-images] - if: needs.changes.outputs.e2e == 'true' + if: needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-bots-any == 'true' # amd64 only: this boots the external itzg/minecraft-server (multi-arch) AND # passivelemon/terraria-docker, whose arm64 support is unverified upstream — # so an arm64 leg would fail on the Terraria pull. Add an arch axis here once @@ -2110,7 +2152,7 @@ jobs: contents: read strategy: fail-fast: false - matrix: ${{ fromJSON(needs.changes.outputs.bot-matrix) }} + matrix: ${{ fromJSON(needs.changes.outputs.e2e-bot-matrix) }} env: BOT_TEST: ${{ matrix.test }} CLUSTER: gameplane-e2e @@ -2164,7 +2206,7 @@ jobs: needs: [changes, build-images] if: >- (github.event_name == 'push' && github.ref == 'refs/heads/master' && needs.changes.outputs.e2e == 'true') || - (github.event_name == 'pull_request' && needs.changes.outputs.capture_overhead == 'true') || + (github.event_name == 'pull_request' && needs.changes.outputs.capture_overhead == 'true' && needs.changes.outputs.e2e-capture-overhead == 'true') || (github.event_name == 'workflow_dispatch' && needs.changes.outputs.e2e == 'true') runs-on: ubuntu-latest timeout-minutes: 45 diff --git a/hack/ci_scope.py b/hack/ci_scope.py index cdbc87e22..eb4deba2c 100644 --- a/hack/ci_scope.py +++ b/hack/ci_scope.py @@ -29,6 +29,32 @@ # Mirrors the e2e-go matrix buckets in .github/workflows/ci.yaml. E2E_BUCKETS = ["operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", "telemetry", "optional-components"] +SUITES_FILE = "test/e2e/suite-components.json" +# Paths no Go e2e test reads. Their own jobs keep their paths-filter gates. +NO_E2E_FILES = { + "design.pen", "LICENSE", ".claudeignore", ".coderabbit.yaml", ".editorconfig", + ".gitattributes", ".gitignore", ".gitmodules", ".hadolint.yaml", + "deploy/kind/up.sh", "deploy/kind/down.sh", + "test/e2e/joincoverage.sh", "test/e2e/joincoverage_test.sh", + "test/e2e/lint-gate-verify.sh", "test/e2e/lint-gate-verify_test.sh", +} +NO_E2E_DIRS = ("docs", "specs", "assets", "website", "hack", ".github", ".claude", ".devcontainer", + ".githooks", ".jules", ".specify", "deploy/standalone", "test/e2e/testdata") +# The dashboard live run drives these components through a real cluster. +WEB_LIVE_COMPONENTS = {"web", "api", "operator", "agent"} +# Component ids beside the go.work modules (data-model.md Component). +EXTRA_COMPONENTS = ("web", "charts", "deploy-kind", "modules", "e2e-harness") +# Top-level areas a selection rule knows; any other path is unmapped and forces full scope. +KNOWN_DIRS = ("charts", "containers", "deploy/kind", "signing", "test/e2e", "web", "modules") + + +def is_unmapped(path, modules): + """Return True when no selection rule knows path; such a path forces full scope.""" + if (path.endswith(".md") or path in NO_E2E_FILES or path in FORCE_ALL_FILES + or path == ".dockerignore"): + return False + return not any(path == d or path.startswith(d + "/") + for d in NO_E2E_DIRS + KNOWN_DIRS + tuple(modules)) def workspace_modules(root): @@ -235,12 +261,265 @@ def table_cell(value): return str(value).replace("\n", " ").replace("|", "\\|") +def decoded(values, key, default=None): + """Return a step output decoded from the JSON that emit() wrote, or default.""" + value = values.get(key) + if value is None or value == "": + return default + if not isinstance(value, str): + return value + try: + return json.loads(value) + except ValueError: + return value + + +def under(path, base): + """Return True when path equals base or lies beneath it.""" + return path == base or path.startswith(base + "/") + + +def prefixed(path, prefix): + """Match a suite-components prefix: a trailing slash means a directory, else one path.""" + return path.startswith(prefix) if prefix.endswith("/") else path == prefix + + +def load_suites(root): + """Return the parsed test/e2e/suite-components.json.""" + return json.loads((root / SUITES_FILE).read_text(encoding="utf-8")) + + +def bucket_lists(root): + """Return {bucket: [test, ...]} from test/e2e/buckets.sh, in its listed order.""" + def run(*args): + """Run buckets.sh with args and return its output words.""" + return subprocess.run(["bash", "test/e2e/buckets.sh", *args], cwd=root, check=True, + capture_output=True, text=True).stdout.split() + return {name: run("list", name) for name in run("buckets")} + + +def go_tests(text): + """Return the Test functions a Go file defines, without TestMain.""" + return [name for name in re.findall(r"^func (Test[A-Za-z0-9_]*)\(", text, re.MULTILINE) + if name != "TestMain"] + + +def helper_pattern(text): + """Return a regex matching uses of a Go file's top-level names, or None if it has none. + + Functions match as calls; types, vars, consts and the receiver types of methods + match as words, so a changed method selects the users of its type. + """ + funcs = set(re.findall(r"^func ([A-Za-z_]\w*)", text, re.MULTILINE)) + funcs -= set(go_tests(text)) | {"TestMain"} + words = set(re.findall(r"^func \(\s*\w*\s*\*?\s*([A-Za-z_]\w*)", text, re.MULTILINE)) + words |= set(re.findall(r"^(?:type|var|const)\s+([A-Za-z_]\w*)", text, re.MULTILINE)) + for block in re.findall(r"^(?:type|var|const)\s*\((.*?)^\)", text, + re.MULTILINE | re.DOTALL): + words |= set(re.findall(r"^\s+([A-Za-z_]\w*)", block, re.MULTILINE)) + parts = ([re.escape(name) + r"\(" for name in sorted(funcs)] + + [re.escape(name) + r"\b" for name in sorted(words)]) + return re.compile(r"\b(?:" + "|".join(parts) + ")") if parts else None + + +def e2e_file_tests(root, path): + """Return the tests a changed test/e2e/.go can affect, or None for every test. + + That is the file's own tests plus every test in a file that uses one of its + top-level names, following pure helper files (no tests) transitively. A + deleted file returns None. + """ + texts = {source.name: source.read_text(encoding="utf-8") + for source in sorted((root / "test/e2e").glob("*.go"))} + start = Path(path).name + if start not in texts: + return None + seen, queue, tests = {start}, [start], set() + while queue: + name = queue.pop() + own = set(go_tests(texts[name])) + tests |= own + if own and name != start: + continue + pattern = helper_pattern(texts[name]) + if pattern is None: + continue + for other, text in texts.items(): + if other not in seen and pattern.search(text): + seen.add(other) + queue.append(other) + return tests + + +def select_e2e(root, paths, reason, suites, buckets): + """Select e2e tests for a change (spec 023 US1). + + Returns (selected, unmapped, reason, touched): selected maps each chosen test + to the changed paths that chose it, reason is set when every test runs because + of the event or an unmapped path, and touched holds the changed component ids + ("*" when a universal path changed). + """ + tests = list(dict.fromkeys(name for names in buckets.values() for name in names)) + if reason is None and paths is None: + reason = "unknown base" + if reason: + return {name: [] for name in tests}, [], reason, {"*"} + modules = workspace_modules(root) + entries = suites["suites"] + picked, components, every, unmapped = {}, {}, [], [] + + def matching(entry): + """Return the tests one suite entry's pattern matches.""" + return [name for name in tests if re.search(entry["tests"], name)] + + def pick(names, path): + """Record that path selects each test in names.""" + for name in names: + picked.setdefault(name, []).append(path) + + for path in paths: + if path.endswith(".md") or path in NO_E2E_FILES: + continue + if any(prefixed(path, prefix) for prefix in suites["universal"]): + every.append(path) + continue + named = [entry for entry in entries + if any(prefixed(path, prefix) for prefix in entry.get("paths", []))] + for entry in named: + pick(matching(entry), path) + if under(path, "test/e2e"): + if path.count("/") == 2 and path.endswith(".go"): + own = e2e_file_tests(root, path) + if own is None: + every.append(path) + else: + pick([name for name in tests if name in own], path) + elif not named: + every.append(path) + continue + if named or any(under(path, directory) for directory in NO_E2E_DIRS): + continue + owner = next((module for module in modules if under(path, module)), None) + if owner: + components.setdefault(owner, []).append(path) + elif under(path, "web"): + components.setdefault("web", []).append(path) + elif under(path, "modules"): + components.setdefault("modules", []).append(path) + else: + unmapped.append(path) + if unmapped: + return {name: [] for name in tests}, unmapped, f"unmapped path {unmapped[0]}", {"*"} + if every: + return {name: list(every) for name in tests}, [], None, {"*"} + module_paths = [path for component, owned in components.items() + if component in modules or component == "modules" for path in owned] + if module_paths: + # Library changes reach the binaries that import them (research R3). + for module in affected_modules(root, module_paths): + components.setdefault(module, module_paths) + for entry in entries: + wanted = set(entry.get("components", [])) + if "*" in wanted: + hit = [path for owned in components.values() for path in owned] + else: + hit = [path for component in sorted(wanted & components.keys()) + for path in components[component]] + for path in dict.fromkeys(hit): + pick(matching(entry), path) + selected = {name: list(dict.fromkeys(picked[name])) for name in tests if name in picked} + return selected, [], None, set(components) + + +def e2e_outputs(root, paths, reason): + """Return the outputs of mode e2e: per-bucket -run regexes, job switches, bot matrix.""" + buckets = bucket_lists(root) + selected, unmapped, reason, touched = select_e2e(root, paths, reason, load_suites(root), + buckets) + + def chosen(bucket): + """Return the selected tests of one bucket, in buckets.sh order.""" + return [name for name in buckets.get(bucket, []) if name in selected] + + def describe(names, total): + """Return 'N of M tests' plus the changed paths that chose them.""" + files = list(dict.fromkeys(path for name in names for path in selected[name])) + text = f"{len(names)} of {total} tests" + return text + (f"; {cite(files)}" if files else "") + + run, why = {}, {} + for bucket in E2E_BUCKETS + ["ratelimit", "multicluster", "upgrade", "capture-overhead"]: + names = chosen(bucket) + if names: + why[bucket] = describe(names, len(buckets[bucket])) + if bucket in E2E_BUCKETS or bucket == "ratelimit": + run[bucket] = "^(" + "|".join(names) + ")$" + bots = chosen("bot-fast") + for name in bots: + why[name] = describe([name], 1) + return { + "all": bool(reason), + "reason": reason or "", + "unmapped": unmapped, + "e2e-run": run, + "e2e-why": why, + "e2e-multicluster": bool(chosen("multicluster")), + "e2e-upgrade": bool(chosen("upgrade")), + "e2e-capture-overhead": bool(chosen("capture-overhead")), + "e2e-web-live": bool(reason) or "*" in touched or bool(touched & WEB_LIVE_COMPONENTS), + "bot-matrix": {"test": bots}, + "bots-any": bool(bots), + "universal": not reason and "*" in touched, + } + + +def verify_suites(suites, buckets, modules): + """Return the violations of contracts/suite-components.md rules 1, 2 and 5.""" + errors = [] + known = set(modules) | set(EXTRA_COMPONENTS) | {"*"} + entries = suites.get("suites", []) + patterns = [] + for entry in entries: + try: + patterns.append(re.compile(entry["tests"])) + except (KeyError, TypeError, re.error) as exc: + errors.append(f"entry {entry!r}: bad tests pattern: {exc}") + continue + if not entry.get("components") and not entry.get("paths"): + errors.append(f"entry {entry['tests']}: names no component and no path") + for component in entry.get("components", []): + if component not in known: + errors.append(f"entry {entry['tests']}: unknown component {component}") + for bucket, names in buckets.items(): + for name in names: + if not any(pattern.search(name) for pattern in patterns): + errors.append(f"test {name} (bucket {bucket}) matches no entry") + named = {component for entry in entries for component in entry.get("components", [])} + named |= {item.get("component") for item in suites.get("unexercised", [])} + if any(prefix.startswith("charts/") for prefix in suites.get("universal", [])): + named.add("charts") + for component in sorted(set(modules) | {"web", "charts"}): + if component not in named: + errors.append(f"component {component} is named by no entry and not listed " + "in unexercised") + for item in suites.get("unexercised", []): + if item.get("component") not in known or not item.get("reason"): + errors.append(f"unexercised entry {item!r} needs a known component and a reason") + return errors + + def summarize(env, paths, reason): """Return (rows, markdown) describing which CI suites run and why.""" scope = load_json(env, "SCOPE_OUTPUTS") filters = load_json(env, "FILTER_OUTPUTS") gates = load_json(env, "GATE_OUTPUTS") bots = load_json(env, "BOT_MATRIX") + e2e = load_json(env, "E2E_OUTPUTS") + # Without the e2e step's outputs every e2e suite follows the coarse e2e gate. + narrowed = bool(e2e) + e2e_run = decoded(e2e, "e2e-run", {}) + e2e_why = decoded(e2e, "e2e-why", {}) + e2e_bots = decoded(e2e, "bot-matrix", {}).get("test", []) event = env.get("GITHUB_EVENT_NAME", "") modules = sorted(workspace_modules(ROOT)) selected = set(jlist(scope, "modules")) @@ -317,19 +596,31 @@ def add(suite, run, feed=(), run_reason="", skip_reason="not affected"): add("build e2e images (amd64, arm64)", gate("images"), images_feed) add("vuln (images)", gate("images"), images_feed) add("e2e bucket coverage", gate("e2e"), e2e_feed()) + def e2e_row(suite, gate_on, picked, key, feed): + """Add an e2e row from the coarse gate and, when present, the e2e step's pick.""" + if narrowed and gate_on and not picked: + add(suite, False, skip_reason="no test in this suite is affected") + else: + add(suite, gate_on, feed, run_reason=e2e_why.get(key, "") if narrowed else "") + for bucket in E2E_BUCKETS: name = "e2e api-auth + ratelimit tail" if bucket == "api-auth" else f"e2e {bucket}" - add(f"{name} (amd64, arm64)", gate("e2e"), e2e_feed()) - add("e2e multicluster (amd64, arm64)", gate("e2e"), e2e_feed()) - add("e2e upgrade (amd64, arm64)", gate("e2e"), e2e_feed()) + picked = bucket in e2e_run or (bucket == "api-auth" and "ratelimit" in e2e_run) + e2e_row(f"{name} (amd64, arm64)", gate("e2e"), picked, bucket, e2e_feed()) + e2e_row("e2e multicluster (amd64, arm64)", gate("e2e"), flag(e2e, "e2e-multicluster"), + "multicluster", e2e_feed()) + e2e_row("e2e upgrade (amd64, arm64)", gate("e2e"), flag(e2e, "e2e-upgrade"), "upgrade", + e2e_feed()) add("e2e postgres (amd64)", gate("e2e"), e2e_feed()) for test in jlist(bots, "test"): - add(f"e2e game bot / {test}", gate("e2e"), e2e_feed()) - add("e2e web live (amd64, arm64)", gate("weblive"), e2e_feed() + filtered("web")) - add("e2e capture overhead", - (event == "pull_request" and gate("capture_overhead")) - or (event in ("push", "workflow_dispatch") and gate("e2e")), - filtered("capture_overhead")) + e2e_row(f"e2e game bot / {test}", gate("e2e"), test in e2e_bots, test, e2e_feed()) + e2e_row("e2e web live (amd64, arm64)", gate("weblive"), flag(e2e, "e2e-web-live"), "", + e2e_feed() + filtered("web")) + e2e_row("e2e capture overhead", + (event == "pull_request" and gate("capture_overhead")) + or (event in ("push", "workflow_dispatch") and gate("e2e")), + event != "pull_request" or flag(e2e, "e2e-capture-overhead"), "capture-overhead", + filtered("capture_overhead")) add("ingress-nginx smoke (kind)", gate("kindsmoke"), filtered("kindsmoke")) kind = "full" if reason else "partial" @@ -359,7 +650,8 @@ def bot_matrix(names): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("mode", choices=("modules", "images", "dockerfiles", "bots", "summary")) + parser.add_argument("mode", choices=("modules", "images", "dockerfiles", "bots", "summary", + "e2e", "verify-suites")) parser.add_argument("--base", default="") parser.add_argument("--head", default="HEAD") args = parser.parse_args() @@ -382,7 +674,8 @@ def main(): env = os.environ try: paths = None if full_scope_reason(env, []) else changed_paths(ROOT, args.base, args.head) - reason = full_scope_reason(env, paths) + reason = full_scope_reason(env, paths) or decoded(load_json(env, "E2E_OUTPUTS"), + "reason") or None if reason is None and flag(load_json(env, "SCOPE_OUTPUTS"), "all"): reason = "selection fallback (see detect changes log)" rows, markdown = summarize(env, paths, reason) @@ -395,12 +688,33 @@ def main(): with open(env["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary: summary.write(markdown) return + if args.mode == "verify-suites": + errors = verify_suites(load_suites(ROOT), bucket_lists(ROOT), workspace_modules(ROOT)) + for error in errors: + print(f"::error::{error}") + if errors: + raise SystemExit(1) + print("suite components OK") + return + if args.mode == "e2e": + env = os.environ + try: + event_reason = full_scope_reason(env, []) + paths = None if event_reason else changed_paths(ROOT, args.base, args.head) + values = e2e_outputs(ROOT, paths, full_scope_reason(env, paths)) + except Exception as exc: # any doubt selects every e2e test (FR-002) + print(f"::warning::e2e selection failed, running every e2e test: {exc}") + values = e2e_outputs(ROOT, None, f"error: {exc}") + emit(values) + return # PRs use affected work; every master push, manual run and ci: full-run # label is the full-suite backstop. env = os.environ try: event_reason = full_scope_reason(env, []) paths = None if event_reason else changed_paths(ROOT, args.base, args.head) + if paths is not None and any(is_unmapped(p, workspace_modules(ROOT)) for p in paths): + paths = None # mode e2e reports the path; every module runs (FR-002) selected = affected_modules(ROOT, paths) except Exception as exc: # any doubt selects the full suite (FR-002) print(f"::warning::test selection failed, running everything: {exc}") diff --git a/hack/test_ci_scope.py b/hack/test_ci_scope.py index de23c5be6..015257971 100644 --- a/hack/test_ci_scope.py +++ b/hack/test_ci_scope.py @@ -1,14 +1,18 @@ """Behavioral coverage for CI module and image selection (run in CI).""" import json +import subprocess import tempfile import unittest from pathlib import Path from unittest.mock import patch from subprocess import CompletedProcess -from ci_scope import (E2E_BUCKETS, FORCE_ALL_FILES, affected_modules, bot_matrix, changed_paths, - full_scope_reason, image_inputs, main, publish_paths, select_images, summarize) +from ci_scope import (E2E_BUCKETS, FORCE_ALL_FILES, ROOT, affected_modules, bot_matrix, + bucket_lists, changed_paths, e2e_outputs, full_scope_reason, image_inputs, + is_unmapped, load_suites, main, publish_paths, select_images, summarize, + verify_suites, + workspace_modules) class ScopeTests(unittest.TestCase): @@ -395,5 +399,161 @@ def test_full_scope_marks_run_rows_and_header(self): self.assertEqual(skipped, {"dependency review"}) +class E2ESelectionTests(unittest.TestCase): + """Spec 023 US1 acceptance scenarios 1-4 against this repository's suite map (T014).""" + + def select(self, *paths): + """Return the e2e outputs for a partial-scope change to paths.""" + return e2e_outputs(ROOT, list(paths), None) + + def test_agent_only_change_selects_agent_tests_and_skips_auth_and_telemetry(self): + values = self.select("agent/internal/console/console.go") + self.assertEqual(values["reason"], "") + self.assertIn("TestAPI_ConsolePTYRoundTrip", values["e2e-run"]["api-agent"]) + for bucket in ("api-auth", "ratelimit", "telemetry"): + self.assertNotIn(bucket, values["e2e-run"]) + + def test_sentinel_only_change_skips_api_suites_and_keeps_wake_tests(self): + values = self.select("sentinel/main.go") + for bucket in ("api-auth", "api-roles", "api-rbac", "api-agent", "telemetry"): + self.assertNotIn(bucket, values["e2e-run"]) + self.assertIn("TestGameServer_WakeOnConnect_LoginWakes", values["bot-matrix"]["test"]) + self.assertNotIn("TestGameServer_MinecraftJavaBot_Joined", values["bot-matrix"]["test"]) + + def test_chart_change_selects_every_suite(self): + values = self.select("charts/gameplane/values.yaml") + buckets = bucket_lists(ROOT) + for bucket in E2E_BUCKETS: + self.assertEqual(values["e2e-run"][bucket], "^(" + "|".join(buckets[bucket]) + ")$") + for key in ("e2e-multicluster", "e2e-upgrade", "e2e-capture-overhead", "e2e-web-live", + "bots-any"): + self.assertTrue(values[key], key) + + def test_changed_e2e_test_file_selects_its_own_tests(self): + values = self.select("test/e2e/telemetry_e2e_test.go") + self.assertEqual(values["e2e-run"].get("telemetry"), "^(TestTelemetryLifecycle)$") + self.assertNotIn("api-rbac", values["e2e-run"]) + + def test_unmapped_path_forces_full_scope(self): + values = self.select("tools/x.sh") + self.assertEqual(values["reason"], "unmapped path tools/x.sh") + self.assertEqual(values["unmapped"], ["tools/x.sh"]) + self.assertTrue(values["all"]) + + def test_docs_only_change_selects_no_e2e_test(self): + values = self.select("docs/architecture.md", "specs/023-change-based-test-selection/plan.md") + self.assertEqual(values["e2e-run"], {}) + self.assertFalse(values["bots-any"]) + + def test_universal_path_change_selects_every_suite_and_turns_on_the_gate(self): + values = self.select(".dockerignore") + buckets = bucket_lists(ROOT) + self.assertFalse(values["all"]) + self.assertTrue(values["universal"]) + for bucket in E2E_BUCKETS: + self.assertEqual(values["e2e-run"][bucket], "^(" + "|".join(buckets[bucket]) + ")$") + + def test_component_change_leaves_the_universal_gate_off(self): + self.assertFalse(self.select("agent/internal/console/console.go")["universal"]) + self.assertFalse(self.select("tools/x.sh")["universal"]) + + +class VerifySuitesTests(unittest.TestCase): + """verify-suites rules 1, 2 and 5 from contracts/suite-components.md (T015).""" + + MODULES = {"api": "example.com/api", "agent": "example.com/agent"} + + def suites(self, **overrides): + """Return a valid minimal suite map, with top-level keys replaced by overrides.""" + value = { + "universal": ["charts/"], + "suites": [{"tests": "^TestAPI_", "components": ["api"]}, + {"tests": "^TestAgent_", "components": ["agent"]}], + "unexercised": [{"component": "web", "reason": "dashboard"}], + } + value.update(overrides) + return value + + def test_valid_map_has_no_errors(self): + self.assertEqual(verify_suites(self.suites(), {"b": ["TestAPI_X", "TestAgent_Y"]}, + self.MODULES), []) + + def test_unmatched_test_is_reported(self): + errors = verify_suites(self.suites(), {"b": ["TestAPI_X", "TestOther"]}, self.MODULES) + self.assertTrue(any("TestOther" in error for error in errors), errors) + + def test_unknown_component_is_reported(self): + suites = self.suites(suites=[{"tests": "^Test", "components": ["api", "agent", "nope"]}]) + errors = verify_suites(suites, {"b": ["TestAPI_X"]}, self.MODULES) + self.assertTrue(any("unknown component nope" in error for error in errors), errors) + + def test_unnamed_code_component_is_reported_unless_unexercised(self): + suites = self.suites(suites=[{"tests": "^Test", "components": ["api"]}]) + errors = verify_suites(suites, {"b": ["TestAPI_X"]}, self.MODULES) + self.assertTrue(any("component agent" in error for error in errors), errors) + suites["unexercised"].append({"component": "agent", "reason": "no e2e image"}) + self.assertEqual(verify_suites(suites, {"b": ["TestAPI_X"]}, self.MODULES), []) + + def test_repository_suite_map_is_valid(self): + self.assertEqual(verify_suites(load_suites(ROOT), bucket_lists(ROOT), + workspace_modules(ROOT)), []) + + +class E2ERunTests(unittest.TestCase): + """The e2e-run regexes that replace buckets.sh regex in the e2e jobs (T016).""" + + def test_full_scope_regex_equals_buckets_sh_regex(self): + values = e2e_outputs(ROOT, None, "master push") + self.assertEqual(values["reason"], "master push") + for bucket in E2E_BUCKETS + ["ratelimit"]: + expected = subprocess.run(["bash", "test/e2e/buckets.sh", "regex", bucket], cwd=ROOT, + check=True, capture_output=True, text=True).stdout.strip() + self.assertEqual(values["e2e-run"][bucket], expected) + + def test_partial_regex_lists_only_selected_tests_of_its_bucket(self): + values = e2e_outputs(ROOT, ["agent/internal/console/console.go"], None) + buckets = bucket_lists(ROOT) + for bucket, regex in values["e2e-run"].items(): + self.assertTrue(regex.startswith("^(") and regex.endswith(")$"), regex) + self.assertLessEqual(set(regex[2:-2].split("|")), set(buckets[bucket])) + operator = values["e2e-run"]["operator"][2:-2].split("|") + self.assertLess(len(operator), len(buckets["operator"])) + self.assertNotIn("TestCRD_Validation_GameServerWithoutTemplate", operator) + + def test_ratelimit_tail_runs_only_when_its_test_is_selected(self): + self.assertNotIn("ratelimit", + e2e_outputs(ROOT, ["agent/internal/console/console.go"], None)["e2e-run"]) + self.assertIn("ratelimit", e2e_outputs(ROOT, ["api/cmd/main.go"], None)["e2e-run"]) + + +class UnmappedPathTests(unittest.TestCase): + """An unmapped path widens mode modules to every module (spec 023 FR-002).""" + + def setUp(self): + """Create the shared Go workspace fixture for each test.""" + make_go_workspace(self) + + def test_known_and_unknown_paths(self): + modules = ["lib", "api", "test/e2e", "other"] + for path in ("README.md", "api/x.go", "charts/gameplane/values.yaml", "docs/a.txt", + "hack/ci_scope.py", ".dockerignore", "web/src/App.tsx", "modules"): + with self.subTest(path=path): + self.assertFalse(is_unmapped(path, modules)) + self.assertTrue(is_unmapped("tools/x.sh", modules)) + + def test_modules_mode_selects_every_module_for_an_unmapped_path(self): + output = self.root / "output" + output.write_text("", encoding="utf-8") + with patch("ci_scope.ROOT", self.root), patch("sys.argv", ["ci_scope.py", "modules"]), \ + patch.dict("os.environ", {"GITHUB_EVENT_NAME": "pull_request", + "GITHUB_REF": "refs/pull/7/merge", "CI_FULL_RUN": "false", + "GITHUB_OUTPUT": str(output)}), \ + patch("ci_scope.changed_paths", return_value=["tools/x.sh"]): + main() + values = {key: json.loads(value) for key, value in + (line.split("=", 1) for line in output.read_text(encoding="utf-8").splitlines())} + self.assertEqual(values, FULL_VALUES) + + if __name__ == "__main__": unittest.main() diff --git a/specs/023-change-based-test-selection/OPEN-DECISIONS.md b/specs/023-change-based-test-selection/OPEN-DECISIONS.md index 4bbf00751..ad3777937 100644 --- a/specs/023-change-based-test-selection/OPEN-DECISIONS.md +++ b/specs/023-change-based-test-selection/OPEN-DECISIONS.md @@ -17,4 +17,7 @@ OD-9 (plan R6, R7): on partial runs only Vitest is narrowed (`vitest related`); OD-10 (plan Phasing): rollout is five PRs: summary table, e2e selection, unit selection, CI-config mapping, docs. OD-11 (tasks T030, T041, T047; quickstart): keep the throwaway draft PRs from quickstart.md, including scenario 6's deliberate compile error. valgul confirmed this twice on 2026-10-11 as an explicit exception to his no-demo-PR rule, for spec 023 only. +## Settled (user, 2026-10-22) +OD-12 (US1-1, US1-2, quickstart rows 1-2; found in T030, see validation.md): keep the `suite-components.json` mapping. An agent-only change runs every test that starts a game server, and a sentinel-only change runs the `TestHelmInstall_` smoke tests; US1-1, US1-2 and quickstart rows 1-2 were reworded to match. + No open decisions remain. diff --git a/specs/023-change-based-test-selection/briefs/e2e-selection.md b/specs/023-change-based-test-selection/briefs/e2e-selection.md new file mode 100644 index 000000000..4393950b2 --- /dev/null +++ b/specs/023-change-based-test-selection/briefs/e2e-selection.md @@ -0,0 +1,1277 @@ +# Scout brief: e2e selection, PR 2 (T014–T030) + +Written on branch `claude/project-thread-63yei6-pr2` (stacked on PR 1, #752) on +2026-10-22. Fix agents edit from this brief only (CLAUDE.md rule 18). Every code block +below is final text: paste it exactly, do not re-derive it. + +## Design decisions in this PR (and why) + +1. **New mode `e2e`, not new keys on mode `modules`.** Two existing tests in + `hack/test_ci_scope.py` assert mode `modules`' exact output dict, so adding keys there + breaks them. This is the same reason valgul chose option A (a new mode) for the PR 1 + summary. The new step `e2e` runs right after `bots` in the `changes` job. +2. **The e2e-go matrices are not trimmed with `exclude`.** GitHub runs `include` after + `exclude`, and an `include` entry that matches no remaining combination becomes a new + one. Every bucket has an `include` entry (its `parallel`/timeouts), so an excluded + bucket would come back as a leg without `arch`. Instead each leg reads its bucket's + regex from `e2e-run` into the job env `RUN_REGEX`; when it is empty every step is + skipped, so the leg finishes in seconds without a cluster. Both arches share the + `&e2e-go-steps` anchor, so FR-010 holds by construction. +3. **Mode `bots` is unchanged.** Mode `e2e` emits its own `bot-matrix` and `bots-any`; + `e2e-game-bot` reads those. The summary still lists every `bot-fast` test from mode + `bots`. +4. **Changed e2e Go files.** A changed `test/e2e/.go` selects its own `Test` + functions plus every test in a file that uses one of its top-level names (funcs by + `name(`, types/vars/consts and method receiver types by `\bname\b`), following pure + helper files transitively but stopping at files that define tests. A deleted file + selects every e2e test. Any other `test/e2e/` path that no `paths` entry names selects + every e2e test. +5. **Unmapped paths.** A path that is not `.md`, not in a known no-e2e set, not under a + `universal` prefix, not named by a `paths` entry and not inside a component forces + full scope with reason `unmapped path

`. `combine` folds `steps.e2e.outputs.all` + into `CI`, so every job runs. The `e2e bucket coverage` job fails on a non-empty + `unmapped` list (the "selection-rules" check in quickstart scenario 12). +6. **Kept as is:** `e2e-postgres` (still on the `e2e` gate), `e2e-multicluster` and + `e2e-upgrade` run their whole bucket when selected, and on PRs `e2e-capture-overhead` + still needs its spec 007 OD-2 path filter (selection only narrows it further). +7. **Agent is part of every test whose GameServer pod starts** (the agent sidecar runs + in it, and backups quiesce through it). So an agent-only change also selects + `upgrade`, `api-mods` and the server-owning tests in `api-roles`, unlike the rough + expectation in quickstart scenario 1. Correctness wins over the expected skip. + +## 1. New file `test/e2e/suite-components.json` + +```json +{ + "version": 1, + "universal": [ + "charts/", + "containers/", + "operator/api/v1alpha1/", + "operator/config/", + "deploy/kind/", + "signing/", + ".dockerignore", + "test/e2e/env.go", + "test/e2e/internal/fakeoidc/", + "test/e2e/go.mod", + "test/e2e/go.sum", + "test/e2e/buckets.sh", + "test/e2e/suite-components.json" + ], + "suites": [ + { "tests": "^TestHelmInstall_", "components": ["*"] }, + { "tests": "^TestCRD_Validation_", "components": ["operator"] }, + { "tests": "^TestGameServer_", "components": ["operator", "api", "agent"] }, + { "tests": "^TestGameServer_TunnelCredentialRefusalSurfaced$", "components": ["tunnel"] }, + { "tests": "^TestGameServer_IdleNeverSleepsWithoutAPlayerCount$", "components": ["sentinel"] }, + { "tests": "^(TestGameServer_NetworkCapture|TestNetworkCapture_)", + "components": ["operator", "api", "agent", "capture-sidecar"] }, + { "tests": "^TestGameTemplate_", "components": ["operator", "api", "agent"] }, + { "tests": "^(TestModule_|TestModuleSource)", "components": ["operator", "api", "modules"] }, + { "tests": "^TestModule_FinalizerBlocksWhileTemplateInUse$", "components": ["agent"] }, + { "tests": "^TestModule_(ScaffoldAndPackage|SteamcmdScaffoldKeepsNonRootSecurity)$", + "components": ["gp-module"] }, + { "tests": "^TestModuleSource_RejectsSSRFTarget$", "components": ["netguard"] }, + { "tests": "^(TestBackup|TestRestore_|TestE2E_ResticRepoReady$)", + "components": ["operator", "api", "agent"] }, + { "tests": "^TestAddressPool_", "components": ["operator", "api", "agent"] }, + { "tests": "^TestAPI_", "components": ["operator", "api"] }, + { "tests": "^TestAPI_(Agent|ConsolePTYRoundTrip|LogsTailWS|Lifecycle|ModManifestInstallUpgrade|ModUpload|ModArchiveConfinement_|OwnerCollaboratorAccess|OwnerOnlyServerOperations_|EventStreamAndRoleEdits_)", + "components": ["agent", "gameaction"] }, + { "tests": "^TestAPI_I18n_", "components": ["i18n"] }, + { "tests": "^TestAPI_I18n_ModuleLocales$", "components": ["modules"] }, + { "tests": "^TestAPI_ModuleUpload_ExtractionStaysWithinBudget$", "components": ["gp-module", "modules"] }, + { "tests": "^TestGameServer_.*(Bot_|_Persistence$|_RCON$|_Workshop$|_Mods$)", + "components": ["agent", "modules"], + "paths": ["test/e2e/internal/", "test/e2e/gamebot_helpers_e2e_test.go", "test/e2e/gameprobe_job.go"] }, + { "tests": "^TestGameServer_WakeOnConnect_", "components": ["sentinel", "gameproto"] }, + { "tests": "^TestGameServer_CaptureOverhead_Joined$", + "components": ["capture-sidecar", "gameproto", "modules"], + "paths": ["test/e2e/internal/minecraft-java/", "test/e2e/internal/probe/", "test/e2e/gameprobe_job.go"] }, + { "tests": "^TestMultiCluster_", "components": ["operator", "api", "agent", "capture-sidecar"] }, + { "tests": "^TestUpgrade_", "components": ["operator", "api", "agent"] }, + { "tests": "^TestTelemetry", "components": ["operator", "api", "telemetry-receiver", "telemetryschema"], + "paths": ["test/e2e/telemetry_e2e_helpers_test.go", "test/e2e/telemetry_client_test.go", "test/e2e/telemetry_pow_e2e_test.go"] }, + { "tests": "^TestOptionalComponents_", + "components": ["operator", "api", "agent", "mcp-server", "audit-syslog-bridge", "tunnel", "capture-sidecar"], + "paths": ["test/e2e/fixtures/frps.yaml", "test/e2e/fixtures/syslog-sink.yaml"] }, + { "tests": "^TestCISelection_", "components": [], + "paths": ["hack/ci_scope.py", ".github/workflows/ci.yaml", "test/e2e/buckets.sh", "test/e2e/suite-components.json"] } + ], + "unexercised": [ + { "component": "svcutil", "reason": "library only; the suites of every service that imports it run through the module closure" }, + { "component": "test/e2e", "reason": "the e2e suite itself; a changed test file selects its own tests and the tests that use its helpers" }, + { "component": "web", "reason": "dashboard; exercised by e2e web live (Playwright), not by a Go e2e test" } + ] +} +``` + +## 2. `hack/ci_scope.py` + +### 2a. After the `E2E_BUCKETS = [...]` constant (line 30–31), insert + +```python +SUITES_FILE = "test/e2e/suite-components.json" +# Paths no Go e2e test reads. Their own jobs keep their paths-filter gates. +NO_E2E_FILES = { + "design.pen", "LICENSE", ".claudeignore", ".coderabbit.yaml", ".editorconfig", + ".gitattributes", ".gitignore", ".gitmodules", ".hadolint.yaml", + "deploy/kind/up.sh", "deploy/kind/down.sh", + "test/e2e/joincoverage.sh", "test/e2e/joincoverage_test.sh", + "test/e2e/lint-gate-verify.sh", "test/e2e/lint-gate-verify_test.sh", +} +NO_E2E_DIRS = ("docs", "specs", "assets", "website", "hack", ".github", ".claude", ".devcontainer", + ".githooks", ".jules", ".specify", "deploy/standalone", "test/e2e/testdata") +# The dashboard live run drives these components through a real cluster. +WEB_LIVE_COMPONENTS = {"web", "api", "operator", "agent"} +# Component ids beside the go.work modules (data-model.md Component). +EXTRA_COMPONENTS = ("web", "charts", "deploy-kind", "modules", "e2e-harness") +``` + +### 2b. After `table_cell()` (ends just before `def summarize`), insert + +```python +def decoded(values, key, default=None): + """Return a step output decoded from the JSON that emit() wrote, or default.""" + value = values.get(key) + if value is None or value == "": + return default + if not isinstance(value, str): + return value + try: + return json.loads(value) + except ValueError: + return value + + +def under(path, base): + """Return True when path equals base or lies beneath it.""" + return path == base or path.startswith(base + "/") + + +def prefixed(path, prefix): + """Match a suite-components prefix: a trailing slash means a directory, else one path.""" + return path.startswith(prefix) if prefix.endswith("/") else path == prefix + + +def load_suites(root): + """Return the parsed test/e2e/suite-components.json.""" + return json.loads((root / SUITES_FILE).read_text(encoding="utf-8")) + + +def bucket_lists(root): + """Return {bucket: [test, ...]} from test/e2e/buckets.sh, in its listed order.""" + def run(*args): + """Run buckets.sh with args and return its output words.""" + return subprocess.run(["bash", "test/e2e/buckets.sh", *args], cwd=root, check=True, + capture_output=True, text=True).stdout.split() + return {name: run("list", name) for name in run("buckets")} + + +def go_tests(text): + """Return the Test functions a Go file defines, without TestMain.""" + return [name for name in re.findall(r"^func (Test[A-Za-z0-9_]*)\(", text, re.MULTILINE) + if name != "TestMain"] + + +def helper_pattern(text): + """Return a regex matching uses of a Go file's top-level names, or None if it has none. + + Functions match as calls; types, vars, consts and the receiver types of methods + match as words, so a changed method selects the users of its type. + """ + funcs = set(re.findall(r"^func ([A-Za-z_]\w*)", text, re.MULTILINE)) + funcs -= set(go_tests(text)) | {"TestMain"} + words = set(re.findall(r"^func \(\s*\w*\s*\*?\s*([A-Za-z_]\w*)", text, re.MULTILINE)) + words |= set(re.findall(r"^(?:type|var|const)\s+([A-Za-z_]\w*)", text, re.MULTILINE)) + for block in re.findall(r"^(?:type|var|const)\s*\((.*?)^\)", text, + re.MULTILINE | re.DOTALL): + words |= set(re.findall(r"^\s+([A-Za-z_]\w*)", block, re.MULTILINE)) + parts = ([re.escape(name) + r"\(" for name in sorted(funcs)] + + [re.escape(name) + r"\b" for name in sorted(words)]) + return re.compile(r"\b(?:" + "|".join(parts) + ")") if parts else None + + +def e2e_file_tests(root, path): + """Return the tests a changed test/e2e/.go can affect, or None for every test. + + That is the file's own tests plus every test in a file that uses one of its + top-level names, following pure helper files (no tests) transitively. A + deleted file returns None. + """ + texts = {source.name: source.read_text(encoding="utf-8") + for source in sorted((root / "test/e2e").glob("*.go"))} + start = Path(path).name + if start not in texts: + return None + seen, queue, tests = {start}, [start], set() + while queue: + name = queue.pop() + own = set(go_tests(texts[name])) + tests |= own + if own and name != start: + continue + pattern = helper_pattern(texts[name]) + if pattern is None: + continue + for other, text in texts.items(): + if other not in seen and pattern.search(text): + seen.add(other) + queue.append(other) + return tests + + +def select_e2e(root, paths, reason, suites, buckets): + """Select e2e tests for a change (spec 023 US1). + + Returns (selected, unmapped, reason, touched): selected maps each chosen test + to the changed paths that chose it, reason is set when every test runs because + of the event or an unmapped path, and touched holds the changed component ids + ("*" when a universal path changed). + """ + tests = list(dict.fromkeys(name for names in buckets.values() for name in names)) + if reason is None and paths is None: + reason = "unknown base" + if reason: + return {name: [] for name in tests}, [], reason, {"*"} + modules = workspace_modules(root) + entries = suites["suites"] + picked, components, every, unmapped = {}, {}, [], [] + + def matching(entry): + """Return the tests one suite entry's pattern matches.""" + return [name for name in tests if re.search(entry["tests"], name)] + + def pick(names, path): + """Record that path selects each test in names.""" + for name in names: + picked.setdefault(name, []).append(path) + + for path in paths: + if path.endswith(".md") or path in NO_E2E_FILES: + continue + if any(prefixed(path, prefix) for prefix in suites["universal"]): + every.append(path) + continue + named = [entry for entry in entries + if any(prefixed(path, prefix) for prefix in entry.get("paths", []))] + for entry in named: + pick(matching(entry), path) + if under(path, "test/e2e"): + if path.count("/") == 2 and path.endswith(".go"): + own = e2e_file_tests(root, path) + if own is None: + every.append(path) + else: + pick([name for name in tests if name in own], path) + elif not named: + every.append(path) + continue + if named or any(under(path, directory) for directory in NO_E2E_DIRS): + continue + owner = next((module for module in modules if under(path, module)), None) + if owner: + components.setdefault(owner, []).append(path) + elif under(path, "web"): + components.setdefault("web", []).append(path) + elif under(path, "modules"): + components.setdefault("modules", []).append(path) + else: + unmapped.append(path) + if unmapped: + return {name: [] for name in tests}, unmapped, f"unmapped path {unmapped[0]}", {"*"} + if every: + return {name: list(every) for name in tests}, [], None, {"*"} + module_paths = [path for component, owned in components.items() + if component in modules or component == "modules" for path in owned] + if module_paths: + # Library changes reach the binaries that import them (research R3). + for module in affected_modules(root, module_paths): + components.setdefault(module, module_paths) + for entry in entries: + wanted = set(entry.get("components", [])) + if "*" in wanted: + hit = [path for owned in components.values() for path in owned] + else: + hit = [path for component in sorted(wanted & components.keys()) + for path in components[component]] + for path in dict.fromkeys(hit): + pick(matching(entry), path) + selected = {name: list(dict.fromkeys(picked[name])) for name in tests if name in picked} + return selected, [], None, set(components) + + +def e2e_outputs(root, paths, reason): + """Return the outputs of mode e2e: per-bucket -run regexes, job switches, bot matrix.""" + buckets = bucket_lists(root) + selected, unmapped, reason, touched = select_e2e(root, paths, reason, load_suites(root), + buckets) + + def chosen(bucket): + """Return the selected tests of one bucket, in buckets.sh order.""" + return [name for name in buckets.get(bucket, []) if name in selected] + + def describe(names, total): + """Return 'N of M tests' plus the changed paths that chose them.""" + files = list(dict.fromkeys(path for name in names for path in selected[name])) + text = f"{len(names)} of {total} tests" + return text + (f"; {cite(files)}" if files else "") + + run, why = {}, {} + for bucket in E2E_BUCKETS + ["ratelimit", "multicluster", "upgrade", "capture-overhead"]: + names = chosen(bucket) + if names: + why[bucket] = describe(names, len(buckets[bucket])) + if bucket in E2E_BUCKETS or bucket == "ratelimit": + run[bucket] = "^(" + "|".join(names) + ")$" + bots = chosen("bot-fast") + for name in bots: + why[name] = describe([name], 1) + return { + "all": bool(reason), + "reason": reason or "", + "unmapped": unmapped, + "e2e-run": run, + "e2e-why": why, + "e2e-multicluster": bool(chosen("multicluster")), + "e2e-upgrade": bool(chosen("upgrade")), + "e2e-capture-overhead": bool(chosen("capture-overhead")), + "e2e-web-live": bool(reason) or "*" in touched or bool(touched & WEB_LIVE_COMPONENTS), + "bot-matrix": {"test": bots}, + "bots-any": bool(bots), + } + + +def verify_suites(suites, buckets, modules): + """Return the violations of contracts/suite-components.md rules 1, 2 and 5.""" + errors = [] + known = set(modules) | set(EXTRA_COMPONENTS) | {"*"} + entries = suites.get("suites", []) + patterns = [] + for entry in entries: + try: + patterns.append(re.compile(entry["tests"])) + except (KeyError, TypeError, re.error) as exc: + errors.append(f"entry {entry!r}: bad tests pattern: {exc}") + continue + if not entry.get("components") and not entry.get("paths"): + errors.append(f"entry {entry['tests']}: names no component and no path") + for component in entry.get("components", []): + if component not in known: + errors.append(f"entry {entry['tests']}: unknown component {component}") + for bucket, names in buckets.items(): + for name in names: + if not any(pattern.search(name) for pattern in patterns): + errors.append(f"test {name} (bucket {bucket}) matches no entry") + named = {component for entry in entries for component in entry.get("components", [])} + named |= {item.get("component") for item in suites.get("unexercised", [])} + if any(prefix.startswith("charts/") for prefix in suites.get("universal", [])): + named.add("charts") + for component in sorted(set(modules) | {"web", "charts"}): + if component not in named: + errors.append(f"component {component} is named by no entry and not listed " + "in unexercised") + for item in suites.get("unexercised", []): + if item.get("component") not in known or not item.get("reason"): + errors.append(f"unexercised entry {item!r} needs a known component and a reason") + return errors +``` + +### 2c. In `summarize()` + +Replace + +```python + bots = load_json(env, "BOT_MATRIX") +``` + +with + +```python + bots = load_json(env, "BOT_MATRIX") + e2e = load_json(env, "E2E_OUTPUTS") + # Without the e2e step's outputs every e2e suite follows the coarse e2e gate. + narrowed = bool(e2e) + e2e_run = decoded(e2e, "e2e-run", {}) + e2e_why = decoded(e2e, "e2e-why", {}) + e2e_bots = decoded(e2e, "bot-matrix", {}).get("test", []) +``` + +Replace the block from ` for bucket in E2E_BUCKETS:` through the end of the +`add("e2e capture overhead", ...)` call (the five lines that end with +`filtered("capture_overhead"))`), currently: + +```python + for bucket in E2E_BUCKETS: + name = "e2e api-auth + ratelimit tail" if bucket == "api-auth" else f"e2e {bucket}" + add(f"{name} (amd64, arm64)", gate("e2e"), e2e_feed()) + add("e2e multicluster (amd64, arm64)", gate("e2e"), e2e_feed()) + add("e2e upgrade (amd64, arm64)", gate("e2e"), e2e_feed()) + add("e2e postgres (amd64)", gate("e2e"), e2e_feed()) + for test in jlist(bots, "test"): + add(f"e2e game bot / {test}", gate("e2e"), e2e_feed()) + add("e2e web live (amd64, arm64)", gate("weblive"), e2e_feed() + filtered("web")) + add("e2e capture overhead", + (event == "pull_request" and gate("capture_overhead")) + or (event in ("push", "workflow_dispatch") and gate("e2e")), + filtered("capture_overhead")) +``` + +with + +```python + def e2e_row(suite, gate_on, picked, key, feed): + """Add an e2e row from the coarse gate and, when present, the e2e step's pick.""" + if narrowed and gate_on and not picked: + add(suite, False, skip_reason="no test in this suite is affected") + else: + add(suite, gate_on, feed, run_reason=e2e_why.get(key, "") if narrowed else "") + + for bucket in E2E_BUCKETS: + name = "e2e api-auth + ratelimit tail" if bucket == "api-auth" else f"e2e {bucket}" + picked = bucket in e2e_run or (bucket == "api-auth" and "ratelimit" in e2e_run) + e2e_row(f"{name} (amd64, arm64)", gate("e2e"), picked, bucket, e2e_feed()) + e2e_row("e2e multicluster (amd64, arm64)", gate("e2e"), flag(e2e, "e2e-multicluster"), + "multicluster", e2e_feed()) + e2e_row("e2e upgrade (amd64, arm64)", gate("e2e"), flag(e2e, "e2e-upgrade"), "upgrade", + e2e_feed()) + add("e2e postgres (amd64)", gate("e2e"), e2e_feed()) + for test in jlist(bots, "test"): + e2e_row(f"e2e game bot / {test}", gate("e2e"), test in e2e_bots, test, e2e_feed()) + e2e_row("e2e web live (amd64, arm64)", gate("weblive"), flag(e2e, "e2e-web-live"), "", + e2e_feed() + filtered("web")) + e2e_row("e2e capture overhead", + (event == "pull_request" and gate("capture_overhead")) + or (event in ("push", "workflow_dispatch") and gate("e2e")), + event != "pull_request" or flag(e2e, "e2e-capture-overhead"), "capture-overhead", + filtered("capture_overhead")) +``` + +### 2d. In `main()` + +Replace + +```python + parser.add_argument("mode", choices=("modules", "images", "dockerfiles", "bots", "summary")) +``` + +with + +```python + parser.add_argument("mode", choices=("modules", "images", "dockerfiles", "bots", "summary", + "e2e", "verify-suites")) +``` + +In the `summary` branch, replace + +```python + reason = full_scope_reason(env, paths) + if reason is None and flag(load_json(env, "SCOPE_OUTPUTS"), "all"): +``` + +with + +```python + reason = full_scope_reason(env, paths) or decoded(load_json(env, "E2E_OUTPUTS"), + "reason") or None + if reason is None and flag(load_json(env, "SCOPE_OUTPUTS"), "all"): +``` + +Directly before the line ` # PRs use affected work; every master push, manual run and ci: full-run` +insert + +```python + if args.mode == "verify-suites": + errors = verify_suites(load_suites(ROOT), bucket_lists(ROOT), workspace_modules(ROOT)) + for error in errors: + print(f"::error::{error}") + if errors: + raise SystemExit(1) + print("suite components OK") + return + if args.mode == "e2e": + env = os.environ + try: + event_reason = full_scope_reason(env, []) + paths = None if event_reason else changed_paths(ROOT, args.base, args.head) + values = e2e_outputs(ROOT, paths, full_scope_reason(env, paths)) + except Exception as exc: # any doubt selects every e2e test (FR-002) + print(f"::warning::e2e selection failed, running every e2e test: {exc}") + values = e2e_outputs(ROOT, None, f"error: {exc}") + emit(values) + return +``` + +## 3. `test/e2e/buckets.sh` + +In `bucket_operator()`, after the line `TestE2E_ResticRepoReady` (line 108) and before +`EOF`, add: + +``` +TestCISelection_AgentOnlyChange +TestCISelection_SentinelOnlyChange +TestCISelection_ChartChangeRunsEverySuite +TestCISelection_E2ETestFileSelectsItsTests +TestCISelection_UnmappedPathRunsEverything +TestCISelection_FullRunLabel +``` + +No other `buckets.sh` change. + +## 4. New file `test/e2e/ci_selection_e2e_test.go` + +```go +//go:build e2e + +package e2e + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// The TestCISelection_* tests prove that `hack/ci_scope.py e2e` turns a +// one-commit change into the expected e2e selection (spec 023 OD-4, research +// R13). Each one copies this checkout with `git archive HEAD`, commits a base +// and a head, and runs the selector on them. They need git and python3 only, +// create no cluster resources and never log in, so they live in the operator +// bucket. + +// ciSelectionBuckets mirrors E2E_BUCKETS in hack/ci_scope.py (the e2e-go matrix). +var ciSelectionBuckets = []string{ + "operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", + "telemetry", "optional-components", +} + +// ciSelection is the decoded output of one `ci_scope.py e2e` run. +type ciSelection struct { + Reason string + Run map[string]string + Bots []string + Flags map[string]bool +} + +// ciGit runs git in dir with a fixed identity and returns trimmed stdout. +func ciGit(t *testing.T, dir string, args ...string) string { + t.Helper() + base := []string{"-C", dir, "-c", "user.name=ci-selection", + "-c", "user.email=ci-selection@example.invalid", "-c", "commit.gpgsign=false"} + out, err := exec.Command("git", append(base, args...)...).CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) +} + +// ciSelectionCopy exports HEAD into a fresh repository and commits it as the +// base. Large paths no selection rule reads are left out to keep it fast. +func ciSelectionCopy(t *testing.T) string { + t.Helper() + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatalf("resolve repository root: %v", err) + } + dir := t.TempDir() + script := `set -o pipefail; git -C "$1" archive --format=tar HEAD -- . ` + + `':(exclude)design.pen' ':(exclude)assets' ':(exclude)docs' ':(exclude)specs' ` + + `':(exclude)web' | tar -x -C "$2"` + if out, err := exec.Command("bash", "-c", script, "bash", root, dir).CombinedOutput(); err != nil { + t.Fatalf("git archive HEAD: %v\n%s", err, out) + } + ciGit(t, dir, "init", "-q") + ciGit(t, dir, "add", "-A") + ciGit(t, dir, "commit", "-q", "-m", "base") + return dir +} + +// ciSelectionAppend appends text to a file in the copy, creating it if needed. +func ciSelectionAppend(t *testing.T, path, text string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + defer func() { _ = f.Close() }() + if _, err := f.WriteString(text); err != nil { + t.Fatalf("append to %s: %v", path, err) + } +} + +// ciSelectionRun commits edit as the head and returns the selector's output. +func ciSelectionRun(t *testing.T, dir string, fullRunLabel bool, edit func(dir string)) ciSelection { + t.Helper() + base := ciGit(t, dir, "rev-parse", "HEAD") + if edit != nil { + edit(dir) + } + ciGit(t, dir, "add", "-A") + ciGit(t, dir, "commit", "-q", "--allow-empty", "-m", "head") + head := ciGit(t, dir, "rev-parse", "HEAD") + output := filepath.Join(t.TempDir(), "github_output") + cmd := exec.Command("python3", "hack/ci_scope.py", "e2e", "--base", base, "--head", head) + cmd.Dir = dir + // A clean environment: the e2e job's own GITHUB_EVENT_NAME and GITHUB_REF + // would turn a master push into a full-scope answer here. + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME"), + "GITHUB_OUTPUT=" + output} + if fullRunLabel { + cmd.Env = append(cmd.Env, "CI_FULL_RUN=true") + } + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("ci_scope.py e2e: %v\n%s", err, out) + } + return ciSelectionParse(t, output) +} + +// ciSelectionParse decodes the key= lines ci_scope.py writes to GITHUB_OUTPUT. +func ciSelectionParse(t *testing.T, path string) ciSelection { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + raw := map[string]json.RawMessage{} + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + key, value, ok := strings.Cut(line, "=") + if !ok { + t.Fatalf("malformed output line %q", line) + } + raw[key] = json.RawMessage(value) + } + decode := func(key string, into any) { + if err := json.Unmarshal(raw[key], into); err != nil { + t.Fatalf("decode %s=%s: %v", key, raw[key], err) + } + } + sel := ciSelection{Flags: map[string]bool{}} + decode("reason", &sel.Reason) + decode("e2e-run", &sel.Run) + var bots struct { + Test []string `json:"test"` + } + decode("bot-matrix", &bots) + sel.Bots = bots.Test + for _, key := range []string{"all", "e2e-multicluster", "e2e-upgrade", "e2e-web-live", + "e2e-capture-overhead", "bots-any"} { + var value bool + decode(key, &value) + sel.Flags[key] = value + } + return sel +} + +// ciSelectionBucket runs buckets.sh in this package directory. +func ciSelectionBucket(t *testing.T, args ...string) []string { + t.Helper() + out, err := exec.Command("bash", append([]string{"buckets.sh"}, args...)...).Output() + if err != nil { + t.Fatalf("buckets.sh %s: %v", strings.Join(args, " "), err) + } + return strings.Fields(string(out)) +} + +// ciSelectionSuiteTests lists the Test functions compiled into this package. +func ciSelectionSuiteTests(t *testing.T) []string { + t.Helper() + files, err := filepath.Glob("*_test.go") + if err != nil { + t.Fatalf("glob test files: %v", err) + } + pattern := regexp.MustCompile(`(?m)^func (Test[A-Za-z0-9_]+)\(`) + var names []string + for _, file := range files { + data, err := os.ReadFile(file) + if err != nil { + t.Fatalf("read %s: %v", file, err) + } + for _, match := range pattern.FindAllStringSubmatch(string(data), -1) { + if match[1] != "TestMain" { + names = append(names, match[1]) + } + } + } + return names +} + +// ciSelectionCheckRegexes asserts every emitted -run regex runs at least one +// test of this package and nothing outside its own bucket. +func ciSelectionCheckRegexes(t *testing.T, sel ciSelection) { + t.Helper() + suite := ciSelectionSuiteTests(t) + for bucket, expr := range sel.Run { + re, err := regexp.Compile(expr) + if err != nil { + t.Errorf("bucket %s: bad regex %q: %v", bucket, expr, err) + continue + } + members := map[string]bool{} + for _, name := range ciSelectionBucket(t, "list", bucket) { + members[name] = true + } + matched := 0 + for _, name := range suite { + if !re.MatchString(name) { + continue + } + matched++ + if !members[name] { + t.Errorf("bucket %s regex %q matches %s, which is not in that bucket", bucket, expr, name) + } + } + if matched == 0 { + t.Errorf("bucket %s regex %q matches no test in this package", bucket, expr) + } + } +} + +// ciSelectionRequireAll asserts every e2e-go bucket runs its whole buckets.sh regex. +func ciSelectionRequireAll(t *testing.T, sel ciSelection) { + t.Helper() + for _, bucket := range ciSelectionBuckets { + want := strings.Join(ciSelectionBucket(t, "regex", bucket), "") + if got := sel.Run[bucket]; got != want { + t.Errorf("bucket %s regex = %q, want the full bucket %q", bucket, got, want) + } + } + for _, key := range []string{"e2e-multicluster", "e2e-upgrade", "e2e-web-live", "bots-any"} { + if !sel.Flags[key] { + t.Errorf("%s = false, want true when every suite runs", key) + } + } +} + +func TestCISelection_AgentOnlyChange(t *testing.T) { + t.Parallel() + dir := ciSelectionCopy(t) + sel := ciSelectionRun(t, dir, false, func(dir string) { + ciSelectionAppend(t, filepath.Join(dir, "agent/internal/console/console.go"), "\n// ci selection probe\n") + }) + if sel.Reason != "" { + t.Fatalf("reason = %q, want a partial selection", sel.Reason) + } + if !strings.Contains(sel.Run["api-agent"], "TestAPI_ConsolePTYRoundTrip") { + t.Errorf("api-agent regex %q lacks TestAPI_ConsolePTYRoundTrip", sel.Run["api-agent"]) + } + for _, bucket := range []string{"api-auth", "ratelimit", "telemetry"} { + if expr, ok := sel.Run[bucket]; ok { + t.Errorf("%s selected for an agent-only change: %s", bucket, expr) + } + } + ciSelectionCheckRegexes(t, sel) +} + +func TestCISelection_SentinelOnlyChange(t *testing.T) { + t.Parallel() + dir := ciSelectionCopy(t) + sel := ciSelectionRun(t, dir, false, func(dir string) { + ciSelectionAppend(t, filepath.Join(dir, "sentinel/main.go"), "\n// ci selection probe\n") + }) + if sel.Reason != "" { + t.Fatalf("reason = %q, want a partial selection", sel.Reason) + } + for _, bucket := range []string{"api-auth", "api-roles", "api-rbac", "api-agent", "telemetry"} { + if expr, ok := sel.Run[bucket]; ok { + t.Errorf("%s selected for a sentinel-only change: %s", bucket, expr) + } + } + wake := false + for _, name := range sel.Bots { + if name == "TestGameServer_WakeOnConnect_LoginWakes" { + wake = true + } + } + if !wake { + t.Errorf("bot matrix %v lacks TestGameServer_WakeOnConnect_LoginWakes", sel.Bots) + } + ciSelectionCheckRegexes(t, sel) +} + +func TestCISelection_ChartChangeRunsEverySuite(t *testing.T) { + t.Parallel() + dir := ciSelectionCopy(t) + sel := ciSelectionRun(t, dir, false, func(dir string) { + ciSelectionAppend(t, filepath.Join(dir, "charts/gameplane/values.yaml"), "\n# ci selection probe\n") + }) + if sel.Reason != "" { + t.Fatalf("reason = %q, want a partial scope that still selects every suite", sel.Reason) + } + ciSelectionRequireAll(t, sel) + ciSelectionCheckRegexes(t, sel) +} + +func TestCISelection_E2ETestFileSelectsItsTests(t *testing.T) { + t.Parallel() + dir := ciSelectionCopy(t) + sel := ciSelectionRun(t, dir, false, func(dir string) { + ciSelectionAppend(t, filepath.Join(dir, "test/e2e/telemetry_e2e_test.go"), "\n// ci selection probe\n") + }) + if got, want := sel.Run["telemetry"], "^(TestTelemetryLifecycle)$"; got != want { + t.Errorf("telemetry regex = %q, want %q", got, want) + } + if expr, ok := sel.Run["api-rbac"]; ok { + t.Errorf("api-rbac selected for a telemetry test edit: %s", expr) + } + ciSelectionCheckRegexes(t, sel) +} + +func TestCISelection_UnmappedPathRunsEverything(t *testing.T) { + t.Parallel() + dir := ciSelectionCopy(t) + sel := ciSelectionRun(t, dir, false, func(dir string) { + ciSelectionAppend(t, filepath.Join(dir, "tools/x.sh"), "#!/bin/sh\n") + }) + if want := "unmapped path tools/x.sh"; sel.Reason != want { + t.Fatalf("reason = %q, want %q", sel.Reason, want) + } + if !sel.Flags["all"] { + t.Errorf("all = false, want true for an unmapped path") + } + ciSelectionRequireAll(t, sel) +} + +func TestCISelection_FullRunLabel(t *testing.T) { + t.Parallel() + dir := ciSelectionCopy(t) + sel := ciSelectionRun(t, dir, true, nil) + if want := "label ci: full-run"; sel.Reason != want { + t.Fatalf("reason = %q, want %q", sel.Reason, want) + } + ciSelectionRequireAll(t, sel) +} +``` + +## 5. `hack/test_ci_scope.py` + +Replace the import lines 1–11 header imports: + +```python +import json +import tempfile +``` + +with + +```python +import json +import subprocess +import tempfile +``` + +and + +```python +from ci_scope import (E2E_BUCKETS, FORCE_ALL_FILES, affected_modules, bot_matrix, changed_paths, + full_scope_reason, image_inputs, main, publish_paths, select_images, summarize) +``` + +with + +```python +from ci_scope import (E2E_BUCKETS, FORCE_ALL_FILES, ROOT, affected_modules, bot_matrix, + bucket_lists, changed_paths, e2e_outputs, full_scope_reason, image_inputs, + load_suites, main, publish_paths, select_images, summarize, verify_suites, + workspace_modules) +``` + +Insert these three classes directly before the final `if __name__ == "__main__":` line +(keep two blank lines between classes): + +```python +class E2ESelectionTests(unittest.TestCase): + """Spec 023 US1 acceptance scenarios 1-4 against this repository's suite map (T014).""" + + def select(self, *paths): + """Return the e2e outputs for a partial-scope change to paths.""" + return e2e_outputs(ROOT, list(paths), None) + + def test_agent_only_change_selects_agent_tests_and_skips_auth_and_telemetry(self): + values = self.select("agent/internal/console/console.go") + self.assertEqual(values["reason"], "") + self.assertIn("TestAPI_ConsolePTYRoundTrip", values["e2e-run"]["api-agent"]) + for bucket in ("api-auth", "ratelimit", "telemetry"): + self.assertNotIn(bucket, values["e2e-run"]) + + def test_sentinel_only_change_skips_api_suites_and_keeps_wake_tests(self): + values = self.select("sentinel/main.go") + for bucket in ("api-auth", "api-roles", "api-rbac", "api-agent", "telemetry"): + self.assertNotIn(bucket, values["e2e-run"]) + self.assertIn("TestGameServer_WakeOnConnect_LoginWakes", values["bot-matrix"]["test"]) + self.assertNotIn("TestGameServer_MinecraftJavaBot_Joined", values["bot-matrix"]["test"]) + + def test_chart_change_selects_every_suite(self): + values = self.select("charts/gameplane/values.yaml") + buckets = bucket_lists(ROOT) + for bucket in E2E_BUCKETS: + self.assertEqual(values["e2e-run"][bucket], "^(" + "|".join(buckets[bucket]) + ")$") + for key in ("e2e-multicluster", "e2e-upgrade", "e2e-capture-overhead", "e2e-web-live", + "bots-any"): + self.assertTrue(values[key], key) + + def test_changed_e2e_test_file_selects_its_own_tests(self): + values = self.select("test/e2e/telemetry_e2e_test.go") + self.assertEqual(values["e2e-run"].get("telemetry"), "^(TestTelemetryLifecycle)$") + self.assertNotIn("api-rbac", values["e2e-run"]) + + def test_unmapped_path_forces_full_scope(self): + values = self.select("tools/x.sh") + self.assertEqual(values["reason"], "unmapped path tools/x.sh") + self.assertEqual(values["unmapped"], ["tools/x.sh"]) + self.assertTrue(values["all"]) + + def test_docs_only_change_selects_no_e2e_test(self): + values = self.select("docs/architecture.md", "specs/023-change-based-test-selection/plan.md") + self.assertEqual(values["e2e-run"], {}) + self.assertFalse(values["bots-any"]) + + +class VerifySuitesTests(unittest.TestCase): + """verify-suites rules 1, 2 and 5 from contracts/suite-components.md (T015).""" + + MODULES = {"api": "example.com/api", "agent": "example.com/agent"} + + def suites(self, **overrides): + """Return a valid minimal suite map, with top-level keys replaced by overrides.""" + value = { + "universal": ["charts/"], + "suites": [{"tests": "^TestAPI_", "components": ["api"]}, + {"tests": "^TestAgent_", "components": ["agent"]}], + "unexercised": [{"component": "web", "reason": "dashboard"}], + } + value.update(overrides) + return value + + def test_valid_map_has_no_errors(self): + self.assertEqual(verify_suites(self.suites(), {"b": ["TestAPI_X", "TestAgent_Y"]}, + self.MODULES), []) + + def test_unmatched_test_is_reported(self): + errors = verify_suites(self.suites(), {"b": ["TestAPI_X", "TestOther"]}, self.MODULES) + self.assertTrue(any("TestOther" in error for error in errors), errors) + + def test_unknown_component_is_reported(self): + suites = self.suites(suites=[{"tests": "^Test", "components": ["api", "agent", "nope"]}]) + errors = verify_suites(suites, {"b": ["TestAPI_X"]}, self.MODULES) + self.assertTrue(any("unknown component nope" in error for error in errors), errors) + + def test_unnamed_code_component_is_reported_unless_unexercised(self): + suites = self.suites(suites=[{"tests": "^Test", "components": ["api"]}]) + errors = verify_suites(suites, {"b": ["TestAPI_X"]}, self.MODULES) + self.assertTrue(any("component agent" in error for error in errors), errors) + suites["unexercised"].append({"component": "agent", "reason": "no e2e image"}) + self.assertEqual(verify_suites(suites, {"b": ["TestAPI_X"]}, self.MODULES), []) + + def test_repository_suite_map_is_valid(self): + self.assertEqual(verify_suites(load_suites(ROOT), bucket_lists(ROOT), + workspace_modules(ROOT)), []) + + +class E2ERunTests(unittest.TestCase): + """The e2e-run regexes that replace buckets.sh regex in the e2e jobs (T016).""" + + def test_full_scope_regex_equals_buckets_sh_regex(self): + values = e2e_outputs(ROOT, None, "master push") + self.assertEqual(values["reason"], "master push") + for bucket in E2E_BUCKETS + ["ratelimit"]: + expected = subprocess.run(["bash", "test/e2e/buckets.sh", "regex", bucket], cwd=ROOT, + check=True, capture_output=True, text=True).stdout.strip() + self.assertEqual(values["e2e-run"][bucket], expected) + + def test_partial_regex_lists_only_selected_tests_of_its_bucket(self): + values = e2e_outputs(ROOT, ["agent/internal/console/console.go"], None) + buckets = bucket_lists(ROOT) + for bucket, regex in values["e2e-run"].items(): + self.assertTrue(regex.startswith("^(") and regex.endswith(")$"), regex) + self.assertLessEqual(set(regex[2:-2].split("|")), set(buckets[bucket])) + operator = values["e2e-run"]["operator"][2:-2].split("|") + self.assertLess(len(operator), len(buckets["operator"])) + self.assertNotIn("TestCRD_Validation_GameServerWithoutTemplate", operator) + + def test_ratelimit_tail_runs_only_when_its_test_is_selected(self): + self.assertNotIn("ratelimit", + e2e_outputs(ROOT, ["agent/internal/console/console.go"], None)["e2e-run"]) + self.assertIn("ratelimit", e2e_outputs(ROOT, ["api/cmd/main.go"], None)["e2e-run"]) +``` + +## 6. `.github/workflows/ci.yaml` + +6a. `changes` job `outputs:` — after the line +` selection: ${{ steps.summary.outputs.selection }}` add: + +```yaml + e2e-run: ${{ steps.e2e.outputs.e2e-run }} + e2e-unmapped: ${{ steps.e2e.outputs.unmapped }} + e2e-multicluster: ${{ steps.e2e.outputs.e2e-multicluster }} + e2e-upgrade: ${{ steps.e2e.outputs.e2e-upgrade }} + e2e-web-live: ${{ steps.e2e.outputs.e2e-web-live }} + e2e-capture-overhead: ${{ steps.e2e.outputs.e2e-capture-overhead }} + e2e-bot-matrix: ${{ steps.e2e.outputs.bot-matrix }} + e2e-bots-any: ${{ steps.e2e.outputs.bots-any }} +``` + +6b. After the `bots` step (the two lines +` id: bots` / ` run: python3 hack/ci_scope.py bots`) insert: + +```yaml + - name: select the e2e tests the change affects + id: e2e + # Per-test e2e selection from test/e2e/suite-components.json (spec 023 US1). + # An unmapped path sets `all`, which `combine` folds into CI below. + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + HEAD_SHA: ${{ github.sha }} + CI_FULL_RUN: "${{ contains(github.event.pull_request.labels.*.name, 'ci: full-run') }}" + run: python3 hack/ci_scope.py e2e --base "$BASE_SHA" --head "$HEAD_SHA" +``` + +6c. In the `combine` step env, replace + +```yaml + CI: ${{ steps.filter.outputs.ci == 'true' || steps.scope.outputs.all == 'true' }} +``` + +with + +```yaml + CI: ${{ steps.filter.outputs.ci == 'true' || steps.scope.outputs.all == 'true' || steps.e2e.outputs.all == 'true' }} +``` + +6d. In the `summarize test selection` step env, after +` BOT_MATRIX: ${{ steps.bots.outputs.matrix }}` add: + +```yaml + E2E_OUTPUTS: ${{ toJSON(steps.e2e.outputs) }} +``` + +6e. `e2e-buckets` job: after the step +` - name: verify buckets are disjoint + exhaustive` / ` run: ./test/e2e/buckets.sh verify` +insert: + +```yaml + - name: verify every e2e test declares what it exercises + run: python3 hack/ci_scope.py verify-suites + - name: selection rules map every changed path + # An unmapped path already forced the full suite; this makes the gap visible + # so a rule gets added to hack/ci_scope.py (spec 023 quickstart scenario 12). + env: + UNMAPPED: ${{ needs.changes.outputs.e2e-unmapped }} + run: | + if [ -n "$UNMAPPED" ] && [ "$UNMAPPED" != "[]" ]; then + echo "::error::no selection rule maps these paths: $UNMAPPED" + exit 1 + fi +``` + +6f. `e2e-go` job: its `env:` block is + +```yaml + env: + CLUSTER: gameplane-e2e + CTX: kind-gameplane-e2e + steps: &e2e-go-steps +``` + +Replace it with + +```yaml + env: + CLUSTER: gameplane-e2e + CTX: kind-gameplane-e2e + # Only this bucket's selected tests (spec 023 OD-6); empty skips the leg. + RUN_REGEX: ${{ fromJSON(needs.changes.outputs.e2e-run)[matrix.bucket] || '' }} + TAIL_REGEX: ${{ matrix.tail == 'ratelimit' && fromJSON(needs.changes.outputs.e2e-run).ratelimit || '' }} + steps: &e2e-go-steps +``` + +In the `e2e-go-arm64` job, replace + +```yaml + env: + CLUSTER: gameplane-e2e + CTX: kind-gameplane-e2e + steps: *e2e-go-steps +``` + +with + +```yaml + env: + CLUSTER: gameplane-e2e + CTX: kind-gameplane-e2e + # Same selection as e2e-go, so both arches always run the same tests (FR-010). + RUN_REGEX: ${{ fromJSON(needs.changes.outputs.e2e-run)[matrix.bucket] || '' }} + TAIL_REGEX: ${{ matrix.tail == 'ratelimit' && fromJSON(needs.changes.outputs.e2e-run).ratelimit || '' }} + steps: *e2e-go-steps +``` + +Inside `&e2e-go-steps`, add `if: env.RUN_REGEX != '' || env.TAIL_REGEX != ''` as the +first key of each of these six steps: the `actions/checkout` step, the +`./.github/actions/go-cache` step, the `azure/setup-helm` step, `install kind`, +`load images`, and `bring up cluster + helm install`. For a `- uses:` step put the `if:` +line right after the `- uses:` line at the same indent as `with:`. Example: + +```yaml + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' + - uses: ./.github/actions/go-cache + if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' + with: + key-suffix: e2e +``` + +Replace the two run steps: + +```yaml + - name: run go e2e (${{ matrix.bucket }}) + run: | + RUN_REGEX="$(./test/e2e/buckets.sh regex '${{ matrix.bucket }}')" + cd test/e2e && \ +``` + +with + +```yaml + - name: run go e2e (${{ matrix.bucket }}) + if: env.RUN_REGEX != '' + run: | + cd test/e2e && \ +``` + +and + +```yaml + if: matrix.tail == 'ratelimit' + run: | + RUN_REGEX="$(./test/e2e/buckets.sh regex ratelimit)" + cd test/e2e && \ + GAMEPLANE_E2E_REUSE_CLUSTER=1 \ + GAMEPLANE_E2E_CLUSTER="$CLUSTER" \ + go test -tags=e2e -timeout 5m -v -run "$RUN_REGEX" ./... +``` + +with + +```yaml + if: env.TAIL_REGEX != '' + run: | + cd test/e2e && \ + GAMEPLANE_E2E_REUSE_CLUSTER=1 \ + GAMEPLANE_E2E_CLUSTER="$CLUSTER" \ + go test -tags=e2e -timeout 5m -v -run "$TAIL_REGEX" ./... +``` + +6g. Job gates. Replace the `if:` line of each job: + +| Job | New `if:` | +|---|---| +| `e2e-multicluster`, `e2e-multicluster-arm64` | `needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-multicluster == 'true'` | +| `e2e-upgrade`, `e2e-upgrade-arm64` | `needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-upgrade == 'true'` | +| `e2e-web-live`, `e2e-web-live-arm64` | `needs.changes.outputs.weblive == 'true' && needs.changes.outputs.e2e-web-live == 'true'` | +| `e2e-game-bot` | `needs.changes.outputs.e2e == 'true' && needs.changes.outputs.e2e-bots-any == 'true'` | + +In `e2e-game-bot` also replace +` matrix: ${{ fromJSON(needs.changes.outputs.bot-matrix) }}` with +` matrix: ${{ fromJSON(needs.changes.outputs.e2e-bot-matrix) }}`. + +In `e2e-capture-overhead` replace the line +` (github.event_name == 'pull_request' && needs.changes.outputs.capture_overhead == 'true') ||` +with +` (github.event_name == 'pull_request' && needs.changes.outputs.capture_overhead == 'true' && needs.changes.outputs.e2e-capture-overhead == 'true') ||`. + +## 7. Specs + +- `contracts/ci-scope-cli.md`: add a `## Mode e2e — new` section listing the outputs of + `e2e_outputs()` (`all`, `reason`, `unmapped`, `e2e-run`, `e2e-why`, `e2e-multicluster`, + `e2e-upgrade`, `e2e-capture-overhead`, `e2e-web-live`, `bot-matrix`, `bots-any`); in + the "Mode modules — added keys" table mark `e2e-exclude`, `e2e-run`, the four booleans + and `unmapped` as "emitted by mode `e2e`"; mark `e2e-exclude` as not emitted (decision 2 + above); in "Mode bots — changed" say the selected matrix comes from mode `e2e`. +- `contracts/suite-components.md` rule 3: append "A changed `test/e2e/.go` also + selects the tests in files that use its top-level names (helpers)." Rule list: + `components` may be empty when `paths` is set (the `TestCISelection_` entry). +- `tasks.md`: tick T014–T029 `[X]` after the PR opens; reword T020/T022–T025 to say mode + `e2e`, and T025 to say "skip unselected legs with `RUN_REGEX`" instead of `e2e-exclude`. + +## 8. Follow-up fix: an unmapped path must also widen mode `modules` + +Mode `e2e` sets `all` for an unmapped path, and `combine` folds it into `CI`, which turns +the `go` and `lint` gates on. But mode `modules` still emits `go-exclude`/`lint-exclude` +from its own (narrow) selection, so for a change like `tools/x.sh` every module is +excluded and the `go` matrix has no legs. Mode `modules` must treat an unmapped path as +an unknown diff (every module), like any other doubt (FR-002). + +8a. `hack/ci_scope.py`: after the `EXTRA_COMPONENTS = (...)` line insert + +```python +# Top-level areas a selection rule knows; any other path is unmapped and forces full scope. +KNOWN_DIRS = ("charts", "containers", "deploy/kind", "signing", "test/e2e", "web", "modules") + + +def is_unmapped(path, modules): + """Return True when no selection rule knows path; such a path forces full scope.""" + if (path.endswith(".md") or path in NO_E2E_FILES or path in FORCE_ALL_FILES + or path == ".dockerignore"): + return False + return not any(path == d or path.startswith(d + "/") + for d in NO_E2E_DIRS + KNOWN_DIRS + tuple(modules)) +``` + +8b. In `main()`, mode `modules` branch, replace + +```python + paths = None if event_reason else changed_paths(ROOT, args.base, args.head) + selected = affected_modules(ROOT, paths) +``` + +with + +```python + paths = None if event_reason else changed_paths(ROOT, args.base, args.head) + if paths is not None and any(is_unmapped(p, workspace_modules(ROOT)) for p in paths): + paths = None # mode e2e reports the path; every module runs (FR-002) + selected = affected_modules(ROOT, paths) +``` + +(Only the occurrence directly followed by `selected = affected_modules(ROOT, paths)`.) + +8c. `hack/test_ci_scope.py`: add `is_unmapped` to the `from ci_scope import (...)` list +(alphabetical, after `image_inputs`), and insert this class directly before the final +`if __name__ == "__main__":` line, with two blank lines around it: + +```python +class UnmappedPathTests(unittest.TestCase): + """An unmapped path widens mode modules to every module (spec 023 FR-002).""" + + def setUp(self): + """Create the shared Go workspace fixture for each test.""" + make_go_workspace(self) + + def test_known_and_unknown_paths(self): + modules = ["lib", "api", "test/e2e", "other"] + for path in ("README.md", "api/x.go", "charts/gameplane/values.yaml", "docs/a.txt", + "hack/ci_scope.py", ".dockerignore", "web/src/App.tsx", "modules"): + with self.subTest(path=path): + self.assertFalse(is_unmapped(path, modules)) + self.assertTrue(is_unmapped("tools/x.sh", modules)) + + def test_modules_mode_selects_every_module_for_an_unmapped_path(self): + output = self.root / "output" + output.write_text("", encoding="utf-8") + with patch("ci_scope.ROOT", self.root), patch("sys.argv", ["ci_scope.py", "modules"]), \ + patch.dict("os.environ", {"GITHUB_EVENT_NAME": "pull_request", + "GITHUB_REF": "refs/pull/7/merge", "CI_FULL_RUN": "false", + "GITHUB_OUTPUT": str(output)}), \ + patch("ci_scope.changed_paths", return_value=["tools/x.sh"]): + main() + values = {key: json.loads(value) for key, value in + (line.split("=", 1) for line in output.read_text(encoding="utf-8").splitlines())} + self.assertEqual(values, FULL_VALUES) +``` diff --git a/specs/023-change-based-test-selection/contracts/ci-scope-cli.md b/specs/023-change-based-test-selection/contracts/ci-scope-cli.md index 4affdc484..2572d915f 100644 --- a/specs/023-change-based-test-selection/contracts/ci-scope-cli.md +++ b/specs/023-change-based-test-selection/contracts/ci-scope-cli.md @@ -23,11 +23,11 @@ path forces everything (research R8). | `scope` | `"full"` \| `"partial"` | Run-level scope (FR-009 switch). Emitted by mode `summary`, not `modules` (see below) | | `go-packages` | JSON object: module → list of package patterns | `["./..."]` on full scope or when a module's own `go.mod`/`go.sum` changed | | `web-tests` | JSON: `"all"` or list of `web/src/...` source paths for `vitest related` | | -| `e2e-exclude` | JSON matrix-exclude list of `{bucket}` | For `e2e-go` and `e2e-go-arm64` | -| `e2e-run` | JSON object: bucket → anchored `-run` regex of the selected tests | Replaces `buckets.sh regex ` in the e2e jobs (OD-6); equals it on full scope | -| `e2e-multicluster`, `e2e-upgrade`, `e2e-web-live`, `e2e-capture-overhead` | `"true"`/`"false"` | Folded into the existing `combine` step booleans | +| `e2e-exclude` | Not emitted (decision 2). Unselected legs skip through `RUN_REGEX` instead | Would have been read by `e2e-go` and `e2e-go-arm64` | +| `e2e-run` | JSON object: bucket → anchored `-run` regex of the selected tests | Emitted by mode `e2e`. Replaces `buckets.sh regex ` in the e2e jobs (OD-6); equals it on full scope | +| `e2e-multicluster`, `e2e-upgrade`, `e2e-web-live`, `e2e-capture-overhead` | `"true"`/`"false"` | Emitted by mode `e2e`. Folded into the existing `combine` step booleans | | `ci-jobs` | JSON list of job ids selected by the `ci.yaml` line mapper | Folded into each job's gate | -| `unmapped` | JSON list of paths | Non-empty forces full scope and fails `selection-rules` | +| `unmapped` | JSON list of paths | Emitted by mode `e2e`. Non-empty forces full scope and fails `selection-rules` | | `selection` | JSON list of SelectionResult | Emitted by mode `summary`; read by the summary table and `report` | ## Mode `summary` — new @@ -54,10 +54,28 @@ Outputs (written to `$GITHUB_OUTPUT`): | `reason` | string | The full-scope reason (`master push`, `manual run`, `label ci: full-run`, `unknown base`, `selection-rule change`, `error: `, `selection fallback (see detect changes log)`), or empty on partial scope | | `selection` | JSON list of SelectionResult | One record per suite and job gate | +## Mode `e2e` (new) + +Emits the e2e selection that `e2e_outputs()` computes from +[suite-components.md](./suite-components.md) and the changed paths. The `e2e` step of the +`changes` job runs right after `bots`. Outputs (written to `$GITHUB_OUTPUT`): + +| Key | Type | Meaning | +|---|---|---| +| `all` | boolean | True when `reason` is set; `combine` folds it into `CI` | +| `reason` | string | Why every e2e test runs (a full-scope reason or `unmapped path

`), or empty | +| `unmapped` | JSON list of paths | Changed paths that no rule maps; non-empty forces full scope | +| `e2e-run` | JSON object: bucket → `-run` regex | Selected tests of each bucket | +| `e2e-why` | JSON object: bucket or test → string | "N of M tests" and the changed paths that chose them, for the summary | +| `e2e-multicluster`, `e2e-upgrade`, `e2e-web-live`, `e2e-capture-overhead` | boolean | Gates for the matching jobs | +| `bot-matrix` | JSON object `{"test": [...]}` | Selected `bot-fast` tests, read by `e2e-game-bot` | +| `bots-any` | boolean | False skips `e2e-game-bot` | +| `universal` | boolean | True when a universal path selected every test without a full-scope reason; `combine` folds it into the e2e and image gates, because universal paths such as `.dockerignore`, `containers/` and `signing/` match no e2e path filter | + ## Mode `bots` — changed Emits only the selected `bot-fast` tests. When none is selected it emits an empty -matrix and `bots-any=false`, and `e2e-game-bot` is gated on `bots-any`. +matrix and `bots-any=false`. The matrix that `e2e-game-bot` reads comes from mode `e2e`, which emits its own `bot-matrix` and `bots-any`. ## Mode `verify-suites` — new diff --git a/specs/023-change-based-test-selection/contracts/suite-components.md b/specs/023-change-based-test-selection/contracts/suite-components.md index 71a59feed..d7e7f7b2b 100644 --- a/specs/023-change-based-test-selection/contracts/suite-components.md +++ b/specs/023-change-based-test-selection/contracts/suite-components.md @@ -37,10 +37,12 @@ from the per-test map in [research.md](../research.md). `buckets.sh list `. Every test in every bucket, including `bot-heavy`, matches at least one entry; a test matching several entries takes the union. 2. `components` holds component ids from [data-model.md](../data-model.md#component), - or `"*"` for "every change selects this test". + or `"*"` for "every change selects this test". `components` may be empty when `paths` + is set, as in the `^TestCISelection_` entry. 3. `paths` are repo-relative prefixes. A changed path under one of them selects the test. A changed `test/e2e/_test.go` always selects the tests defined in that file - (US1 scenario 4), without needing a `paths` entry. + (US1 scenario 4), without needing a `paths` entry. A changed `test/e2e/.go` also selects the tests + in files that use its top-level names (helpers). 4. A changed path under any `universal` prefix selects every suite. 5. Every component with code that is not listed in `unexercised` is named by at least one entry. `unexercised` entries carry a one-line reason. diff --git a/specs/023-change-based-test-selection/quickstart.md b/specs/023-change-based-test-selection/quickstart.md index 8a8039e90..227f7bdec 100644 --- a/specs/023-change-based-test-selection/quickstart.md +++ b/specs/023-change-based-test-selection/quickstart.md @@ -14,8 +14,8 @@ results go in `validation.md` in this folder. | # | Draft PR changes | Expect in the `detect changes` summary | Expect in the run | |---|---|---|---| -| 1 | One line in `agent/internal/console/` | scope partial; agent suites `run` with only their agent tests in the `-run` regex, `telemetry`, `upgrade`, `api-rbac`, `api-auth`, `api-roles`, `api-mods`, `capture-overhead` `skip` | Full result in ≤ 20 min (SC-001) | -| 2 | One line in `sentinel/` | operator RBAC and auth suites `skip` (US1-2) | | +| 1 | One line in `agent/internal/console/` | scope partial; suites with agent-mapped tests `run` with only those tests in the `-run` regex; `api-auth`, `telemetry`, `ratelimit` `skip` (US1-1) | Full result in ≤ 20 min (SC-001) | +| 2 | One line in `sentinel/` | RBAC and auth suites `skip`; `operator` runs only the `TestHelmInstall_` tests and the sentinel's own test (US1-2) | | | 3 | One line in `charts/gameplane/values.yaml` | `full`-equivalent: every e2e suite `run` (US1-3) | | | 4 | One line in an existing `test/e2e/*_e2e_test.go` | the bucket holding that file's tests `run` (US1-4) | | | 5 | One leaf package in `api/internal/` | `go api` runs only that package and its importers (US2-1, US2-2) | changed-lines coverage line in summary | diff --git a/specs/023-change-based-test-selection/spec.md b/specs/023-change-based-test-selection/spec.md index 80eefc87d..815cf0236 100644 --- a/specs/023-change-based-test-selection/spec.md +++ b/specs/023-change-based-test-selection/spec.md @@ -49,10 +49,13 @@ end-to-end suite as skipped with a reason. **Acceptance Scenarios**: 1. **Given** a PR that changes only the agent's console code, **When** CI runs, **Then** - only the end-to-end suites mapped to the agent run, and the unrelated suites - (for example multicluster or upgrade) show as skipped with "not affected by this change". -2. **Given** a PR that changes only the sentinel, **When** CI runs, **Then** the - operator, RBAC and authentication suites do not run. + only the end-to-end tests mapped to the agent run (every test that starts a game + server, since the agent runs in that pod), and the unrelated suites (for example + authentication, telemetry or rate limiting) show as skipped with "not affected by + this change". +2. **Given** a PR that changes only the sentinel, **When** CI runs, **Then** the RBAC + and authentication suites do not run, and the operator suite runs only the sentinel's + tests plus the `TestHelmInstall_` smoke tests that every component selects. 3. **Given** a PR that changes a component every suite depends on (the operator's resource definitions or the Helm chart), **When** CI runs, **Then** every end-to-end suite runs, exactly as today. diff --git a/specs/023-change-based-test-selection/tasks.md b/specs/023-change-based-test-selection/tasks.md index 12987c852..191699657 100644 --- a/specs/023-change-based-test-selection/tasks.md +++ b/specs/023-change-based-test-selection/tasks.md @@ -81,26 +81,26 @@ Repository root of `GameplanePanel/Gameplane`. Main files: `hack/ci_scope.py`, ` ### Tests for User Story 1 -- [ ] T014 [P] [US1] Add `hack/test_ci_scope.py` cases for US1 acceptance scenarios 1–4: agent-only change, sentinel-only change, chart change (all suites), and a changed `test/e2e/*_test.go` selecting its own tests -- [ ] T015 [P] [US1] Add `hack/test_ci_scope.py` cases for `verify-suites`: unmatched test, unknown component id, code component named by no entry and not in `unexercised` -- [ ] T016 [P] [US1] Add `hack/test_ci_scope.py` cases for `e2e-run`: the regex lists only selected tests of a bucket, equals `buckets.sh regex ` on full scope, and the `ratelimit` tail is dropped when none of its tests is selected -- [ ] T017 [US1] Write `test/e2e/ci_selection_e2e_test.go` (`//go:build e2e`, package `e2e`) per research R13: one `TestCISelection_` per scenario in T014 plus unmapped-path and full-run-label cases; each uses `t.Parallel()`, its own `t.TempDir()`, `git archive HEAD` for the base commit, runs `python3 hack/ci_scope.py modules` with `GITHUB_OUTPUT` in the temp dir, and asserts `selection`, `e2e-exclude`, `e2e-run`; then asserts every emitted regex matches at least one compiled `Test` function and nothing outside that bucket's `buckets.sh list`. Follow the `e2e-test-authoring` skill; no logins -- [ ] T018 [US1] Register every `TestCISelection_*` name in `bucket_operator` in `test/e2e/buckets.sh` +- [X] T014 [P] [US1] Add `hack/test_ci_scope.py` cases for US1 acceptance scenarios 1–4: agent-only change, sentinel-only change, chart change (all suites), and a changed `test/e2e/*_test.go` selecting its own tests +- [X] T015 [P] [US1] Add `hack/test_ci_scope.py` cases for `verify-suites`: unmatched test, unknown component id, code component named by no entry and not in `unexercised` +- [X] T016 [P] [US1] Add `hack/test_ci_scope.py` cases for `e2e-run`: the regex lists only selected tests of a bucket, equals `buckets.sh regex ` on full scope, and the `ratelimit` tail is dropped when none of its tests is selected +- [X] T017 [US1] Write `test/e2e/ci_selection_e2e_test.go` (`//go:build e2e`, package `e2e`) per research R13: one `TestCISelection_` per scenario in T014 plus unmapped-path and full-run-label cases; each uses `t.Parallel()`, its own `t.TempDir()`, `git archive HEAD` for the base commit, runs `python3 hack/ci_scope.py modules` with `GITHUB_OUTPUT` in the temp dir, and asserts `selection`, `e2e-exclude`, `e2e-run`; then asserts every emitted regex matches at least one compiled `Test` function and nothing outside that bucket's `buckets.sh list`. Follow the `e2e-test-authoring` skill; no logins — Withdrawn in part (2026-10-22): the test runs mode `e2e`, not `modules`, and asserts `reason` and `e2e-run` only; `selection` and `e2e-exclude` are not outputs of mode `e2e` (contracts/ci-scope-cli.md, Mode `e2e`; PR #778 deviations 1 and 2) +- [X] T018 [US1] Register every `TestCISelection_*` name in `bucket_operator` in `test/e2e/buckets.sh` ### Implementation for User Story 1 -- [ ] T019 [US1] Write `test/e2e/suite-components.json` per contracts/suite-components.md from the per-test map in research.md: `universal` prefixes (research R3), one entry per test-name pattern covering every test of every bucket including `bot-heavy`, and `unexercised` entries for audit-syslog-bridge, mcp-server and the tunnel runtime with reasons; add an entry for `^TestCISelection_` with `paths` per research R13 -- [ ] T020 [US1] In `hack/ci_scope.py`, add mode `verify-suites` implementing contracts/suite-components.md rules 1, 2 and 5, reading bucket lists by running `test/e2e/buckets.sh list ` -- [ ] T021 [US1] Add a `python3 hack/ci_scope.py verify-suites` step after `buckets.sh verify` in the `e2e-buckets` job of `.github/workflows/ci.yaml` -- [ ] T022 [US1] In `hack/ci_scope.py`, map changed paths to component ids (data-model.md Component), reusing `affected_modules()` for the Go reverse-dependency closure, and collect `unmapped` paths; a non-empty `unmapped` forces full scope with reason `unmapped path

` -- [ ] T023 [US1] In `hack/ci_scope.py`, select tests from `suite-components.json` (union over matching entries; a changed `test/e2e/_test.go` selects the tests defined in that file by scanning `func Test` names) and emit `e2e-exclude`, `e2e-run` (OD-6) and the `e2e-multicluster`, `e2e-upgrade`, `e2e-web-live`, `e2e-capture-overhead` booleans -- [ ] T024 [US1] In mode `bots` of `hack/ci_scope.py`, emit only selected `bot-fast` tests and a `bots-any` output (contracts/ci-scope-cli.md "Mode bots") -- [ ] T025 [US1] In `.github/workflows/ci.yaml`, apply `e2e-exclude` to the `e2e-go` and `e2e-go-arm64` matrices with one shared expression so both architectures always get the same list (FR-010) -- [ ] T026 [US1] In `.github/workflows/ci.yaml`, replace `RUN_REGEX="$(./test/e2e/buckets.sh regex '${{ matrix.bucket }}')"` (line ~1377 and its arm64 twin) with the bucket's value from `fromJSON(needs.changes.outputs.e2e-run)`, and gate the `ratelimit` tail step (line ~1389) on its presence -- [ ] T027 [US1] In `.github/workflows/ci.yaml`, fold the four e2e booleans from T023 into the gates of `e2e-multicluster*`, `e2e-upgrade*`, `e2e-web-live*` and `e2e-capture-overhead`, and gate `e2e-game-bot` on `bots-any` -- [ ] T028 [US1] Extend the `selection` records (T010) with the per-bucket and per-test decisions and reasons from T023 -- [ ] T029 [US1] Compile checks only: `python3 -m py_compile hack/ci_scope.py`, `go vet -tags e2e ./...` in `test/e2e`; open PR 2 with labels `type: ci`, `area: shared`, `area: e2e` -- [ ] T030 [US1] Run quickstart scenarios 1–4 and 12 as draft PRs based on PR 2's branch, record the summaries and wall times in `specs/023-change-based-test-selection/validation.md`, close the draft PRs and delete their branches +- [X] T019 [US1] Write `test/e2e/suite-components.json` per contracts/suite-components.md from the per-test map in research.md: `universal` prefixes (research R3), one entry per test-name pattern covering every test of every bucket including `bot-heavy`, and `unexercised` entries for audit-syslog-bridge, mcp-server and the tunnel runtime with reasons; add an entry for `^TestCISelection_` with `paths` per research R13 +- [X] T020 [US1] In `hack/ci_scope.py`, add mode `verify-suites` implementing contracts/suite-components.md rules 1, 2 and 5, reading bucket lists by running `test/e2e/buckets.sh list ` +- [X] T021 [US1] Add a `python3 hack/ci_scope.py verify-suites` step after `buckets.sh verify` in the `e2e-buckets` job of `.github/workflows/ci.yaml` +- [X] T022 [US1] In `hack/ci_scope.py` mode `e2e`, map changed paths to component ids (data-model.md Component), reusing `affected_modules()` for the Go reverse-dependency closure, and collect `unmapped` paths; a non-empty `unmapped` forces full scope with reason `unmapped path

` +- [X] T023 [US1] In `hack/ci_scope.py` mode `e2e`, select tests from `suite-components.json` (union over matching entries; a changed `test/e2e/_test.go` selects the tests defined in that file by scanning `func Test` names) and emit `e2e-run` (OD-6) and the `e2e-multicluster`, `e2e-upgrade`, `e2e-web-live`, `e2e-capture-overhead` booleans +- [X] T024 [US1] In mode `e2e` of `hack/ci_scope.py`, emit only selected `bot-fast` tests as `bot-matrix` and a `bots-any` output (contracts/ci-scope-cli.md "Mode bots") +- [X] T025 [US1] In `.github/workflows/ci.yaml`, skip unselected legs with `RUN_REGEX` in the `e2e-go` and `e2e-go-arm64` jobs, both reading mode `e2e`'s `e2e-run` through one shared `&e2e-go-steps` anchor so both architectures always get the same tests (FR-010) +- [X] T026 [US1] In `.github/workflows/ci.yaml`, replace `RUN_REGEX="$(./test/e2e/buckets.sh regex '${{ matrix.bucket }}')"` (line ~1377 and its arm64 twin) with the bucket's value from `fromJSON(needs.changes.outputs.e2e-run)`, and gate the `ratelimit` tail step (line ~1389) on its presence +- [X] T027 [US1] In `.github/workflows/ci.yaml`, fold the four e2e booleans from T023 into the gates of `e2e-multicluster*`, `e2e-upgrade*`, `e2e-web-live*` and `e2e-capture-overhead`, and gate `e2e-game-bot` on `bots-any` +- [X] T028 [US1] Extend the `selection` records (T010) with the per-bucket and per-test decisions and reasons from T023 +- [X] T029 [US1] Compile checks only: `python3 -m py_compile hack/ci_scope.py`, `go vet -tags e2e ./...` in `test/e2e`; open PR 2 with labels `type: ci`, `area: shared`, `area: e2e` +- [X] T030 [US1] Run quickstart scenarios 1–4 and 12 as draft PRs based on PR 2's branch, record the summaries and wall times in `specs/023-change-based-test-selection/validation.md`, close the draft PRs and delete their branches **Checkpoint**: MVP. A single-component PR boots only its suites (SC-001). diff --git a/specs/023-change-based-test-selection/validation.md b/specs/023-change-based-test-selection/validation.md new file mode 100644 index 000000000..04dfb5c5c --- /dev/null +++ b/specs/023-change-based-test-selection/validation.md @@ -0,0 +1,77 @@ +# Validation results + +Results of the [quickstart](./quickstart.md) scenarios. Each scenario was a throwaway +draft PR based on PR 2's branch (`claude/project-thread-63yei6-pr2`, head `82b77ab`), +allowed by OD-11. The drafts were closed and their branches deleted after the check. + +## T030: scenarios 1–4 and 12 (2026-10-22) + +All five runs were queued for about 65 minutes behind other PRs' runs before any job +started, and every later job waited for a runner too. Wall time therefore says nothing +about SC-001 yet; it is measured again after rollout on an idle queue. Runner-minutes +are the sum of the durations of jobs that ran (success or failure). + +| # | Draft PR | Change | Scope | Runner-min | Result | +|---|---|---|---|---|---| +| 1 | #780 | comment in `agent/internal/console/console.go` | partial | 170 | selection broader than the quickstart row, see below | +| 2 | #781 | comment in `sentinel/main.go` | partial | 61 | RBAC and auth suites skipped (US1-2 met for those); 11 operator tests ran | +| 3 | #782 | comment in `charts/gameplane/values.yaml` | partial, every suite selected | 83 (arm64 only) | every e2e suite selected, 78 of 78 operator tests (US1-3 met) | +| 4 | #783 | comment in `test/e2e/api_agent_e2e_test.go` | partial | 104 | `api-agent` ran 7 of 7 (US1-4 met) | +| 12 | #784 | new file `tools/x.sh` | full: `unmapped path tools/x.sh` | 146 (still running at close) | everything selected; `e2e bucket coverage` failed with `no selection rule maps these paths: ["tools/x.sh"]` (FR-012 met) | + +Every summary listed each suite with a decision and a reason (SC-005 met). + +### Selections in detail + +**Scenario 1 (agent).** Selected: `operator` 55 of 78, `api-agent` 7 of 7, `api-mods` +6 of 7, `api-rbac` 2 of 8, `api-roles` 1 of 7, `optional-components` 1 of 1, +`multicluster` 2 of 6, `upgrade` 1 of 1, web live, all six game-bot joins. Skipped: +`api-auth`, `telemetry`, `ratelimit`, `postgres`. `capture-overhead` was selected in +the summary but its job stayed skipped, because its PR gate still uses the +`capture_overhead` path filter. + +The quickstart row expects `upgrade`, `api-rbac`, `api-roles`, `api-mods` and +`capture-overhead` to skip, and spec US1-1 names multicluster and upgrade as examples +of skipped suites. They ran because `suite-components.json` maps `TestMultiCluster_`, +`TestUpgrade_`, `TestAPI_Lifecycle*`, `TestAPI_OwnerCollaboratorAccess` and the mod +tests to the `agent` component: each of them starts a game server, and the agent runs +in that pod. The rows and the mapping disagree; see "Open item" below. + +**Scenario 2 (sentinel).** Selected: `operator` 11 of 78 (the ten `TestHelmInstall_` +tests, mapped to every component, plus `TestGameServer_IdleNeverSleepsWithoutAPlayerCount`) +and the three `WakeOnConnect` game-bot joins. Skipped: `api-auth`, `api-rbac`, +`api-roles`, `api-agent`, `api-mods`, `telemetry`, `ratelimit`, `multicluster`, +`upgrade`, web live, the other three bots. Spec US1-2 says the operator suite does not +run; 11 of its tests ran, because `TestHelmInstall_` is mapped to `*`. + +**Scenario 3 (chart).** Every suite was selected with "changed: +charts/gameplane/values.yaml". The scope reads `partial` because the chart is a +`universal` path rather than a full-run trigger, which gives the same suites. + +**Scenario 4 (e2e test file).** Selected: `api-agent` 7 of 7 plus the tests that share +the file's helpers: `operator` 16 of 78, `api-rbac` 2 of 8, `api-mods` 5 of 7, +`optional-components`, three game-bot joins. + +**Scenario 12 (unmapped path).** `all: true`, reason `unmapped path tools/x.sh`, +every gate on, and the `e2e bucket coverage` job failed on the unmapped path as +designed. + +### Failures unrelated to selection + +- `lint` failed on scenarios 1, 2 and 4 with `gofmt`: the throwaway comment line was + appended without a blank line. It does not affect the selection result. +- Scenario 2: `e2e operator / amd64` failed in `TestHelmInstall_MetricsNotOnPublicPort` + (`helm_install_e2e_test.go:239`, metrics port not serving within 1m30s). The change + was a comment in `sentinel/main.go`, and the same test passed on arm64 in the same run. +- Scenarios 3 and 12: the amd64 `build e2e images` job was cancelled at its 15-minute + timeout (layer-cache miss), so only the arm64 e2e jobs ran. +- `TestCISelection_*` ran in scenarios 3 and 12 (operator, arm64) and passed in + scenario 3; scenario 12's operator job was still running at close. + +### OD-12 + +Scenarios 1 and 2 selected more than the quickstart rows and spec US1-1/US1-2 said, and +the cause is the agreed `suite-components.json` mapping, not a code defect. Settled as +OD-12 in [OPEN-DECISIONS.md](./OPEN-DECISIONS.md): the mapping stays, and US1-1, US1-2 +and quickstart rows 1-2 were reworded to match it. With the new wording, scenarios 1 and +2 pass. diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index d60a26f14..d664cea6d 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -106,6 +106,12 @@ TestAddressPool_AddressInUseConflict TestAddressPool_PoolExhausted TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions TestE2E_ResticRepoReady +TestCISelection_AgentOnlyChange +TestCISelection_SentinelOnlyChange +TestCISelection_ChartChangeRunsEverySuite +TestCISelection_E2ETestFileSelectsItsTests +TestCISelection_UnmappedPathRunsEverything +TestCISelection_FullRunLabel EOF } diff --git a/test/e2e/ci_selection_e2e_test.go b/test/e2e/ci_selection_e2e_test.go new file mode 100644 index 000000000..a62906016 --- /dev/null +++ b/test/e2e/ci_selection_e2e_test.go @@ -0,0 +1,325 @@ +//go:build e2e + +package e2e + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// The TestCISelection_* tests prove that `hack/ci_scope.py e2e` turns a +// one-commit change into the expected e2e selection (spec 023 OD-4, research +// R13). Each one copies this checkout with `git archive HEAD`, commits a base +// and a head, and runs the selector on them. They need git and python3 only, +// create no cluster resources and never log in, so they live in the operator +// bucket. + +// ciSelectionBuckets mirrors E2E_BUCKETS in hack/ci_scope.py (the e2e-go matrix). +var ciSelectionBuckets = []string{ + "operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", + "telemetry", "optional-components", +} + +// ciSelection is the decoded output of one `ci_scope.py e2e` run. +type ciSelection struct { + Reason string + Run map[string]string + Bots []string + Flags map[string]bool +} + +// ciGit runs git in dir with a fixed identity and returns trimmed stdout. +func ciGit(t *testing.T, dir string, args ...string) string { + t.Helper() + base := []string{"-C", dir, "-c", "user.name=ci-selection", + "-c", "user.email=ci-selection@example.invalid", "-c", "commit.gpgsign=false"} + out, err := exec.CommandContext(t.Context(), "git", append(base, args...)...).CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) +} + +// ciSelectionCopy exports HEAD into a fresh repository and commits it as the +// base. Large paths no selection rule reads are left out to keep it fast. +func ciSelectionCopy(t *testing.T) string { + t.Helper() + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatalf("resolve repository root: %v", err) + } + dir := t.TempDir() + script := `set -o pipefail; git -C "$1" archive --format=tar HEAD -- . ` + + `':(exclude)design.pen' ':(exclude)assets' ':(exclude)docs' ':(exclude)specs' ` + + `':(exclude)web' | tar -x -C "$2"` + if out, err := exec.CommandContext(t.Context(), "bash", "-c", script, "bash", root, dir).CombinedOutput(); err != nil { + t.Fatalf("git archive HEAD: %v\n%s", err, out) + } + ciGit(t, dir, "init", "-q") + ciGit(t, dir, "add", "-A") + ciGit(t, dir, "commit", "-q", "-m", "base") + return dir +} + +// ciSelectionAppend appends text to a file in the copy, creating it if needed. +func ciSelectionAppend(t *testing.T, path, text string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + defer func() { _ = f.Close() }() + if _, err := f.WriteString(text); err != nil { + t.Fatalf("append to %s: %v", path, err) + } +} + +// ciSelectionRun commits edit as the head and returns the selector's output. +func ciSelectionRun(t *testing.T, dir string, fullRunLabel bool, edit func(dir string)) ciSelection { + t.Helper() + base := ciGit(t, dir, "rev-parse", "HEAD") + if edit != nil { + edit(dir) + } + ciGit(t, dir, "add", "-A") + ciGit(t, dir, "commit", "-q", "--allow-empty", "-m", "head") + head := ciGit(t, dir, "rev-parse", "HEAD") + output := filepath.Join(t.TempDir(), "github_output") + cmd := exec.CommandContext(t.Context(), "python3", "hack/ci_scope.py", "e2e", "--base", base, "--head", head) + cmd.Dir = dir + // A clean environment: the e2e job's own GITHUB_EVENT_NAME and GITHUB_REF + // would turn a master push into a full-scope answer here. + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME"), + "GITHUB_OUTPUT=" + output} + if fullRunLabel { + cmd.Env = append(cmd.Env, "CI_FULL_RUN=true") + } + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("ci_scope.py e2e: %v\n%s", err, out) + } + return ciSelectionParse(t, output) +} + +// ciSelectionParse decodes the key= lines ci_scope.py writes to GITHUB_OUTPUT. +func ciSelectionParse(t *testing.T, path string) ciSelection { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + raw := map[string]json.RawMessage{} + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + key, value, ok := strings.Cut(line, "=") + if !ok { + t.Fatalf("malformed output line %q", line) + } + raw[key] = json.RawMessage(value) + } + decode := func(key string, into any) { + if err := json.Unmarshal(raw[key], into); err != nil { + t.Fatalf("decode %s=%s: %v", key, raw[key], err) + } + } + sel := ciSelection{Flags: map[string]bool{}} + decode("reason", &sel.Reason) + decode("e2e-run", &sel.Run) + var bots struct { + Test []string `json:"test"` + } + decode("bot-matrix", &bots) + sel.Bots = bots.Test + for _, key := range []string{"all", "e2e-multicluster", "e2e-upgrade", "e2e-web-live", + "e2e-capture-overhead", "bots-any"} { + var value bool + decode(key, &value) + sel.Flags[key] = value + } + return sel +} + +// ciSelectionBucket runs buckets.sh in this package directory. +func ciSelectionBucket(t *testing.T, args ...string) []string { + t.Helper() + out, err := exec.CommandContext(t.Context(), "bash", append([]string{"buckets.sh"}, args...)...).Output() + if err != nil { + t.Fatalf("buckets.sh %s: %v", strings.Join(args, " "), err) + } + return strings.Fields(string(out)) +} + +// ciSelectionSuiteTests lists the Test functions compiled into this package. +func ciSelectionSuiteTests(t *testing.T) []string { + t.Helper() + files, err := filepath.Glob("*_test.go") + if err != nil { + t.Fatalf("glob test files: %v", err) + } + pattern := regexp.MustCompile(`(?m)^func (Test[A-Za-z0-9_]+)\(`) + var names []string + for _, file := range files { + data, err := os.ReadFile(file) + if err != nil { + t.Fatalf("read %s: %v", file, err) + } + for _, match := range pattern.FindAllStringSubmatch(string(data), -1) { + if match[1] != "TestMain" { + names = append(names, match[1]) + } + } + } + return names +} + +// ciSelectionCheckRegexes asserts every emitted -run regex runs at least one +// test of this package and nothing outside its own bucket. +func ciSelectionCheckRegexes(t *testing.T, sel ciSelection) { + t.Helper() + suite := ciSelectionSuiteTests(t) + for bucket, expr := range sel.Run { + re, err := regexp.Compile(expr) + if err != nil { + t.Errorf("bucket %s: bad regex %q: %v", bucket, expr, err) + continue + } + members := map[string]bool{} + for _, name := range ciSelectionBucket(t, "list", bucket) { + members[name] = true + } + matched := 0 + for _, name := range suite { + if !re.MatchString(name) { + continue + } + matched++ + if !members[name] { + t.Errorf("bucket %s regex %q matches %s, which is not in that bucket", bucket, expr, name) + } + } + if matched == 0 { + t.Errorf("bucket %s regex %q matches no test in this package", bucket, expr) + } + } +} + +// ciSelectionRequireAll asserts every e2e-go bucket runs its whole buckets.sh regex. +func ciSelectionRequireAll(t *testing.T, sel ciSelection) { + t.Helper() + for _, bucket := range ciSelectionBuckets { + want := strings.Join(ciSelectionBucket(t, "regex", bucket), "") + if got := sel.Run[bucket]; got != want { + t.Errorf("bucket %s regex = %q, want the full bucket %q", bucket, got, want) + } + } + for _, key := range []string{"e2e-multicluster", "e2e-upgrade", "e2e-web-live", "bots-any"} { + if !sel.Flags[key] { + t.Errorf("%s = false, want true when every suite runs", key) + } + } +} + +func TestCISelection_AgentOnlyChange(t *testing.T) { + t.Parallel() + dir := ciSelectionCopy(t) + sel := ciSelectionRun(t, dir, false, func(dir string) { + ciSelectionAppend(t, filepath.Join(dir, "agent/internal/console/console.go"), "\n// ci selection probe\n") + }) + if sel.Reason != "" { + t.Fatalf("reason = %q, want a partial selection", sel.Reason) + } + if !strings.Contains(sel.Run["api-agent"], "TestAPI_ConsolePTYRoundTrip") { + t.Errorf("api-agent regex %q lacks TestAPI_ConsolePTYRoundTrip", sel.Run["api-agent"]) + } + for _, bucket := range []string{"api-auth", "ratelimit", "telemetry"} { + if expr, ok := sel.Run[bucket]; ok { + t.Errorf("%s selected for an agent-only change: %s", bucket, expr) + } + } + ciSelectionCheckRegexes(t, sel) +} + +func TestCISelection_SentinelOnlyChange(t *testing.T) { + t.Parallel() + dir := ciSelectionCopy(t) + sel := ciSelectionRun(t, dir, false, func(dir string) { + ciSelectionAppend(t, filepath.Join(dir, "sentinel/main.go"), "\n// ci selection probe\n") + }) + if sel.Reason != "" { + t.Fatalf("reason = %q, want a partial selection", sel.Reason) + } + for _, bucket := range []string{"api-auth", "api-roles", "api-rbac", "api-agent", "telemetry"} { + if expr, ok := sel.Run[bucket]; ok { + t.Errorf("%s selected for a sentinel-only change: %s", bucket, expr) + } + } + wake := false + for _, name := range sel.Bots { + if name == "TestGameServer_WakeOnConnect_LoginWakes" { + wake = true + } + } + if !wake { + t.Errorf("bot matrix %v lacks TestGameServer_WakeOnConnect_LoginWakes", sel.Bots) + } + ciSelectionCheckRegexes(t, sel) +} + +func TestCISelection_ChartChangeRunsEverySuite(t *testing.T) { + t.Parallel() + dir := ciSelectionCopy(t) + sel := ciSelectionRun(t, dir, false, func(dir string) { + ciSelectionAppend(t, filepath.Join(dir, "charts/gameplane/values.yaml"), "\n# ci selection probe\n") + }) + if sel.Reason != "" { + t.Fatalf("reason = %q, want a partial scope that still selects every suite", sel.Reason) + } + ciSelectionRequireAll(t, sel) + ciSelectionCheckRegexes(t, sel) +} + +func TestCISelection_E2ETestFileSelectsItsTests(t *testing.T) { + t.Parallel() + dir := ciSelectionCopy(t) + sel := ciSelectionRun(t, dir, false, func(dir string) { + ciSelectionAppend(t, filepath.Join(dir, "test/e2e/telemetry_e2e_test.go"), "\n// ci selection probe\n") + }) + if got, want := sel.Run["telemetry"], "^(TestTelemetryLifecycle)$"; got != want { + t.Errorf("telemetry regex = %q, want %q", got, want) + } + if expr, ok := sel.Run["api-rbac"]; ok { + t.Errorf("api-rbac selected for a telemetry test edit: %s", expr) + } + ciSelectionCheckRegexes(t, sel) +} + +func TestCISelection_UnmappedPathRunsEverything(t *testing.T) { + t.Parallel() + dir := ciSelectionCopy(t) + sel := ciSelectionRun(t, dir, false, func(dir string) { + ciSelectionAppend(t, filepath.Join(dir, "tools/x.sh"), "#!/bin/sh\n") + }) + if want := "unmapped path tools/x.sh"; sel.Reason != want { + t.Fatalf("reason = %q, want %q", sel.Reason, want) + } + if !sel.Flags["all"] { + t.Errorf("all = false, want true for an unmapped path") + } + ciSelectionRequireAll(t, sel) +} + +func TestCISelection_FullRunLabel(t *testing.T) { + t.Parallel() + dir := ciSelectionCopy(t) + sel := ciSelectionRun(t, dir, true, nil) + if want := "label ci: full-run"; sel.Reason != want { + t.Fatalf("reason = %q, want %q", sel.Reason, want) + } + ciSelectionRequireAll(t, sel) +} diff --git a/test/e2e/suite-components.json b/test/e2e/suite-components.json new file mode 100644 index 000000000..017d1f88f --- /dev/null +++ b/test/e2e/suite-components.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "universal": [ + "charts/", + "containers/", + "operator/api/v1alpha1/", + "operator/config/", + "deploy/kind/", + "signing/", + ".dockerignore", + "test/e2e/env.go", + "test/e2e/internal/fakeoidc/", + "test/e2e/go.mod", + "test/e2e/go.sum", + "test/e2e/buckets.sh", + "test/e2e/suite-components.json" + ], + "suites": [ + { "tests": "^TestHelmInstall_", "components": ["*"] }, + { "tests": "^TestCRD_Validation_", "components": ["operator"] }, + { "tests": "^TestGameServer_", "components": ["operator", "api", "agent"] }, + { "tests": "^TestGameServer_TunnelCredentialRefusalSurfaced$", "components": ["tunnel"] }, + { "tests": "^TestGameServer_IdleNeverSleepsWithoutAPlayerCount$", "components": ["sentinel"] }, + { "tests": "^(TestGameServer_NetworkCapture|TestNetworkCapture_)", + "components": ["operator", "api", "agent", "capture-sidecar"] }, + { "tests": "^TestGameTemplate_", "components": ["operator", "api", "agent"] }, + { "tests": "^(TestModule_|TestModuleSource)", "components": ["operator", "api", "modules"] }, + { "tests": "^TestModule_FinalizerBlocksWhileTemplateInUse$", "components": ["agent"] }, + { "tests": "^TestModule_(ScaffoldAndPackage|SteamcmdScaffoldKeepsNonRootSecurity)$", + "components": ["gp-module"] }, + { "tests": "^TestModuleSource_RejectsSSRFTarget$", "components": ["netguard"] }, + { "tests": "^(TestBackup|TestRestore_|TestE2E_ResticRepoReady$)", + "components": ["operator", "api", "agent"] }, + { "tests": "^TestAddressPool_", "components": ["operator", "api", "agent"] }, + { "tests": "^TestAPI_", "components": ["operator", "api"] }, + { "tests": "^TestAPI_(Agent|ConsolePTYRoundTrip|LogsTailWS|Lifecycle|ModManifestInstallUpgrade|ModUpload|ModArchiveConfinement_|OwnerCollaboratorAccess|OwnerOnlyServerOperations_|EventStreamAndRoleEdits_)", + "components": ["agent", "gameaction"] }, + { "tests": "^TestAPI_I18n_", "components": ["i18n"] }, + { "tests": "^TestAPI_I18n_ModuleLocales$", "components": ["modules"] }, + { "tests": "^TestAPI_ModuleUpload_ExtractionStaysWithinBudget$", "components": ["gp-module", "modules"] }, + { "tests": "^TestGameServer_.*(Bot_|_Persistence$|_RCON$|_Workshop$|_Mods$)", + "components": ["agent", "modules"], + "paths": ["test/e2e/internal/", "test/e2e/gamebot_helpers_e2e_test.go", "test/e2e/gameprobe_job.go"] }, + { "tests": "^TestGameServer_WakeOnConnect_", "components": ["sentinel", "gameproto"] }, + { "tests": "^TestGameServer_CaptureOverhead_Joined$", + "components": ["capture-sidecar", "gameproto", "modules"], + "paths": ["test/e2e/internal/minecraft-java/", "test/e2e/internal/probe/", "test/e2e/gameprobe_job.go"] }, + { "tests": "^TestMultiCluster_", "components": ["operator", "api", "agent", "capture-sidecar"] }, + { "tests": "^TestUpgrade_", "components": ["operator", "api", "agent"] }, + { "tests": "^TestTelemetry", "components": ["operator", "api", "telemetry-receiver", "telemetryschema"], + "paths": ["test/e2e/telemetry_e2e_helpers_test.go", "test/e2e/telemetry_client_test.go", "test/e2e/telemetry_pow_e2e_test.go"] }, + { "tests": "^TestOptionalComponents_", + "components": ["operator", "api", "agent", "mcp-server", "audit-syslog-bridge", "tunnel", "capture-sidecar"], + "paths": ["test/e2e/fixtures/frps.yaml", "test/e2e/fixtures/syslog-sink.yaml"] }, + { "tests": "^TestCISelection_", "components": [], + "paths": ["hack/ci_scope.py", ".github/workflows/ci.yaml", "test/e2e/buckets.sh", "test/e2e/suite-components.json"] } + ], + "unexercised": [ + { "component": "svcutil", "reason": "library only; the suites of every service that imports it run through the module closure" }, + { "component": "test/e2e", "reason": "the e2e suite itself; a changed test file selects its own tests and the tests that use its helpers" }, + { "component": "web", "reason": "dashboard; exercised by e2e web live (Playwright), not by a Go e2e test" } + ] +}