diff --git a/.github/workflows/selftests.yml b/.github/workflows/selftests.yml index 1d58b06..0c29fa6 100644 --- a/.github/workflows/selftests.yml +++ b/.github/workflows/selftests.yml @@ -100,5 +100,63 @@ jobs: # MUTATION_TARGETS it satisfied `selftests-cover` -- which asks make what # that list would run -- while never executing here. Covered on paper, # unrun in fact (Bugbot, .github#300). `make mutations` is the list. + # THE INVENTORY'S CITATIONS, in the SAME already-required context + # (backend#2449). + # + # `repo-inventory.yml` exemption reasons cite tickets, and nothing read + # them: `caller-drift.py` enforces staleness on the ENTRY and never on the + # sentence explaining it. So a reason could say "sequenced behind + # backend#1408" for the eighteen days after that issue closed, with the + # audit green throughout. `make reason-citations` fails when a cited issue + # is closed, or a cited PR was closed without merging. + # + # HERE RATHER THAN A NEW JOB OR A NEW WORKFLOW, for the two reasons the + # steps above were: `selftests` is ALREADY a required context on + # develop/staging/main, so this arms the check with no branch-protection + # edit, and a new context would sit unrequired until somebody armed it -- + # which is how a guard ends up advisory. A new REUSABLE would also need its + # own repo-inventory.yml row in every repo, serialising against every other + # guarded-file PR in this single-writer repo. + # + # BOTH HALVES RUN, exactly as for mint-scope: `make selftests` runs the + # FIXTURE suite, which proves the rule catches, and this proves the live + # inventory complies. Either can pass while the other fails. + # + # THE TOKEN IS ORG-WIDE ON PURPOSE, which is the opposite of the narrowing + # `add-to-kanban.yml` argues for and for a stated reason: the reasons in + # repo-inventory.yml cite tickets in `backend`, `client`, `client-runtime`, + # `model-zoo`, `data-ingestors`, `release-train` and `.github`. A token + # scoped with `repositories:` to this repo would resolve NONE of those, and + # the check would report "cannot tell" on almost every citation. Read-only + # in both scopes, and it writes nothing. + # + # FAIL CLOSED ON A MISSING SECRET, which also means this step fails on a + # PR from a FORK, where GitHub withholds secrets. That is the correct + # answer for a required check whose whole contract is that it never reports + # clean from a read it could not make -- and `add-to-kanban.yml` already + # carries the same property in this repo. + - name: Mint a read-only token for the citation check + id: citation-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.RELEASE_TRAIN_APP_ID }} + private-key: ${{ secrets.RELEASE_TRAIN_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + permission-issues: read + permission-pull-requests: read + + - name: reason-citations (the live inventory, not fixtures) + env: + GH_TOKEN: ${{ steps.citation-token.outputs.token }} + run: | + set -euo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "::error::the citation token is empty. Without it every citation" \ + "reads as unresolvable, and this check must not report clean from" \ + "a read it could not make." >&2 + exit 1 + fi + make reason-citations + - name: mutation-check (every runner in MUTATION_TARGETS) run: make mutations diff --git a/Makefile b/Makefile index 6ff370f..afae389 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,8 @@ # bricked-prs-selftest.yml -> `selftest-bricked-prs` # kanban-columns.yml -> `selftest-kanban-columns` # kanban-deploy-state-selftest.yml -> `selftest-kanban-deploy-state` -# selftests.yml -> `selftests`, `mint-scope` AND `selftest-house-rules` +# selftests.yml -> `selftests`, `mint-scope`, `reason-citations` +# AND `selftest-house-rules` # (the `selftests` required context runs all three. `selftests` + the fixture # suites prove the rules CATCH; `mint-scope` proves the real workflows COMPLY. # They disagree in either direction, so neither substitutes for the other. @@ -105,6 +106,7 @@ help: @echo " selftests all $(words $(SELFTEST_FILES)) gate selftests (+ the coverage assertion)" @echo " credential-scan gitleaks over the whole history, as code-quality.yml runs it" @echo " audit caller-drift.py against the live org — needs a token" + @echo " reason-citations the live inventory's ticket citations — needs a token" @echo @echo " Not reproducible locally, by construction:" @echo " conformance-gate.yml polls the API for caller-drift's verdict on a" @@ -213,6 +215,25 @@ selftest-mint-scope: guard-pyyaml mint-scope: guard-pyyaml $(PYTHON) scripts/mint-scope.py +# reason-citations: a ticket cited by a repo-inventory exemption reason must still +# be live (backend#2449). THREE targets, in three different tiers, because they +# answer three different questions: +# +# selftest-reason-citations does the rule CATCH? fixtures, offline, in `check` +# mutation-reason-citations would the suite NOTICE? `mutations`, in `check-all` +# reason-citations does the INVENTORY comply? the live file, needs a token +# +# The third is NOT in `check` or `lint`, unlike `mint-scope` -- and that is the +# only reason the two are wired differently. mint-scope reads workflow files off +# disk; this one reads issue state from the API, so it needs `gh` authenticated +# and the network. `make check` is the offline pre-push tier with an ~18 s budget, +# and a target that can fail on somebody's train wifi does not belong in it. It +# runs in CI from selftests.yml, where the App token is minted, and on demand +# here -- the same split `audit` gets, for the same reason. +.PHONY: reason-citations +reason-citations: guard-pyyaml + $(PYTHON) scripts/reason-citations.py + action-pins: @set -e; \ @@ -276,7 +297,7 @@ SELFTEST_FILES := $(sort $(wildcard scripts/tests/*-selftest.py scripts/tests/*- MUTATION_FILES := $(sort $(wildcard scripts/tests/*-mutations.py)) MUTATION_TARGETS := mutation-house-rules mutation-pipefail-early-close \ mutation-bugbot-gate mutation-closing-ref-gate mutation-bug-to-ready \ - mutation-branch-owner + mutation-branch-owner mutation-reason-citations # THE WHOLE MUTATION TIER, BY NAME OF THE LIST. Every entry point -- CI, # `check-all`, `lint` -- depends on one of these two rather than on any @@ -306,6 +327,7 @@ SELFTEST_TARGETS := selftest-caller-drift selftest-blocked-marker selftest-stand selftest-version-bump-gate selftest-bricked-prs selftest-kanban-columns \ selftest-kanban-deploy-state selftest-git-reap \ selftest-mint-scope selftest-house-rules \ + selftest-reason-citations \ selftest-pipefail-early-close \ selftest-bugbot-gate \ selftest-closing-ref-gate \ @@ -512,6 +534,25 @@ mutation-closing-ref-gate: mutation-closing-ref-gate-dry: $(PYTHON) scripts/tests/closing-ref-gate-mutations.py --dry + +# The reason-citation check (backend#2449). guard-pyyaml: it parses +# repo-inventory.yml. The suite stubs `gh` on PATH, so neither of these two needs +# a token or the network -- only the `reason-citations` target above does. +.PHONY: selftest-reason-citations +selftest-reason-citations: guard-pyyaml + $(PYTHON) scripts/tests/reason-citations-selftest.py + +# Measured on a laptop: the suite alone ~4 s (every case runs the checker as a +# subprocess against a fixture inventory), the full mutation pass ~95 s for 23 +# mutations. Same split as every other runner -- the full pass rides the required +# `selftests` context via `make mutations`, and `--dry` (anchor resolution only, +# milliseconds) rides `make check`. +.PHONY: mutation-reason-citations mutation-reason-citations-dry +mutation-reason-citations: + $(PYTHON) scripts/tests/reason-citations-mutations.py + +mutation-reason-citations-dry: + $(PYTHON) scripts/tests/reason-citations-mutations.py --dry # The bug-label promotion (backend#2348). guard-pyyaml: the suite parses THREE # workflows -- it extracts the decision out of `customer-priority-bump.yml` by its # `# selftest:` markers, asserts `col_index` byte-identical to the router's, and diff --git a/scripts/reason-citations.py b/scripts/reason-citations.py new file mode 100755 index 0000000..2027aaa --- /dev/null +++ b/scripts/reason-citations.py @@ -0,0 +1,558 @@ +#!/usr/bin/env python3 +"""A citation inside a repo-inventory exemption reason must still be live. + +WHY THIS EXISTS (tracebloc/backend#2449) +---------------------------------------- +`repo-inventory.yml` entries carry a free-prose `reason`. `caller-drift.py` +enforces staleness on the ENTRY -- an exemption whose caller turned up is a +finding (`caller-drift.py:2309`) -- and NOTHING reads the reason. So the entry +stays legitimately exempt, the audit stays green, and the sentence a human reads +before deciding whether the exemption still applies can be false for as long as +nobody re-reads it. + +Measured false twice on 2026-08-24, independently: + + * `customer_priority_bump_caller_missing` said the wiring was "sequenced + behind" backend#1408. That issue closed COMPLETED on 2026-08-06. Three repos + sat behind a sentence describing a ticket that had been shut for eighteen + days; the wiring went ahead the moment somebody read the ticket instead of + the reason. + * `rfcs`' `advance-deploy-env.yml` reason said "this repo has no `develop`". + It has one. + +WHAT THIS CHECKS, AND WHAT IT DELIBERATELY DOES NOT +--------------------------------------------------- +ONE of the three mechanisms backend#2449 sketches: **where a reason names an +issue, fail when that issue is CLOSED.** The reason may well still be valid -- a +closed citation is not proof of anything -- but it is exactly the case a human +has to re-read, and it is checkable from the citation alone. That alone would +have caught case 1 the day #1408 closed. + +The other two are NOT built here, on purpose: + + (2) date expiry -- "a reason older than N days is a finding" -- needs a + measurement date on every reason first, which is an edit to + repo-inventory.yml and a schema decision, not a checker. + (3) "assert the falsifiable half" -- decide whether a reason states a FACT or + a JUDGEMENT and check the facts -- is a much larger design problem: it + needs prose understood well enough to know which half is falsifiable, and + the wrong answer either invents findings or teaches people to phrase + reasons so the check cannot see them. + +Splitting them keeps this one small enough to be obviously correct. + +DERIVED, NEVER RESTATED (CLAUDE.md rule 1) +------------------------------------------ +The citations come from PARSING the inventory: every `exempt:` string, every +`divergent:`'s `reason:`, and every anchor body in `shared_reasons:`. There is no +hand-written list here of which reason cites which ticket -- a checker holding +its own copy of the answer agrees with itself while disagreeing with reality, and +this very file warns about that in prose ("an anchor cannot state its own reach +-- trust `grep` over this sentence"). Write a new reason citing a new ticket and +it is covered the moment it lands. + +COMMENTS ARE NOT REASONS. The scan runs over the PARSED YAML, so the ~40 ticket +numbers in this file's header comments are out of scope. They are documentation +about the file, not the written justification for an exemption, and nothing reads +them to decide whether an exemption still applies. + +WHICH STATES ARE FINDINGS, AND WHY A MERGED PR IS NOT ONE +---------------------------------------------------------- +GitHub answers "issue or pull request" in one field, so the distinction is READ, +not assumed: + + Issue OPEN fine + Issue CLOSED FINDING -- re-read the reason (this is backend#2449 case 1) + PullRequest OPEN fine + PullRequest MERGED fine + PullRequest CLOSED FINDING -- a plan that never landed + +A merged PR is the one closed thing whose terminal state is SUCCESS. Reasons cite +PRs as provenance ("remediated under model-zoo#115", "Bugbot, .github#196"), and +that sentence stays true forever. Flagging them would have made 9 of the 23 live +citations findings on day one for describing history correctly -- noise that +teaches people to stop reading the report, which is the failure mode a gate can +least afford. A CLOSED-unmerged PR is the opposite: the reason is leaning on +something that did not happen. + +WHICH REPO A CITATION MEANS -- STATED, NOT ASSUMED +--------------------------------------------------- + `owner/repo#N` as written. + `repo#N` `/repo#N`, org read from this inventory's own `org:` key. + `#N` `/#N` -- the repo this inventory LIVES IN, + which is what GitHub itself renders a bare `#N` as in a file in + that repo. Both values come from the inventory (`org:`, + `source_repo:`); neither is typed here. + +That last rule is the one worth stating out loud, because guessing it wrong is a +defect this org has already shipped and fixed: `closing-ref-gate.py` resolved a +bare number against `backend` and so advised `Closes tracebloc/backend#N` on a +`release-train` PR (.github#314). So this file does not assume `backend`, and it +does not stay quiet either -- when `GITHUB_REPOSITORY` is set and disagrees with +`/`, the premise of the bare rule is false and a bare citation +becomes "cannot tell" rather than a guess. + +FAIL CLOSED (rule 3), AND "CANNOT TELL" IS A FINDING ABOUT THE CHECK +--------------------------------------------------------------------- +Exit 2, never a pass: an unreadable or unparseable inventory, a missing `org:` or +`source_repo:`, a `gh` call that fails, a GraphQL response that is not JSON, a +citation the API will not resolve (404, 403, rate limit), and ZERO CITATIONS +FOUND. Zero is the important one and the reason it is checked at all: this file's +whole premise is that reasons cite tickets, so finding none means the MATCHER +broke, not that the inventory got clean. + +Exit 1 is a real finding about the inventory. Both fail; they are separated so a +red run says whether to fix a reason or fix this script. + +EXEMPTIONS ARE TEMPORARY, AND STALENESS IS A FINDING +----------------------------------------------------- +Same shape as `mint-scope.py`'s `EXEMPT`, for the same reason: this lands GREEN +over the citations that were ALREADY dead when it was written, rather than as a +red gate in a REQUIRED context that nobody can merge past (rule 4 -- never land a +red gate). And the other half is what stops the list becoming cover: +a row whose citation is no longer a finding -- reopened, or edited out of the +inventory -- is itself reported, so the list has to be pruned. +""" +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +try: + import yaml +except ModuleNotFoundError: # pragma: no cover - guarded by `make guard-pyyaml` + sys.stderr.write("::error::PyYAML is required: python3 -m pip install pyyaml\n") + raise SystemExit(2) + +ROOT = Path(__file__).resolve().parents[1] + +# The suite drives this against fixture inventories. Same seam `mint-scope.py` +# uses for MINT_SCOPE_DIR: a guard that can only run against production state is +# a guard no test can pin. +INVENTORY = Path(os.environ.get("REASON_CITATIONS_INVENTORY") or (ROOT / "repo-inventory.yml")) + +# Overridable for the same reason and ONLY that reason: a suite that had to name +# the real exempt rows would redden every time one is burnt down. +_EXEMPT_OVERRIDE = os.environ.get("REASON_CITATIONS_EXEMPT") + +# WHICH INVENTORY KEYS CARRY A WRITTEN REASON. Taken from `caller-drift.py`'s +# schema, which accepts THREE spellings and this org uses all three: +# `exempt: ""` every family (`_reason_entry`) +# `divergent: ""` copies (`_reason_entry`) +# `divergent: {reason: "...", ...}` protection (`_protection_entry`) +# All three are read, so a reason cannot escape the scan by being written in +# another one of them -- which `divergent` nearly did: the first draft here read +# only `exempt` and `reason`, and the bare-string `divergent` at line 1159 of the +# inventory would have gone unscanned. +REASON_KEYS = ("exempt", "divergent", "reason") + +# The anchor: a `#` followed by digits is somebody citing a ticket. Everything +# before it is then parsed as a repo, rather than the repo being part of the +# match -- so `owner/repo/extra#5` is REPORTED as malformed instead of quietly +# matching its tail, and a bare `#5` is seen rather than skipped. +ANCHOR_RE = re.compile(r"#(\d+)") + +# The characters a citation prefix may be built from, scanned leftwards from the +# `#`. `.` is in the set because `.github` is a real repo name in this org and an +# alnum-first class silently dropped every citation to it. +PREFIX_CHARS = re.compile(r"[A-Za-z0-9._/-]") + +# A fully-formed prefix: `owner/repo` or `repo`. GitHub owners are alnum + `-`; +# repo names additionally allow `.` and `_`. Anything else is malformed, INCLUDING +# a second slash -- this is also what keeps the repo name safe to interpolate into +# the GraphQL document below. +PREFIX_RE = re.compile(r"\A(?:([A-Za-z0-9][A-Za-z0-9-]*)/)?([A-Za-z0-9.][A-Za-z0-9._-]*)\Z") + +# CITATIONS THAT WERE ALREADY CLOSED WHEN THIS GUARD LANDED (2026-08-24), each +# with what it is doing in the inventory. Burn this down; do not grow it. +# +# This is the state the check was written to stop GROWING, not to fix in the +# commit that adds the check -- backend#2449 says so explicitly, because +# re-reading these reasons is its own work and rewriting them here would bury the +# mechanism in a prose diff. Every row below is a citation to a CLOSED issue that +# a human still has to judge. +# +# NO TALLY IN THIS COMMENT, deliberately. `mint-scope.py` shipped with one and it +# said 13 in two places while the list held 12 (saadqbal, #287); the run prints +# the number from `len(_exempt())`, which is the only place it should exist. +# +# TWO OF THEM ARE THE TICKET'S OWN CASE 1, FOUND ON THE FIRST RUN, and they are +# marked UNREMEDIATED rather than explained away. Both lean FORWARD on a ticket +# that is shut -- the same shape as the backend#1408 sentence that held three +# repos for eighteen days. Do not let the row read as permission. +# +# The rest are cited in the PAST TENSE, as the ticket under which something +# already landed or was already decided. That distinction is a judgement about +# prose, which is why it is written here as a note to whoever burns these down +# and is NOT something this script tries to infer (see mechanism (3) in the +# header). +EXEMPT = { + "tracebloc/backend#1408": ( + "already re-read, and the prose says so: `stale_backlog_exemption_needs_redeciding` " + "states that this ticket's basis is gone and records the exemption as UNDECIDED" + ), + "tracebloc/backend#1415": ( + "UNREMEDIATED -- `wip_limit_check_has_no_callers` defers to it in the FUTURE tense " + "(\"the decision to wire it up or delete it is backend#1415 follow-up work\") and the " + "ticket is closed. Backend#2449 case 1 exactly; found by this guard's first run" + ), + "tracebloc/backend#1729": ( + "UNREMEDIATED -- `blocked_gate_rollout_pending` says \"the callers follow in the " + "rollout PR (backend#1729)\" and the ticket is closed, so the rollout it is waiting on " + "is not tracked by anything open. Backend#2449 case 1 shape" + ), + "tracebloc/backend#1276": ( + "cited as the decision record its D-numbers are quoted from; closed-completed is a " + "decision record's terminal state" + ), + "tracebloc/backend#1420": "cited in the past tense: the ticket two repos were remediated under", + "tracebloc/backend#1816": "cited in the past tense: the ticket that made add-to-kanban v2.0.0 the fleet norm", + "tracebloc/backend#1563": "cited in the past tense: the ticket version-bump-pr.yml was deleted under", + "tracebloc/backend#1752": ( + "cited in the past tense, as the provenance of the two live PRs blocked-gate would redden" + ), + "tracebloc/backend#1975": "cited in the past tense: the ticket that armed frontend-app's Vitest contexts", + "tracebloc/backend#1976": "cited in the past tense: the ticket that raised the action-pins baseline", + "tracebloc/backend#1979": "cited in the past tense: the ticket that removed stale-backlog's column-blindness", + "tracebloc/backend#2243": "cited in the past tense: the ticket that flipped release-train to `required`", +} + +# The GraphQL field that answers "issue or PR?" in one read. `state` is +# OPEN/CLOSED on an Issue and OPEN/CLOSED/MERGED on a PullRequest, which is the +# whole reason a merged PR can be told apart from an abandoned one. +NODE_FIELDS = ( + "{ __typename ... on Issue { state } ... on PullRequest { state } }" +) + + +class Finding(Exception): + """A malfunction of the check, or something it cannot tell. Always exit 2.""" + + +def _exempt() -> dict: + """The live exemption map, or the suite's override. + + Read through a function rather than mutating the module global, so the map + above stays the single written-down answer and a case cannot leave it + modified for the next one. + """ + if _EXEMPT_OVERRIDE is None: + return EXEMPT + return {c.strip(): "test override" for c in _EXEMPT_OVERRIDE.split(",") if c.strip()} + + +# ------------------------------------------------------------------ parsing --- + + +def load_inventory(path: Path) -> dict: + """The inventory as a mapping. Anything else is a finding about the check.""" + try: + text = path.read_text(encoding="utf-8") + except OSError as exc: + raise Finding(f"{path} could not be read ({exc}) -- refusing to report clean") + try: + doc = yaml.safe_load(text) + except yaml.YAMLError as exc: + raise Finding(f"{path} could not be parsed ({exc}) -- refusing to report clean") + if not isinstance(doc, dict): + raise Finding(f"{path} did not parse to a mapping -- refusing to report clean") + return doc + + +def reason_strings(doc: dict) -> "list[tuple[str, str]]": + """Every written reason in the inventory, as (where, text). + + Two sources, because a reason can be written in either place and the guard + must not be escapable by choosing the other: + + * `shared_reasons:` -- the anchor bodies. Walking `repos:` alone would find + every ALIASED one (safe_load resolves an alias to the same string), but + an anchor defined and not yet referenced would be invisible, and that is + the state a reason is in on the PR that introduces it. + * `repos:` -- every `exempt:` string and every `divergent:`'s `reason:`, at + whatever depth, so a new property FAMILY is covered without an edit here. + """ + out = [] + shared = doc.get("shared_reasons") + if isinstance(shared, dict): + for name, text in shared.items(): + if isinstance(text, str): + out.append((f"shared_reasons.{name}", text)) + + def walk(node, where: str) -> None: + if isinstance(node, dict): + for key, value in node.items(): + if key in REASON_KEYS and isinstance(value, str): + out.append((f"{where}.{key}", value)) + else: + walk(value, f"{where}.{key}") + elif isinstance(node, list): + for i, value in enumerate(node): + walk(value, f"{where}[{i}]") + + walk(doc.get("repos"), "repos") + return out + + +class Citation: + """One `#N` an author wrote, and what could be made of what precedes it. + + `legal` is False when a prefix WAS written and is not a valid `owner/repo` -- + reported as malformed rather than dropped, so a typo cannot make a citation + invisible to the check. That is the difference between this and a regex that + matches only well-formed citations: the well-formed regex is quiet about + exactly the input a human needs to see. + """ + + __slots__ = ("owner", "repo", "number", "raw", "legal") + + def __init__(self, owner, repo, number, raw, legal): + self.owner, self.repo, self.number = owner, repo, number + self.raw, self.legal = raw, legal + + +def parse_citations(text: str) -> "list[Citation]": + """Every ticket citation in one reason.""" + out = [] + for match in ANCHOR_RE.finditer(text): + start = match.start() + i = start + while i > 0 and PREFIX_CHARS.match(text[i - 1]): + i -= 1 + prefix = text[i:start] + raw = prefix + match.group(0) + number = int(match.group(1)) + if not prefix: + out.append(Citation(None, None, number, raw, True)) + continue + parsed = PREFIX_RE.match(prefix) + if parsed is None: + out.append(Citation(None, None, number, raw, False)) + continue + out.append(Citation(parsed.group(1), parsed.group(2), number, raw, True)) + return out + + +def host(doc: dict) -> "tuple[str, str]": + """(org, repo-this-inventory-lives-in), both READ from the inventory. + + `source_repo` is the inventory's own name for the repo that hosts the + reusables, which is the repo this file sits in. Nothing here types + "tracebloc" or ".github" -- move the inventory and the bare-citation rule + moves with it, or fails loudly (see `resolve`). + """ + org, src = doc.get("org"), doc.get("source_repo") + if not isinstance(org, str) or not org.strip(): + raise Finding("the inventory declares no `org:` -- cannot resolve any citation") + if not isinstance(src, str) or not src.strip(): + raise Finding( + "the inventory declares no `source_repo:` -- cannot resolve a bare `#N`, " + "and guessing one is the defect .github#314 fixed" + ) + return org.strip(), src.strip() + + +def resolve(owner, repo, number: int, raw: str, org: str, source_repo: str) -> str: + """Canonical `owner/repo#N`, or a Finding when it cannot be resolved.""" + if repo is None: + # A bare `#N`. The rule is GitHub's own -- this repo -- and it is only + # sound while this file actually lives in `/`. When the + # runner tells us otherwise, say so instead of resolving it anyway. + here = os.environ.get("GITHUB_REPOSITORY") + if here and here.strip().lower() != f"{org}/{source_repo}".lower(): + raise Finding( + f"{raw!r} is a bare citation, which this check resolves against the repo the " + f"inventory lives in ({org}/{source_repo} per `source_repo:`). GITHUB_REPOSITORY " + f"says {here!r}, so that premise is false and the repo cannot be told. Write " + "`owner/repo#N` in the reason." + ) + repo = source_repo + return f"{owner or org}/{repo}#{number}" + + +# --------------------------------------------------------------------- reads --- + + +def _run_gh(args, env): + return subprocess.run(args, capture_output=True, text=True, env=env, check=False) + + +def fetch_states(keys: "list[str]", env=None, runner=_run_gh) -> "dict[str, tuple[str, str]]": + """(typename, state) per citation, in ONE GraphQL call. + + One request rather than one per citation on purpose: this runs inside a + REQUIRED context on every PR, and N sequential API calls is N chances for a + blip to fail a merge closed. + + ANY failure raises. A partial read is not a clean read -- GitHub answers a + bad alias with a null node AND an `errors[]` entry while still returning 200 + and data for the others, so a caller that only looked at the exit code would + silently score an unresolvable citation as fine. + """ + env = dict(os.environ if env is None else env) + parts = [] + for i, key in enumerate(keys): + repo_part, _, number = key.rpartition("#") + owner, _, name = repo_part.partition("/") + parts.append( + f'c{i}: repository(owner: "{owner}", name: "{name}") ' + f"{{ issueOrPullRequest(number: {int(number)}) {NODE_FIELDS} }}" + ) + query = "query {" + " ".join(parts) + "}" + proc = runner(["gh", "api", "graphql", "-f", "query=" + query], env) + try: + payload = json.loads(proc.stdout) + except (ValueError, TypeError): + raise Finding( + f"the GraphQL read returned no JSON (exit {proc.returncode}): " + f"{(proc.stderr or proc.stdout or '').strip()[:300]}" + ) + data = payload.get("data") + if not isinstance(data, dict): + raise Finding( + f"the GraphQL response carried no data (exit {proc.returncode}): " + f"{json.dumps(payload.get('errors'))[:300]}" + ) + out = {} + for i, key in enumerate(keys): + repo = data.get(f"c{i}") + node = repo.get("issueOrPullRequest") if isinstance(repo, dict) else None + if not isinstance(node, dict) or not node.get("__typename") or not node.get("state"): + # 404, 403, a rate limit, a renamed repo: all the same answer, which + # is CANNOT TELL. Never "open". + raise Finding( + f"{key} could not be read (no issue or pull request came back). That is " + "'cannot tell', not 'still open' -- fix the citation or the token's reach" + ) + out[key] = (node["__typename"], node["state"]) + return out + + +# ------------------------------------------------------------------ verdicts --- + +# (typename, state) pairs that are NOT a finding. Written down as the whole +# allowed set rather than as "not closed", so a state GitHub adds later lands in +# the unknown branch below and is refused rather than silently passing. +LIVE = { + ("Issue", "OPEN"), + ("PullRequest", "OPEN"), + ("PullRequest", "MERGED"), +} +DEAD = { + ("Issue", "CLOSED"): "the issue is CLOSED -- re-read the reason that cites it", + ("PullRequest", "CLOSED"): "the pull request was CLOSED WITHOUT MERGING -- the reason " + "leans on something that never landed", +} + + +def classify(state: "tuple[str, str]") -> "str | None": + """None when the citation is live, else why it is a finding.""" + if state in LIVE: + return None + if state in DEAD: + return DEAD[state] + raise Finding( + f"unrecognised citation state {state!r}. GitHub returned something this check has " + "no verdict for; refusing to guess whether it is live" + ) + + +def audit(env=None, runner=_run_gh): + """Returns (findings, malformed, citations, reasons_scanned). + + `findings` and `malformed` are lists of (key, why, [where, ...]). + """ + doc = load_inventory(INVENTORY) + org, source_repo = host(doc) + reasons = reason_strings(doc) + + seen: "dict[str, list[str]]" = {} + malformed: "dict[str, list[str]]" = {} + for where, text in reasons: + for c in parse_citations(text): + # `#0` is nobody's issue: GitHub numbers from 1. A citation that + # cannot name a real ticket is a defect in the prose, reported rather + # than sent to the API to 404. + if not c.legal or c.number < 1: + malformed.setdefault(c.raw, []).append(where) + continue + key = resolve(c.owner, c.repo, c.number, c.raw, org, source_repo) + seen.setdefault(key, []).append(where) + + if not seen and not malformed: + # The premise of this file is that reasons cite tickets. None found means + # the matcher broke -- a schema change, a key rename, a reason moved -- + # not that the inventory got clean. + raise Finding( + f"no ticket citation found in any of the {len(reasons)} written reason(s) in " + f"{INVENTORY.name}. Either the schema moved or the matcher is broken; a check " + "that finds nothing to check must not report success" + ) + + states = fetch_states(sorted(seen), env=env, runner=runner) if seen else {} + findings = [] + for key in sorted(seen): + why = classify(states[key]) + if why: + findings.append((key, why, sorted(set(seen[key])))) + bad = [(raw, "not a legal `owner/repo#N`, `repo#N` or `#N` citation", sorted(set(w))) + for raw, w in sorted(malformed.items())] + return findings, bad, seen, len(reasons) + + +def stale_exemptions(findings) -> "list[str]": + """Exempted citations that are no longer a finding. + + An exemption list nobody prunes stops being a burn-down and becomes cover: a + reason rewritten to cite a NEWLY closed ticket would be admitted by a row + written years earlier about a different one. So a row that is no longer + needed is reported too. + """ + return sorted(set(_exempt()) - {key for key, _, _ in findings}) + + +def main() -> int: + try: + findings, malformed, citations, reasons = audit() + except Finding as exc: + sys.stderr.write(f"::error::{exc}\n") + return 2 + + exempt = _exempt() + offenders = [f for f in findings if f[0] not in exempt] + \ + [m for m in malformed if m[0] not in exempt] + stale = stale_exemptions(findings) + + print(f"reason-citations: {len(citations)} distinct citation(s) across " + f"{reasons} written reason(s) in {INVENTORY.name}") + print(f" {len(findings)} dead, {len(malformed)} malformed, {len(exempt)} exempted, " + f"{len(offenders)} finding(s)") + + rc = 0 + for key, why, where in offenders: + sys.stderr.write( + f"::error file={INVENTORY.name}::{key} is cited by a repo-inventory reason and " + f"{why}. Re-read the reason and either restate it against something still true, " + f"or add {key} to EXEMPT in scripts/reason-citations.py with what it is doing " + f"there. Cited by: {', '.join(where[:4])}" + f"{' (+%d more)' % (len(where) - 4) if len(where) > 4 else ''}\n" + ) + rc = 1 + for key in stale: + sys.stderr.write( + f"::error::{key} is listed in EXEMPT but is no longer a finding -- it reopened, or " + "no reason cites it any more. Remove the row: a stale exemption is cover for the " + "next dead citation that lands in the same reason.\n" + ) + rc = 1 + if rc == 0: + print(" no findings (every citation is live, and every exemption still applies)") + return rc + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/reason-citations-mutations.py b/scripts/tests/reason-citations-mutations.py new file mode 100644 index 0000000..5df2199 --- /dev/null +++ b/scripts/tests/reason-citations-mutations.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Mutation harness for the reason-citation check (tracebloc/backend#2449). + +`reason-citations-selftest.py` asserts the check's behaviour; this asserts the +SELFTEST. Break a rule in `scripts/reason-citations.py`, watch the suite redden, +restore. A case that stays green while the rule it names is deleted is vacuous, +and a green selftest log cannot tell you which of its assertions are +load-bearing. + +THE MUTATION CALLS THE CODE UNDER TEST (CLAUDE.md rule 9). It edits +`scripts/reason-citations.py` on disk and re-runs the real suite, which executes +that same file as a subprocess. There is no second copy of the rule in here -- +the alternative shape, re-implementing the rule inline and mutating the copy, +is indistinguishable from real coverage in a log and has bitten this org twice. + +EVERY ANCHOR MUST MATCH EXACTLY ONCE. An anchor matching twice mutates an +arbitrary one, so the run reports "uncaught" for the wrong reason; an anchor +matching zero times is stale and fails the run exactly like an uncaught +mutation. That is the assertion that the mutation ACTUALLY APPLIED -- an inert +mutation and good coverage look identical in a log otherwise. `--dry` resolves +every anchor without running the suite, which is what belongs in the fast tier. + + reason-citations-mutations.py run them all + reason-citations-mutations.py --dry resolve anchors only +""" +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +GUARD = ROOT / "scripts" / "reason-citations.py" +SUITE = ROOT / "scripts" / "tests" / "reason-citations-selftest.py" + +# (label, old, new) +MUTATIONS = [ + # --- the load-bearing claim: a CLOSED issue is a finding ---------------- + ("a CLOSED issue counts as live, so backend#2449 case 1 goes unreported", + 'LIVE = {\n ("Issue", "OPEN"),', + 'LIVE = {\n ("Issue", "CLOSED"),\n ("Issue", "OPEN"),'), + ("the dead-state table is emptied, so nothing is ever a finding", + ' if state in DEAD:\n return DEAD[state]', + ' if state in DEAD:\n return None'), + + # --- the issue / pull-request distinction ------------------------------- + ("a MERGED pull request is treated as staleness, so provenance reads as a finding", + ' ("PullRequest", "MERGED"),\n}', + '}'), + ("a pull request CLOSED WITHOUT MERGING counts as live", + ' ("PullRequest", "OPEN"),', + ' ("PullRequest", "OPEN"),\n ("PullRequest", "CLOSED"),'), + ("a state with no verdict passes instead of being refused", + ' raise Finding(\n f"unrecognised citation state {state!r}.', + ' return None\n raise Finding(\n f"unrecognised citation state {state!r}.'), + + # --- which repo a bare `#N` means: the .github#314 defect --------------- + ("a bare `#N` is resolved against `backend` again (.github#314)", + " repo = source_repo", + ' repo = "backend"'), + ("the bare-citation premise is never checked against GITHUB_REPOSITORY", + ' if here and here.strip().lower() != f"{org}/{source_repo}".lower():', + " if False:"), + ("`org:` and `source_repo:` fall back to hardcoded values instead of refusing", + ' org, src = doc.get("org"), doc.get("source_repo")', + ' org, src = doc.get("org") or "tracebloc", doc.get("source_repo") or ".github"'), + + # --- the matcher -------------------------------------------------------- + ("a dot is dropped from the prefix charset, so `.github#N` stops being parsed", + r'PREFIX_CHARS = re.compile(r"[A-Za-z0-9._/-]")', + r'PREFIX_CHARS = re.compile(r"[A-Za-z0-9_/-]")'), + ("a malformed citation is silently dropped instead of reported", + " if not c.legal or c.number < 1:", + " if False and (not c.legal or c.number < 1):"), + ("`#0` is accepted as a real ticket number", + "c.number < 1", + "c.number < 0"), + ("`divergent:` stops being a place a reason can live", + 'REASON_KEYS = ("exempt", "divergent", "reason")', + 'REASON_KEYS = ("exempt", "reason")'), + ("`shared_reasons:` is not scanned, so an anchor not yet aliased is invisible", + ' shared = doc.get("shared_reasons")', + " shared = None"), + ("the walk stops descending, so every nested reason is missed", + ' else:\n walk(value, f"{where}.{key}")', + " else:\n pass"), + + # --- fail closed -------------------------------------------------------- + ("ZERO citations found reports clean instead of refusing", + " if not seen and not malformed:", + " if False and not seen and not malformed:"), + ("an unreadable inventory is swallowed instead of refused", + ' raise Finding(f"{path} could not be read ({exc}) -- refusing to report clean")', + " return {}"), + ("an unparseable inventory is swallowed instead of refused", + ' raise Finding(f"{path} could not be parsed ({exc}) -- refusing to report clean")', + " return {}"), + ("an inventory that is not a mapping is accepted", + " if not isinstance(doc, dict):", + " if False and not isinstance(doc, dict):"), + # NOTE: this one must REPLACE the raise, not precede it. The first draft + # inserted the assignment above the `raise` and reported UNCAUGHT -- correctly, + # because the mutation was inert. That is the harness doing its job: an inert + # mutation and a missing case look identical until the anchor is read. + ("a citation the API will not resolve is read as OPEN -- the guess rule 3 forbids", + ''' raise Finding( + f"{key} could not be read (no issue or pull request came back). That is " + "\'cannot tell\', not \'still open\' -- fix the citation or the token\'s reach" + )''', + ' node = {"__typename": "Issue", "state": "OPEN"}'), + ("a GraphQL payload with no data is treated as an empty read", + ' data = payload.get("data")\n if not isinstance(data, dict):', + ' data = payload.get("data") or {}\n if False and not isinstance(data, dict):'), + ("a response that is not JSON is treated as no citations at all", + " except (ValueError, TypeError):\n raise Finding(", + " except (ValueError, TypeError):\n return {}\n raise Finding("), + + # --- the exemption map, both halves ------------------------------------- + ("the exemption map is ignored, so this lands as a red gate", + " offenders = [f for f in findings if f[0] not in exempt] + \\\n" + " [m for m in malformed if m[0] not in exempt]", + " offenders = list(findings) + list(malformed)"), + ("a stale exemption stops being reported, so the list becomes cover", + " return sorted(set(_exempt()) - {key for key, _, _ in findings})", + " return []"), +] + + +def apply_one(src, old, new): + n = src.count(old) + if n != 1: + raise LookupError("anchor matched %d times, expected exactly 1: %r" % (n, old[:80])) + out = src.replace(old, new, 1) + return None if out == src else out + + +def main(): + dry = "--dry" in sys.argv + pristine = GUARD.read_text(encoding="utf-8") + stale, uncaught = [], [] + + for label, old, new in MUTATIONS: + try: + mutated = apply_one(pristine, old, new) + except LookupError as exc: + stale.append((label, str(exc))) + continue + if mutated is None: + stale.append((label, "NO-OP: the mutation changed nothing")) + continue + if dry: + print(" anchor ok %s" % label) + continue + GUARD.write_text(mutated, encoding="utf-8") + env = dict(os.environ) + env["PYTHONDONTWRITEBYTECODE"] = "1" + try: + run = subprocess.run( + [sys.executable, "-B", str(SUITE)], + capture_output=True, text=True, cwd=str(ROOT), env=env, + ) + finally: + # ALWAYS restore, including on a crash. A mutation left on disk makes + # every later run measure the wrong script, and the tell is a suite + # that reddens for reasons nobody typed. + GUARD.write_text(pristine, encoding="utf-8") + caught = [line.strip()[6:].strip() for line in run.stdout.splitlines() + if line.strip().startswith("FAIL ")] + # A crash counts as caught ONLY if the suite actually ran and reported. A + # bare traceback with no case output means the mutation broke the harness + # rather than being detected by a case, which is not coverage. + reported = "reason-citations-selftest:" in run.stdout + if reported and run.returncode != 0: + print(" caught %s\n by: %s" % (label, ", ".join(caught)[:140])) + elif not reported: + uncaught.append((label, "the suite did not report -- mutation broke the harness")) + print(" UNCAUGHT %s (harness broke, not detected)" % label) + else: + uncaught.append((label, "the suite passed with this broken")) + print(" UNCAUGHT %s" % label) + + if GUARD.read_text(encoding="utf-8") != pristine: + sys.stderr.write("::error::%s was left mutated. Restore it from git.\n" % GUARD.name) + return 2 + + print("\n%d mutation(s): %d stale, %d uncaught" % (len(MUTATIONS), len(stale), len(uncaught))) + for label, why in stale: + sys.stderr.write("::error::STALE mutation `%s`: %s\n" % (label, why)) + for label, why in uncaught: + sys.stderr.write( + "::error::UNCAUGHT `%s`: %s. Add a case that fails under it, or delete the " + "mutation if the rule is genuinely not worth pinning.\n" % (label, why)) + return 1 if (stale or uncaught) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/reason-citations-selftest.py b/scripts/tests/reason-citations-selftest.py new file mode 100644 index 0000000..60cdf2c --- /dev/null +++ b/scripts/tests/reason-citations-selftest.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +"""Suite for scripts/reason-citations.py (tracebloc/backend#2449). + +The check says "a citation inside a repo-inventory reason must still be live". +Everything below drives it against FIXTURE inventories and a STUBBED `gh`, +because a suite that named the real inventory would redden every time a reason is +rewritten or a ticket is closed -- and the burn-down is the point. + +HERMETIC, WITH THE REAL SEAM EXERCISED. There is no network and no token: a +throwaway `gh` executable is put first on PATH, so the subprocess call, the +exit-code path and the JSON decoding are all covered by the same cases rather +than being the part nobody tests. + +INPUTS ARE WRITTEN DOWN INDEPENDENTLY OF THE MATCHER (CLAUDE.md rule 9's +corollary). The citation strings, the typenames and the states below are +LITERALS. Iterating the module's own `LIVE`/`DEAD` sets to check the module would +be self-consistent and therefore blind -- typo one and the fixture carries the +same typo and still passes. + +Each case pins a behaviour a mutation would break. `reason-citations-mutations.py` +breaks each one and asserts this suite reddens; a case that survives its own +mutation is vacuous and worse than absent. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +GUARD = HERE.parent / "reason-citations.py" + +RESULTS = [] + + +def record(ok: bool, name: str, detail: str) -> None: + RESULTS.append((ok, name)) + print(f"{'PASS' if ok else 'FAIL'} {name}\n {detail}") + + +# --- the stub `gh` --------------------------------------------------------- +# Answers the ONE GraphQL document the guard sends, from a table the case hands +# it. Modes cover the failure shapes a real `gh` produces: a nonzero exit with a +# message, a 200 whose body is not JSON, and GitHub's own partial answer -- a +# null node plus an `errors[]` entry, at exit 0, alongside good data. +STUB = r'''#!/usr/bin/env python3 +import json, os, re, sys + +mode = os.environ.get("STUB_MODE", "ok") +if mode == "exit": + sys.stderr.write("gh: HTTP 401 Bad credentials\n") + sys.exit(1) +if mode == "garbage": + sys.stdout.write("rate limited") + sys.exit(0) +if mode == "nodata": + print(json.dumps({"errors": [{"message": "boom"}]})) + sys.exit(1) + +query = "" +for arg in sys.argv: + if arg.startswith("query="): + query = arg[len("query="):] +states = json.loads(os.environ.get("STUB_STATES", "{}")) +data = {} +errors = [] +pattern = r'(c\d+): repository\(owner: "([^"]*)", name: "([^"]*)"\) \{ issueOrPullRequest\(number: (\d+)\)' +for alias, owner, name, number in re.findall(pattern, query): + key = "%s/%s#%s" % (owner, name, number) + hit = states.get(key) + if hit is None: + data[alias] = {"issueOrPullRequest": None} + errors.append({"type": "NOT_FOUND", "path": [alias]}) + continue + data[alias] = {"issueOrPullRequest": {"__typename": hit[0], "state": hit[1]}} +out = {"data": data} +if errors: + out["errors"] = errors +print(json.dumps(out)) +''' + + +def _stub_dir() -> str: + d = tempfile.mkdtemp() + gh = Path(d, "gh") + gh.write_text(STUB, encoding="utf-8") + gh.chmod(0o755) + return d + + +STUB_DIR = _stub_dir() + + +def run(inventory: str, *, states: "dict | None" = None, exempt: "str | None" = "", + mode: str = "ok", extra_env: "dict | None" = None, write: bool = True): + """Write a fixture inventory, run the guard over it, return (rc, out, err). + + PASS `exempt` ON EVERY CASE, including the default `""`. Letting the LIVE + exemption map apply would mean every real row reads as stale against a + fixture -- so the suite would redden on production state rather than on the + case under test. `mint-scope-selftest.py` learned that the hard way. + """ + d = tempfile.mkdtemp() + path = Path(d, "repo-inventory.yml") + if write: + path.write_text(inventory, encoding="utf-8") + env = dict(os.environ) + env["PATH"] = STUB_DIR + os.pathsep + env.get("PATH", "") + env["REASON_CITATIONS_INVENTORY"] = str(path) + env["STUB_STATES"] = json.dumps(states or {}) + env["STUB_MODE"] = mode + env.pop("GITHUB_REPOSITORY", None) + if exempt is not None: + env["REASON_CITATIONS_EXEMPT"] = exempt + env.update(extra_env or {}) + p = subprocess.run([sys.executable, str(GUARD)], capture_output=True, text=True, env=env) + return p.returncode, p.stdout, p.stderr + + +def inv(reason: str, *, org: str = "tracebloc", source_repo: str = ".github", + head: str = "") -> str: + """One inventory carrying exactly one written reason.""" + return f"""org: {org} +source_repo: {source_repo} +{head}repos: + demo: + callers: + thing.yml: + exempt: >- + {reason} +""" + + +OPEN_ISSUE = ["Issue", "OPEN"] +CLOSED_ISSUE = ["Issue", "CLOSED"] +OPEN_PR = ["PullRequest", "OPEN"] +MERGED_PR = ["PullRequest", "MERGED"] +CLOSED_PR = ["PullRequest", "CLOSED"] + +# --- the finding this check exists for ------------------------------------ +rc, out, err = run(inv("sequenced behind backend#1408, which is still being worked"), + states={"tracebloc/backend#1408": CLOSED_ISSUE}) +record(rc == 1 and "tracebloc/backend#1408" in err and "is CLOSED" in err, + "a citation to a CLOSED issue is a finding", + f"rc={rc} err={err.strip()[:130]!r}") + +rc, out, err = run(inv("sequenced behind backend#1408, which is still being worked"), + states={"tracebloc/backend#1408": OPEN_ISSUE}) +record(rc == 0 and not err.strip(), + "a citation to an OPEN issue is clean", + f"rc={rc} out={out.strip()[:120]!r}") + +# The report must name WHERE the reason lives, or the remedy is a grep. +rc, out, err = run(inv("blocked on backend#1408"), + states={"tracebloc/backend#1408": CLOSED_ISSUE}) +record("repos.demo.callers.thing.yml.exempt" in err, + "the finding names the inventory path of the reason that cites it", + f"err={err.strip()[-160:]!r}") + +# --- pull requests: merged is not staleness, abandoned is ------------------- +rc, out, err = run(inv("remediated under model-zoo#115"), + states={"tracebloc/model-zoo#115": MERGED_PR}) +record(rc == 0, + "a MERGED pull request is not a finding (its terminal state is success)", + f"rc={rc} err={err.strip()[:120]!r}") + +rc, out, err = run(inv("goes in with model-zoo#115"), + states={"tracebloc/model-zoo#115": OPEN_PR}) +record(rc == 0, "an OPEN pull request is not a finding", f"rc={rc}") + +rc, out, err = run(inv("goes in with model-zoo#115"), + states={"tracebloc/model-zoo#115": CLOSED_PR}) +record(rc == 1 and "never landed" in err, + "a pull request CLOSED WITHOUT MERGING is a finding", + f"rc={rc} err={err.strip()[:130]!r}") + +# --- which repo a citation means ------------------------------------------- +# A BARE `#N` RESOLVES AGAINST THE REPO THE INVENTORY LIVES IN, never `backend`. +# Assuming `backend` is a defect this org shipped and fixed (.github#314), so the +# stub is given ONLY the `.github` answer: a guard that guessed `backend` would +# get a null node and exit 2 instead of the clean 0 asserted here. +rc, out, err = run(inv("raised in the staging baseline (Bugbot, #277)"), + states={"tracebloc/.github#277": MERGED_PR}) +record(rc == 0, + "a bare `#N` resolves against `source_repo`, not `backend`", + f"rc={rc} err={err.strip()[:130]!r}") + +rc, out, err = run(inv("raised in the staging baseline (Bugbot, #277)", + source_repo="rfcs"), + states={"tracebloc/rfcs#277": MERGED_PR}) +record(rc == 0, + "the bare-`#N` repo is READ from `source_repo`, not typed into the guard", + f"rc={rc} err={err.strip()[:130]!r}") + +# `repo#N` takes its owner from the inventory's own `org:`. +rc, out, err = run(inv("see model-zoo#115", org="someorg"), + states={"someorg/model-zoo#115": MERGED_PR}) +record(rc == 0, + "`repo#N` takes its owner from the inventory's `org:`", + f"rc={rc} err={err.strip()[:130]!r}") + +rc, out, err = run(inv("see tracebloc/backend#1408", org="someorg"), + states={"tracebloc/backend#1408": OPEN_ISSUE}) +record(rc == 0, + "`owner/repo#N` is taken as written, overriding `org:`", + f"rc={rc} err={err.strip()[:130]!r}") + +# A repo name starting with a dot is real in this org and must not be dropped. +rc, out, err = run(inv("(Bugbot, .github#196)"), + states={"tracebloc/.github#196": CLOSED_PR}) +record(rc == 1 and "tracebloc/.github#196" in err, + "a leading-dot repo name (`.github#N`) is parsed, not skipped", + f"rc={rc} err={err.strip()[:130]!r}") + +# The bare rule's PREMISE is that this file lives in /. When the +# runner says otherwise, the answer is "cannot tell" -- not a guess. +rc, out, err = run(inv("raised in the staging baseline (Bugbot, #277)"), + states={"tracebloc/.github#277": MERGED_PR}, + extra_env={"GITHUB_REPOSITORY": "tracebloc/backend"}) +record(rc == 2 and "bare citation" in err, + "a bare `#N` is refused when GITHUB_REPOSITORY contradicts `source_repo`", + f"rc={rc} err={err.strip()[:150]!r}") + +# --- malformed --------------------------------------------------------------- +rc, out, err = run(inv("see tracebloc/backend/extra#12"), + states={"tracebloc/backend#12": OPEN_ISSUE}) +record(rc == 1 and "not a legal" in err and "extra#12" in err, + "a citation whose prefix is not a legal owner/repo is reported, not dropped", + f"rc={rc} err={err.strip()[:150]!r}") + +rc, out, err = run(inv("see backend#0"), states={}) +record(rc == 1 and "not a legal" in err, + "`#0` is malformed: GitHub numbers issues from 1", + f"rc={rc} err={err.strip()[:150]!r}") + +# --- fail closed -------------------------------------------------------------- +# The premise is that reasons cite tickets. Finding none means the matcher broke. +rc, out, err = run(inv("no citation anywhere in this sentence"), states={}) +record(rc == 2 and "no ticket citation found" in err, + "ZERO citations found is a hard error, not a clean run", + f"rc={rc} err={err.strip()[:150]!r}") + +rc, out, err = run("", write=False) +record(rc == 2 and "could not be read" in err, + "an inventory that does not exist is a hard error", + f"rc={rc} err={err.strip()[:130]!r}") + +rc, out, err = run("repos: [this is: not: valid\n") +record(rc == 2 and "could not be parsed" in err, + "an unparseable inventory is a hard error, not a skip", + f"rc={rc} err={err.strip()[:130]!r}") + +rc, out, err = run("- just\n- a list\n") +record(rc == 2 and "did not parse to a mapping" in err, + "an inventory that is not a mapping is a hard error", + f"rc={rc} err={err.strip()[:130]!r}") + +# CANNOT TELL, in every shape a real `gh` produces it. +rc, out, err = run(inv("blocked on backend#1408"), states={}) +record(rc == 2 and "could not be read" in err and "cannot tell" in err, + "a citation the API will not resolve is CANNOT TELL, never 'still open'", + f"rc={rc} err={err.strip()[:170]!r}") + +rc, out, err = run(inv("blocked on backend#1408"), + states={"tracebloc/backend#1408": OPEN_ISSUE}, mode="exit") +record(rc == 2, "a failing `gh` call is a hard error", f"rc={rc} err={err.strip()[:130]!r}") + +rc, out, err = run(inv("blocked on backend#1408"), + states={"tracebloc/backend#1408": OPEN_ISSUE}, mode="garbage") +record(rc == 2 and "no JSON" in err, + "a non-JSON GraphQL response is a hard error", + f"rc={rc} err={err.strip()[:130]!r}") + +rc, out, err = run(inv("blocked on backend#1408"), + states={"tracebloc/backend#1408": OPEN_ISSUE}, mode="nodata") +record(rc == 2 and "carried no data" in err, + "a GraphQL payload with errors and no data is a hard error", + f"rc={rc} err={err.strip()[:130]!r}") + +# A PARTIAL read is not a clean read: one good node, one null, exit 0. +rc, out, err = run(inv("blocked on backend#1408 and on backend#9999"), + states={"tracebloc/backend#1408": OPEN_ISSUE}) +record(rc == 2 and "backend#9999" in err, + "one unresolvable citation among good ones still refuses the whole run", + f"rc={rc} err={err.strip()[:150]!r}") + +rc, out, err = run(inv("blocked on backend#1408"), + states={"tracebloc/backend#1408": ["Issue", "TRIAGED"]}) +record(rc == 2 and "unrecognised citation state" in err, + "a state the check has no verdict for is refused, not passed", + f"rc={rc} err={err.strip()[:150]!r}") + +rc, out, err = run("source_repo: .github\nrepos:\n demo:\n callers:\n" + " t.yml:\n exempt: see backend#1408\n") +record(rc == 2 and "no `org:`" in err, + "an inventory with no `org:` is a hard error", + f"rc={rc} err={err.strip()[:130]!r}") + +rc, out, err = run("org: tracebloc\nrepos:\n demo:\n callers:\n" + " t.yml:\n exempt: see #277\n") +record(rc == 2 and "no `source_repo:`" in err, + "a bare `#N` with no `source_repo:` is a hard error, not a guess", + f"rc={rc} err={err.strip()[:150]!r}") + +# --- where reasons live ------------------------------------------------------- +# A `divergent:` written as a BARE STRING is the copies-family spelling, and the +# first draft of the guard read only `exempt`/`reason` -- so this shape went +# unscanned. Both `divergent` spellings are pinned here. +rc, out, err = run("""org: tracebloc +source_repo: .github +repos: + demo: + copies: + add-to-kanban.yml: + divergent: >- + the pin was the other half of this entry until backend#1816 landed +""", states={"tracebloc/backend#1816": CLOSED_ISSUE}) +record(rc == 1 and "backend#1816" in err, + "a `divergent:` written as a bare string is scanned", + f"rc={rc} err={err.strip()[:150]!r}") + +rc, out, err = run("""org: tracebloc +source_repo: .github +repos: + demo: + protection: + develop: + divergent: + reason: >- + armed on top of the baseline under backend#1975 + min_reviews: 2 +""", states={"tracebloc/backend#1975": CLOSED_ISSUE}) +record(rc == 1 and "backend#1975" in err, + "a `divergent:` mapping's `reason:` is scanned", + f"rc={rc} err={err.strip()[:150]!r}") + +# An anchor DEFINED BUT NOT YET ALIASED is invisible to a walk of `repos:` alone, +# and that is the state a new reason is in on the PR that introduces it. +rc, out, err = run("""org: tracebloc +source_repo: .github +shared_reasons: + brand_new: &brand_new >- + staged behind backend#1408 +repos: + demo: + callers: + thing.yml: + exempt: nothing cited here at all, deliberately +""", states={"tracebloc/backend#1408": CLOSED_ISSUE}) +record(rc == 1 and "shared_reasons.brand_new" in err, + "an anchor defined in `shared_reasons:` but not yet aliased is scanned", + f"rc={rc} err={err.strip()[:170]!r}") + +# COMMENTS ARE NOT REASONS. The scan runs over parsed YAML on purpose -- the +# inventory's header carries dozens of ticket numbers that justify nothing. +rc, out, err = run(inv("blocked on backend#1408", + head="# a header comment citing backend#9999\n"), + states={"tracebloc/backend#1408": OPEN_ISSUE}) +record(rc == 0, + "a ticket number in a YAML COMMENT is not treated as a citation", + f"rc={rc} err={err.strip()[:130]!r}") + +# --- the exemption map, both halves ------------------------------------------ +rc, out, err = run(inv("blocked on backend#1408"), + states={"tracebloc/backend#1408": CLOSED_ISSUE}, + exempt="tracebloc/backend#1408") +record(rc == 0, + "an EXEMPTED dead citation is not a finding", + f"rc={rc} err={err.strip()[:130]!r}") + +rc, out, err = run(inv("blocked on backend#1408"), + states={"tracebloc/backend#1408": OPEN_ISSUE}, + exempt="tracebloc/backend#1408") +record(rc == 1 and "no longer a finding" in err, + "a STALE exemption is a finding too", + f"rc={rc} err={err.strip()[:150]!r}") + +# --- only the offender is named ---------------------------------------------- +rc, out, err = run(inv("landed under model-zoo#115; blocked on backend#1408"), + states={"tracebloc/model-zoo#115": MERGED_PR, + "tracebloc/backend#1408": CLOSED_ISSUE}) +record(rc == 1 and "backend#1408" in err and "model-zoo#115" not in err, + "only the dead citation is named, not its live neighbour", + f"rc={rc} err={err.strip()[:150]!r}") + +failed = [r for r in RESULTS if not r[0]] +print(f"\nreason-citations-selftest: {len(RESULTS) - len(failed)} passed, {len(failed)} failed") +sys.exit(1 if failed else 0)