From c57dc21fd908ed9a6c007247c030ef5415e4380a Mon Sep 17 00:00:00 2001 From: Josui Date: Tue, 17 Nov 2026 21:50:30 +0000 Subject: [PATCH 1/2] feat: scripts/fleet_wave.py, the driver for the last fleet PR wave Runs convert_repo.py across every public, non-archived crunchtools repo (or a --repos subset). A dry run by default: shallow clone at the default branch the API names, convert, verify (a second converter run is a no-op, the bootstrap is committed executable, nothing outside the converter's files changed), and record the plan in a TSV ledger. Preflight skips a repo whose SDLC_IMAGE the bootstrap would refuse or that has an open chore/python-sdlc PR. --apply pushes chore/sdlc-floating (one per repo, reused), opens or updates one PR with a fixed body, and queues a squash auto-merge via the REST and GraphQL APIs. The token comes from the environment variable --token-env names (GH_TOKEN), goes in a header, and is never printed or an argument. Sequential; a re-run skips what is done. tests/test_fleet_wave.py runs against a fake API and bare repos on disk standing in for GitHub, and is wired into the image's unit-test step. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01R7Y1jg3ZJ6BsAEfhrPWFRh --- .github/workflows/container.yml | 3 +- CHANGELOG.md | 12 + README.md | 4 + gourmand-exceptions.toml | 17 + pyproject.toml | 2 + scripts/fleet_wave.py | 571 ++++++++++++++++++++++++++++++++ tests/test_fleet_wave.py | 499 ++++++++++++++++++++++++++++ 7 files changed, 1107 insertions(+), 1 deletion(-) create mode 100755 scripts/fleet_wave.py create mode 100644 tests/test_fleet_wave.py diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 5ffc52f..ef65e7d 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -126,7 +126,8 @@ jobs: set -eu for t in tests/test_plan.py tests/test_codeql_globs.py \ tests/test_pycache_prefix.py tests/test_gourmand_sidecar.py \ - tests/test_codeql_resilience.py tests/test_convert_repo.py; do + tests/test_codeql_resilience.py tests/test_convert_repo.py \ + tests/test_fleet_wave.py; do podman run --rm -v "$PWD:/src:ro,z" -w /src \ --entrypoint /opt/constitution-venv/bin/python3 \ "localhost/python-sdlc:$GITHUB_SHA" "$t" diff --git a/CHANGELOG.md b/CHANGELOG.md index fc7adab..2922206 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,18 @@ All notable changes to this project are documented here, following `.specify/`; refuses with exit 2 a shape it would have to guess at. Every file it would write is parsed back and held to the floating wiring before anything is written, and a second run is a no-op. +- `scripts/fleet_wave.py`: runs the converter across the org, the last + fleet-wide PR wave. A dry run by default: each public, non-archived repo (or + each named in `--repos FILE`) is shallow-cloned at the default branch the API + names, converted and verified -- a second converter run must be a no-op, the + bootstrap committed executable, nothing outside the converter's files + changed -- and the plan goes to a TSV ledger. Preflight skips a repo whose + `SDLC_IMAGE` the bootstrap would refuse or that has an open + `chore/python-sdlc` PR. `--apply` pushes `chore/sdlc-floating` (reused, one + per repo), opens or updates one PR with a fixed body, and queues a squash + auto-merge, through the REST and GraphQL APIs with `GH_TOKEN` from the + environment, never argv. Sequential; a re-run skips what the ledger says is + done. - `container-smoke` SKIPS an image whose PID 1 is systemd (`/sbin/init`, `/usr/sbin/init`, `systemd`, as in ubi-init and bootc images): systemd needs diff --git a/README.md b/README.md index cfb85fa..d5d5b94 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,10 @@ stub covers every job in it (`sdlc ci`, Gourmand), refreshes `gatehouse.yml` from [`examples/gatehouse.yml`](examples/gatehouse.yml) when it differs only in comments, keeps everything else (publish, release, container builds), and refuses with exit 2 a shape it would have to guess at. +`scripts/fleet_wave.py` runs it across the org: a plan by default, one +auto-merging pull request per repo with `--apply`, a TSV ledger either way. +It reads `GH_TOKEN` from the environment for the GitHub API (required with +`--apply`), and pushes with the credentials git already has. **What floating trusts.** Whoever can move `quay.io/crunchtools/sdlc:latest` now decides the host-side code every seat and runner runs, and that code is no diff --git a/gourmand-exceptions.toml b/gourmand-exceptions.toml index c521cf1..f69bb44 100644 --- a/gourmand-exceptions.toml +++ b/gourmand-exceptions.toml @@ -38,3 +38,20 @@ justification = """ values explicitly so that raising one would be a visible change. """ classification = "by_design" + +# A shape match, not a copy. CH012 pairs convert_hooks (rewrites the hooks +# file) with fleet_wave.py's verify (checks a conversion after --apply): no +# line, call or name is shared, only a run of assign/if/return statements of +# similar length. Reshaping verify to dodge it was tried and changed nothing. +[[exceptions]] +check = "CH012-copy_paste_detection" +path = "scripts/convert_repo.py" +classification = "gourmand_bug" +justification = """ + convert_hooks and scripts/fleet_wave.py's verify share no line, call or \ + identifier: one rewrites .pre-commit-config.yaml text, the other runs the \ + converter a second time and inspects the git index. The match is a run of \ + nine assign/if/return statements, which is the shape of most short Python \ + functions, not duplicated logic. Two restructurings of verify left it in \ + place, so there is no shared helper to extract. +""" diff --git a/pyproject.toml b/pyproject.toml index 1bc18a4..fc7f69b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,3 +58,5 @@ max-complexity = 10 # Its fixtures are the fleet's files byte for byte, long lines included: the # script matches the pinned CI step exactly, so a wrapped copy tests nothing. "tests/test_convert_repo.py" = ["S101", "E501"] +# The same fixtures, and counts asserted as the numbers they are. +"tests/test_fleet_wave.py" = ["S101", "E501", "PLR2004"] diff --git a/scripts/fleet_wave.py b/scripts/fleet_wave.py new file mode 100755 index 0000000..d692b48 --- /dev/null +++ b/scripts/fleet_wave.py @@ -0,0 +1,571 @@ +#!/usr/bin/env python3 +"""Run scripts/convert_repo.py across the org: the last fleet-wide PR wave. + +Every public, non-archived repo in the org (or the ones a --repos file names) +is cloned at its default branch, converted, and verified. Without --apply that +is all: the plan goes to the ledger and nothing leaves this machine. With +--apply each converted repo gets one commit on chore/sdlc-floating, pushed, a +pull request with a fixed body, and auto-merge (squash) -- and its own CI +decides whether it lands. + +Per repo, in order, and a repo that fails a step is skipped with the reason: + + preflight SDLC_IMAGE, the org's and the repo's, is unset or a value the + bootstrap takes (main, latest, X.Y.Z, @sha256:<64 hex>); no pull + request is open from an earlier wave's branch, whose `rev:` line + this rewrites out from under it + convert a shallow clone of the default branch, read from the API, never + assumed; convert_repo.py's exit 2 is a refusal, an empty diff a + no-op + verify convert_repo.py --apply, then a second run must find nothing to + do, the bootstrap must be committed executable, and nothing + outside the files the converter owns may have changed + +The branch is one per repo and reused: a re-run force-pushes it and updates +the pull request already open on it rather than opening a second. + +THE LEDGER is a TSV (repo, action, pr_url, reason), appended one row per repo +as it finishes, so a run that dies leaves what it did. A re-run reads it and +skips every repo already done (opened, updated, noop, or would-* for a dry +run); refused, skipped and failed repos are tried again. Dry runs and applied +runs keep separate ledgers by default, so a plan never reads as done. + +Sequential, with a pause between repos: no parallel pushes. + +Credentials: the API token comes from the environment variable --token-env +names (GH_TOKEN by default), goes in a header, and is never printed or taken as +an argument. Without it a dry run reads what anonymous API calls can and says +which checks it could not make. git pushes with whatever credentials git +already has (on a seat, the gateway's git route). + +Exit 0 when every repo reached an outcome, 1 when one failed or was left +without auto-merge (rerun to retry them), 2 when the wave could not run. + +Usage: scripts/fleet_wave.py [--repos FILE] [--apply] [--ledger TSV] [--sleep S] + [--token-env NAME] +""" + +from __future__ import annotations + +import argparse +import http.client +import importlib.util +import json +import os +import re +import subprocess +import sys +import tempfile +import time +import urllib.parse +from dataclasses import dataclass, field +from http import HTTPStatus +from pathlib import Path +from typing import Any + +ORG = "crunchtools" +SELF = "python-sdlc" # the converter's own repo: the source, not a consumer +BRANCH = "chore/sdlc-floating" +# Branches of earlier waves whose open pull requests edit the `rev:` line this +# wave deletes: one merging after the conversion would conflict, and a +# conflicting PR runs no checks, so it would sit looking green forever. +EARLIER_WAVES = ("chore/python-sdlc",) +CONVERTER = Path(__file__).resolve().parent / "convert_repo.py" +API_HOST = "api.github.com" +API_TIMEOUT = 30 # seconds, per call +CLONE_BASE = "https://github.com" +# What convert_repo.py may write. Anything else changed in the checkout is a +# converter that has grown a new reach, which a fleet wave is not the place to +# find out about. +CONVERTER_OWNS = re.compile( + r"^(\.sdlc/bin/sdlc|\.pre-commit-config\.yaml|\.github/workflows/[^/]+\.ya?ml" + r"|pyproject\.toml|sdlc\.toml)$" +) +# bin/sdlc-bootstrap's own rule, which refuses anything else with exit 2. +FLEET_IMAGE = "quay.io/crunchtools/sdlc" +VALID_IMAGE = re.compile(r"^(main|latest|\d+\.\d+\.\d+|@sha256:[0-9a-f]{64})$") + +REFUSED = 2 # convert_repo.py: a shape it will not guess at +PAGE = 100 +UNANSWERED = {HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN, HTTPStatus.TOO_MANY_REQUESTS} + +DONE = {"opened", "updated", "noop", "would-open", "would-update"} +LEDGER_HEADER = ("repo", "action", "pr_url", "reason") + +COMMIT_MESSAGE = """\ +chore: float the sdlc gates on quay.io/crunchtools/sdlc + +Converted by python-sdlc's scripts/convert_repo.py. The hooks and the +`sdlc ci` job run .sdlc/bin/sdlc, which takes its launcher from +quay.io/crunchtools/sdlc:latest, so nothing in this repo pins a python-sdlc +release and no version-bump pull request follows this one. + +Co-Authored-By: Claude Opus 5.5 +""" +PR_TITLE = COMMIT_MESSAGE.splitlines()[0] +PR_BODY = """\ +The last fleet-wide PR wave: this repo's gates move to the floating sdlc, so \ +what they run changes when the image does and no pull request has to bump a pin. + +## What changed + +- `.sdlc/bin/sdlc` (new, executable): python-sdlc's `bin/sdlc-bootstrap`. It \ +asks `quay.io/crunchtools/sdlc` for its launcher and runs it. It names no \ +release, so nothing ever bumps it. +- `.pre-commit-config.yaml`: the rev-pinned python-sdlc hooks (or the \ +standalone Gourmand and Gatehouse hooks) become two local hooks running \ +`.sdlc/bin/sdlc commit` and `.sdlc/bin/sdlc push`. Other hooks are kept. +- `.github/workflows/`: the `sdlc ci` job runs `.sdlc/bin/sdlc ci` with \ +`SDLC_IMAGE: ${{ vars.SDLC_IMAGE }}`. Its name, and so the required check, is \ +unchanged. Workflows the stub does not cover are kept; \ +`quay.io/crunchtools/python-sdlc` becomes `quay.io/crunchtools/sdlc`. +- `.specify/` is not touched. + +## How to roll back + +- Before it merges: close this pull request. +- A bad image, not this change: set the repository variable `SDLC_IMAGE` to \ +the last good release (`X.Y.Z`). CI uses it on its next run, with no pull \ +request; the organization variable does the same fleet-wide. Locally, \ +`SDLC_IMAGE=X.Y.Z git push`. +- This change itself: `git revert` its squash commit on the default branch. \ +That restores the pinned hooks and workflows exactly as they were. + +It merges itself (squash) when its required checks pass. Opened by \ +python-sdlc's `scripts/fleet_wave.py`; a re-run of the wave reuses the branch \ +`chore/sdlc-floating` and this pull request. +""" + + +class ApiError(Exception): + """The API could not answer: network, auth, rate limit, or a 5xx.""" + + +class LeftAloneError(Exception): + """A repo this wave leaves alone, with the action and reason to record.""" + + def __init__(self, action: str, reason: str) -> None: + super().__init__(reason) + self.action = action + self.reason = reason + + +# ------------------------------------------------------------------- the API + + +class Api: + """GitHub's REST and GraphQL APIs. The token lives in a header, nowhere else.""" + + def __init__(self, token: str | None, connect: Any = None) -> None: + self._token = token + self._connect = connect or ( + lambda: http.client.HTTPSConnection(API_HOST, timeout=API_TIMEOUT) + ) + + @property + def authenticated(self) -> bool: + return bool(self._token) + + def redact(self, text: str) -> str: + return text.replace(self._token, "***") if self._token else text + + def call(self, method: str, path: str, body: Any = None) -> tuple[int, Any]: + """(status, decoded body). 4xx other than 401/403/429 is returned, not raised.""" + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "crunchtools-fleet-wave", + **({"Authorization": f"Bearer {self._token}"} if self._token else {}), + } + conn = self._connect() + try: + conn.request(method, path, None if body is None else json.dumps(body), headers) + response = conn.getresponse() + status, raw = response.status, response.read() + except (OSError, http.client.HTTPException) as exc: + raise ApiError(self.redact(f"{method} {path}: {exc}")) from exc + finally: + conn.close() + decoded = json.loads(raw) if raw.strip() else None + if status in UNANSWERED or status >= HTTPStatus.INTERNAL_SERVER_ERROR: + message = decoded.get("message", "") if isinstance(decoded, dict) else "" + raise ApiError(self.redact(f"{method} {path}: HTTP {status} {message}".rstrip())) + return status, decoded + + +def org_repos(api: Api) -> dict[str, str]: + """Every public, non-archived repo: name -> its default branch, as the API says.""" + repos: dict[str, str] = {} + for page in range(1, 50): + status, page_repos = api.call( + "GET", f"/orgs/{ORG}/repos?type=public&per_page={PAGE}&page={page}" + ) + if status != HTTPStatus.OK or not isinstance(page_repos, list): + raise ApiError(f"listing {ORG}'s repos: HTTP {status}") + for repo in page_repos: + if not repo["archived"] and not repo["private"]: + repos[repo["name"]] = repo["default_branch"] + if len(page_repos) < PAGE: + break + return repos + + +def open_prs_from(api: Api, branch: str) -> dict[str, dict]: + """Open pull requests from `branch`, by repo: one search for the whole org.""" + found: dict[str, dict] = {} + query = urllib.parse.quote(f"org:{ORG} is:pr is:open head:{branch}") + for page in range(1, 11): + status, hits = api.call("GET", f"/search/issues?q={query}&per_page={PAGE}&page={page}") + if status != HTTPStatus.OK or not isinstance(hits, dict): + raise ApiError(f"searching for open pull requests from {branch}: HTTP {status}") + for item in hits["items"]: + repo = item["repository_url"].rsplit("/", 1)[1] + found[repo] = {"number": item["number"], "url": item["html_url"]} + if len(hits["items"]) < PAGE: + break + return found + + +def sdlc_image(api: Api, path: str) -> str | None: + """The SDLC_IMAGE variable at `path`, or None when it is unset.""" + status, variable = api.call("GET", f"{path}/actions/variables/SDLC_IMAGE") + if status == HTTPStatus.NOT_FOUND: + return None + if status != HTTPStatus.OK or not isinstance(variable, dict): + raise ApiError(f"reading SDLC_IMAGE at {path}: HTTP {status}") + return str(variable.get("value", "")) + + +def image_problem(value: str | None) -> str | None: + """Why the bootstrap would refuse this SDLC_IMAGE, or None when it would not.""" + if not value: + return None # unset, or empty: `${SDLC_IMAGE:-latest}` reads both as latest + # The bootstrap's two spellings with the repository named in full. + bare = value.removeprefix(f"{FLEET_IMAGE}:") + if bare.startswith(f"{FLEET_IMAGE}@"): + bare = bare.removeprefix(FLEET_IMAGE) + if VALID_IMAGE.match(bare): + return None + return f"SDLC_IMAGE={value!r} is not main, latest, X.Y.Z or @sha256:<64 hex>" + + +# ------------------------------------------------------------------- the repo + + +@dataclass +class Wave: + api: Api + apply: bool + clone_base: str = CLONE_BASE + org_image: str | None = None + earlier: dict[str, str] = field(default_factory=dict) # repo -> open PR url + ours: dict[str, dict] = field(default_factory=dict) # repo -> open PR on BRANCH + notes: list[str] = field(default_factory=list) + + +def git(cwd: Path, *args: str, stdin: str | None = None) -> str: + proc = subprocess.run( + ["git", "-C", str(cwd), *args], input=stdin, capture_output=True, text=True, check=False + ) + if proc.returncode: + verb = next(a for a in args if not a.startswith("-") and "=" not in a) + raise RuntimeError(f"git {verb}: {proc.stderr.strip() or proc.stdout.strip()}") + return proc.stdout + + +def converter(checkout: Path, *args: str) -> tuple[int, str, str]: + proc = subprocess.run( + [sys.executable, str(CONVERTER), str(checkout), *args], + capture_output=True, + text=True, + check=False, + ) + return proc.returncode, proc.stdout, proc.stderr + + +def refusal(stderr: str) -> str: + """convert_repo.py's own reason, without its prefix.""" + for line in stderr.splitlines(): + if "NOT CONVERTED:" in line: + return line.split("NOT CONVERTED:", 1)[1].strip() + return stderr.strip().splitlines()[-1] if stderr.strip() else "refused, no reason given" + + +def preflight(wave: Wave, repo: str) -> None: + if repo == SELF: + raise LeftAloneError("skipped", "the converter's own repo, not a consumer") + if repo in wave.earlier: + raise LeftAloneError( + "skipped", f"open pull request from an earlier wave: {wave.earlier[repo]}" + ) + problem = image_problem(wave.org_image) + if problem: + raise LeftAloneError("skipped", f"organization {problem}") + if not wave.api.authenticated: + return # a dry run without GH_TOKEN; main() has said which checks it skipped + problem = image_problem(sdlc_image(wave.api, f"/repos/{ORG}/{repo}")) + if problem: + raise LeftAloneError("skipped", f"repository {problem}") + + +def convert(wave: Wave, repo: str, default: str, checkout: Path) -> list[str]: + """Clone, convert, verify. The paths changed, staged in the checkout.""" + source = f"{wave.clone_base}/{ORG}/{repo}" + git( + checkout.parent, + "clone", + "-q", + "--depth", + "1", + "--single-branch", + "--branch", + default, + source, + checkout.name, + ) + code, diff, err = converter(checkout) + if code == REFUSED: + raise LeftAloneError("refused", refusal(err)) + if code: + raise RuntimeError(f"convert_repo.py exited {code}: {err.strip()[-300:]}") + # The diff shows no modes: a bootstrap committed without +x is still work. + if not diff and bootstrap_mode(checkout) == "100755": + raise LeftAloneError("noop", "already floating") + code, _, err = converter(checkout, "--apply") + if code: + raise RuntimeError(f"convert_repo.py --apply exited {code}: {err.strip()[-300:]}") + return verify(checkout) + + +def verify(checkout: Path) -> list[str]: + """The paths the conversion changed, staged; or every way it is not fit to ship.""" + code, diff, _ = converter(checkout) + git(checkout, "add", "-A") + changed = git(checkout, "diff", "--cached", "--name-only").split() + mode = bootstrap_mode(checkout) + problems = [ + f"changed {p}, which it does not own" for p in changed if not CONVERTER_OWNS.match(p) + ] + if code or diff: + problems.append(f"is not a fixed point: a second convert_repo.py run exits {code}") + if mode != "100755": + problems.append(f"would commit .sdlc/bin/sdlc as mode {mode or 'absent'}") + if problems: + raise RuntimeError("the conversion " + "; ".join(problems)) + return changed + + +def bootstrap_mode(checkout: Path) -> str: + """The bootstrap's mode in the index, or "" when it is not there.""" + return (git(checkout, "ls-files", "-s", "--", ".sdlc/bin/sdlc").split() or [""])[0] + + +def ship(wave: Wave, repo: str, default: str, checkout: Path) -> tuple[str, str, str]: + """Commit, push, open or update the pull request, queue auto-merge.""" + git(checkout, "checkout", "-q", "-B", BRANCH) + # No hooks: whatever the operator's git config installs is not this repo's + # gate; the pull request's own CI is. + git( + checkout, + "-c", + "core.hooksPath=/dev/null", + "-c", + "user.name=crunchtools-ci", + "-c", + "user.email=ci@crunchtools.com", + "commit", + "-q", + "-F", + "-", + stdin=COMMIT_MESSAGE, + ) + # Forced: the branch is this wave's own, one per repo, and carries one + # commit rebuilt from the default branch on every run. + git(checkout, "push", "-q", "--force", "origin", f"HEAD:refs/heads/{BRANCH}") + pr, action = open_or_update(wave.api, repo, default) + problem = auto_merge(wave.api, pr["node_id"]) + if problem: + return "no-automerge", pr["html_url"], f"auto-merge not enabled: {problem}" + return action, pr["html_url"], "" + + +def message(reply: Any) -> str: + if not isinstance(reply, dict): + return "" + details = [e.get("message", "") for e in reply.get("errors") or [] if isinstance(e, dict)] + return "; ".join(filter(None, [reply.get("message", ""), *details])) + + +def open_or_update(api: Api, repo: str, default: str) -> tuple[dict, str]: + """The pull request from BRANCH: opened, or the open one updated in place.""" + pulls = f"/repos/{ORG}/{repo}/pulls" + fields = {"title": PR_TITLE, "body": PR_BODY} + status, pull = api.call("POST", pulls, {**fields, "head": BRANCH, "base": default}) + if status == HTTPStatus.CREATED: + return pull, "opened" + if status == HTTPStatus.UNPROCESSABLE_ENTITY and "already exists" in message(pull): + head = urllib.parse.quote(f"{ORG}:{BRANCH}") + status, found = api.call("GET", f"{pulls}?head={head}&state=open") + if status == HTTPStatus.OK and found: + status, pull = api.call("PATCH", f"{pulls}/{found[0]['number']}", fields) + if status == HTTPStatus.OK: + return pull, "updated" + raise RuntimeError(f"opening the pull request: HTTP {status} {message(pull)}".rstrip()) + + +AUTO_MERGE = """\ +mutation($id: ID!) { + enablePullRequestAutoMerge(input: {pullRequestId: $id, mergeMethod: SQUASH}) { + pullRequest { autoMergeRequest { mergeMethod } } + } +}""" + + +def auto_merge(api: Api, node_id: str) -> str | None: + """Queue a squash merge; None when queued, else why not.""" + try: + status, reply = api.call( + "POST", "/graphql", {"query": AUTO_MERGE, "variables": {"id": node_id}} + ) + except ApiError as exc: + return str(exc) + if status != HTTPStatus.OK or not isinstance(reply, dict): + return f"HTTP {status} {message(reply)}".rstrip() + if reply.get("errors"): + return message(reply) + return None + + +def process(wave: Wave, repo: str, default: str) -> tuple[str, str, str]: + """One repo, start to finish: (action, pr_url, reason). Never raises for the repo's sake.""" + ours = wave.ours.get(repo, {}).get("url", "") + try: + preflight(wave, repo) + with tempfile.TemporaryDirectory(prefix="fleet-wave-") as workdir: + checkout = Path(workdir) / repo + changed = convert(wave, repo, default, checkout) + if wave.apply: + return ship(wave, repo, default, checkout) + return "would-update" if ours else "would-open", ours, ", ".join(changed) + except LeftAloneError as left: + return left.action, ours, left.reason + except (RuntimeError, ApiError, OSError) as exc: + return "failed", ours, wave.api.redact(str(exc)) + + +# ----------------------------------------------------------------- the ledger + + +def read_ledger(path: Path) -> dict[str, str]: + """Each repo's last recorded action.""" + if not path.exists(): + return {} + last: dict[str, str] = {} + for line in path.read_text().splitlines()[1:]: + repo, _, rest = line.partition("\t") + if rest: + last[repo] = rest.split("\t", 1)[0] + return last + + +def record(path: Path, row: tuple[str, str, str, str]) -> None: + clean = [re.sub(r"[\t\r\n]+", " ", cell) for cell in row] + new = not path.exists() + with path.open("a") as ledger: + if new: + ledger.write("\t".join(LEDGER_HEADER) + "\n") + ledger.write("\t".join(clean) + "\n") + + +# ------------------------------------------------------------------- the wave + + +def wanted(path: Path) -> list[str]: + """The repo names a --repos file lists: one per line, `#` comments, org/ optional.""" + names = [] + for line in path.read_text().splitlines(): + name = line.split("#", 1)[0].strip().removeprefix(f"{ORG}/") + if name: + names.append(name) + return names + + +def survey(api: Api, apply: bool) -> Wave: + """What the whole wave needs to know before the first repo: a few calls, not one per repo.""" + wave = Wave(api, apply) + wave.earlier = { + repo: pr["url"] + for branch in EARLIER_WAVES + for repo, pr in open_prs_from(api, branch).items() + } + wave.ours = open_prs_from(api, BRANCH) + if api.authenticated: + wave.org_image = sdlc_image(api, f"/orgs/{ORG}") + else: + wave.notes.append("no API token: SDLC_IMAGE (organization and repository) not checked") + return wave + + +def run( + wave: Wave, names: list[str], available: dict[str, str], ledger: Path, pause: float +) -> dict: + done = read_ledger(ledger) + counts: dict[str, int] = {} + for n, repo in enumerate(names): + if done.get(repo) in DONE: + counts["already done"] = counts.get("already done", 0) + 1 + continue + if repo not in available: + row = ("skipped", "", f"not a public, non-archived repo in {ORG}") + else: + row = process(wave, repo, available[repo]) + if n < len(names) - 1: + time.sleep(pause) + record(ledger, (repo, *row)) + counts[row[0]] = counts.get(row[0], 0) + 1 + print("\t".join((repo, *row)).rstrip("\t"), flush=True) + return counts + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--repos", type=Path, help="a file of repo names, one per line; default all") + ap.add_argument("--apply", action="store_true", help="push and open PRs; default plans only") + ap.add_argument("--ledger", type=Path, help="TSV; default fleet-wave[-plan].tsv here") + ap.add_argument("--sleep", type=float, default=3.0, help="seconds between repos") + ap.add_argument( + "--token-env", + default="GH_TOKEN", + metavar="NAME", + help="the environment variable holding the GitHub API token (default %(default)s; " + "required with --apply). The token itself is never an argument", + ) + args = ap.parse_args() + ledger = args.ledger or Path("fleet-wave.tsv" if args.apply else "fleet-wave-plan.tsv") + token = os.environ.get(args.token_env, "") + if args.apply and not token: + print(f"fleet_wave: --apply needs {args.token_env} in the environment", file=sys.stderr) + return 2 + # convert_repo.py runs under this interpreter. Without PyYAML every repo + # would come back refused, which reads as a plan. + if importlib.util.find_spec("yaml") is None: + print(f"fleet_wave: convert_repo.py needs PyYAML in {sys.executable}", file=sys.stderr) + return 2 + api = Api(token) + try: + available = org_repos(api) + names = wanted(args.repos) if args.repos else sorted(available) + wave = survey(api, args.apply) + except (ApiError, OSError) as exc: + print(f"fleet_wave: CANNOT RUN: {api.redact(str(exc))}", file=sys.stderr) + return 2 + for note in wave.notes: + print(f"fleet_wave: note: {note}", file=sys.stderr) + counts = run(wave, names, available, ledger, args.sleep) + summary = ", ".join(f"{v} {k}" for k, v in sorted(counts.items())) + print(f"fleet_wave: {len(names)} repo(s): {summary or 'nothing to do'}; ledger {ledger}") + return 1 if counts.get("failed") or counts.get("no-automerge") else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_fleet_wave.py b/tests/test_fleet_wave.py new file mode 100644 index 0000000..1bffbd8 --- /dev/null +++ b/tests/test_fleet_wave.py @@ -0,0 +1,499 @@ +#!/usr/bin/env python3 +"""scripts/fleet_wave.py, against a fake API and bare repos standing in for GitHub. + +The wave is the last fleet-wide one, so what it does to a repo is pinned: + + - a dry run converts and verifies in a throwaway clone and writes nothing + anywhere else: no push, no pull request, no API call but reads + - the default branch is the one the API names, master included + - a refusal, a no-op, an earlier wave's open PR and a bad SDLC_IMAGE are + each skipped with a reason, and the run goes on + - --apply commits the canonical message, pushes chore/sdlc-floating with + the bootstrap executable, opens the PR with the fixed body, queues a + squash auto-merge -- and updates the PR already open rather than a second + - the ledger makes a re-run skip what is done and retry what is not + - the token goes in a header and nowhere else + +Run inside the image, which has the interpreter, git and PyYAML it needs: + podman run --rm -v .:/src:ro,z -w /src --entrypoint /opt/constitution-venv/bin/python3 \ + quay.io/crunchtools/sdlc: tests/test_fleet_wave.py +""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "fleet_wave.py" +_SPEC = importlib.util.spec_from_file_location("fleet_wave", SCRIPT) +W = importlib.util.module_from_spec(_SPEC) +sys.modules["fleet_wave"] = W # dataclasses look their module up there +_SPEC.loader.exec_module(W) + +FAILED: list[str] = [] +# A stand-in GH_CANARY, asserted never to appear in anything the wave prints or writes. +CANARY = "ghs_fake0123456789abcdef" + + +def check(name: str, cond: bool, detail: str = "") -> None: + print(f"{'ok ' if cond else 'FAIL'} {name}" + (f" -- {detail}" if detail and not cond else "")) + if not cond: + FAILED.append(name) + + +PINNED_HOOKS = """\ +default_install_hook_types: [pre-commit, pre-push] +repos: + - repo: https://github.com/crunchtools/python-sdlc + rev: v0.8.0 + hooks: + - id: sdlc-commit + - id: sdlc-push +""" + +PINNED_CI = """\ +name: CI +on: + pull_request: +jobs: + sdlc: + name: sdlc ci + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: sdlc ci + run: | + set -eu + rev=$(awk '/repo: https:\\/\\/github.com\\/crunchtools\\/python-sdlc/ {f=1; next} + f && /rev:/ {print $2; exit}' .pre-commit-config.yaml) + [ -n "$rev" ] || { echo "no python-sdlc rev in .pre-commit-config.yaml" >&2; exit 2; } + git clone -q --depth 1 --branch "$rev" https://github.com/crunchtools/python-sdlc "$RUNNER_TEMP/sdlc" + $RUNNER_TEMP/sdlc/bin/sdlc ci +""" + +CONVERTIBLE = {".pre-commit-config.yaml": PINNED_HOOKS, ".github/workflows/ci.yml": PINNED_CI} +# Neither the python-sdlc pin nor both standalone hooks: convert_repo.py refuses. +UNKNOWN_SHAPE = {".pre-commit-config.yaml": "repos: []\n"} + + +def git(cwd: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(cwd), *args], capture_output=True, text=True, check=True + ).stdout + + +def origin(base: Path, name: str, files: dict[str, str], branch: str = "main") -> Path: + """A bare repo at base/crunchtools/name, its `branch` holding `files`.""" + work = base / "work" / name + work.mkdir(parents=True) + git(work, "init", "-q", "-b", branch) + for path, text in files.items(): + (work / path).parent.mkdir(parents=True, exist_ok=True) + (work / path).write_text(text) + git(work, "add", "-A") + git(work, "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "init") + bare = base / "crunchtools" / name + bare.parent.mkdir(parents=True, exist_ok=True) + subprocess.run(["git", "clone", "-q", "--bare", str(work), str(bare)], check=True) + return bare + + +class FakeApi(W.Api): + """Answers from a table; records every call. Writes answer as GitHub does.""" + + def __init__(self, token: str | None, routes: dict, pr_exists: set | None = None) -> None: + super().__init__(token) + self.routes = routes + self.calls: list[tuple[str, str, object]] = [] + self.pr_exists = pr_exists or set() + self.auto_merge_error: str | None = None + + def call(self, method: str, path: str, body: object = None) -> tuple[int, object]: + self.calls.append((method, path, body)) + if path == "/graphql": + return self.graphql() + if method == "GET": + return self.read(path) + return self.write(method, path) + + def read(self, path: str) -> tuple[int, object]: + for prefix, answer in self.routes.items(): + if path.startswith(prefix): + return answer + return 404, {"message": "Not Found"} + + def graphql(self) -> tuple[int, object]: + errors = [{"message": self.auto_merge_error}] if self.auto_merge_error else [] + return 200, {"errors": errors} if errors else {"data": {}} + + def write(self, method: str, path: str) -> tuple[int, object]: + repo = path.split("/")[3] + if method == "POST": + if repo in self.pr_exists: + return 422, { + "message": "Validation Failed", + "errors": [{"message": "A pull request already exists for x."}], + } + return 201, { + "html_url": f"https://github.com/crunchtools/{repo}/pull/1", + "node_id": f"PR_{repo}", + "number": 1, + } + return 200, { + "html_url": f"https://github.com/crunchtools/{repo}/pull/7", + "node_id": f"PR_{repo}", + "number": 7, + } + + +def listing(**repos: str) -> tuple[int, list]: + return 200, [ + {"name": name, "default_branch": branch, "archived": False, "private": False} + for name, branch in repos.items() + ] + + +def search(*hits: str) -> tuple[int, dict]: + return 200, { + "items": [ + { + "repository_url": f"https://api.github.com/repos/crunchtools/{h}", + "number": 3, + "html_url": f"https://github.com/crunchtools/{h}/pull/3", + } + for h in hits + ] + } + + +def routes(earlier: tuple = (), ours: tuple = (), **repos: str) -> dict: + q = "/search/issues?q=org%3Acrunchtools%20is%3Apr%20is%3Aopen%20head%3A" + return { + "/orgs/crunchtools/repos?type=public&per_page=100&page=1": listing(**repos), + "/orgs/crunchtools/repos": (200, []), + q + "chore/python-sdlc": search(*earlier), + q + "chore/sdlc-floating": search(*ours), + } + + +def wave_run(api: FakeApi, base: Path, names: list[str], apply: bool) -> tuple[dict, Path, str]: + """W.run over names, its stdout captured: (counts, ledger, output).""" + available = W.org_repos(api) + wave = W.survey(api, apply) + wave.clone_base = f"file://{base}" + ledger = base / "ledger.tsv" + out = io.StringIO() + with contextlib.redirect_stdout(out): + counts = W.run(wave, names, available, ledger, 0) + return counts, ledger, out.getvalue() + + +def rows(ledger: Path) -> dict[str, list[str]]: + lines = ledger.read_text().splitlines() + return {line.split("\t")[0]: line.split("\t") for line in lines[1:]} + + +# -------------------------------------------------------------- SDLC_IMAGE + +for value, ok in ( + (None, True), + ("", True), + ("main", True), + ("latest", True), + ("0.9.0", True), + ("@sha256:" + "a" * 64, True), + ("quay.io/crunchtools/sdlc:0.9.0", True), + ("quay.io/crunchtools/sdlc@sha256:" + "0" * 64, True), + ("localhost/python-sdlc:dev", False), + ("0.9", False), + ("v0.9.0", False), + ("@sha256:" + "A" * 64, False), + ("quay.io/crunchtools/sdlcx:main", False), + ("quay.io/crunchtools/python-sdlc:0.8.0", False), +): + check( + f"SDLC_IMAGE={value!r} is {'taken' if ok else 'refused'}", + (W.image_problem(value) is None) == ok, + ) + +check( + "a refusal is read from convert_repo.py's stderr", + W.refusal("convert_repo: x: NOT CONVERTED: ci.yml has jobs\n") == "ci.yml has jobs", +) + +# ----------------------------------------------------------------- dry run + +base = Path(tempfile.mkdtemp()) +origin(base, "pinned", CONVERTIBLE) +origin(base, "old", CONVERTIBLE, branch="master") +origin(base, "odd", UNKNOWN_SHAPE) +origin(base, "bumping", CONVERTIBLE) +floating = origin(base, "floating", CONVERTIBLE) +# A repo already converted: convert it the way --apply would, then push. +tmp = Path(tempfile.mkdtemp()) / "floating" +subprocess.run(["git", "clone", "-q", str(floating), str(tmp)], check=True) +subprocess.run( + [sys.executable, str(W.CONVERTER), str(tmp), "--apply"], check=True, capture_output=True +) +git(tmp, "add", "-A") +git(tmp, "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "floated") +git(tmp, "push", "-q", "origin", "HEAD:main") +# The same, but its bootstrap committed without +x: the diff is empty, the work is not. +origin(base, "noexec", CONVERTIBLE) +tmp = Path(tempfile.mkdtemp()) / "noexec" +subprocess.run(["git", "clone", "-q", str(base / "crunchtools" / "noexec"), str(tmp)], check=True) +subprocess.run( + [sys.executable, str(W.CONVERTER), str(tmp), "--apply"], check=True, capture_output=True +) +(tmp / ".sdlc/bin/sdlc").chmod(0o644) +git(tmp, "add", "-A") +git(tmp, "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "floated, no +x") +git(tmp, "push", "-q", "origin", "HEAD:main") + +api = FakeApi( + None, + routes( + earlier=("bumping",), + pinned="main", + old="master", + odd="main", + bumping="main", + floating="main", + noexec="main", + **{"python-sdlc": "main"}, + ), +) +names = ["pinned", "old", "odd", "bumping", "floating", "noexec", "python-sdlc", "gone"] +counts, ledger, out = wave_run(api, base, names, apply=False) +got = rows(ledger) +check("dry run: a pinned repo would open", got["pinned"][1] == "would-open", str(got["pinned"])) +check( + "dry run: the plan names the files it changes", + ".sdlc/bin/sdlc" in got["pinned"][3] and ".pre-commit-config.yaml" in got["pinned"][3], +) +check( + "dry run: a master default branch is cloned as master", + got["old"][1] == "would-open", + str(got["old"]), +) +check( + "dry run: a refusal is skipped with convert_repo.py's reason", + got["odd"][1] == "refused" and "nothing this script knows" in got["odd"][3], + str(got["odd"]), +) +check( + "dry run: an earlier wave's open PR is skipped and named", + got["bumping"][1] == "skipped" and "/bumping/pull/3" in got["bumping"][3], +) +check( + "dry run: an already floating repo is a no-op", + got["floating"][1] == "noop", + str(got["floating"]), +) +check( + "dry run: a floating repo whose bootstrap lacks +x is not a no-op", + got["noexec"][1] == "would-open", + str(got["noexec"]), +) +check("dry run: the converter's own repo is skipped", got["python-sdlc"][1] == "skipped") +check("dry run: a name not in the org is skipped", got["gone"][1] == "skipped") +check("dry run: only reads", all(m == "GET" for m, _, _ in api.calls), str(api.calls)) +check( + "dry run: nothing is pushed", + "chore/sdlc-floating" not in git(base / "crunchtools" / "pinned", "branch", "-a"), +) +check( + "dry run: SDLC_IMAGE is not read without a token", + not any("variables" in p for _, p, _ in api.calls), +) +check("dry run: counts add up", sum(counts.values()) == len(names), str(counts)) + +# The ledger: a re-run skips what is done and retries the rest. +counts, _, _ = wave_run(api, base, names, apply=False) +check( + "re-run: done repos are skipped", counts.get("already done") == 4, str(counts) +) # pinned, old, floating, noexec +check( + "re-run: refused and skipped repos are tried again", + counts.get("refused") == 1 and counts.get("skipped") == 3, + str(counts), +) + +# ------------------------------------------------------------------- apply + +base = Path(tempfile.mkdtemp()) +bare = origin(base, "pinned", CONVERTIBLE) +origin(base, "reused", CONVERTIBLE) +origin(base, "badvar", CONVERTIBLE) +r = routes(ours=("reused",), pinned="main", reused="main", badvar="main") +r["/repos/crunchtools/badvar/actions/variables/SDLC_IMAGE"] = (200, {"value": "localhost/x"}) +r["/repos/crunchtools/reused/pulls?head="] = (200, [{"number": 7}]) +api = FakeApi(CANARY, r, pr_exists={"reused"}) +counts, ledger, out = wave_run(api, base, ["pinned", "reused", "badvar"], apply=True) +got = rows(ledger) +check("apply: a pinned repo is opened", got["pinned"][1] == "opened", str(got["pinned"])) +check("apply: the ledger has the PR URL", got["pinned"][2].endswith("/pinned/pull/1")) +check( + "apply: the open PR on the wave branch is updated, not doubled", + got["reused"][1] == "updated" and got["reused"][2].endswith("/pull/7"), + str(got["reused"]), +) +check( + "apply: a repository SDLC_IMAGE the bootstrap refuses is skipped", + got["badvar"][1] == "skipped" and "localhost/x" in got["badvar"][3], + str(got["badvar"]), +) +check( + "apply: the org variable is read once", + sum(p == "/orgs/crunchtools/actions/variables/SDLC_IMAGE" for _, p, _ in api.calls) == 1, +) + +log = git(bare, "log", "-1", "--format=%an%n%B", "chore/sdlc-floating") +check( + "apply: the branch is pushed with the canonical message", + log.startswith("crunchtools-ci\n" + W.COMMIT_MESSAGE.splitlines()[0]), + log, +) +check( + "apply: the commit carries the Co-Authored-By trailer", + "Co-Authored-By: Claude" + in git(bare, "log", "-1", "--format=%(trailers)", "chore/sdlc-floating"), +) +check( + "apply: the bootstrap is committed executable", + git(bare, "ls-tree", "chore/sdlc-floating", ".sdlc/bin/sdlc").startswith("100755"), +) +check( + "apply: the default branch is untouched", + ".sdlc" not in git(bare, "ls-tree", "-r", "--name-only", "main"), +) + +opened = [b for m, p, b in api.calls if m == "POST" and p == "/repos/crunchtools/pinned/pulls"] +check( + "apply: the PR is opened from the wave branch to the default branch", + opened and opened[0]["head"] == W.BRANCH and opened[0]["base"] == "main", +) +check( + "apply: the PR body is the fixed one, with a rollback", + opened and opened[0]["body"] == W.PR_BODY and "How to roll back" in W.PR_BODY, +) +merges = [b for m, p, b in api.calls if p == "/graphql"] +check( + "apply: auto-merge is queued as a squash for each PR", + len(merges) == 2 + and all("SQUASH" in b["query"] for b in merges) + and merges[0]["variables"]["id"] == "PR_pinned", +) +check("apply: the token is never printed", CANARY not in out and CANARY not in ledger.read_text()) + +# A PR that opens but cannot auto-merge is recorded, retried, and fails the run. +base = Path(tempfile.mkdtemp()) +origin(base, "nomerge", CONVERTIBLE) +api = FakeApi(CANARY, routes(nomerge="main")) +api.auto_merge_error = "Auto merge is not allowed for this repository" +counts, ledger, _ = wave_run(api, base, ["nomerge"], apply=True) +got = rows(ledger) +check( + "apply: a PR auto-merge refused is recorded as no-automerge, with why", + got["nomerge"][1] == "no-automerge" and "not allowed" in got["nomerge"][3], + str(got), +) +check("apply: no-automerge is not done", "no-automerge" not in W.DONE) + +# ------------------------------------------------------------------- verify + +base = Path(tempfile.mkdtemp()) +bare = origin(base, "stray", CONVERTIBLE) +checkout = base / "stray-checkout" +subprocess.run(["git", "clone", "-q", str(bare), str(checkout)], check=True) +subprocess.run( + [sys.executable, str(W.CONVERTER), str(checkout), "--apply"], check=True, capture_output=True +) +(checkout / "README.md").write_text("not the converter's\n") +try: + W.verify(checkout) + stray = "" +except RuntimeError as exc: + stray = str(exc) +check("verify: a change outside the converter's files is refused", "README.md" in stray, stray) +(checkout / "README.md").unlink() +git(checkout, "rm", "-q", "--cached", "README.md") +(checkout / ".sdlc/bin/sdlc").chmod(0o644) +try: + W.verify(checkout) + mode = "" +except RuntimeError as exc: + mode = str(exc) +check("verify: a bootstrap that is not executable is refused", "100644" in mode, mode) + +# ---------------------------------------------------------------------- Api + + +class FakeConnection: + """http.client's half the Api uses: one request, one 403 echoing the credential.""" + + def __init__(self) -> None: + self.sent: list[tuple[str, str, object, dict]] = [] + self.status = 403 + + def request(self, method: str, path: str, body: object, headers: dict) -> None: + self.sent.append((method, path, body, headers)) + + def getresponse(self) -> FakeConnection: + return self + + def read(self) -> bytes: + return b'{"message": "Bad credentials ' + CANARY.encode() + b'"}' + + def close(self) -> None: + pass + + +conn = FakeConnection() +try: + W.Api(CANARY, connect=lambda: conn).call("GET", "/orgs/crunchtools/repos") + error = "" +except W.ApiError as exc: + error = str(exc) +check( + "Api: the token is sent as a bearer header", + bool(conn.sent) and conn.sent[0][3].get("Authorization") == f"Bearer {CANARY}", +) +check("Api: the token is not in the path", bool(conn.sent) and CANARY not in conn.sent[0][1]) +check( + "Api: a 403 raises, and its message is redacted", + "HTTP 403" in error and CANARY not in error, + error, +) + +# --------------------------------------------------------------------- main + +env = {k: v for k, v in os.environ.items() if k != "GH_TOKEN"} +proc = subprocess.run( + [sys.executable, str(SCRIPT), "--apply"], capture_output=True, text=True, env=env, check=False +) +check( + "main: --apply without GH_TOKEN cannot run (exit 2)", + proc.returncode == 2 and "GH_TOKEN" in proc.stderr, + proc.stderr, +) + + +def main() -> int: + print() + if FAILED: + print(f"{len(FAILED)} FAILED") + return 1 + print("all passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 63eef8761a29936afde0f8ffa4ab9d42a2b2c3e8 Mon Sep 17 00:00:00 2001 From: Josui Date: Tue, 17 Nov 2026 21:53:48 +0000 Subject: [PATCH 2/2] docs: say what fleet_wave.run takes, records and returns Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01R7Y1jg3ZJ6BsAEfhrPWFRh --- scripts/fleet_wave.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/fleet_wave.py b/scripts/fleet_wave.py index d692b48..d146ec9 100755 --- a/scripts/fleet_wave.py +++ b/scripts/fleet_wave.py @@ -508,7 +508,15 @@ def survey(api: Api, apply: bool) -> Wave: def run( wave: Wave, names: list[str], available: dict[str, str], ledger: Path, pause: float -) -> dict: +) -> dict[str, int]: + """Process `names` in order, one at a time, `pause` seconds apart. + + `available` is the org listing (name -> default branch); a name not in it is + skipped. Each repo's outcome is appended to `ledger` as it finishes, and a + repo whose last ledger action is in DONE is not processed again. Returns + how many repos ended in each action, plus "already done" for those passed + over -- so the counts always add up to len(names). + """ done = read_ledger(ledger) counts: dict[str, int] = {} for n, repo in enumerate(names):