From 5693c04ed246b0223a089a0f36290f3853f1aa00 Mon Sep 17 00:00:00 2001 From: Scott McCarty Date: Wed, 4 Nov 2026 21:30:09 -0500 Subject: [PATCH 1/3] fix: one bump pull request per repo, and a run that fails when one cannot merge v0.4.0 and v0.5.0 released two minutes apart and left 10 repos with a bump pull request that can never merge. The branch name was the cause and the silence was the damage. Per-version branches -- `chore/python-sdlc-v0.4.0`, `chore/python-sdlc-v0.5.0` -- look tidier and guarantee this. Both are cut from a default branch that still pins the old version, so both rewrite the same `rev:` line. When the OLDER one merges it moves that line out from under the newer branch, which goes CONFLICTING. Rebasing before opening would not have helped: the conflict appears afterwards, created by the other pull request landing. One branch per repo, `chore/python-sdlc`, force-updated each release and its pull request reused in place. A second release then supersedes the first on the same branch rather than racing it -- which is correct, since the later image contains the earlier one's code. The clone is always the current default branch, so the rewrite is always against what the repo pins now. The silence is the part worth fixing properly. A CONFLICTING pull request runs no `pull_request` workflows AT ALL, so its required check is ABSENT rather than red: auto-merge never fires, nothing reports a failure, and it sits looking green indefinitely. All 10 would have. That is the same failure this whole release has been about -- a check that did not run reading exactly like a check that passed -- so the script now asks GitHub whether each pull request can actually merge and FAILS the run on CONFLICTING, naming the repos to rerun. UNKNOWN is not a failure: GitHub computes mergeability asynchronously and a release must not go red because the API was slow. `bump-consumers` also gets a fixed concurrency group, like `promote`'s. One branch per repo means two releases at once would fight over it and the loser's version would be what the branch was left pointing at. Verified with `--dry-run` against the fleet: 33 repos pin python-sdlc, 23 already on v0.5.0, and the 10 it names are exactly the 10 that conflicted. --- .github/workflows/container.yml | 9 +++ scripts/bump_consumers.py | 116 +++++++++++++++++++++++++++----- 2 files changed, 109 insertions(+), 16 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 70afd4c..2d49ed7 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -261,6 +261,15 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + # A FIXED group, like promote's, and for the same kind of reason. Each repo + # gets ONE bump branch which this force-pushes, so two releases running at + # once would fight over it and the loser's version would be the one left on + # the branch. Serialized, the later release simply supersedes the earlier -- + # which is correct, since its image contains the earlier one's code. + # Not cancel-in-progress: a half-finished fleet bump is worse than a slow one. + concurrency: + group: bump-consumers-python-sdlc + cancel-in-progress: false steps: - uses: actions/checkout@v7 with: diff --git a/scripts/bump_consumers.py b/scripts/bump_consumers.py index 6a6d261..956a6ac 100755 --- a/scripts/bump_consumers.py +++ b/scripts/bump_consumers.py @@ -7,6 +7,20 @@ The same shape as crunchtools/constitution's scripts/fleet-bump.py: clone, rewrite, push, open, queue, and name the repos that failed so they can be rerun. +ONE BRANCH PER REPO, not one per version, and reused in place. Per-version +branches look tidier and are a trap: two releases close together open two PRs +against the same `rev:` line, and when the OLDER one merges it moves that line +out from under the newer branch, which goes CONFLICTING. v0.4.0 and v0.5.0 were +two minutes apart and left 10 repos exactly there. + +What makes that bad rather than merely untidy is the shape of the failure: a +conflicting pull request runs no `pull_request` workflows AT ALL, so its +required check is ABSENT rather than red. Auto-merge never fires, nothing +reports a failure, and the PR sits looking green forever. So this also asks +GitHub whether each pull request can actually merge, and a run that leaves one +unmergeable FAILS -- the same reason this script already refuses to finish +quietly (RT#1553, below). + Usage: scripts/bump_consumers.py vX.Y.Z [--only repo,repo] [--dry-run] Needs GH_TOKEN able to push branches and open PRs across the org. """ @@ -20,10 +34,13 @@ import subprocess import sys import tempfile +import time from pathlib import Path ORG = "crunchtools" SELF = "python-sdlc" +# One per repo, reused and force-updated each release. See the module docstring. +BRANCH = f"chore/{SELF}" PIN = re.compile( r"(-\s*repo:\s*https://github\.com/crunchtools/python-sdlc(?:\.git)?\s*\n\s*rev:\s*)(v\d+\.\d+\.\d+)" ) @@ -65,8 +82,48 @@ def pinned_rev(repo: str) -> str | None: return match.group(2) if match else None +def open_bump(repo: str) -> str | None: + """The bump pull request already open on this repo's branch, or None.""" + return ( + gh( + "pr", + "list", + "-R", + f"{ORG}/{repo}", + "--head", + BRANCH, + "--state", + "open", + "--json", + "url", + "--jq", + ".[0].url // empty", + ).strip() + or None + ) + + +def can_merge(repo: str, url: str) -> str: + """MERGEABLE, CONFLICTING, or UNKNOWN if GitHub will not say in time. + + GitHub computes this asynchronously after a push, so the first answer is + almost always UNKNOWN. UNKNOWN is NOT treated as a failure: a release must + not go red because the API was slow. CONFLICTING is, because nothing else + will ever report it -- see the module docstring. + """ + for attempt in range(6): + if attempt: + time.sleep(5) + state = gh( + "pr", "view", url, "-R", f"{ORG}/{repo}", "--json", "mergeable", "--jq", ".mergeable" + ).strip() + if state and state != "UNKNOWN": + return state + return "UNKNOWN" + + def bump(repo: str, version: str) -> str: - branch = f"chore/python-sdlc-{version}" + branch = BRANCH with tempfile.TemporaryDirectory() as workdir: root = Path(workdir) / repo gh("repo", "clone", f"{ORG}/{repo}", str(root), "--", "--depth", "1", "-q") @@ -79,7 +136,10 @@ def bump(repo: str, version: str) -> str: def git(*a: str) -> None: subprocess.run(["git", "-C", str(root), *a], check=True, capture_output=True) - git("checkout", "-q", "-b", branch) + # -B, not -b: the branch is this script's own and may already exist from + # an earlier release. The clone is the CURRENT default branch, so the + # rewrite is always against what the repo pins right now. + git("checkout", "-q", "-B", branch) git( "-c", "user.name=crunchtools-ci", @@ -89,25 +149,49 @@ def git(*a: str) -> None: "-qam", f"chore: python-sdlc {version}", ) - git("push", "-q", "-u", "origin", branch) - url = gh( - "pr", - "create", - "-R", - f"{ORG}/{repo}", - "--head", - branch, - "--title", - f"chore: python-sdlc {version}", - "--body", - f"Moves the python-sdlc pin to {version}. Its own CI " - f"decides whether it lands; it merges itself when green.", - ).strip() + # Forced, because this branch carries one commit that is rewritten each + # release rather than a history worth keeping. + git("push", "-q", "-u", "--force", "origin", branch) + + title = f"chore: python-sdlc {version}" + body = ( + f"Moves the python-sdlc pin to {version}. Its own CI decides whether " + f"it lands; it merges itself when green.\n\n" + f"One branch per repo, reused each release, so two releases close " + f"together cannot leave two pull requests fighting over the same line." + ) + url = open_bump(repo) + if url: + # Reused rather than opened again. The force-push above has already + # moved it to this version; this only makes it say so. + gh("pr", "edit", url, "-R", f"{ORG}/{repo}", "--title", title, "--body", body) + print(f" updated in place: {url}") + else: + url = gh( + "pr", + "create", + "-R", + f"{ORG}/{repo}", + "--head", + branch, + "--title", + title, + "--body", + body, + ).strip() try: gh("pr", "merge", url, "--auto", "--merge") except subprocess.CalledProcessError as exc: reason = exc.stderr.strip() print(f" warning: {repo}: auto-merge not queued: {reason}", file=sys.stderr) + + state = can_merge(repo, url) + if state == "CONFLICTING": + raise RuntimeError( + f"{url} cannot merge. Nothing else will say so: a conflicting " + f"pull request runs no workflows, so its required check is absent " + f"rather than failing and auto-merge never fires" + ) return url From d00f0470e11b2fedc8f7847524d60df9d36f88b6 Mon Sep 17 00:00:00 2001 From: Scott McCarty Date: Wed, 4 Nov 2026 21:36:05 -0500 Subject: [PATCH 2/3] fix: ask once for the whole fleet whether the bumps can merge, not once per repo Gatehouse was right, and its suggestion was better than what it reviewed. The mergeability check sat inside bump(), so each repo got up to six questions with five-second waits between them. GitHub computes mergeability asynchronously, so asked immediately after a push the answer is ALWAYS UNKNOWN -- which means the first ask is guaranteed to be wasted and the waiting is the normal case, not the exception. Across 33 repos that is minutes of a release run doing nothing at all, serially, for an answer that arrives for free once the rest of the fleet has been bumped. Now it is one pass after the loop, and each round asks only the repos still unanswered. Three rounds, five seconds apart. Same guard, same failure behaviour, without spending a release's time on a question that answers itself. It also reads better: the conflicts are reported together, as a fleet state, rather than one repo at a time as a side effect of bumping it. And a counting bug of my own in the same block: the summary said `len(opened) - len(failed)` bumped, but `failed` also carries repos whose config could not be READ and which were never opened, so a single unreadable repo would have understated the bump count. It subtracts the conflicts, which is what it meant. Verified with --dry-run: 33 repos pin python-sdlc, 32 already on v0.5.0 as the rebases land, and an unreadable repo still reports honestly rather than silently. --- scripts/bump_consumers.py | 101 ++++++++++++++++++++++++++++---------- 1 file changed, 74 insertions(+), 27 deletions(-) diff --git a/scripts/bump_consumers.py b/scripts/bump_consumers.py index 956a6ac..5bccafc 100755 --- a/scripts/bump_consumers.py +++ b/scripts/bump_consumers.py @@ -103,23 +103,52 @@ def open_bump(repo: str) -> str | None: ) -def can_merge(repo: str, url: str) -> str: - """MERGEABLE, CONFLICTING, or UNKNOWN if GitHub will not say in time. - - GitHub computes this asynchronously after a push, so the first answer is - almost always UNKNOWN. UNKNOWN is NOT treated as a failure: a release must - not go red because the API was slow. CONFLICTING is, because nothing else - will ever report it -- see the module docstring. +# How many times, and how long apart, the mergeability pass asks again. Asked +# once per ROUND for the repos still unanswered, not once per repo: see +# conflicting(). +ROUNDS = 3 +PAUSE = 5 + + +def conflicting(opened: dict[str, str]) -> tuple[list[str], list[str]]: + """Which of these pull requests cannot merge, and which GitHub would not say. + + Asked AFTER every repo has been bumped rather than inside bump(), and this + is the difference between a question and a wait. GitHub computes + mergeability asynchronously, so asked immediately after a push the answer is + always UNKNOWN; polling per repo would spend PAUSE seconds of a release run + per unanswered repo, serially, for an answer that arrives for free once the + rest of the fleet has been bumped. Across 33 repos that is minutes of a + release doing nothing. Here each round asks only the repos still unanswered. + + Returns (cannot merge, no answer). UNKNOWN is deliberately not a failure: a + release must not go red because the API was slow. """ - for attempt in range(6): + pending = dict(opened) + cannot: list[str] = [] + for attempt in range(ROUNDS): if attempt: - time.sleep(5) - state = gh( - "pr", "view", url, "-R", f"{ORG}/{repo}", "--json", "mergeable", "--jq", ".mergeable" - ).strip() - if state and state != "UNKNOWN": - return state - return "UNKNOWN" + time.sleep(PAUSE) + for repo, url in list(pending.items()): + state = gh( + "pr", + "view", + url, + "-R", + f"{ORG}/{repo}", + "--json", + "mergeable", + "--jq", + ".mergeable", + ).strip() + if not state or state == "UNKNOWN": + continue + del pending[repo] + if state == "CONFLICTING": + cannot.append(repo) + if not pending: + break + return cannot, sorted(pending) def bump(repo: str, version: str) -> str: @@ -184,14 +213,8 @@ def git(*a: str) -> None: except subprocess.CalledProcessError as exc: reason = exc.stderr.strip() print(f" warning: {repo}: auto-merge not queued: {reason}", file=sys.stderr) - - state = can_merge(repo, url) - if state == "CONFLICTING": - raise RuntimeError( - f"{url} cannot merge. Nothing else will say so: a conflicting " - f"pull request runs no workflows, so its required check is absent " - f"rather than failing and auto-merge never fires" - ) + # Whether it can actually merge is asked once, for the whole fleet, + # after this loop has finished. See conflicting(). return url @@ -227,22 +250,46 @@ def main() -> int: repos = args.only.split(",") if args.only else org_repos() current, stale, failed = survey(repos, args.version) - opened = [] + opened: dict[str, str] = {} if not args.dry_run: for repo in stale: try: - print(f" {bump(repo, args.version)}") - opened.append(repo) + url = bump(repo, args.version) + print(f" {url}") + opened[repo] = url except (subprocess.CalledProcessError, RuntimeError) as bump_error: detail = getattr(bump_error, "stderr", "") or "" print(f" FAILED: {bump_error}: {detail}".rstrip(": "), file=sys.stderr) failed.append(repo) + # One pass over everything just opened. A pull request that cannot merge is + # the one failure here that NOTHING else reports: a conflicting pull request + # runs no `pull_request` workflows, so its required check is absent rather + # than red, auto-merge never fires, and it sits looking green indefinitely. + # v0.4.0 and v0.5.0 two minutes apart left 10 repos exactly there. + cannot: list[str] = [] + if opened: + cannot, unanswered = conflicting(opened) + for repo in cannot: + print( + f" CANNOT MERGE: {opened[repo]} conflicts with its base. " + f"Nothing else will report this.", + file=sys.stderr, + ) + failed.append(repo) + if unanswered: + print( + f" note: GitHub would not say whether these can merge: {', '.join(unanswered)}", + file=sys.stderr, + ) + # Always say what happened. A run with nothing to do and a run that could # not open a pull request both used to print nothing at all, which is how # RT#1553 -- the App had no `pull_requests: write` -- survived two releases # looking like a job that had half worked. - did = f"{len(stale)} would be bumped" if args.dry_run else f"{len(opened)} bumped" + # `failed` also carries repos whose config could not be READ, which were + # never opened, so it is not what to subtract here. + did = f"{len(stale)} would be bumped" if args.dry_run else f"{len(opened) - len(cannot)} bumped" print( f"{len(current) + len(stale)} repo(s) pin python-sdlc: " f"{did} to {args.version}, {len(current)} already there, {len(failed)} failed" From f974ba29b332434b911e00f4e63ff386a65f1f70 Mon Sep 17 00:00:00 2001 From: Scott McCarty Date: Wed, 4 Nov 2026 21:41:17 -0500 Subject: [PATCH 3/3] fix: a mergeability query that fails names its repo instead of ending the run Gatehouse caught a regression I introduced in the previous commit. Moving the check out of the per-repo loop also moved it outside that loop's error handling: `gh()` raises on a non-zero exit, and from `conflicting()` there is nothing to catch it, so one refused API call would have ended the whole run with a traceback and printed no summary at all. That is the specific failure this script already carries a comment about (RT#1553): a run that could not finish its job looking like a job that half worked. Reintroducing it while adding a guard against silent failure would have been funny in the wrong way. A repo the API will not answer for is now named and treated as unanswered, the same as UNKNOWN. Not a failure: a release must not go red because GitHub was slow or briefly unavailable. Both kinds of non-answer are printed by name, so a guard that quietly answered for nobody is visible rather than assumed -- which matters more here than usual, since a guard that silently does nothing is exactly the shape it exists to catch. Verified with --dry-run: the fleet is now 33 of 33 pinned at v0.5.0. --- scripts/bump_consumers.py | 42 +++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/scripts/bump_consumers.py b/scripts/bump_consumers.py index 5bccafc..e89ac24 100755 --- a/scripts/bump_consumers.py +++ b/scripts/bump_consumers.py @@ -121,8 +121,10 @@ def conflicting(opened: dict[str, str]) -> tuple[list[str], list[str]]: rest of the fleet has been bumped. Across 33 repos that is minutes of a release doing nothing. Here each round asks only the repos still unanswered. - Returns (cannot merge, no answer). UNKNOWN is deliberately not a failure: a - release must not go red because the API was slow. + Returns (cannot merge, no answer). UNKNOWN is deliberately not a failure, + and neither is a query the API refuses: a release must not go red because + GitHub was slow or momentarily unavailable. Both are reported by name, so a + guard that quietly answered for nobody is visible rather than assumed. """ pending = dict(opened) cannot: list[str] = [] @@ -130,17 +132,31 @@ def conflicting(opened: dict[str, str]) -> tuple[list[str], list[str]]: if attempt: time.sleep(PAUSE) for repo, url in list(pending.items()): - state = gh( - "pr", - "view", - url, - "-R", - f"{ORG}/{repo}", - "--json", - "mergeable", - "--jq", - ".mergeable", - ).strip() + try: + state = gh( + "pr", + "view", + url, + "-R", + f"{ORG}/{repo}", + "--json", + "mergeable", + "--jq", + ".mergeable", + ).strip() + except subprocess.CalledProcessError as exc: + # Named, and survived. Outside the main loop's per-repo handler, + # an exception here would end the whole run with a traceback and + # no summary -- and the summary is the thing this script exists + # to print (RT#1553, below). One repo the API will not answer + # for is reported as unanswered, like UNKNOWN: a release must not + # go red because a query failed. + print( + f" warning: {repo}: cannot read whether it can merge: {exc.stderr.strip()}", + file=sys.stderr, + ) + del pending[repo] + continue if not state or state == "UNKNOWN": continue del pending[repo]