diff --git a/.github/workflows/bot-sweep.yml b/.github/workflows/bot-sweep.yml index 6f17718a..81739f03 100644 --- a/.github/workflows/bot-sweep.yml +++ b/.github/workflows/bot-sweep.yml @@ -1,9 +1,18 @@ name: Bot sweep -# Hourly upkeep of the bot's PRs (branches autobump/*, opened by github-actions[bot] -# with GITHUB_TOKEN, so their own pull_request runs wait for approval). +# Upkeep of the bot's PRs (branches autobump/*, opened by github-actions[bot] with +# GITHUB_TOKEN, so their own pull_request runs wait for approval). # -# ci-fill: hack/bot-ci-dispatch runs ci.yml by workflow_dispatch on each bot PR +# When it runs: each ci.yml run dispatched on autobump/* or main and each canary.yml +# run dispatched on autobump/* ends with job sweep, which dispatches this workflow +# with after_run set to its own run id: those are the runs whose results the sweep +# acts on, and a run dispatched with GITHUB_TOKEN starts no workflow_run. The +# hourly cron is the backstop: GitHub started it only 3.7 to 9.8 h apart here +# (22 scheduled runs, 2026-11-01 to 11-07). +# +# ci-fill: with after_run set, it first waits (5 min at most) for that run to +# complete, so the sweep reads it finished. +# hack/bot-ci-dispatch runs ci.yml by workflow_dispatch on each bot PR # branch with no run at its head, and re-runs a first-attempt failure # once, one bot run in flight at a time and none while a human PR's CI # is queued. It fills in what the nightly deferred. Then the same for @@ -24,6 +33,11 @@ on: schedule: - cron: '13 * * * *' workflow_dispatch: + inputs: + after_run: + description: 'a run id: wait for that run to complete first (job sweep of ci.yml and canary.yml sets it)' + type: string + default: '' permissions: {} @@ -35,12 +49,34 @@ jobs: ci-fill: name: CI on bot PRs runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 15 # the wait for after_run (5 min at most), then the dispatches permissions: actions: write contents: read pull-requests: read steps: + - name: Wait for the run that dispatched this sweep to complete + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + AFTER_RUN: ${{ inputs.after_run }} + run: | + [ -n "$AFTER_RUN" ] || exit 0 + if ! [[ $AFTER_RUN =~ ^[0-9]+$ ]]; then + echo "::error::after_run must be a run id (digits only)" + exit 1 + fi + st=unread + for _ in $(seq 60); do + st=$(gh run view "$AFTER_RUN" --json status -q .status 2>/dev/null) || st=unreadable + if [ "$st" = completed ]; then + echo "run $AFTER_RUN completed" + exit 0 + fi + sleep 5 + done + echo "::warning::run $AFTER_RUN is still ${st:-unread} after 5 min; this sweep reads its evidence as it stands" + - name: Checkout main uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 8be93728..7ebca996 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -15,8 +15,9 @@ name: Canary # every selected engine's verdict is pass with hard true. # # Read-only: no secrets, no registry login (public images), GitHub-hosted runners only. -# The re-dispatch job alone may write: it starts this workflow again (attempt + 1, at most -# 3) when an engine was isa-skip, the runner's CPU lacking an instruction set it needs. +# Two jobs may write, each only to dispatch a workflow: re-dispatch starts this workflow +# again (attempt + 1, at most 3) when an engine was isa-skip, the runner's CPU lacking an +# instruction set it needs; sweep starts bot-sweep.yml after a run dispatched on autobump/*. on: schedule: @@ -235,3 +236,41 @@ jobs: exit 0 fi gh workflow run canary.yml --repo "$REPO" --ref "$REF" -f engines="$ENGINES" -f attempt="$((ATTEMPT + 1))" + + # A run dispatched on autobump/* (a bot PR's canary) is evidence bot-sweep.yml acts + # on, and a run dispatched with GITHUB_TOKEN starts no workflow_run, so the sweep + # would wait for its hourly cron, which GitHub starts hours apart here. After + # every other job this one dispatches the sweep with after_run set to this run's id; + # the sweep waits for this run to complete before it reads it. It never fails the + # canary (continue-on-error, exit 0). The step is ci.yml's job sweep's step. + sweep: + name: Hand off to the bot sweep + needs: [plan, runtime, surface, node, report, redispatch] + if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' && startsWith(github.ref, 'refs/heads/autobump/') }} + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + permissions: + actions: write + steps: + - name: Dispatch bot-sweep.yml, which waits for this run to complete + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + run: | + # GITHUB_TOKEN: 1,000 REST requests an hour for the repository, every workflow's; + # a sweep reads about 10 per open bot PR. GET /rate_limit costs none. + left=$(gh api rate_limit --jq .resources.core.remaining 2>/dev/null) || left= + if [[ $left =~ ^[0-9]+$ ]] && [ "$left" -lt 500 ]; then + echo "::notice::$left GITHUB_TOKEN requests left this hour; no hand-off, the next scheduled sweep reads this run" + exit 0 + fi + for n in 1 2 3; do + if gh workflow run bot-sweep.yml --ref main -f after_run="$RUN_ID"; then + echo "dispatched bot-sweep.yml with after_run=$RUN_ID" + exit 0 + fi + if [ "$n" -lt 3 ]; then sleep $((10 * n)); fi + done + echo "::warning::could not dispatch bot-sweep.yml; its next scheduled run reads this run" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42b2e5bf..a1c18cd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -649,6 +649,54 @@ jobs: persist-credentials: false - run: tests/bot_merge_test.sh + # ─── Hand-off to the bot sweep ─────────────────────────────────────────── + # A run dispatched on autobump/* (a bot PR's CI) or on main (the CI the sweep or the + # watchdog dispatches) is evidence bot-sweep.yml acts on, and a run dispatched with + # GITHUB_TOKEN starts no workflow_run, so the sweep would wait for its hourly cron, + # which GitHub starts hours apart here (bot-sweep.yml). After every other job this + # one dispatches the sweep with after_run set to this run's id; the sweep waits for + # this run to complete before it reads it. It never fails CI (continue-on-error, + # exit 0). + # needs lists every other job (tests/ci_wiring_test.sh checks it). + sweep: + name: Hand off to the bot sweep + needs: [shellcheck, actionlint, detect-tests, quickstart-tests, fuzz-props, remote-tests, telemetry-tests, + agent-manifests, intel-gpu-dra-manifests, share-manifests, console-tests, images-lock, report-html, + oracle, claims-check, bench-tests, fit-tests, tune-tests, capacity-tests, mcp-tests, kuberay-example, + chat-ui-manifest, sim-fixtures, sim-matrix, agent-examples, register-model-test, render, + packaging-tests, deb-rpm, homebrew, nix-flake, hadolint, arch-image, pkgbuild-lint, kserve-example, + ops-tests, known-issues, canary-tests, pin-guard, pins, bot-merge-tests, alarms] + if: >- + ${{ !cancelled() && github.event_name == 'workflow_dispatch' + && (startsWith(github.ref, 'refs/heads/autobump/') || github.ref == 'refs/heads/main') }} + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + permissions: + actions: write + steps: + - name: Dispatch bot-sweep.yml, which waits for this run to complete + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + run: | + # GITHUB_TOKEN: 1,000 REST requests an hour for the repository, every workflow's; + # a sweep reads about 10 per open bot PR. GET /rate_limit costs none. + left=$(gh api rate_limit --jq .resources.core.remaining 2>/dev/null) || left= + if [[ $left =~ ^[0-9]+$ ]] && [ "$left" -lt 500 ]; then + echo "::notice::$left GITHUB_TOKEN requests left this hour; no hand-off, the next scheduled sweep reads this run" + exit 0 + fi + for n in 1 2 3; do + if gh workflow run bot-sweep.yml --ref main -f after_run="$RUN_ID"; then + echo "dispatched bot-sweep.yml with after_run=$RUN_ID" + exit 0 + fi + if [ "$n" -lt 3 ]; then sleep $((10 * n)); fi + done + echo "::warning::could not dispatch bot-sweep.yml; its next scheduled run reads this run" + # ─── Bot PR CI and alarm surfacing ─────────────────────────────────────── # The watchdog files a ci-alarm (an issue; with Issues disabled an entry on branch # ci-status) for a red or missing scheduled run or a broken release (email went diff --git a/README.md b/README.md index 97236f2c..663df92c 100644 --- a/README.md +++ b/README.md @@ -1762,8 +1762,9 @@ Freshness workflow (`pins/registry.toml`, group `nri`) runs `--latest` for the components and opens one PR for them (branch `autobump/nri`); merging it changes `share/`, so auto-release cuts a launcher release and the node image is rebuilt for it. GITHUB_TOKEN opens these PRs, so their own pull_request CI runs wait for approval; `hack/bot-ci-dispatch` runs `ci.yml` on -the branch by workflow_dispatch instead, and that run is the PR's CI. The hourly bot sweep -(`hack/bot-merge`) merges one such PR at a time when its pins policy says auto (or a +the branch by workflow_dispatch instead, and that run is the PR's CI. The bot sweep +(`hack/bot-merge`), which each such CI or canary run starts as it ends (its hourly cron is +the backstop), merges one such PR at a time when its pins policy says auto (or a maintainer approved its head), its CI and canary are green, its base is current and main's CI is green; it then dispatches CI on main and a batched release. Any other bot PR stays open with one comment that says what blocks it. diff --git a/tests/canary_spec_test.sh b/tests/canary_spec_test.sh index 4b3e43fc..327b0053 100755 --- a/tests/canary_spec_test.sh +++ b/tests/canary_spec_test.sh @@ -298,7 +298,8 @@ assert jobs["runtime"]["timeout-minutes"] == 45 and jobs["runtime"]["strategy"][ assert jobs["surface"]["timeout-minutes"] == 30 and jobs["node"]["timeout-minutes"] == 40 assert "always()" in jobs["report"]["if"] writes = {n for n, j in jobs.items() if any(v == "write" for v in (j["permissions"] or {}).values())} -assert writes <= {"redispatch"}, writes +assert writes <= {"redispatch", "sweep"}, writes +assert jobs["sweep"]["permissions"] == {"actions": "write"}, jobs["sweep"]["permissions"] assert "secrets." not in src, "no secrets" for u in re.findall(r"uses:\s*(\S+)", src): assert re.fullmatch(r"[\w.-]+/[\w./-]+@[0-9a-f]{40}", u), "not pinned by SHA: " + u diff --git a/tests/ci_wiring_test.sh b/tests/ci_wiring_test.sh index 014dd499..c5ba16e1 100755 --- a/tests/ci_wiring_test.sh +++ b/tests/ci_wiring_test.sh @@ -603,6 +603,140 @@ got=$(ki_case none '' '[]' | tr '\n' ';') if [ -z "${got}" ]; then ok "bot-sweep known-issues: nothing newly fixed, no dispatch" else fail "bot-sweep known-issues with nothing to do: ${got}"; fi +# ─── Hand-off to the bot sweep ─────────────────────────────────────────────── +# GitHub delivers bot-sweep.yml's hourly cron every 4 to 10 h on this repository, +# and a run dispatched with GITHUB_TOKEN (every bot PR's CI and canary, and the CI +# the sweep dispatches on main) starts no workflow_run, so nothing else follows it. +# ci.yml (dispatched on autobump/* or main) and canary.yml (dispatched on +# autobump/*) therefore end with job sweep: after every other job, never failing, +# actions: write only, it dispatches bot-sweep.yml on main with after_run set to +# its own run id. The sweep's ci-fill job first waits (5 min at most) for that run +# to complete, so the run it reads as evidence is finished. +python3 - "${WF}" "${T}" > "${T}/handoff.out" 2>&1 <<'PY' +import os, sys, yaml +wf, out = sys.argv[1], sys.argv[2] +def on(d): + v = d.get("on", d.get(True)) + return v if isinstance(v, dict) else {} +texts = {} +for f, refs in (("ci.yml", ("refs/heads/autobump/", "refs/heads/main")), ("canary.yml", ("refs/heads/autobump/",))): + jobs = yaml.safe_load(open(os.path.join(wf, f)))["jobs"] + job = jobs.get("sweep") + if not job: + print("FAIL %s has no job sweep" % f); continue + needs = job.get("needs") or [] + needs = [needs] if isinstance(needs, str) else needs + cond = str(job.get("if", "")) + steps = job.get("steps") or [] + env = (steps[0].get("env") or {}) if len(steps) == 1 else {} + others = set(jobs) - {"sweep"} + print(("ok %s job sweep needs every other job" % f) if set(needs) == others and len(needs) == len(others) + else "FAIL %s job sweep needs %s, not every other job (missing %s)" % (f, sorted(needs), sorted(others - set(needs)))) + want = ["!cancelled()", "github.event_name == 'workflow_dispatch'"] + ["'%s'" % r for r in refs] + extra = "refs/heads/main" in cond and "refs/heads/main" not in refs + print(("ok %s job sweep runs, unless cancelled, on a dispatch on %s" % (f, " or ".join(refs))) + if all(w in cond for w in want) and not extra else "FAIL %s job sweep if: %s" % (f, cond)) + good = (job.get("continue-on-error") is True and isinstance(job.get("timeout-minutes"), int) + and job.get("permissions") == {"actions": "write"} and len(steps) == 1 and "run" in steps[0] + and env == {"GH_TOKEN": "${{ github.token }}", "GH_REPO": "${{ github.repository }}", + "RUN_ID": "${{ github.run_id }}"}) + print(("ok %s job sweep: continue-on-error, timeout, actions: write only, one step, no checkout" % f) if good + else "FAIL %s job sweep: %s" % (f, yaml.safe_dump(job).replace("\n", " "))) + if good: + texts[f] = steps[0]["run"] +if len(texts) == 2: + print(("ok ci.yml and canary.yml hand off with the same step") if len(set(texts.values())) == 1 + else "FAIL ci.yml and canary.yml hand off with different steps") + open(os.path.join(out, "handoff_step.sh"), "w").write(texts["ci.yml"]) +bs = yaml.safe_load(open(os.path.join(wf, "bot-sweep.yml"))) +inp = ((on(bs).get("workflow_dispatch") or {}).get("inputs") or {}).get("after_run") or {} +print(("ok bot-sweep.yml takes the dispatch input after_run (string, default empty)") + if inp.get("type") == "string" and inp.get("default") == "" else "FAIL bot-sweep.yml input after_run: %s" % inp) +steps = (bs["jobs"].get("ci-fill") or {}).get("steps") or [] +wait = [i for i, s in enumerate(steps) if (s.get("env") or {}).get("AFTER_RUN") == "${{ inputs.after_run }}"] +fill = [i for i, s in enumerate(steps) if "hack/bot-ci-dispatch" in s.get("run", "")] +if len(wait) == 1 and fill and wait[0] < min(fill) and "run" in steps[wait[0]] \ + and (steps[wait[0]].get("env") or {}).get("GH_TOKEN") == "${{ github.token }}" \ + and (steps[wait[0]].get("env") or {}).get("GH_REPO") == "${{ github.repository }}": + print("ok bot-sweep.yml ci-fill waits for the after_run run before it dispatches CI") + open(os.path.join(out, "wait_step.sh"), "w").write(steps[wait[0]]["run"]) +else: + print("FAIL bot-sweep.yml ci-fill has no wait step on AFTER_RUN before hack/bot-ci-dispatch") +PY +while IFS= read -r line; do + case "${line}" in "ok "*) ok "${line#ok }" ;; *) fail "${line#FAIL }" ;; esac +done < "${T}/handoff.out" +# The two steps run as Actions runs them (bash -eo pipefail) against a fake gh that +# logs each call and gives the Nth call the Nth answer of a script (the last one +# repeats): fail (a 502), ok (exit 0, no output) or a line to print. sleep is a no-op. +mkdir -p "${T}/hbin" +cat > "${T}/hbin/gh" <<'STUB' +#!/usr/bin/env bash +echo "$*" >> "$HD/calls" +n=$(wc -l < "$HD/calls") +a=$(sed -n "${n}p" "$HD/answers"); [ -n "$a" ] || a=$(tail -n 1 "$HD/answers") +case $a in fail) echo "HTTP 502: Bad Gateway" >&2; exit 1;; ok) exit 0;; *) echo "$a";; esac +STUB +printf '#!/bin/sh\nexit 0\n' > "${T}/hbin/sleep" +chmod +x "${T}/hbin/gh" "${T}/hbin/sleep" +# step_case NAME STEPFILE ANSWERS VAR=VALUE...: runs the step; sets HRC, HOUT and HCALLS. +step_case() { + local name=$1 step=$2 answers=$3; shift 3 + HD="${T}/step-${name}"; export HD; mkdir -p "$HD"; : > "$HD/calls" + printf '%b' "$answers" > "$HD/answers" + HOUT=$(env PATH="${T}/hbin:$PATH" GH_TOKEN=t GH_REPO=o/r "$@" \ + bash --noprofile --norc -eo pipefail "$step" 2>&1); HRC=$? + HCALLS=$(cat "$HD/calls") +} +# The repository's GITHUB_TOKEN gets 1,000 REST requests an hour, shared by every +# workflow, and a sweep reads about 10 per open bot PR: the hand-off reads the +# budget first (GET /rate_limit costs none) and leaves the run to the cron when +# fewer than 500 remain. +if [ -f "${T}/handoff_step.sh" ]; then + step_case hand-ok "${T}/handoff_step.sh" '999\nok\n' RUN_ID=4242 + if [ "$HRC" -eq 0 ] && [ "$(sed -n 1p <<<"$HCALLS")" = "api rate_limit --jq .resources.core.remaining" ] \ + && [ "$(sed -n '2,$p' <<<"$HCALLS")" = "workflow run bot-sweep.yml --ref main -f after_run=4242" ]; then + ok "hand-off: reads the token budget, then dispatches bot-sweep.yml on main with after_run=" + else fail "hand-off ok: rc=$HRC calls=$(tr '\n' ';' <<<"$HCALLS") out=$(tr '\n' ';' <<<"$HOUT")"; fi + step_case hand-retry "${T}/handoff_step.sh" '999\nfail\nok\n' RUN_ID=4242 + if [ "$HRC" -eq 0 ] && [ "$(grep -c '^workflow run bot-sweep.yml' <<<"$HCALLS")" -eq 2 ] && ! grep -q '::warning::' <<<"$HOUT"; then + ok "hand-off: a failed dispatch is sent again" + else fail "hand-off retry: rc=$HRC calls=$(tr '\n' ';' <<<"$HCALLS") out=$(tr '\n' ';' <<<"$HOUT")"; fi + step_case hand-down "${T}/handoff_step.sh" '999\nfail\n' RUN_ID=4242 + if [ "$HRC" -eq 0 ] && [ "$(grep -c '^workflow run bot-sweep.yml' <<<"$HCALLS")" -eq 3 ] && grep -q '^::warning::' <<<"$HOUT"; then + ok "hand-off: 3 failed dispatches warn and exit 0 (the run's result stands)" + else fail "hand-off down: rc=$HRC calls=$(tr '\n' ';' <<<"$HCALLS") out=$(tr '\n' ';' <<<"$HOUT")"; fi + step_case hand-low "${T}/handoff_step.sh" '499\nok\n' RUN_ID=4242 + if [ "$HRC" -eq 0 ] && ! grep -q '^workflow run' <<<"$HCALLS" && grep -q '^::notice::.*499' <<<"$HOUT"; then + ok "hand-off: under 500 requests left this hour, no dispatch (a notice; the cron sweeps)" + else fail "hand-off low budget: rc=$HRC calls=$(tr '\n' ';' <<<"$HCALLS") out=$(tr '\n' ';' <<<"$HOUT")"; fi + step_case hand-edge "${T}/handoff_step.sh" '500\nok\n' RUN_ID=4242 + if [ "$HRC" -eq 0 ] && [ "$(grep -c '^workflow run bot-sweep.yml' <<<"$HCALLS")" -eq 1 ]; then + ok "hand-off: exactly 500 left still dispatches" + else fail "hand-off 500 left: rc=$HRC calls=$(tr '\n' ';' <<<"$HCALLS") out=$(tr '\n' ';' <<<"$HOUT")"; fi + step_case hand-unread "${T}/handoff_step.sh" 'fail\nok\n' RUN_ID=4242 + if [ "$HRC" -eq 0 ] && [ "$(grep -c '^workflow run bot-sweep.yml' <<<"$HCALLS")" -eq 1 ]; then + ok "hand-off: an unreadable budget still dispatches" + else fail "hand-off unreadable budget: rc=$HRC calls=$(tr '\n' ';' <<<"$HCALLS") out=$(tr '\n' ';' <<<"$HOUT")"; fi +else fail "hand-off step not extracted, its behaviour is untested"; fi +if [ -f "${T}/wait_step.sh" ]; then + step_case wait-none "${T}/wait_step.sh" 'completed\n' AFTER_RUN= + if [ "$HRC" -eq 0 ] && [ -z "$HCALLS" ]; then ok "sweep wait: no after_run (the cron, a manual dispatch) waits for nothing" + else fail "sweep wait none: rc=$HRC calls=$(tr '\n' ';' <<<"$HCALLS") out=$(tr '\n' ';' <<<"$HOUT")"; fi + step_case wait-bad "${T}/wait_step.sh" 'completed\n' 'AFTER_RUN=4242;id' + if [ "$HRC" -ne 0 ] && [ -z "$HCALLS" ] && grep -q '^::error::' <<<"$HOUT"; then ok "sweep wait: an after_run that is not a run id fails the step" + else fail "sweep wait bad: rc=$HRC calls=$(tr '\n' ';' <<<"$HCALLS") out=$(tr '\n' ';' <<<"$HOUT")"; fi + step_case wait-done "${T}/wait_step.sh" 'in_progress\nfail\nqueued\ncompleted\n' AFTER_RUN=4242 + if [ "$HRC" -eq 0 ] && [ "$(grep -c '^run view 4242 ' <<<"$HCALLS")" -eq 4 ] && [ "$(wc -l <<<"$HCALLS")" -eq 4 ] \ + && ! grep -q '::warning::' <<<"$HOUT"; then + ok "sweep wait: polls the run (through a failed read) until it completes" + else fail "sweep wait done: rc=$HRC calls=$(tr '\n' ';' <<<"$HCALLS") out=$(tr '\n' ';' <<<"$HOUT")"; fi + step_case wait-ceiling "${T}/wait_step.sh" 'in_progress\n' AFTER_RUN=4242 + if [ "$HRC" -eq 0 ] && [ "$(wc -l <<<"$HCALLS")" -eq 60 ] && grep -q '^::warning::.*4242' <<<"$HOUT"; then + ok "sweep wait: a run still open after 60 reads warns and the sweep goes on" + else fail "sweep wait ceiling: rc=$HRC reads=$(wc -l <<<"$HCALLS") out=$(tr '\n' ';' <<<"$HOUT")"; fi +else fail "sweep wait step not extracted, its behaviour is untested"; fi + # ─── GitHub's "Latest" release ─────────────────────────────────────────────── # /releases/latest leads users to the packages, so "Latest" is the newest aur-v* # package release (.deb, .rpm, SHA256SUMS). An image release (-ndp,