diff --git a/.github/workflows/constitution.yml b/.github/workflows/constitution.yml index 25788b8..a1e0004 100644 --- a/.github/workflows/constitution.yml +++ b/.github/workflows/constitution.yml @@ -18,4 +18,4 @@ permissions: jobs: constitution: name: Constitution - uses: crunchtools/constitution/.github/workflows/validate.yml@v1.31.0 + uses: crunchtools/constitution/.github/workflows/validate.yml@v1.35.0 diff --git a/.github/workflows/dependabot-automerge.yml b/.github/workflows/dependabot-automerge.yml index 794a085..f10e8e3 100644 --- a/.github/workflows/dependabot-automerge.yml +++ b/.github/workflows/dependabot-automerge.yml @@ -15,4 +15,4 @@ permissions: jobs: automerge: name: Dependabot - uses: crunchtools/constitution/.github/workflows/dependabot-automerge.yml@v1.31.0 + uses: crunchtools/constitution/.github/workflows/dependabot-automerge.yml@v1.35.0 diff --git a/.specify/memory/constitution.md b/.specify/memory/constitution.md index c0b764f..93f3200 100644 --- a/.specify/memory/constitution.md +++ b/.specify/memory/constitution.md @@ -4,7 +4,7 @@ > **Ratified:** 2026-11-04 > **Amended:** 2026-11-04 > **Status:** Active -> **Inherits:** [crunchtools/constitution](https://github.com/crunchtools/constitution) v1.31.0 +> **Inherits:** [crunchtools/constitution](https://github.com/crunchtools/constitution) v1.35.0 > **Profile:** Container Image This is a manifest. It carries only what is unique to this repo; the fleet rules diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ee8524..d318f2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,31 @@ All notable changes to this project are documented here, following ## [Unreleased] +### Added + +- **Exit 3: could not run, deferred to CI** (constitution 1.35.0, XII). When + CodeQL itself breaks at the push tier, outside CI, in a repo whose workflow + RUNS `bin/sdlc ci`, the push goes through with a loud `DEFERRED TO CI` and the + record shows exit 3. Before this, a broken CodeQL blocked every push and + `--no-verify`, which skips every gate, was the only way past it. `sdlc ci`, + `sdlc codeql` and a config error are still 2. A tool's own exit 3 is mapped to + 2, so only the gate can defer, and it is granted per check -- today only + CodeQL. The text `sdlc ci` alone does not unlock it: a comment, a YAML `name:` + and an echoed string are mentions, not runs, and trentina has two of those. + The question is answered from the PARSED workflow: an unconditional + `steps[].run` invoking a path ending in `bin/sdlc` with `ci` as its argument, + in a workflow whose `push`, `pull_request`, `pull_request_target` or + `merge_group` trigger is unfiltered or reaches the default branch under + GitHub's own glob rules. A job name, + a step name, an `env:` value, a comment and an echoed string are mentions; + a job or step behind an `if:` may be skipped on this commit; `on: push` + filtered to another branch judges nothing; a file that does not parse grants + nothing (#33, constitution#99). + +### Changed + +- Inherits constitution v1.35.0, the release that amended XII to allow exit 3. + ### Fixed - **CodeQL's evaluator gets the memory the gate budgets.** CodeQL ignored the @@ -18,8 +43,8 @@ All notable changes to this project are documented here, following other's database mid-run. The previous database is removed, read-only parts included, and checked to be gone before `create` runs (#33). - `run_plan` read run_serial's per-step `_rc` as its own result, because shell - variables are global. Harmless today, because every non-zero step fails the - run, but one new status away from a wrong exit code. + variables are global. It had no visible effect while every non-zero step + failed the run, but it turned a deferred push's 0 into a 3. ## [0.5.3] - 2026-11-08 diff --git a/bin/sdlc b/bin/sdlc index dd7846d..88f42fe 100755 --- a/bin/sdlc +++ b/bin/sdlc @@ -19,6 +19,10 @@ # 0 ran, found nothing # 1 ran, found something # 2 COULD NOT RUN -- never to be worked around +# 3 COULD NOT RUN, deferred to CI -- push only, granted by the image when +# CodeQL itself broke and `sdlc ci` will judge the same commit. Printed +# loudly, recorded, and not a refusal (python-sdlc#33). Anywhere but a +# push it is a 2. # # Podman's own 125/126/127 mean the container never started, so they map to 2. # trentina's gates.sh maps them to 1 via `|| fail`, which reports "found @@ -271,6 +275,8 @@ run_in_image() { -e "SDLC_PROJECT=$PROJECT" \ -e "SDLC_JOBS=$GATE_CPUS" \ -e "SDLC_MEMORY=$GATE_MEMORY" \ + -e "SDLC_DEFAULT_BRANCH=$(default_branch 2>/dev/null || :)" \ + ${SDLC_TIER:+-e SDLC_TIER} \ -e "SDLC_HOST_REPO=$REPO" \ -e "SDLC_CACHE_VOLUME=$CACHE_VOLUME" \ ${SDLC_SKIP:+-e SDLC_SKIP} \ @@ -485,6 +491,20 @@ run_host() { } run_step() { # mode payload -> the step's exit status + _rs=0 + _run_step "$@" || _rs=$? + # Only a push may defer to CI. Anything else that reports 3 could not run. + if [ "$_rs" -eq 3 ] && [ "${SDLC_TIER:-}" != push ]; then return 2; fi + return "$_rs" +} + +deferred() { # check -> the warning a deferral prints, every time + printf '\n%s: %s COULD NOT RUN -- DEFERRED TO CI.\n' "$ME" "$1" >&2 + printf '%s: This push is not blocked, and nothing was judged here.\n' "$ME" >&2 + printf '%s: The "sdlc ci" job is the verdict on this commit.\n' "$ME" >&2 +} + +_run_step() { case "$1" in image-ro) run_in_image ro nostdin "$IMAGE" "$2" ;; image-rw) run_in_image rw nostdin "$IMAGE" "$2" ;; @@ -542,7 +562,11 @@ run_serial() { # the plan -> its worst step status _rc=0 run_step "$_mode" "$_payload" || _rc=$? _scur_took=$(( _scur_took + $(now_s) - _t0 )) - if [ "$_rc" -ne 0 ]; then + if [ "$_rc" -eq 3 ]; then + _failed="$_failed $_check" + _scur_rc=3 + deferred "$_check" + elif [ "$_rc" -ne 0 ]; then _failed="$_failed $_check" _scur_rc=$_rc printf '\n%s: %s %s after %ss\n' "$ME" "$_check" \ @@ -607,6 +631,8 @@ _close_check() { # the open check's block is complete # and then the run -- ended with two, the first of them "passed in 0s". if [ "$_cur_rc" -eq 0 ]; then printf '\n== %s passed in %ss\n' "$_cur_name" "$_cur_took" >> "$OUT/out.$_cur" + elif [ "$_cur_rc" -eq 3 ]; then + deferred "$_cur_name" 2>> "$OUT/out.$_cur" else printf '\n%s: %s %s after %ss\n' "$ME" "$_cur_name" \ "$([ "$_cur_rc" -eq 2 ] && echo 'COULD NOT RUN' || echo 'FAILED')" \ @@ -662,7 +688,10 @@ run_lane() { # lane-index steps-file lane-cpus -> the lane's worst _rc=0 run_step "$_mode" "$_payload" >> "$OUT/out.$_i" 2>&1 || _rc=$? _cur_took=$(( _cur_took + $(now_s) - _t0 )) - if [ "$_rc" -ne 0 ]; then + if [ "$_rc" -eq 3 ]; then + _failed="$_failed $_check" + _cur_rc=3 + elif [ "$_rc" -ne 0 ]; then _failed="$_failed $_check" _cur_rc=$_rc # A later 1 must not mask an earlier 2, in this lane or any other. @@ -872,6 +901,9 @@ run_lanes() { # the plan -> the worst status of every lane } run_plan() { # commit | push | ci | + # The image asks this to decide whether CodeQL may defer to CI. + SDLC_TIER=$1 + export SDLC_TIER _plan=$(run_in_image ro nostdin "$IMAGE" plan "$1") \ || die "the image could not plan '$1'; nothing was checked" [ -n "$_plan" ] || die "the image planned nothing for '$1'" @@ -889,8 +921,8 @@ run_plan() { # commit | push | ci | _oncpu=$(printf '%s\n' "$_plan" \ | awk -F"$TAB" '$3 != "local" && !seen[$2]++ {n++} END {print n + 0}') # Its own name: shell variables are global, and run_serial's loop sets `_rc` - # for every step, so the last step's status leaked in here whenever - # run_serial itself returned 0. + # for every step. A deferred step's 3 survived into here and became the + # exit status of a push that run_serial had passed. _plan_rc=0 if [ "${_n:-1}" -le 1 ]; then run_serial "$_plan" || _plan_rc=$? @@ -931,6 +963,8 @@ usage: $ME commit | push | ci | | tiers | list | versions | codeql-basel Configure each command in pyproject.toml: [tool.sdlc.commit], [tool.sdlc.push], [tool.sdlc.ci]. Skip one check once: SDLC_SKIP=name. Exit 2 means a check COULD NOT RUN and is never to be worked around. +A push prints DEFERRED TO CI, and goes through, only when CodeQL itself broke +and the repo's \`sdlc ci\` job will judge the same commit. Working a change, in the order that costs least: diff --git a/docs/codeql.md b/docs/codeql.md index cd1cfb0..fe83e9f 100644 --- a/docs/codeql.md +++ b/docs/codeql.md @@ -34,7 +34,7 @@ fails. ## When it cannot run -Two things went wrong on josui-skills at once (#33). +Three things went wrong on josui-skills at once (#33), and each has its own fix. - **Memory.** CodeQL sizes its evaluator itself and does not look at the container: `codeql resolve ram` gave a 1509 MB heap under `--memory 8g` and @@ -49,6 +49,40 @@ Two things went wrong on josui-skills at once (#33). database mid-run. The gate now takes a lock beside it and waits. It clears the previous database before building and checks that it is gone. A half-built one is a cache miss, not something a person deletes by hand. +- **Deferring to CI.** When CodeQL itself breaks — `create` or `analyze` exits + non-zero — at the **push** tier, outside CI, in a repo whose CI really runs + the gate, the check exits **3**. The launcher prints `DEFERRED TO CI`, + records it in `last-push.json`, and lets the push through. Anywhere else (`sdlc ci`, + `sdlc codeql`, a repo with no such workflow) it is still a 2. A config the + repo got wrong is still a 2 everywhere, because CI fails on it the same way. + Constitution XII, v1.35.0, is what permits this; nothing else may defer. + +"Really runs the gate" is answered from the parsed workflow, never by +searching it as text, because every loose answer hands a push a verdict nobody +reaches: + +- an **unconditional `steps[].run`** that invokes a path ending in `bin/sdlc` + with `ci` as its argument. A job name, a step name, an `env:` value, a + comment and an echoed string are mentions — trentina carries two of them. + A job or step behind an `if:` may be skipped on this very commit, so it + grants nothing: ashigaru's `sdlc ci` job sits behind `ASHIGARU_ENABLED`, and + a broken CodeQL there is correctly still a 2. +- a **trigger that reaches the commit**: `push`, `pull_request`, + `pull_request_target` or `merge_group`, unfiltered or naming the default + branch — the branch XII's required status check guards, matched with + GitHub's own glob rules, where `*` stops at a `/` and `**` does not. + `workflow_dispatch` alone judges nothing anybody pushed, and `on: push` + filtered to another branch is the same gap wearing the right event name. + Two filters are not evaluated at all and grant nothing: `branches-ignore`, + which needs every branch name to judge, and any `branches` list carrying a + negation, because GitHub applies those in order and `['**', '!main']` + excludes `main` while matching everything. + +A workflow this cannot parse grants nothing, and neither does a run where the +default branch is unknown and the trigger is filtered. + +`--no-verify` skips every gate, so refusing the push left it as the only way +past one broken tool. A deferral leaves the other gates running. ## The config file diff --git a/sdlc b/sdlc index 45daa40..46232ae 100755 --- a/sdlc +++ b/sdlc @@ -10,6 +10,14 @@ THE EXIT CODE CONTRACT, which is the whole point of this file: 0 the check ran and found nothing 1 the check ran and found something 2 the check COULD NOT RUN + 3 the check COULD NOT RUN, and CI will judge this same commit + +3 exists for one case and is granted by this file, never returned by a tool: +CodeQL at the push tier, when the analysis itself broke and the repo's CI runs +`sdlc ci`. It is not a pass. It is a loud "nothing was judged here, CI is the +judge", which the launcher lets through instead of refusing the push -- because +refusing left `--no-verify` as the only door, and that door skips every other +gate too (python-sdlc#33). 2 is never to be worked around. A gate that cannot run and reports 0 is worse than no gate, because the pipeline goes green and nobody looks again. The @@ -98,12 +106,17 @@ def self_reported(name: str) -> str: CLEAN = 0 FOUND = 1 CANNOT_RUN = 2 +DEFERRED = 3 class CannotRunError(Exception): """Raised for anything that is not the tool's own verdict. Becomes exit CANNOT_RUN.""" +class DeferredError(Exception): + """Could not run, and CI will judge the same commit. Becomes exit DEFERRED.""" + + def cannot(msg: str) -> NoReturn: raise CannotRunError(msg) @@ -1285,18 +1298,196 @@ def codeql_ram_mb() -> int: return max(2048, limit * 3 // 4) if limit else 2048 +def ci_judges() -> bool: + """Is a push's CodeQL also judged by CI on this same commit? + + True only at the push tier, outside CI, with codeql in the ci tier and a + workflow that RUNS `bin/sdlc ci`. Anywhere else -- `sdlc ci` itself, the + inner loop's `sdlc codeql`, a repo without that workflow -- nobody else + will judge the commit, and a check that could not run stays a 2. + """ + if os.environ.get("SDLC_TIER") != "push" or os.environ.get("CI"): + return False + if "codeql" not in resolve(sdlc_config(), "ci"): + return False + workflows = REPO / ".github" / "workflows" + try: + files = [*workflows.glob("*.yml"), *workflows.glob("*.yaml")] + except OSError as exc: + print(f"codeql: cannot read {workflows}, so nothing may be deferred: {exc}", flush=True) + return False + # The launcher knows it; 10 of the 69 checkouts in this fleet are not main. + default = (os.environ.get("SDLC_DEFAULT_BRANCH") or "").removeprefix("origin/") + for f in files: + try: + if runs_sdlc_ci(f.read_text(encoding="utf-8", errors="replace"), default): + return True + except OSError as exc: + print(f"codeql: cannot read {f}, so it cannot grant a deferral: {exc}", flush=True) + return False + + +JUDGING_EVENTS = ("pull_request", "pull_request_target", "merge_group", "push") + +# A path ending in `bin/sdlc`, then `ci` as its argument: how a workflow +# actually invokes the gate (`$RUNNER_TEMP/sdlc/bin/sdlc ci`). +_RUNS_SDLC_CI = re.compile(r"""(?:^|[\s;&|("'=])\S*bin/sdlc\s+ci\b""") + + +def runs_sdlc_ci(text: str, default_branch: str) -> bool: + """Does this workflow RUN the gate, in a job that this commit reaches? + + Not "does the text `sdlc ci` appear in it". Every way of answering that + question loosely hands a push a verdict nobody ever reaches, and each of + these was a real finding against an earlier version of this function: + + - trentina carries `VERIFIER_JOB: sdlc ci` as a job name and `sdlc ci` + inside an error message. So the invocation is looked for in parsed + `steps[].run` scripts only -- never in a comment, a `name:`, an `env:` + value or an echoed string (python-sdlc#34). + - a job or step behind an `if:` may be skipped on this very commit, so a + conditional one grants nothing. + - a workflow triggered only by `workflow_dispatch` or `schedule` judges + nothing anybody pushed, and `on: push` filtered to another branch is the + same gap wearing the right event name. The trigger has to reach + `default_branch`, which is the branch XII's required status check guards. + + A workflow this cannot parse grants nothing. + """ + try: + doc = yaml.safe_load(text) + except yaml.YAMLError: + return False + if not isinstance(doc, dict): + return False + return judges_a_commit(doc, default_branch) and any_step_runs_the_gate(doc) + + +def judges_a_commit(doc: dict, default_branch: str) -> bool: + """Does a trigger of this workflow cover the default branch? + + `on:` is YAML 1.1's boolean `True` once parsed, which is why both keys are + looked for. + """ + on = doc.get("on", doc.get(True)) + if isinstance(on, str): + events: dict[str, object] = {on: None} + elif isinstance(on, list): + events = {e: None for e in on if isinstance(e, str)} + elif isinstance(on, dict): + events = {k: v for k, v in on.items() if isinstance(k, str)} + else: + return False + return any(covers_branch(events[e], default_branch) for e in JUDGING_EVENTS if e in events) + + +def covers_branch(filters: object, branch: str) -> bool: + """Does this event's branch filter reach `branch`? + + GitHub applies `branches` in order and lets a later `!pattern` take a + branch back out, which a set of names cannot express: `['**', '!main']` + excludes `main` while containing a pattern that matches everything + (Gatehouse, on the first version of this). Rather than reimplement that + ordering, a filter carrying any negation grants nothing, and so does + `branches-ignore`, which cannot be evaluated without knowing every branch. + """ + if not isinstance(filters, dict): + return True + if "branches-ignore" in filters: + return False + branches = filters.get("branches") + if branches is None: + return True + if not isinstance(branches, list) or not all(isinstance(b, str) for b in branches): + return False + if any(b.startswith("!") for b in branches): + return False + return bool(branch) and any(branch_matches(b, branch) for b in branches) + + +def branch_matches(pattern: str, branch: str) -> bool: + """GitHub's filter-pattern glob, where `*` stops at a `/` and `**` does not.""" + out = ["^"] + i = 0 + while i < len(pattern): + c = pattern[i] + if c == "*": + if pattern[i + 1 : i + 2] == "*": + out.append(".*") + i += 2 + continue + out.append("[^/]*") + elif c == "?": + out.append("[^/]") + elif c == "+": + out.append(".") + else: + out.append(re.escape(c)) + i += 1 + out.append("$") + return re.match("".join(out), branch) is not None + + +def any_step_runs_the_gate(doc: dict) -> bool: + """Is there an unconditional `steps[].run` that invokes `bin/sdlc ci`?""" + jobs = doc.get("jobs") + if not isinstance(jobs, dict): + return False + for job in jobs.values(): + if not isinstance(job, dict) or job.get("if") is not None: + continue + steps = job.get("steps") + if not isinstance(steps, list): + continue + for step in steps: + if not isinstance(step, dict) or step.get("if") is not None: + continue + run = step.get("run") + if isinstance(run, str) and invokes_the_gate(run): + return True + return False + + +def invokes_the_gate(script: str) -> bool: + """Does this shell script run the gate, rather than talk about it?""" + for raw in script.splitlines(): + line = raw.split("#", 1)[0] + if "echo" in line or "printf" in line: + continue + if _RUNS_SDLC_CI.search(line): + return True + return False + + +def codeql_broke(msg: str) -> NoReturn: + """CodeQL itself failed: defer to CI when CI will judge, otherwise a 2. + + Only for the TOOL breaking. A config the repo got wrong stays a 2 wherever + it is found, because CI would fail on it in exactly the same way. + """ + if ci_judges(): + raise DeferredError(msg) + cannot(msg) + + def run_codeql(create: list[str], analyze: list[str], sarif: Path) -> None: - """Build the database and analyse it, or fail as 'could not run'.""" + """Build the database and analyse it, or report that it could not run. + + Either step failing means no verdict was reached, which is a 2 -- except + on a push whose CI will judge the same commit, where `codeql_broke` makes + it a 3 and says so loudly (constitution XII, v1.35.0). + """ step("codeql database create") rc, _ = run(create, env=project_env()) if rc != 0: - cannot(f"codeql could not build a database (exit {rc}); nothing was analysed") + codeql_broke(f"codeql could not build a database (exit {rc}); nothing was analysed") step("codeql database analyze") rc, _ = run(analyze, env=project_env()) if rc != 0: # Analysis failing is not the tool's verdict ON the code -- it means no - # verdict was reached, which the exit-code contract makes a 2. - cannot(f"codeql database analyze failed (exit {rc}); nothing was judged") + # verdict was reached, which the exit-code contract makes a 2, or a 3 + # where CI will judge this commit. + codeql_broke(f"codeql database analyze failed (exit {rc}); nothing was judged") def baseline_path(s: dict) -> Path: @@ -1445,7 +1636,9 @@ def main() -> int: fn = BUILTINS.get(args.what) if fn is None: cannot(f"{args.what} is not a check this image runs itself") - return fn(check_settings(sdlc_config(), args.what)) + rc = fn(check_settings(sdlc_config(), args.what)) + # A tool's own 3 is not a deferral; only DeferredError grants one. + return CANNOT_RUN if rc == DEFERRED else rc if __name__ == "__main__": @@ -1454,5 +1647,14 @@ if __name__ == "__main__": except CannotRunError as exc: print(f"GATE CANNOT RUN: {exc}", file=sys.stderr) sys.exit(CANNOT_RUN) + except DeferredError as exc: + print( + f"GATE CANNOT RUN: {exc}\n" + f"DEFERRED TO CI: this push is NOT blocked, and NOTHING was judged here. " + f"The `sdlc ci` job is the judge of this commit -- read its verdict " + f"before merging. Rerun `sdlc codeql` to see the failure as a 2.", + file=sys.stderr, + ) + sys.exit(DEFERRED) except KeyboardInterrupt: sys.exit(CANNOT_RUN) diff --git a/tests/test_codeql_resilience.py b/tests/test_codeql_resilience.py index 9b5da24..4d23f63 100755 --- a/tests/test_codeql_resilience.py +++ b/tests/test_codeql_resilience.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -"""CodeQL must not fail for want of memory or for a half-built cache. +"""A CodeQL that cannot run must not be what teaches people `--no-verify`. python-sdlc#33: on josui-skills a killed run left a half-built database that a -person had to delete by hand, and the evaluator was sized by CodeQL's own guess -rather than the container's budget, so analyze died and every push was refused. +person had to delete by hand, the evaluator was sized by CodeQL's own guess +rather than the container's budget, and an analysis that died blocked every +push with no door but `--no-verify` -- which skips every gate, not the broken +one. These are the three pieces of that fix that need no CodeQL to test. Run: tests/test_codeql_resilience.py (needs the image's python3 and PyYAML) """ @@ -16,6 +18,7 @@ import stat import sys import tempfile +from collections.abc import Callable from pathlib import Path # Loaded by path, as in test_codeql_globs.py: `sdlc` has no .py extension. @@ -41,6 +44,15 @@ def check(ok: bool, what: str) -> bool: return ok +def raises(exc: type[BaseException], fn: Callable[[], object]) -> bool: + """True when `fn` raises `exc`. Anything else it raises fails the test run.""" + try: + fn() + except exc: + return True + return False + + def database_and_lock() -> bool: ok = True with tempfile.TemporaryDirectory() as tmp: @@ -108,12 +120,103 @@ def failures() -> bool: return ok +def deferral() -> bool: + """Deferring to CI is a push-only, outside-CI privilege.""" + ok = True + os.environ.pop("CI", None) + os.environ["SDLC_TIER"] = "ci" + ok &= check(not _SDLC.ci_judges(), "`sdlc ci` never defers: nothing judges after it") + os.environ["SDLC_TIER"] = "codeql" + ok &= check(not _SDLC.ci_judges(), "the inner loop's `sdlc codeql` never defers") + os.environ["SDLC_TIER"] = "push" + os.environ["CI"] = "true" + ok &= check(not _SDLC.ci_judges(), "a push inside CI never defers") + ok &= check( + raises(_SDLC.CannotRunError, lambda: _SDLC.codeql_broke("x")), + "where CI does not judge, a broken CodeQL is still a 2", + ) + return ok + + +def mentions_are_not_a_run() -> bool: + """Only a workflow that RUNS the gate on this commit unlocks a deferral. + + Every case here was a Gatehouse finding against an earlier version of + `runs_sdlc_ci`, which read the file as text: a mention in a name, an env + value or an echo; a job or step behind an `if:`; a trigger that does not + reach the branch; and a branch filter whose negation a set of names + cannot express. + """ + ok = True + runs = " steps:\n - run: $RUNNER_TEMP/sdlc/bin/sdlc ci\n" + cases = ( + ("on:\n push:\n branches: [main]\njobs:\n ci:\n" + runs, True, "a real run"), + ("on: pull_request\njobs:\n ci:\n" + runs, True, "a PR run"), + ("on: push\njobs:\n ci:\n" + runs, True, "an unfiltered push"), + ("on:\n push:\n branches: ['**']\njobs:\n ci:\n" + runs, True, "a ** filter"), + ("on:\n push:\n branches: [ma*]\njobs:\n ci:\n" + runs, True, "a glob that matches"), + ("on: push\njobs:\n ci:\n name: ./bin/sdlc ci\n", False, "a job name:"), + ("on: push\njobs:\n ci:\n env:\n JOB: bin/sdlc ci\n", False, "an env value"), + ("on: push\njobs:\n ci:\n steps:\n - name: bin/sdlc ci\n", False, "a step name:"), + ( + 'on: push\njobs:\n ci:\n steps:\n - run: echo "bin/sdlc ci did nothing"\n', + False, + "an echo", + ), + ("on: push\njobs:\n ci:\n # bin/sdlc ci is the judge\n", False, "a comment"), + ("on: push\njobs:\n ci:\n if: false\n" + runs, False, "a conditional job"), + ( + "on: push\njobs:\n ci:\n steps:\n - run: ./bin/sdlc ci\n if: false\n", + False, + "a conditional step", + ), + ("on:\n workflow_dispatch:\njobs:\n ci:\n" + runs, False, "a run no push triggers"), + ( + "on:\n push:\n branches: [release]\njobs:\n ci:\n" + runs, + False, + "a push filtered to another branch", + ), + ( + "on:\n push:\n branches: ['**', '!main']\njobs:\n ci:\n" + runs, + False, + "a filter whose negation takes the branch back out", + ), + ( + "on:\n push:\n branches-ignore: [wip]\njobs:\n ci:\n" + runs, + False, + "a branches-ignore filter", + ), + ( + "on:\n push:\n branches: [feature/*]\njobs:\n ci:\n" + runs, + False, + "a `*` that does not cross a slash", + ), + (": : not yaml : [\n", False, "a workflow that does not parse"), + ) + for text, judged, what in cases: + ok &= check( + _SDLC.runs_sdlc_ci(text, "main") is judged, + f"{what} {'does' if judged else 'does not'} judge", + ) + ok &= check( + not _SDLC.runs_sdlc_ci("on:\n push:\n branches: [main]\njobs:\n ci:\n" + runs, ""), + "with no default branch known, a filtered push grants nothing", + ) + ok &= check( + _SDLC.branch_matches("**", "feature/x") and not _SDLC.branch_matches("*", "feature/x"), + "`**` crosses a slash and `*` does not", + ) + return ok + + def main() -> int: saved = dict(os.environ) try: ok = database_and_lock() ok &= memory() ok &= failures() + ok &= deferral() + ok &= mentions_are_not_a_run() finally: os.environ.clear() os.environ.update(saved) diff --git a/tests/test_host_contract.sh b/tests/test_host_contract.sh index 7b8aeb6..b6be1f1 100755 --- a/tests/test_host_contract.sh +++ b/tests/test_host_contract.sh @@ -119,6 +119,8 @@ case " \$* " in printf 'p${TAB}one${TAB}host${TAB}podman${US}build${US}-t${US}localhost/x:sdlc${US}.\n' printf 'q${TAB}two${TAB}image-ro${TAB}quiet-ok\n' ;; *" plan "*) + if [ -f "$SHIM/defer-last" ]; then printf 'codeql${TAB}main${TAB}image-ro${TAB}rc-three\n'; exit 0; fi + if [ -f "$SHIM/defer" ]; then printf 'codeql${TAB}main${TAB}image-ro${TAB}rc-three\n'; fi printf 'mypy${TAB}main${TAB}image-ro${TAB}mypy\n' printf 'ruff-check${TAB}main${TAB}image-rw${TAB}ruff-check\n' printf 'image${TAB}main${TAB}host${TAB}podman${US}build${US}-t${US}localhost/x:sdlc${US}.\n' @@ -131,6 +133,7 @@ case "\$_last" in printf 'DONE %s\n' "\$_last" >> "$SHIM/race.log" ;; rc-one) printf 'RAN rc-one\n'; exit 1 ;; rc-two) printf 'RAN rc-two\n'; exit 2 ;; + rc-three) printf 'RAN rc-three\n'; exit 3 ;; slow-rc-one) printf 'RAN slow-rc-one\n'; sleep 2; exit 1 ;; very-slow-rc-one) printf 'RAN very-slow-rc-one\n'; sleep 6; printf 'LAST LINE\n'; exit 1 ;; step-one) printf 'RAN step-one\n'; sleep 1 ;; @@ -185,18 +188,44 @@ else bad "a host step's word was evaluated as shell" fi -printf '\n== what CodeQL is told\n\n' +printf '\n== a check deferred to CI\n\n' -# CodeQL sizes its evaluator itself unless told, and rootless podman can drop -# --memory silently, so the budget is handed to the image as well (#33). +# 3 is the image saying "could not run, and CI judges this commit". A push lets +# it through, loudly, so --no-verify is not the only door past a broken +# CodeQL; every other command must still treat it as a 2 (python-sdlc#33). +touch "$SHIM/defer" log_mark -PATH="$SHIM" sh bin/sdlc push >/dev/null 2>&1 || true +PATH="$SHIM" sh bin/sdlc push > "$RUN/out" 2>&1 && rc=0 || rc=$? log_since -if grep -q -- '-e SDLC_MEMORY=8g' "$RUN/podman.log"; then - ok "the image is told the memory budget" +if [ "$rc" -eq 0 ] && grep -q 'codeql COULD NOT RUN -- DEFERRED TO CI' "$RUN/out" \ + && grep -q ' mypy$' "$RUN/podman.log"; then + ok "a push lets a deferred check through, says so, and runs the rest" else - bad "the image was not given SDLC_MEMORY" + bad "a deferred check on push exited $rc or said nothing" fi +if grep -q 'SDLC_MEMORY=' "$RUN/podman.log" && grep -q -- '-e SDLC_TIER' "$RUN/podman.log"; then + ok "the image is told the memory budget and the tier" +else + bad "the image was not given SDLC_MEMORY and SDLC_TIER" +fi +for target in ci codeql; do + PATH="$SHIM" sh bin/sdlc "$target" > "$RUN/out" 2>&1 && rc=0 || rc=$? + if [ "$rc" -eq 2 ] && ! grep -q 'DEFERRED' "$RUN/out"; then + ok "$target treats a deferral as could-not-run (exit 2)" + else + bad "$target let a deferral through (exit $rc)" + fi +done +rm -f "$SHIM/defer" +# Alone, so the deferral is the LAST status the executor saw. +touch "$SHIM/defer-last" +PATH="$SHIM" sh bin/sdlc push > "$RUN/out" 2>&1 && rc=0 || rc=$? +if [ "$rc" -eq 0 ]; then + ok "a push whose only check deferred exits 0" +else + bad "a push whose only check deferred exited $rc" +fi +rm -f "$SHIM/defer-last" printf '\n== the CPU budget\n\n'