From 6a11c8aab0191fc42aeeed6a074c4c27e625d20b Mon Sep 17 00:00:00 2001 From: Josui Date: Sun, 8 Nov 2026 13:51:10 +0000 Subject: [PATCH 1/4] fix: a CodeQL that cannot run no longer blocks the push (#33) Three faults behind josui-skills' refused pushes: - CodeQL sized its own evaluator (~1.2-1.5 GB heap) and ignored the container's 8g. `--ram` is now three quarters of the smallest of SDLC_MEMORY, the cgroup limit and MemTotal: 6144 MB at the default. - Runs on one project shared one database directory and could delete each other's mid-run. They now take a lock, and the previous database is cleared and checked gone, so a half-built one rebuilds itself. - When CodeQL itself breaks at the push tier, outside CI, in a repo whose workflow runs `sdlc ci`, the check exits 3: DEFERRED TO CI, printed and recorded, and the push goes through. `sdlc ci`, `sdlc codeql` and config errors stay a 2. Also fixes run_plan reading run_serial's global `_rc` as its own result. Closes #33 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/container.yml | 3 +- CHANGELOG.md | 25 +++++ bin/sdlc | 53 ++++++++-- docs/codeql.md | 27 ++++++ sdlc | 167 +++++++++++++++++++++++++++++++- tests/test_codeql_resilience.py | 129 ++++++++++++++++++++++++ tests/test_host_contract.sh | 42 ++++++++ 7 files changed, 432 insertions(+), 14 deletions(-) create mode 100755 tests/test_codeql_resilience.py diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 3fa93cf..2e5f868 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -116,7 +116,8 @@ jobs: run: | set -eu for t in tests/test_plan.py tests/test_codeql_globs.py \ - tests/test_pycache_prefix.py tests/test_gourmand_sidecar.py; do + tests/test_pycache_prefix.py tests/test_gourmand_sidecar.py \ + tests/test_codeql_resilience.py; do podman run --rm -v "$PWD:/src:ro,z" -w /src \ --entrypoint /opt/constitution-venv/bin/python3 \ "localhost/python-sdlc:$GITHUB_SHA" "$t" diff --git a/CHANGELOG.md b/CHANGELOG.md index e7fc080..52f69fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,31 @@ All notable changes to this project are documented here, following ## [Unreleased] +### Added + +- **Exit 3: could not run, deferred to CI.** When CodeQL itself breaks at the + push tier, outside CI, in a repo whose workflow runs `sdlc ci`, the push goes + through with a loud `DEFERRED TO CI` and the record shows exit 3. Before + this, a broken CodeQL blocked every push and `--no-verify`, which skips every + gate, was the only way past it. `sdlc ci`, `sdlc codeql` and a config error + are still 2. A tool's own exit 3 is mapped to 2, so only the gate can defer + (#33). + +### Fixed + +- **CodeQL's evaluator gets the memory the gate budgets.** CodeQL ignored the + container and gave itself a ~1.2-1.5 GB heap, and analyze died on + josui-skills with status 70. `--ram` is now three quarters of the smallest of + `SDLC_MEMORY`, the cgroup limit and MemTotal: 6144 MB at the default 8g + (#33). +- **A half-built CodeQL database no longer needs a person to delete it.** Runs + on one project take a lock and wait for each other instead of deleting each + other's database mid-run. The previous database is removed, read-only parts + included, and checked to be gone before `create` runs (#33). +- `run_plan` read run_serial's per-step `_rc` as its own result, because shell + variables are global. It had no visible effect while every non-zero step + failed the run, but it turned a deferred push's 0 into a 3. + ## [0.5.3] - 2026-11-08 ### Changed diff --git a/bin/sdlc b/bin/sdlc index 40c7a30..003e51c 100755 --- a/bin/sdlc +++ b/bin/sdlc @@ -19,6 +19,10 @@ # 0 ran, found nothing # 1 ran, found something # 2 COULD NOT RUN -- never to be worked around +# 3 COULD NOT RUN, deferred to CI -- push only, granted by the image when +# CodeQL itself broke and `sdlc ci` will judge the same commit. Printed +# loudly, recorded, and not a refusal (python-sdlc#33). Anywhere but a +# push it is a 2. # # Podman's own 125/126/127 mean the container never started, so they map to 2. # trentina's gates.sh maps them to 1 via `|| fail`, which reports "found @@ -270,6 +274,8 @@ run_in_image() { -e SEMGREP_SETTINGS_FILE=/tmp/semgrep.yml \ -e "SDLC_PROJECT=$PROJECT" \ -e "SDLC_JOBS=$GATE_CPUS" \ + -e "SDLC_MEMORY=$GATE_MEMORY" \ + ${SDLC_TIER:+-e SDLC_TIER} \ -e "SDLC_HOST_REPO=$REPO" \ -e "SDLC_CACHE_VOLUME=$CACHE_VOLUME" \ ${SDLC_SKIP:+-e SDLC_SKIP} \ @@ -484,6 +490,20 @@ run_host() { } run_step() { # mode payload -> the step's exit status + _rs=0 + _run_step "$@" || _rs=$? + # Only a push may defer to CI. Anything else that reports 3 could not run. + if [ "$_rs" -eq 3 ] && [ "${SDLC_TIER:-}" != push ]; then return 2; fi + return "$_rs" +} + +deferred() { # check -> the warning a deferral prints, every time + printf '\n%s: %s COULD NOT RUN -- DEFERRED TO CI.\n' "$ME" "$1" >&2 + printf '%s: This push is not blocked, and nothing was judged here.\n' "$ME" >&2 + printf '%s: The `sdlc ci` job is the verdict on this commit.\n' "$ME" >&2 +} + +_run_step() { case "$1" in image-ro) run_in_image ro nostdin "$IMAGE" "$2" ;; image-rw) run_in_image rw nostdin "$IMAGE" "$2" ;; @@ -541,7 +561,11 @@ run_serial() { # the plan -> its worst step status _rc=0 run_step "$_mode" "$_payload" || _rc=$? _scur_took=$(( _scur_took + $(now_s) - _t0 )) - if [ "$_rc" -ne 0 ]; then + if [ "$_rc" -eq 3 ]; then + _failed="$_failed $_check" + _scur_rc=3 + deferred "$_check" + elif [ "$_rc" -ne 0 ]; then _failed="$_failed $_check" _scur_rc=$_rc printf '\n%s: %s %s after %ss\n' "$ME" "$_check" \ @@ -606,6 +630,8 @@ _close_check() { # the open check's block is complete # and then the run -- ended with two, the first of them "passed in 0s". if [ "$_cur_rc" -eq 0 ]; then printf '\n== %s passed in %ss\n' "$_cur_name" "$_cur_took" >> "$OUT/out.$_cur" + elif [ "$_cur_rc" -eq 3 ]; then + deferred "$_cur_name" 2>> "$OUT/out.$_cur" else printf '\n%s: %s %s after %ss\n' "$ME" "$_cur_name" \ "$([ "$_cur_rc" -eq 2 ] && echo 'COULD NOT RUN' || echo 'FAILED')" \ @@ -661,7 +687,10 @@ run_lane() { # lane-index steps-file lane-cpus -> the lane's worst _rc=0 run_step "$_mode" "$_payload" >> "$OUT/out.$_i" 2>&1 || _rc=$? _cur_took=$(( _cur_took + $(now_s) - _t0 )) - if [ "$_rc" -ne 0 ]; then + if [ "$_rc" -eq 3 ]; then + _failed="$_failed $_check" + _cur_rc=3 + elif [ "$_rc" -ne 0 ]; then _failed="$_failed $_check" _cur_rc=$_rc # A later 1 must not mask an earlier 2, in this lane or any other. @@ -871,6 +900,9 @@ run_lanes() { # the plan -> the worst status of every lane } run_plan() { # commit | push | ci | + # The image asks this to decide whether CodeQL may defer to CI. + SDLC_TIER=$1 + export SDLC_TIER _plan=$(run_in_image ro nostdin "$IMAGE" plan "$1") \ || die "the image could not plan '$1'; nothing was checked" [ -n "$_plan" ] || die "the image planned nothing for '$1'" @@ -887,18 +919,21 @@ run_plan() { # commit | push | ci | # something that is not a CPU, so that one can stream. _oncpu=$(printf '%s\n' "$_plan" \ | awk -F"$TAB" '$3 != "local" && !seen[$2]++ {n++} END {print n + 0}') - _rc=0 + # Its own name: shell variables are global, and run_serial's loop sets `_rc` + # for every step. A deferred step's 3 survived into here and became the + # exit status of a push that run_serial had passed. + _plan_rc=0 if [ "${_n:-1}" -le 1 ]; then - run_serial "$_plan" || _rc=$? + run_serial "$_plan" || _plan_rc=$? elif [ "${_oncpu:-0}" -eq 1 ]; then - run_split "$_plan" || _rc=$? + run_split "$_plan" || _plan_rc=$? else - run_lanes "$_plan" || _rc=$? + run_lanes "$_plan" || _plan_rc=$? fi trap - EXIT HUP INT TERM - record_finish "$_rc" - return "$_rc" + record_finish "$_plan_rc" + return "$_plan_rc" } # ---------------------------------------------------------------------- dispatch @@ -927,6 +962,8 @@ usage: $ME commit | push | ci | | tiers | list | versions | codeql-basel Configure each command in pyproject.toml: [tool.sdlc.commit], [tool.sdlc.push], [tool.sdlc.ci]. Skip one check once: SDLC_SKIP=name. Exit 2 means a check COULD NOT RUN and is never to be worked around. +A push prints DEFERRED TO CI, and goes through, only when CodeQL itself broke +and the repo's \`sdlc ci\` job will judge the same commit. Working a change, in the order that costs least: diff --git a/docs/codeql.md b/docs/codeql.md index 51c1894..15ad342 100644 --- a/docs/codeql.md +++ b/docs/codeql.md @@ -32,6 +32,33 @@ That command is the one way to make a finding stop failing without fixing it, and it lands in a commit a reviewer reads. With no baseline file, every finding fails. +## When it cannot run + +Three things went wrong on josui-skills at once (#33), and each has its own fix. + +- **Memory.** CodeQL sizes its evaluator itself and does not look at the + container: `codeql resolve ram` gave a 1509 MB heap under `--memory 8g` and + under `--memory 2g` alike. The gate now passes `--ram` explicitly, three + quarters of the smallest of `SDLC_MEMORY`, the cgroup's `memory.max` and the + machine's MemTotal. All three, because rootless podman drops `--memory` + silently when the memory controller is not delegated, as in a developer + container. At the default 8g that is `--ram=6144`, a 4.6 GB heap. +- **One database per project, one run at a time.** The database lives at + `/cache/codeql/`, so two checkouts with the same basename, + or a push retried while the last is still running, deleted each other's + database mid-run. The gate now takes a lock beside it and waits. It clears + the previous database before building and checks that it is gone. A + half-built one is a cache miss, not something a person deletes by hand. +- **Deferring to CI.** When CodeQL itself breaks — `create` or `analyze` exits + non-zero — at the **push** tier, outside CI, in a repo with a workflow that + runs `sdlc ci`, the check exits **3**. The launcher prints `DEFERRED TO CI`, + records it in `last-push.json`, and lets the push through. Anywhere else (`sdlc ci`, + `sdlc codeql`, a repo with no such workflow) it is still a 2. A config the + repo got wrong is still a 2 everywhere, because CI fails on it the same way. + +`--no-verify` skips every gate, so refusing the push left it as the only way +past one broken tool. A deferral leaves the other gates running. + ## The config file `.github/codeql/codeql-config.yml` is the **Action's** format, not the CLI's. diff --git a/sdlc b/sdlc index 89a0d8b..8fab285 100755 --- a/sdlc +++ b/sdlc @@ -10,6 +10,14 @@ THE EXIT CODE CONTRACT, which is the whole point of this file: 0 the check ran and found nothing 1 the check ran and found something 2 the check COULD NOT RUN + 3 the check COULD NOT RUN, and CI will judge this same commit + +3 exists for one case and is granted by this file, never returned by a tool: +CodeQL at the push tier, when the analysis itself broke and the repo's CI runs +`sdlc ci`. It is not a pass. It is a loud "nothing was judged here, CI is the +judge", which the launcher lets through instead of refusing the push -- because +refusing left `--no-verify` as the only door, and that door skips every other +gate too (python-sdlc#33). 2 is never to be worked around. A gate that cannot run and reports 0 is worse than no gate, because the pipeline goes green and nobody looks again. The @@ -26,6 +34,7 @@ the tool's own verdict is a 2. from __future__ import annotations import argparse +import fcntl import json import os import re @@ -33,6 +42,7 @@ import shutil import subprocess import sys import tomllib +from collections.abc import Callable from datetime import UTC, datetime from pathlib import Path from typing import NoReturn @@ -96,12 +106,17 @@ def self_reported(name: str) -> str: CLEAN = 0 FOUND = 1 CANNOT_RUN = 2 +DEFERRED = 3 class CannotRunError(Exception): """Raised for anything that is not the tool's own verdict. Becomes exit CANNOT_RUN.""" +class DeferredError(Exception): + """Could not run, and CI will judge the same commit. Becomes exit DEFERRED.""" + + def cannot(msg: str) -> NoReturn: raise CannotRunError(msg) @@ -1050,6 +1065,12 @@ def t_codeql(s: dict) -> int: if config != default_config and not (REPO / config).is_file(): cannot(f"codeql config {config} does not exist") db = CACHE / "codeql" / project_name() + # Held until this process exits. The database directory is named for the + # worktree's basename, so two checkouts that share one -- or a push retried + # while the last is still running -- used ONE directory: each deleted the + # other's database mid-run, which left half-built caches and analyses dying + # with nothing to read (python-sdlc#33). + lock_codeql(db) # Compiled query plans, kept between runs. CodeQL's default is ~/.codeql, # which is /tmp in this image and gone when the container exits, so every # run compiled all of security-extended again: 1m32s-3m13s of trentina's @@ -1059,15 +1080,15 @@ def t_codeql(s: dict) -> int: # a changed query recompiles and nothing stale is reused. caches = CACHE / "codeql-common" sarif = Path("/tmp/codeql-results.sarif") - shutil.rmtree(db, ignore_errors=True) + clear_database(db) # An unwritable cache is a 2, not a traceback and not a 1. The same bug in # `validate` surfaced as a PermissionError stack trace with an exit status # that claimed a finding. try: - db.parent.mkdir(parents=True, exist_ok=True) caches.mkdir(parents=True, exist_ok=True) except OSError as exc: cannot(f"cannot create the CodeQL database directory {db.parent}: {exc}") + ram = f"--ram={codeql_ram_mb()}" create = [ "codeql", @@ -1078,6 +1099,7 @@ def t_codeql(s: dict) -> int: "--source-root=.", "--overwrite", f"--common-caches={caches}", + ram, ] analyze = [ "codeql", @@ -1088,6 +1110,7 @@ def t_codeql(s: dict) -> int: f"--output={sarif}", f"--threads={os.environ.get('SDLC_JOBS', '0')}", f"--common-caches={caches}", + ram, ] # `--codescanning-config` is NOT a CLI flag in 2.27.4, on either subcommand. @@ -1186,18 +1209,141 @@ def read_codeql_config(path: Path) -> tuple[list[str], list[str]]: return queries, ignores +def lock_codeql(db: Path) -> None: + """Take this project's CodeQL lock, waiting for whoever holds it.""" + try: + db.parent.mkdir(parents=True, exist_ok=True) + # Deliberately never closed: the lock is released when the process ends. + fd = os.open(db.parent / f"{db.name}.lock", os.O_RDWR | os.O_CREAT, 0o644) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print(f"codeql: another run holds {db.name}'s database; waiting for it", flush=True) + fcntl.flock(fd, fcntl.LOCK_EX) + except OSError as exc: + cannot(f"cannot lock the CodeQL database directory {db.parent}: {exc}") + + +def clear_database(db: Path) -> None: + """Remove the last run's database, and PROVE it is gone. + + Nothing in it is reused, so a half-built one -- `trap/` and `working/` with + no `db-python`, from a run that was killed -- is just a cache miss. This + used to be `rmtree(ignore_errors=True)`, and when that quietly left anything + behind, `create --overwrite` refused the directory as "not a CodeQL + database" and a person had to delete it by hand. + """ + errors: list[str] = [] + + def retry(func: Callable[[str], object], path: str, _exc: BaseException) -> None: + # A read-only directory cannot have its entries unlinked; open it up + # and try once more before recording the failure. + try: + os.chmod(os.path.dirname(path) or ".", 0o700) + if os.path.isdir(path) and not os.path.islink(path): + os.chmod(path, 0o700) + func(path) + except OSError as again: + errors.append(f"{path}: {again}") + + if db.exists() or db.is_symlink(): + shutil.rmtree(db, onexc=retry) + if db.exists() or db.is_symlink(): + cannot(f"cannot clear the previous CodeQL database {db}: {'; '.join(errors[:3])}") + + +def memory_limit_mb() -> int | None: + """The smallest memory ceiling this container can see, in MB. + + Three sources, because none is reliable alone. `SDLC_MEMORY` is the budget + the launcher gave `podman run --memory`, which rootless podman silently + drops when the cgroup memory controller is not delegated -- a gate inside a + developer container is exactly that. The cgroup's own `memory.max` is the + one the kernel enforces. MemTotal is the machine. + """ + found: list[int] = [] + budget = os.environ.get("SDLC_MEMORY", "").strip().lower() + match = re.fullmatch(r"(\d+)([kmg]?)b?", budget) + if match: + scale = {"": 1 / 2**20, "k": 1 / 1024, "m": 1, "g": 1024}[match.group(2)] + found.append(int(int(match.group(1)) * scale)) + try: + cg = Path("/sys/fs/cgroup/memory.max").read_text(encoding="utf-8").strip() + if cg.isdigit(): + found.append(int(cg) // 2**20) + except OSError: + pass + try: + for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines(): + if line.startswith("MemTotal:"): + found.append(int(line.split()[1]) // 1024) + except OSError, ValueError, IndexError: + pass + return min((f for f in found if f > 0), default=None) + + +def codeql_ram_mb() -> int: + """`--ram` for the evaluator: three quarters of the ceiling, never under 2048. + + Left to itself CodeQL does not look at the container at all. `codeql + resolve ram` gave a 1509 MB heap in a container with 8g and in one with 2g + alike, and 1253 MB on josui-skills' dataset -- where analyze then died with + status 70 (python-sdlc#33). The 8g the launcher budgets was never used. + + The quarter left over is for the JVM outside its heap, file-backed maps of + the dataset, and /tmp, which is a tmpfs and so is charged to this same + cgroup. 2048 is CodeQL's own floor; it rounds anything smaller up. + """ + limit = memory_limit_mb() + return max(2048, limit * 3 // 4) if limit else 2048 + + +def ci_judges() -> bool: + """Is a push's CodeQL also judged by CI on this same commit? + + True only at the push tier, outside CI, with codeql in the ci tier and a + workflow that runs `sdlc ci`. Anywhere else -- `sdlc ci` itself, the inner + loop's `sdlc codeql`, a repo without that workflow -- nobody else will + judge the commit, and a check that could not run stays a 2. + """ + if os.environ.get("SDLC_TIER") != "push" or os.environ.get("CI"): + return False + if "codeql" not in resolve(sdlc_config(), "ci"): + return False + workflows = REPO / ".github" / "workflows" + try: + files = [*workflows.glob("*.yml"), *workflows.glob("*.yaml")] + return any( + re.search(r"\bsdlc\s+ci\b", f.read_text(encoding="utf-8", errors="replace")) + for f in files + ) + except OSError: + return False + + +def codeql_broke(msg: str) -> NoReturn: + """CodeQL itself failed: defer to CI when CI will judge, otherwise a 2. + + Only for the TOOL breaking. A config the repo got wrong stays a 2 wherever + it is found, because CI would fail on it in exactly the same way. + """ + if ci_judges(): + raise DeferredError(msg) + cannot(msg) + + def run_codeql(create: list[str], analyze: list[str], sarif: Path) -> None: """Build the database and analyse it, or fail as 'could not run'.""" step("codeql database create") rc, _ = run(create, env=project_env()) if rc != 0: - cannot("codeql could not build a database; nothing was analysed") + codeql_broke(f"codeql could not build a database (exit {rc}); nothing was analysed") step("codeql database analyze") rc, _ = run(analyze, env=project_env()) if rc != 0: # Analysis failing is not the tool's verdict ON the code -- it means no # verdict was reached, which the exit-code contract makes a 2. - cannot(f"codeql database analyze failed (exit {rc}); nothing was judged") + codeql_broke(f"codeql database analyze failed (exit {rc}); nothing was judged") def baseline_path(s: dict) -> Path: @@ -1346,7 +1492,9 @@ def main() -> int: fn = BUILTINS.get(args.what) if fn is None: cannot(f"{args.what} is not a check this image runs itself") - return fn(check_settings(sdlc_config(), args.what)) + rc = fn(check_settings(sdlc_config(), args.what)) + # A tool's own 3 is not a deferral; only DeferredError grants one. + return CANNOT_RUN if rc == DEFERRED else rc if __name__ == "__main__": @@ -1355,5 +1503,14 @@ if __name__ == "__main__": except CannotRunError as exc: print(f"GATE CANNOT RUN: {exc}", file=sys.stderr) sys.exit(CANNOT_RUN) + except DeferredError as exc: + print( + f"GATE CANNOT RUN: {exc}\n" + f"DEFERRED TO CI: this push is NOT blocked, and NOTHING was judged here. " + f"The `sdlc ci` job is the judge of this commit -- read its verdict " + f"before merging. Rerun `sdlc codeql` to see the failure as a 2.", + file=sys.stderr, + ) + sys.exit(DEFERRED) except KeyboardInterrupt: sys.exit(CANNOT_RUN) diff --git a/tests/test_codeql_resilience.py b/tests/test_codeql_resilience.py new file mode 100755 index 0000000..f1a09c2 --- /dev/null +++ b/tests/test_codeql_resilience.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""A CodeQL that cannot run must not be what teaches people `--no-verify`. + +python-sdlc#33: on josui-skills a killed run left a half-built database that a +person had to delete by hand, the evaluator was sized by CodeQL's own guess +rather than the container's budget, and an analysis that died blocked every +push with no door but `--no-verify` -- which skips every gate, not the broken +one. These are the three pieces of that fix that need no CodeQL to test. + +Run: tests/test_codeql_resilience.py (needs the image's python3 and PyYAML) +""" + +from __future__ import annotations + +import importlib.machinery +import importlib.util +import os +import stat +import sys +import tempfile +from collections.abc import Callable +from pathlib import Path + +# Loaded by path, as in test_codeql_globs.py: `sdlc` has no .py extension. +_SPEC = importlib.util.spec_from_loader( + "sdlc_under_test", + importlib.machinery.SourceFileLoader( + "sdlc_under_test", str(Path(__file__).resolve().parent.parent / "sdlc") + ), +) +assert _SPEC is not None and _SPEC.loader is not None +_SDLC = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_SDLC) + +# CodeQL's own floor, and what an 8g budget should yield: three quarters of it. +FLOOR_MB = 2048 +EIGHT_G_MB = 8192 +FOUR_G_MB = 4096 +EIGHT_G_RAM_MB = 6144 + + +def check(ok: bool, what: str) -> bool: + print(f"{'ok ' if ok else 'FAIL'} {what}") + return ok + + +def raises(exc: type[BaseException], fn: Callable[[], object]) -> bool: + """True when `fn` raises `exc`. Anything else it raises fails the test run.""" + try: + fn() + except exc: + return True + return False + + +def database_and_lock() -> bool: + ok = True + with tempfile.TemporaryDirectory() as tmp: + # A killed run's leftovers: trap/ and working/, no db-python, and a + # read-only directory that a plain rmtree cannot empty. + db = Path(tmp) / "codeql" / "proj" + (db / "trap" / "deep").mkdir(parents=True) + (db / "working").mkdir() + (db / "trap" / "deep" / "x.trap").write_text("x") + os.chmod(db / "trap" / "deep", stat.S_IRUSR | stat.S_IXUSR) + _SDLC.clear_database(db) + ok &= check(not db.exists(), "a half-built database, read-only parts and all, is cleared") + _SDLC.clear_database(db) + ok &= check(not db.exists(), "clearing a database that is not there is not an error") + + # The lock is per project, in a directory that may not exist yet. + _SDLC.lock_codeql(Path(tmp) / "fresh" / "proj") + lock = Path(tmp) / "fresh" / "proj.lock" + ok &= check(lock.is_file(), "the lock file is created beside the database") + return ok + + +def memory() -> bool: + ok = True + os.environ["SDLC_MEMORY"] = "8g" + limit = _SDLC.memory_limit_mb() + ok &= check( + limit is not None and limit <= EIGHT_G_MB, f"an 8g budget caps the limit ({limit} MB)" + ) + os.environ["SDLC_MEMORY"] = "4096m" + ok &= check(_SDLC.memory_limit_mb() <= FOUR_G_MB, "a budget in m is read as MB") + os.environ["SDLC_MEMORY"] = "1g" + ok &= check(_SDLC.codeql_ram_mb() == FLOOR_MB, "never below CodeQL's own 2048 MB floor") + os.environ["SDLC_MEMORY"] = "8g" + ram = _SDLC.codeql_ram_mb() + ok &= check( + FLOOR_MB <= ram <= EIGHT_G_RAM_MB, f"three quarters of 8g to the evaluator ({ram} MB)" + ) + return ok + + +def deferral() -> bool: + """Deferring to CI is a push-only, outside-CI privilege.""" + ok = True + os.environ.pop("CI", None) + os.environ["SDLC_TIER"] = "ci" + ok &= check(not _SDLC.ci_judges(), "`sdlc ci` never defers: nothing judges after it") + os.environ["SDLC_TIER"] = "codeql" + ok &= check(not _SDLC.ci_judges(), "the inner loop's `sdlc codeql` never defers") + os.environ["SDLC_TIER"] = "push" + os.environ["CI"] = "true" + ok &= check(not _SDLC.ci_judges(), "a push inside CI never defers") + ok &= check( + raises(_SDLC.CannotRunError, lambda: _SDLC.codeql_broke("x")), + "where CI does not judge, a broken CodeQL is still a 2", + ) + return ok + + +def main() -> int: + saved = dict(os.environ) + try: + ok = database_and_lock() + ok &= memory() + ok &= deferral() + finally: + os.environ.clear() + os.environ.update(saved) + print("codeql resilience PASSED" if ok else "codeql resilience FAILED") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_host_contract.sh b/tests/test_host_contract.sh index 1b17cca..b6be1f1 100755 --- a/tests/test_host_contract.sh +++ b/tests/test_host_contract.sh @@ -119,6 +119,8 @@ case " \$* " in printf 'p${TAB}one${TAB}host${TAB}podman${US}build${US}-t${US}localhost/x:sdlc${US}.\n' printf 'q${TAB}two${TAB}image-ro${TAB}quiet-ok\n' ;; *" plan "*) + if [ -f "$SHIM/defer-last" ]; then printf 'codeql${TAB}main${TAB}image-ro${TAB}rc-three\n'; exit 0; fi + if [ -f "$SHIM/defer" ]; then printf 'codeql${TAB}main${TAB}image-ro${TAB}rc-three\n'; fi printf 'mypy${TAB}main${TAB}image-ro${TAB}mypy\n' printf 'ruff-check${TAB}main${TAB}image-rw${TAB}ruff-check\n' printf 'image${TAB}main${TAB}host${TAB}podman${US}build${US}-t${US}localhost/x:sdlc${US}.\n' @@ -131,6 +133,7 @@ case "\$_last" in printf 'DONE %s\n' "\$_last" >> "$SHIM/race.log" ;; rc-one) printf 'RAN rc-one\n'; exit 1 ;; rc-two) printf 'RAN rc-two\n'; exit 2 ;; + rc-three) printf 'RAN rc-three\n'; exit 3 ;; slow-rc-one) printf 'RAN slow-rc-one\n'; sleep 2; exit 1 ;; very-slow-rc-one) printf 'RAN very-slow-rc-one\n'; sleep 6; printf 'LAST LINE\n'; exit 1 ;; step-one) printf 'RAN step-one\n'; sleep 1 ;; @@ -185,6 +188,45 @@ else bad "a host step's word was evaluated as shell" fi +printf '\n== a check deferred to CI\n\n' + +# 3 is the image saying "could not run, and CI judges this commit". A push lets +# it through, loudly, so --no-verify is not the only door past a broken +# CodeQL; every other command must still treat it as a 2 (python-sdlc#33). +touch "$SHIM/defer" +log_mark +PATH="$SHIM" sh bin/sdlc push > "$RUN/out" 2>&1 && rc=0 || rc=$? +log_since +if [ "$rc" -eq 0 ] && grep -q 'codeql COULD NOT RUN -- DEFERRED TO CI' "$RUN/out" \ + && grep -q ' mypy$' "$RUN/podman.log"; then + ok "a push lets a deferred check through, says so, and runs the rest" +else + bad "a deferred check on push exited $rc or said nothing" +fi +if grep -q 'SDLC_MEMORY=' "$RUN/podman.log" && grep -q -- '-e SDLC_TIER' "$RUN/podman.log"; then + ok "the image is told the memory budget and the tier" +else + bad "the image was not given SDLC_MEMORY and SDLC_TIER" +fi +for target in ci codeql; do + PATH="$SHIM" sh bin/sdlc "$target" > "$RUN/out" 2>&1 && rc=0 || rc=$? + if [ "$rc" -eq 2 ] && ! grep -q 'DEFERRED' "$RUN/out"; then + ok "$target treats a deferral as could-not-run (exit 2)" + else + bad "$target let a deferral through (exit $rc)" + fi +done +rm -f "$SHIM/defer" +# Alone, so the deferral is the LAST status the executor saw. +touch "$SHIM/defer-last" +PATH="$SHIM" sh bin/sdlc push > "$RUN/out" 2>&1 && rc=0 || rc=$? +if [ "$rc" -eq 0 ]; then + ok "a push whose only check deferred exits 0" +else + bad "a push whose only check deferred exited $rc" +fi +rm -f "$SHIM/defer-last" + printf '\n== the CPU budget\n\n' # Halving the machine protects the person at the keyboard. Two cases have no From 33c9b9ee372204b28a90f4381e3c4f3fd20d826e Mon Sep 17 00:00:00 2001 From: Josui Date: Sun, 8 Nov 2026 14:00:08 +0000 Subject: [PATCH 2/4] fix: shellcheck SC2016 in the deferral warning Co-Authored-By: Claude Opus 5.5 --- bin/sdlc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/sdlc b/bin/sdlc index 003e51c..03fb036 100755 --- a/bin/sdlc +++ b/bin/sdlc @@ -500,7 +500,7 @@ run_step() { # mode payload -> the step's exit status deferred() { # check -> the warning a deferral prints, every time printf '\n%s: %s COULD NOT RUN -- DEFERRED TO CI.\n' "$ME" "$1" >&2 printf '%s: This push is not blocked, and nothing was judged here.\n' "$ME" >&2 - printf '%s: The `sdlc ci` job is the verdict on this commit.\n' "$ME" >&2 + printf '%s: The "sdlc ci" job is the verdict on this commit.\n' "$ME" >&2 } _run_step() { From 80c1f7031d535efd854926cbad6564fa218991cd Mon Sep 17 00:00:00 2001 From: Josui Date: Sun, 8 Nov 2026 14:12:31 +0000 Subject: [PATCH 3/4] revert: take the exit-3 deferral out of this change Constitution XII says every check exits 0, 1 or 2. A fourth status that lets a push through is an amendment to that rule for every Python repo, so it is proposed to the constitution instead of shipped here. What stays is the cause of josui-skills' refused pushes: CodeQL's memory and the shared, half-built database. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 14 ++------ bin/sdlc | 41 +++------------------ docs/codeql.md | 11 +----- sdlc | 64 ++------------------------------- tests/test_codeql_resilience.py | 37 ++----------------- tests/test_host_contract.sh | 43 ++++------------------ 6 files changed, 20 insertions(+), 190 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52f69fe..1ee8524 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,16 +6,6 @@ All notable changes to this project are documented here, following ## [Unreleased] -### Added - -- **Exit 3: could not run, deferred to CI.** When CodeQL itself breaks at the - push tier, outside CI, in a repo whose workflow runs `sdlc ci`, the push goes - through with a loud `DEFERRED TO CI` and the record shows exit 3. Before - this, a broken CodeQL blocked every push and `--no-verify`, which skips every - gate, was the only way past it. `sdlc ci`, `sdlc codeql` and a config error - are still 2. A tool's own exit 3 is mapped to 2, so only the gate can defer - (#33). - ### Fixed - **CodeQL's evaluator gets the memory the gate budgets.** CodeQL ignored the @@ -28,8 +18,8 @@ All notable changes to this project are documented here, following other's database mid-run. The previous database is removed, read-only parts included, and checked to be gone before `create` runs (#33). - `run_plan` read run_serial's per-step `_rc` as its own result, because shell - variables are global. It had no visible effect while every non-zero step - failed the run, but it turned a deferred push's 0 into a 3. + variables are global. Harmless today, because every non-zero step fails the + run, but one new status away from a wrong exit code. ## [0.5.3] - 2026-11-08 diff --git a/bin/sdlc b/bin/sdlc index 03fb036..dd7846d 100755 --- a/bin/sdlc +++ b/bin/sdlc @@ -19,10 +19,6 @@ # 0 ran, found nothing # 1 ran, found something # 2 COULD NOT RUN -- never to be worked around -# 3 COULD NOT RUN, deferred to CI -- push only, granted by the image when -# CodeQL itself broke and `sdlc ci` will judge the same commit. Printed -# loudly, recorded, and not a refusal (python-sdlc#33). Anywhere but a -# push it is a 2. # # Podman's own 125/126/127 mean the container never started, so they map to 2. # trentina's gates.sh maps them to 1 via `|| fail`, which reports "found @@ -275,7 +271,6 @@ run_in_image() { -e "SDLC_PROJECT=$PROJECT" \ -e "SDLC_JOBS=$GATE_CPUS" \ -e "SDLC_MEMORY=$GATE_MEMORY" \ - ${SDLC_TIER:+-e SDLC_TIER} \ -e "SDLC_HOST_REPO=$REPO" \ -e "SDLC_CACHE_VOLUME=$CACHE_VOLUME" \ ${SDLC_SKIP:+-e SDLC_SKIP} \ @@ -490,20 +485,6 @@ run_host() { } run_step() { # mode payload -> the step's exit status - _rs=0 - _run_step "$@" || _rs=$? - # Only a push may defer to CI. Anything else that reports 3 could not run. - if [ "$_rs" -eq 3 ] && [ "${SDLC_TIER:-}" != push ]; then return 2; fi - return "$_rs" -} - -deferred() { # check -> the warning a deferral prints, every time - printf '\n%s: %s COULD NOT RUN -- DEFERRED TO CI.\n' "$ME" "$1" >&2 - printf '%s: This push is not blocked, and nothing was judged here.\n' "$ME" >&2 - printf '%s: The "sdlc ci" job is the verdict on this commit.\n' "$ME" >&2 -} - -_run_step() { case "$1" in image-ro) run_in_image ro nostdin "$IMAGE" "$2" ;; image-rw) run_in_image rw nostdin "$IMAGE" "$2" ;; @@ -561,11 +542,7 @@ run_serial() { # the plan -> its worst step status _rc=0 run_step "$_mode" "$_payload" || _rc=$? _scur_took=$(( _scur_took + $(now_s) - _t0 )) - if [ "$_rc" -eq 3 ]; then - _failed="$_failed $_check" - _scur_rc=3 - deferred "$_check" - elif [ "$_rc" -ne 0 ]; then + if [ "$_rc" -ne 0 ]; then _failed="$_failed $_check" _scur_rc=$_rc printf '\n%s: %s %s after %ss\n' "$ME" "$_check" \ @@ -630,8 +607,6 @@ _close_check() { # the open check's block is complete # and then the run -- ended with two, the first of them "passed in 0s". if [ "$_cur_rc" -eq 0 ]; then printf '\n== %s passed in %ss\n' "$_cur_name" "$_cur_took" >> "$OUT/out.$_cur" - elif [ "$_cur_rc" -eq 3 ]; then - deferred "$_cur_name" 2>> "$OUT/out.$_cur" else printf '\n%s: %s %s after %ss\n' "$ME" "$_cur_name" \ "$([ "$_cur_rc" -eq 2 ] && echo 'COULD NOT RUN' || echo 'FAILED')" \ @@ -687,10 +662,7 @@ run_lane() { # lane-index steps-file lane-cpus -> the lane's worst _rc=0 run_step "$_mode" "$_payload" >> "$OUT/out.$_i" 2>&1 || _rc=$? _cur_took=$(( _cur_took + $(now_s) - _t0 )) - if [ "$_rc" -eq 3 ]; then - _failed="$_failed $_check" - _cur_rc=3 - elif [ "$_rc" -ne 0 ]; then + if [ "$_rc" -ne 0 ]; then _failed="$_failed $_check" _cur_rc=$_rc # A later 1 must not mask an earlier 2, in this lane or any other. @@ -900,9 +872,6 @@ run_lanes() { # the plan -> the worst status of every lane } run_plan() { # commit | push | ci | - # The image asks this to decide whether CodeQL may defer to CI. - SDLC_TIER=$1 - export SDLC_TIER _plan=$(run_in_image ro nostdin "$IMAGE" plan "$1") \ || die "the image could not plan '$1'; nothing was checked" [ -n "$_plan" ] || die "the image planned nothing for '$1'" @@ -920,8 +889,8 @@ run_plan() { # commit | push | ci | _oncpu=$(printf '%s\n' "$_plan" \ | awk -F"$TAB" '$3 != "local" && !seen[$2]++ {n++} END {print n + 0}') # Its own name: shell variables are global, and run_serial's loop sets `_rc` - # for every step. A deferred step's 3 survived into here and became the - # exit status of a push that run_serial had passed. + # for every step, so the last step's status leaked in here whenever + # run_serial itself returned 0. _plan_rc=0 if [ "${_n:-1}" -le 1 ]; then run_serial "$_plan" || _plan_rc=$? @@ -962,8 +931,6 @@ usage: $ME commit | push | ci | | tiers | list | versions | codeql-basel Configure each command in pyproject.toml: [tool.sdlc.commit], [tool.sdlc.push], [tool.sdlc.ci]. Skip one check once: SDLC_SKIP=name. Exit 2 means a check COULD NOT RUN and is never to be worked around. -A push prints DEFERRED TO CI, and goes through, only when CodeQL itself broke -and the repo's \`sdlc ci\` job will judge the same commit. Working a change, in the order that costs least: diff --git a/docs/codeql.md b/docs/codeql.md index 15ad342..cd1cfb0 100644 --- a/docs/codeql.md +++ b/docs/codeql.md @@ -34,7 +34,7 @@ fails. ## When it cannot run -Three things went wrong on josui-skills at once (#33), and each has its own fix. +Two things went wrong on josui-skills at once (#33). - **Memory.** CodeQL sizes its evaluator itself and does not look at the container: `codeql resolve ram` gave a 1509 MB heap under `--memory 8g` and @@ -49,15 +49,6 @@ Three things went wrong on josui-skills at once (#33), and each has its own fix. database mid-run. The gate now takes a lock beside it and waits. It clears the previous database before building and checks that it is gone. A half-built one is a cache miss, not something a person deletes by hand. -- **Deferring to CI.** When CodeQL itself breaks — `create` or `analyze` exits - non-zero — at the **push** tier, outside CI, in a repo with a workflow that - runs `sdlc ci`, the check exits **3**. The launcher prints `DEFERRED TO CI`, - records it in `last-push.json`, and lets the push through. Anywhere else (`sdlc ci`, - `sdlc codeql`, a repo with no such workflow) it is still a 2. A config the - repo got wrong is still a 2 everywhere, because CI fails on it the same way. - -`--no-verify` skips every gate, so refusing the push left it as the only way -past one broken tool. A deferral leaves the other gates running. ## The config file diff --git a/sdlc b/sdlc index 8fab285..45daa40 100755 --- a/sdlc +++ b/sdlc @@ -10,14 +10,6 @@ THE EXIT CODE CONTRACT, which is the whole point of this file: 0 the check ran and found nothing 1 the check ran and found something 2 the check COULD NOT RUN - 3 the check COULD NOT RUN, and CI will judge this same commit - -3 exists for one case and is granted by this file, never returned by a tool: -CodeQL at the push tier, when the analysis itself broke and the repo's CI runs -`sdlc ci`. It is not a pass. It is a loud "nothing was judged here, CI is the -judge", which the launcher lets through instead of refusing the push -- because -refusing left `--no-verify` as the only door, and that door skips every other -gate too (python-sdlc#33). 2 is never to be worked around. A gate that cannot run and reports 0 is worse than no gate, because the pipeline goes green and nobody looks again. The @@ -106,17 +98,12 @@ def self_reported(name: str) -> str: CLEAN = 0 FOUND = 1 CANNOT_RUN = 2 -DEFERRED = 3 class CannotRunError(Exception): """Raised for anything that is not the tool's own verdict. Becomes exit CANNOT_RUN.""" -class DeferredError(Exception): - """Could not run, and CI will judge the same commit. Becomes exit DEFERRED.""" - - def cannot(msg: str) -> NoReturn: raise CannotRunError(msg) @@ -1298,52 +1285,18 @@ def codeql_ram_mb() -> int: return max(2048, limit * 3 // 4) if limit else 2048 -def ci_judges() -> bool: - """Is a push's CodeQL also judged by CI on this same commit? - - True only at the push tier, outside CI, with codeql in the ci tier and a - workflow that runs `sdlc ci`. Anywhere else -- `sdlc ci` itself, the inner - loop's `sdlc codeql`, a repo without that workflow -- nobody else will - judge the commit, and a check that could not run stays a 2. - """ - if os.environ.get("SDLC_TIER") != "push" or os.environ.get("CI"): - return False - if "codeql" not in resolve(sdlc_config(), "ci"): - return False - workflows = REPO / ".github" / "workflows" - try: - files = [*workflows.glob("*.yml"), *workflows.glob("*.yaml")] - return any( - re.search(r"\bsdlc\s+ci\b", f.read_text(encoding="utf-8", errors="replace")) - for f in files - ) - except OSError: - return False - - -def codeql_broke(msg: str) -> NoReturn: - """CodeQL itself failed: defer to CI when CI will judge, otherwise a 2. - - Only for the TOOL breaking. A config the repo got wrong stays a 2 wherever - it is found, because CI would fail on it in exactly the same way. - """ - if ci_judges(): - raise DeferredError(msg) - cannot(msg) - - def run_codeql(create: list[str], analyze: list[str], sarif: Path) -> None: """Build the database and analyse it, or fail as 'could not run'.""" step("codeql database create") rc, _ = run(create, env=project_env()) if rc != 0: - codeql_broke(f"codeql could not build a database (exit {rc}); nothing was analysed") + cannot(f"codeql could not build a database (exit {rc}); nothing was analysed") step("codeql database analyze") rc, _ = run(analyze, env=project_env()) if rc != 0: # Analysis failing is not the tool's verdict ON the code -- it means no # verdict was reached, which the exit-code contract makes a 2. - codeql_broke(f"codeql database analyze failed (exit {rc}); nothing was judged") + cannot(f"codeql database analyze failed (exit {rc}); nothing was judged") def baseline_path(s: dict) -> Path: @@ -1492,9 +1445,7 @@ def main() -> int: fn = BUILTINS.get(args.what) if fn is None: cannot(f"{args.what} is not a check this image runs itself") - rc = fn(check_settings(sdlc_config(), args.what)) - # A tool's own 3 is not a deferral; only DeferredError grants one. - return CANNOT_RUN if rc == DEFERRED else rc + return fn(check_settings(sdlc_config(), args.what)) if __name__ == "__main__": @@ -1503,14 +1454,5 @@ if __name__ == "__main__": except CannotRunError as exc: print(f"GATE CANNOT RUN: {exc}", file=sys.stderr) sys.exit(CANNOT_RUN) - except DeferredError as exc: - print( - f"GATE CANNOT RUN: {exc}\n" - f"DEFERRED TO CI: this push is NOT blocked, and NOTHING was judged here. " - f"The `sdlc ci` job is the judge of this commit -- read its verdict " - f"before merging. Rerun `sdlc codeql` to see the failure as a 2.", - file=sys.stderr, - ) - sys.exit(DEFERRED) except KeyboardInterrupt: sys.exit(CANNOT_RUN) diff --git a/tests/test_codeql_resilience.py b/tests/test_codeql_resilience.py index f1a09c2..0074826 100755 --- a/tests/test_codeql_resilience.py +++ b/tests/test_codeql_resilience.py @@ -1,11 +1,9 @@ #!/usr/bin/env python3 -"""A CodeQL that cannot run must not be what teaches people `--no-verify`. +"""CodeQL must not fail for want of memory or for a half-built cache. python-sdlc#33: on josui-skills a killed run left a half-built database that a -person had to delete by hand, the evaluator was sized by CodeQL's own guess -rather than the container's budget, and an analysis that died blocked every -push with no door but `--no-verify` -- which skips every gate, not the broken -one. These are the three pieces of that fix that need no CodeQL to test. +person had to delete by hand, and the evaluator was sized by CodeQL's own guess +rather than the container's budget, so analyze died and every push was refused. Run: tests/test_codeql_resilience.py (needs the image's python3 and PyYAML) """ @@ -18,7 +16,6 @@ import stat import sys import tempfile -from collections.abc import Callable from pathlib import Path # Loaded by path, as in test_codeql_globs.py: `sdlc` has no .py extension. @@ -44,15 +41,6 @@ def check(ok: bool, what: str) -> bool: return ok -def raises(exc: type[BaseException], fn: Callable[[], object]) -> bool: - """True when `fn` raises `exc`. Anything else it raises fails the test run.""" - try: - fn() - except exc: - return True - return False - - def database_and_lock() -> bool: ok = True with tempfile.TemporaryDirectory() as tmp: @@ -94,30 +82,11 @@ def memory() -> bool: return ok -def deferral() -> bool: - """Deferring to CI is a push-only, outside-CI privilege.""" - ok = True - os.environ.pop("CI", None) - os.environ["SDLC_TIER"] = "ci" - ok &= check(not _SDLC.ci_judges(), "`sdlc ci` never defers: nothing judges after it") - os.environ["SDLC_TIER"] = "codeql" - ok &= check(not _SDLC.ci_judges(), "the inner loop's `sdlc codeql` never defers") - os.environ["SDLC_TIER"] = "push" - os.environ["CI"] = "true" - ok &= check(not _SDLC.ci_judges(), "a push inside CI never defers") - ok &= check( - raises(_SDLC.CannotRunError, lambda: _SDLC.codeql_broke("x")), - "where CI does not judge, a broken CodeQL is still a 2", - ) - return ok - - def main() -> int: saved = dict(os.environ) try: ok = database_and_lock() ok &= memory() - ok &= deferral() finally: os.environ.clear() os.environ.update(saved) diff --git a/tests/test_host_contract.sh b/tests/test_host_contract.sh index b6be1f1..7b8aeb6 100755 --- a/tests/test_host_contract.sh +++ b/tests/test_host_contract.sh @@ -119,8 +119,6 @@ case " \$* " in printf 'p${TAB}one${TAB}host${TAB}podman${US}build${US}-t${US}localhost/x:sdlc${US}.\n' printf 'q${TAB}two${TAB}image-ro${TAB}quiet-ok\n' ;; *" plan "*) - if [ -f "$SHIM/defer-last" ]; then printf 'codeql${TAB}main${TAB}image-ro${TAB}rc-three\n'; exit 0; fi - if [ -f "$SHIM/defer" ]; then printf 'codeql${TAB}main${TAB}image-ro${TAB}rc-three\n'; fi printf 'mypy${TAB}main${TAB}image-ro${TAB}mypy\n' printf 'ruff-check${TAB}main${TAB}image-rw${TAB}ruff-check\n' printf 'image${TAB}main${TAB}host${TAB}podman${US}build${US}-t${US}localhost/x:sdlc${US}.\n' @@ -133,7 +131,6 @@ case "\$_last" in printf 'DONE %s\n' "\$_last" >> "$SHIM/race.log" ;; rc-one) printf 'RAN rc-one\n'; exit 1 ;; rc-two) printf 'RAN rc-two\n'; exit 2 ;; - rc-three) printf 'RAN rc-three\n'; exit 3 ;; slow-rc-one) printf 'RAN slow-rc-one\n'; sleep 2; exit 1 ;; very-slow-rc-one) printf 'RAN very-slow-rc-one\n'; sleep 6; printf 'LAST LINE\n'; exit 1 ;; step-one) printf 'RAN step-one\n'; sleep 1 ;; @@ -188,44 +185,18 @@ else bad "a host step's word was evaluated as shell" fi -printf '\n== a check deferred to CI\n\n' +printf '\n== what CodeQL is told\n\n' -# 3 is the image saying "could not run, and CI judges this commit". A push lets -# it through, loudly, so --no-verify is not the only door past a broken -# CodeQL; every other command must still treat it as a 2 (python-sdlc#33). -touch "$SHIM/defer" +# CodeQL sizes its evaluator itself unless told, and rootless podman can drop +# --memory silently, so the budget is handed to the image as well (#33). log_mark -PATH="$SHIM" sh bin/sdlc push > "$RUN/out" 2>&1 && rc=0 || rc=$? +PATH="$SHIM" sh bin/sdlc push >/dev/null 2>&1 || true log_since -if [ "$rc" -eq 0 ] && grep -q 'codeql COULD NOT RUN -- DEFERRED TO CI' "$RUN/out" \ - && grep -q ' mypy$' "$RUN/podman.log"; then - ok "a push lets a deferred check through, says so, and runs the rest" +if grep -q -- '-e SDLC_MEMORY=8g' "$RUN/podman.log"; then + ok "the image is told the memory budget" else - bad "a deferred check on push exited $rc or said nothing" + bad "the image was not given SDLC_MEMORY" fi -if grep -q 'SDLC_MEMORY=' "$RUN/podman.log" && grep -q -- '-e SDLC_TIER' "$RUN/podman.log"; then - ok "the image is told the memory budget and the tier" -else - bad "the image was not given SDLC_MEMORY and SDLC_TIER" -fi -for target in ci codeql; do - PATH="$SHIM" sh bin/sdlc "$target" > "$RUN/out" 2>&1 && rc=0 || rc=$? - if [ "$rc" -eq 2 ] && ! grep -q 'DEFERRED' "$RUN/out"; then - ok "$target treats a deferral as could-not-run (exit 2)" - else - bad "$target let a deferral through (exit $rc)" - fi -done -rm -f "$SHIM/defer" -# Alone, so the deferral is the LAST status the executor saw. -touch "$SHIM/defer-last" -PATH="$SHIM" sh bin/sdlc push > "$RUN/out" 2>&1 && rc=0 || rc=$? -if [ "$rc" -eq 0 ]; then - ok "a push whose only check deferred exits 0" -else - bad "a push whose only check deferred exited $rc" -fi -rm -f "$SHIM/defer-last" printf '\n== the CPU budget\n\n' From 95a01ebdb7ccf27c4be58a235e23e41b206c212a Mon Sep 17 00:00:00 2001 From: Josui Date: Sun, 8 Nov 2026 14:30:02 +0000 Subject: [PATCH 4/4] test: both CodeQL failure paths are a 2 that names the step Co-Authored-By: Claude Opus 5.5 --- tests/test_codeql_resilience.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_codeql_resilience.py b/tests/test_codeql_resilience.py index 0074826..9b5da24 100755 --- a/tests/test_codeql_resilience.py +++ b/tests/test_codeql_resilience.py @@ -82,11 +82,38 @@ def memory() -> bool: return ok +def failures() -> bool: + """Either CodeQL step exiting non-zero is a 2 that names the step.""" + ok = True + real_run, real_step = _SDLC.run, _SDLC.step + _SDLC.step = lambda _msg: None + try: + for failing, says in ( + ("create", "could not build a database"), + ("analyze", "analyze failed"), + ): + _SDLC.run = lambda cmd, failing=failing, **_kw: (7 if cmd[2] == failing else 0, "") + try: + _SDLC.run_codeql( + ["codeql", "database", "create"], ["codeql", "database", "analyze"], Path("/x") + ) + raised = "" + except _SDLC.CannotRunError as exc: + raised = str(exc) + ok &= check( + says in raised and "exit 7" in raised, f"a failed {failing} is a 2 that says so" + ) + finally: + _SDLC.run, _SDLC.step = real_run, real_step + return ok + + def main() -> int: saved = dict(os.environ) try: ok = database_and_lock() ok &= memory() + ok &= failures() finally: os.environ.clear() os.environ.update(saved)