diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 3fa93cf..2e5f868 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -116,7 +116,8 @@ jobs: run: | set -eu for t in tests/test_plan.py tests/test_codeql_globs.py \ - tests/test_pycache_prefix.py tests/test_gourmand_sidecar.py; do + tests/test_pycache_prefix.py tests/test_gourmand_sidecar.py \ + tests/test_codeql_resilience.py; do podman run --rm -v "$PWD:/src:ro,z" -w /src \ --entrypoint /opt/constitution-venv/bin/python3 \ "localhost/python-sdlc:$GITHUB_SHA" "$t" diff --git a/CHANGELOG.md b/CHANGELOG.md index e7fc080..1ee8524 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,21 @@ All notable changes to this project are documented here, following ## [Unreleased] +### Fixed + +- **CodeQL's evaluator gets the memory the gate budgets.** CodeQL ignored the + container and gave itself a ~1.2-1.5 GB heap, and analyze died on + josui-skills with status 70. `--ram` is now three quarters of the smallest of + `SDLC_MEMORY`, the cgroup limit and MemTotal: 6144 MB at the default 8g + (#33). +- **A half-built CodeQL database no longer needs a person to delete it.** Runs + on one project take a lock and wait for each other instead of deleting each + other's database mid-run. The previous database is removed, read-only parts + included, and checked to be gone before `create` runs (#33). +- `run_plan` read run_serial's per-step `_rc` as its own result, because shell + variables are global. Harmless today, because every non-zero step fails the + run, but one new status away from a wrong exit code. + ## [0.5.3] - 2026-11-08 ### Changed diff --git a/bin/sdlc b/bin/sdlc index 40c7a30..dd7846d 100755 --- a/bin/sdlc +++ b/bin/sdlc @@ -270,6 +270,7 @@ run_in_image() { -e SEMGREP_SETTINGS_FILE=/tmp/semgrep.yml \ -e "SDLC_PROJECT=$PROJECT" \ -e "SDLC_JOBS=$GATE_CPUS" \ + -e "SDLC_MEMORY=$GATE_MEMORY" \ -e "SDLC_HOST_REPO=$REPO" \ -e "SDLC_CACHE_VOLUME=$CACHE_VOLUME" \ ${SDLC_SKIP:+-e SDLC_SKIP} \ @@ -887,18 +888,21 @@ run_plan() { # commit | push | ci | # something that is not a CPU, so that one can stream. _oncpu=$(printf '%s\n' "$_plan" \ | awk -F"$TAB" '$3 != "local" && !seen[$2]++ {n++} END {print n + 0}') - _rc=0 + # Its own name: shell variables are global, and run_serial's loop sets `_rc` + # for every step, so the last step's status leaked in here whenever + # run_serial itself returned 0. + _plan_rc=0 if [ "${_n:-1}" -le 1 ]; then - run_serial "$_plan" || _rc=$? + run_serial "$_plan" || _plan_rc=$? elif [ "${_oncpu:-0}" -eq 1 ]; then - run_split "$_plan" || _rc=$? + run_split "$_plan" || _plan_rc=$? else - run_lanes "$_plan" || _rc=$? + run_lanes "$_plan" || _plan_rc=$? fi trap - EXIT HUP INT TERM - record_finish "$_rc" - return "$_rc" + record_finish "$_plan_rc" + return "$_plan_rc" } # ---------------------------------------------------------------------- dispatch diff --git a/docs/codeql.md b/docs/codeql.md index 51c1894..cd1cfb0 100644 --- a/docs/codeql.md +++ b/docs/codeql.md @@ -32,6 +32,24 @@ That command is the one way to make a finding stop failing without fixing it, and it lands in a commit a reviewer reads. With no baseline file, every finding fails. +## When it cannot run + +Two things went wrong on josui-skills at once (#33). + +- **Memory.** CodeQL sizes its evaluator itself and does not look at the + container: `codeql resolve ram` gave a 1509 MB heap under `--memory 8g` and + under `--memory 2g` alike. The gate now passes `--ram` explicitly, three + quarters of the smallest of `SDLC_MEMORY`, the cgroup's `memory.max` and the + machine's MemTotal. All three, because rootless podman drops `--memory` + silently when the memory controller is not delegated, as in a developer + container. At the default 8g that is `--ram=6144`, a 4.6 GB heap. +- **One database per project, one run at a time.** The database lives at + `/cache/codeql/`, so two checkouts with the same basename, + or a push retried while the last is still running, deleted each other's + database mid-run. The gate now takes a lock beside it and waits. It clears + the previous database before building and checks that it is gone. A + half-built one is a cache miss, not something a person deletes by hand. + ## The config file `.github/codeql/codeql-config.yml` is the **Action's** format, not the CLI's. diff --git a/sdlc b/sdlc index 89a0d8b..45daa40 100755 --- a/sdlc +++ b/sdlc @@ -26,6 +26,7 @@ the tool's own verdict is a 2. from __future__ import annotations import argparse +import fcntl import json import os import re @@ -33,6 +34,7 @@ import shutil import subprocess import sys import tomllib +from collections.abc import Callable from datetime import UTC, datetime from pathlib import Path from typing import NoReturn @@ -1050,6 +1052,12 @@ def t_codeql(s: dict) -> int: if config != default_config and not (REPO / config).is_file(): cannot(f"codeql config {config} does not exist") db = CACHE / "codeql" / project_name() + # Held until this process exits. The database directory is named for the + # worktree's basename, so two checkouts that share one -- or a push retried + # while the last is still running -- used ONE directory: each deleted the + # other's database mid-run, which left half-built caches and analyses dying + # with nothing to read (python-sdlc#33). + lock_codeql(db) # Compiled query plans, kept between runs. CodeQL's default is ~/.codeql, # which is /tmp in this image and gone when the container exits, so every # run compiled all of security-extended again: 1m32s-3m13s of trentina's @@ -1059,15 +1067,15 @@ def t_codeql(s: dict) -> int: # a changed query recompiles and nothing stale is reused. caches = CACHE / "codeql-common" sarif = Path("/tmp/codeql-results.sarif") - shutil.rmtree(db, ignore_errors=True) + clear_database(db) # An unwritable cache is a 2, not a traceback and not a 1. The same bug in # `validate` surfaced as a PermissionError stack trace with an exit status # that claimed a finding. try: - db.parent.mkdir(parents=True, exist_ok=True) caches.mkdir(parents=True, exist_ok=True) except OSError as exc: cannot(f"cannot create the CodeQL database directory {db.parent}: {exc}") + ram = f"--ram={codeql_ram_mb()}" create = [ "codeql", @@ -1078,6 +1086,7 @@ def t_codeql(s: dict) -> int: "--source-root=.", "--overwrite", f"--common-caches={caches}", + ram, ] analyze = [ "codeql", @@ -1088,6 +1097,7 @@ def t_codeql(s: dict) -> int: f"--output={sarif}", f"--threads={os.environ.get('SDLC_JOBS', '0')}", f"--common-caches={caches}", + ram, ] # `--codescanning-config` is NOT a CLI flag in 2.27.4, on either subcommand. @@ -1186,12 +1196,101 @@ def read_codeql_config(path: Path) -> tuple[list[str], list[str]]: return queries, ignores +def lock_codeql(db: Path) -> None: + """Take this project's CodeQL lock, waiting for whoever holds it.""" + try: + db.parent.mkdir(parents=True, exist_ok=True) + # Deliberately never closed: the lock is released when the process ends. + fd = os.open(db.parent / f"{db.name}.lock", os.O_RDWR | os.O_CREAT, 0o644) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print(f"codeql: another run holds {db.name}'s database; waiting for it", flush=True) + fcntl.flock(fd, fcntl.LOCK_EX) + except OSError as exc: + cannot(f"cannot lock the CodeQL database directory {db.parent}: {exc}") + + +def clear_database(db: Path) -> None: + """Remove the last run's database, and PROVE it is gone. + + Nothing in it is reused, so a half-built one -- `trap/` and `working/` with + no `db-python`, from a run that was killed -- is just a cache miss. This + used to be `rmtree(ignore_errors=True)`, and when that quietly left anything + behind, `create --overwrite` refused the directory as "not a CodeQL + database" and a person had to delete it by hand. + """ + errors: list[str] = [] + + def retry(func: Callable[[str], object], path: str, _exc: BaseException) -> None: + # A read-only directory cannot have its entries unlinked; open it up + # and try once more before recording the failure. + try: + os.chmod(os.path.dirname(path) or ".", 0o700) + if os.path.isdir(path) and not os.path.islink(path): + os.chmod(path, 0o700) + func(path) + except OSError as again: + errors.append(f"{path}: {again}") + + if db.exists() or db.is_symlink(): + shutil.rmtree(db, onexc=retry) + if db.exists() or db.is_symlink(): + cannot(f"cannot clear the previous CodeQL database {db}: {'; '.join(errors[:3])}") + + +def memory_limit_mb() -> int | None: + """The smallest memory ceiling this container can see, in MB. + + Three sources, because none is reliable alone. `SDLC_MEMORY` is the budget + the launcher gave `podman run --memory`, which rootless podman silently + drops when the cgroup memory controller is not delegated -- a gate inside a + developer container is exactly that. The cgroup's own `memory.max` is the + one the kernel enforces. MemTotal is the machine. + """ + found: list[int] = [] + budget = os.environ.get("SDLC_MEMORY", "").strip().lower() + match = re.fullmatch(r"(\d+)([kmg]?)b?", budget) + if match: + scale = {"": 1 / 2**20, "k": 1 / 1024, "m": 1, "g": 1024}[match.group(2)] + found.append(int(int(match.group(1)) * scale)) + try: + cg = Path("/sys/fs/cgroup/memory.max").read_text(encoding="utf-8").strip() + if cg.isdigit(): + found.append(int(cg) // 2**20) + except OSError: + pass + try: + for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines(): + if line.startswith("MemTotal:"): + found.append(int(line.split()[1]) // 1024) + except OSError, ValueError, IndexError: + pass + return min((f for f in found if f > 0), default=None) + + +def codeql_ram_mb() -> int: + """`--ram` for the evaluator: three quarters of the ceiling, never under 2048. + + Left to itself CodeQL does not look at the container at all. `codeql + resolve ram` gave a 1509 MB heap in a container with 8g and in one with 2g + alike, and 1253 MB on josui-skills' dataset -- where analyze then died with + status 70 (python-sdlc#33). The 8g the launcher budgets was never used. + + The quarter left over is for the JVM outside its heap, file-backed maps of + the dataset, and /tmp, which is a tmpfs and so is charged to this same + cgroup. 2048 is CodeQL's own floor; it rounds anything smaller up. + """ + limit = memory_limit_mb() + return max(2048, limit * 3 // 4) if limit else 2048 + + def run_codeql(create: list[str], analyze: list[str], sarif: Path) -> None: """Build the database and analyse it, or fail as 'could not run'.""" step("codeql database create") rc, _ = run(create, env=project_env()) if rc != 0: - cannot("codeql could not build a database; nothing was analysed") + cannot(f"codeql could not build a database (exit {rc}); nothing was analysed") step("codeql database analyze") rc, _ = run(analyze, env=project_env()) if rc != 0: diff --git a/tests/test_codeql_resilience.py b/tests/test_codeql_resilience.py new file mode 100755 index 0000000..9b5da24 --- /dev/null +++ b/tests/test_codeql_resilience.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""CodeQL must not fail for want of memory or for a half-built cache. + +python-sdlc#33: on josui-skills a killed run left a half-built database that a +person had to delete by hand, and the evaluator was sized by CodeQL's own guess +rather than the container's budget, so analyze died and every push was refused. + +Run: tests/test_codeql_resilience.py (needs the image's python3 and PyYAML) +""" + +from __future__ import annotations + +import importlib.machinery +import importlib.util +import os +import stat +import sys +import tempfile +from pathlib import Path + +# Loaded by path, as in test_codeql_globs.py: `sdlc` has no .py extension. +_SPEC = importlib.util.spec_from_loader( + "sdlc_under_test", + importlib.machinery.SourceFileLoader( + "sdlc_under_test", str(Path(__file__).resolve().parent.parent / "sdlc") + ), +) +assert _SPEC is not None and _SPEC.loader is not None +_SDLC = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_SDLC) + +# CodeQL's own floor, and what an 8g budget should yield: three quarters of it. +FLOOR_MB = 2048 +EIGHT_G_MB = 8192 +FOUR_G_MB = 4096 +EIGHT_G_RAM_MB = 6144 + + +def check(ok: bool, what: str) -> bool: + print(f"{'ok ' if ok else 'FAIL'} {what}") + return ok + + +def database_and_lock() -> bool: + ok = True + with tempfile.TemporaryDirectory() as tmp: + # A killed run's leftovers: trap/ and working/, no db-python, and a + # read-only directory that a plain rmtree cannot empty. + db = Path(tmp) / "codeql" / "proj" + (db / "trap" / "deep").mkdir(parents=True) + (db / "working").mkdir() + (db / "trap" / "deep" / "x.trap").write_text("x") + os.chmod(db / "trap" / "deep", stat.S_IRUSR | stat.S_IXUSR) + _SDLC.clear_database(db) + ok &= check(not db.exists(), "a half-built database, read-only parts and all, is cleared") + _SDLC.clear_database(db) + ok &= check(not db.exists(), "clearing a database that is not there is not an error") + + # The lock is per project, in a directory that may not exist yet. + _SDLC.lock_codeql(Path(tmp) / "fresh" / "proj") + lock = Path(tmp) / "fresh" / "proj.lock" + ok &= check(lock.is_file(), "the lock file is created beside the database") + return ok + + +def memory() -> bool: + ok = True + os.environ["SDLC_MEMORY"] = "8g" + limit = _SDLC.memory_limit_mb() + ok &= check( + limit is not None and limit <= EIGHT_G_MB, f"an 8g budget caps the limit ({limit} MB)" + ) + os.environ["SDLC_MEMORY"] = "4096m" + ok &= check(_SDLC.memory_limit_mb() <= FOUR_G_MB, "a budget in m is read as MB") + os.environ["SDLC_MEMORY"] = "1g" + ok &= check(_SDLC.codeql_ram_mb() == FLOOR_MB, "never below CodeQL's own 2048 MB floor") + os.environ["SDLC_MEMORY"] = "8g" + ram = _SDLC.codeql_ram_mb() + ok &= check( + FLOOR_MB <= ram <= EIGHT_G_RAM_MB, f"three quarters of 8g to the evaluator ({ram} MB)" + ) + return ok + + +def failures() -> bool: + """Either CodeQL step exiting non-zero is a 2 that names the step.""" + ok = True + real_run, real_step = _SDLC.run, _SDLC.step + _SDLC.step = lambda _msg: None + try: + for failing, says in ( + ("create", "could not build a database"), + ("analyze", "analyze failed"), + ): + _SDLC.run = lambda cmd, failing=failing, **_kw: (7 if cmd[2] == failing else 0, "") + try: + _SDLC.run_codeql( + ["codeql", "database", "create"], ["codeql", "database", "analyze"], Path("/x") + ) + raised = "" + except _SDLC.CannotRunError as exc: + raised = str(exc) + ok &= check( + says in raised and "exit 7" in raised, f"a failed {failing} is a 2 that says so" + ) + finally: + _SDLC.run, _SDLC.step = real_run, real_step + return ok + + +def main() -> int: + saved = dict(os.environ) + try: + ok = database_and_lock() + ok &= memory() + ok &= failures() + finally: + os.environ.clear() + os.environ.update(saved) + print("codeql resilience PASSED" if ok else "codeql resilience FAILED") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_host_contract.sh b/tests/test_host_contract.sh index 1b17cca..7b8aeb6 100755 --- a/tests/test_host_contract.sh +++ b/tests/test_host_contract.sh @@ -185,6 +185,19 @@ else bad "a host step's word was evaluated as shell" fi +printf '\n== what CodeQL is told\n\n' + +# CodeQL sizes its evaluator itself unless told, and rootless podman can drop +# --memory silently, so the budget is handed to the image as well (#33). +log_mark +PATH="$SHIM" sh bin/sdlc push >/dev/null 2>&1 || true +log_since +if grep -q -- '-e SDLC_MEMORY=8g' "$RUN/podman.log"; then + ok "the image is told the memory budget" +else + bad "the image was not given SDLC_MEMORY" +fi + printf '\n== the CPU budget\n\n' # Halving the machine protects the person at the keyboard. Two cases have no