diff --git a/.gitignore b/.gitignore index 7b3690a..353e6e4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,8 @@ # gourmand refuses to cache without this entry and then writes the directory, # which is also why its hook mounts the tree read-write. .gourmand-cache/ + +# `python3 sdlc` and the unit tests import this repo's own `sdlc` module, which +# leaves bytecode beside it. A committed .pyc was tracked until v0.3.0. +__pycache__/ +*.pyc diff --git a/CHANGELOG.md b/CHANGELOG.md index d922535..634d3b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,73 @@ All notable changes to this project are documented here, following ## [Unreleased] +## [0.3.0] - 2026-11-04 + +### Changed + +- **A command's checks run in lanes, at the same time.** `sdlc ci` used to run + every check one after another, so trentina's image build -- which depends on + none of mypy, pytest, semgrep or pip-audit -- waited for all four. The job + took the sum of its checks instead of the length of its longest chain: + 5m09s on the self-hosted runner, 16m05s on a GitHub-hosted one, of which the + image chain was 2m34s and 11m21s. + + Each plan line now carries a lane as its second field, + `check TAB lane TAB mode TAB payload`. Steps of one lane run in sequence, in + plan order; lanes run at the same time. The image assigns them, by two rules + that name no repo: + + - an `image-build` check and every `pytest-in-image` check that runs the + image it tags are one lane, because the build has to finish before the + tests start and two builds writing one tag must not race; + - everything else is the `main` lane, in plan order, which is where `uv sync` + happens -- mypy, pytest and pip-audit each sync the project venv in the + shared cache volume, and one lane makes those sequential. uv does lock the + target environment, but a gate should not bet its venv on a lock it did not + take. + + And one refusal: a command that rewrites the tree gets no lanes at all. ruff + rewrites your source and gourmand writes its cache, and a build context or an + in-image mount reading those files while they change is a check judging a + tree that no longer exists. `sdlc commit` is ruff, so commit stays + sequential, which is what "seconds, offline, free" wanted anyway. + + A repo with no image-build check gets one lane, and one lane is the + launcher's sequential path: the same run, the same output, as before. `sdlc + push` has no image chain, so push is unchanged too. + +- **The CPU budget is for the whole gate, not for each container.** It is + divided between the lanes, floor of one CPU each, so two lanes take the same + share of the machine that one lane took when the checks were sequential. + `sdlc ci` on a laptop is no busier than it was. + +- **Each lane's output is buffered and printed whole, in plan order**, as each + check finishes -- a log that cuts two lanes into each other is a log nobody + can read. Buffering collapses the timestamps people were reading the + `== ` markers for, so every block now ends with `== passed in + Ns`, or with how long it ran before it failed. + +- `sdlc tiers` prints each check's lane when a command has more than one. + +### Added + +- `SDLC_LANE_STALL` (default 3600 seconds): how long the printer waits with no + check finishing before it stops waiting on the order. What it measures is + silence, and two things cause it -- a lane killed outright (SIGKILL, the OOM + killer, a cancelled CI job), which runs no cleanup and so never reports at + all, or one check that honestly takes longer than the threshold. The first is + the one worth guarding, because without the guard the printer polls for a + report that is never coming. Tripping it is cosmetic: the launcher says so, + waits for every lane to stop and prints what is left, possibly out of plan + order, and every lane's real exit status is still collected. + +### Fixed + +- Two lanes starting on a machine with no cache volume could both try to create + it, and the one that lost the race called it a gate that could not run. +- A `__pycache__/*.pyc` was committed, and nothing ignored it. Running + `python3 sdlc` or the unit tests left a modified tracked file behind. + ## [0.2.2] - 2026-11-04 ### Fixed @@ -182,7 +249,8 @@ cached tag was never refreshed — that gate had been running a two-week-old reviewer while CI pulled fresh. Local and CI were different reviewers on the same diff. -[Unreleased]: https://github.com/crunchtools/python-sdlc/compare/v0.2.2...HEAD +[Unreleased]: https://github.com/crunchtools/python-sdlc/compare/v0.3.0...HEAD +[0.3.0]: https://github.com/crunchtools/python-sdlc/releases/tag/v0.3.0 [0.2.2]: https://github.com/crunchtools/python-sdlc/releases/tag/v0.2.2 [0.2.1]: https://github.com/crunchtools/python-sdlc/releases/tag/v0.2.1 [0.2.0]: https://github.com/crunchtools/python-sdlc/releases/tag/v0.2.0 diff --git a/Containerfile b/Containerfile index 2586ab5..49c4431 100644 --- a/Containerfile +++ b/Containerfile @@ -44,7 +44,7 @@ LABEL description="Every CrunchTools Python SDLC check in one image: ruff, mypy, LABEL org.opencontainers.image.source="https://github.com/crunchtools/python-sdlc" LABEL org.opencontainers.image.description="Every CrunchTools Python SDLC check in one image." LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later" -LABEL version="0.2.2" +LABEL version="0.3.0" # What the image carries, readable without starting anything. `sdlc versions` # prints these and a consuming repo's CI pins them. diff --git a/IMAGE_TAG b/IMAGE_TAG index ee1372d..0d91a54 100644 --- a/IMAGE_TAG +++ b/IMAGE_TAG @@ -1 +1 @@ -0.2.2 +0.3.0 diff --git a/README.md b/README.md index 44f835d..5a35b57 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ default_install_hook_types: [pre-commit, pre-push] repos: - repo: https://github.com/crunchtools/python-sdlc - rev: v0.2.2 + rev: v0.3.0 hooks: - id: sdlc-commit - id: sdlc-push @@ -69,7 +69,47 @@ laptop and CI both. One pin. | `push` | mypy, pytest, gourmand, gatehouse, semgrep, pip-audit, constitution, codeql | | `ci` | mypy, pytest, semgrep, pip-audit, constitution, codeql | -`sdlc tiers` prints what each command runs in the repo you are in. +`sdlc tiers` prints what each command runs in the repo you are in, and which +lane each check runs in. + +## Lanes + +A command's checks run in lanes. Steps of one lane run in sequence; lanes run +at the same time, so a command takes about as long as its longest chain rather +than the sum of everything in it. + +The image decides the lanes, by two rules: + +- An `image-build` check and every `pytest-in-image` check that runs the image + it tags are **one lane** — the build has to finish before the tests start. +- Everything else is the `main` lane, in the order `tiers` prints. That is + where `uv sync` happens, and one lane keeps two syncs out of one venv. + +A command that rewrites the tree — anything running `ruff-check`, +`ruff-format` or `gourmand` — gets no lanes at all, because a build context +reading those files while they change is judging a tree that no longer exists. +That makes `sdlc commit` sequential by definition. + +A repo with no `image-build` check has one lane and runs exactly as it did +before lanes existed. So does `sdlc push`, which never builds a container. + +The CPU budget is for the whole gate, not for each container: it is divided +between the lanes, floor of one CPU each. Two lanes are no busier on your +laptop than one was. + +Each lane's output is buffered and printed whole, in the order `tiers` lists +the checks, so two lanes never cut into each other's log. Each block ends with +how long its check took, because buffering is what the `== ` timestamps +used to tell you. + +If no check finishes for `SDLC_LANE_STALL` seconds (default 3600) the printer +stops waiting on the order, says so, waits for every lane to stop and then +prints what is left. That guards against a lane killed outright — SIGKILL, the +OOM killer, a cancelled CI job — which runs no cleanup and so never reports at +all, leaving the printer waiting for a report that is never coming. One check +honestly taking longer than the threshold reaches it too; raise the variable if +that is your repo. Either way only the order is given up, never the verdict: +every lane's real exit status is still collected. ## Configuring it @@ -192,7 +232,10 @@ then runs the reviewer CI runs. A developer machine needs `git`, `podman`, `pre-commit` and `gh`. `bin/sdlc` is POSIX shell, runs only `git` and `podman`, and refuses any step that is not a `podman build` or `podman run`. `tests/test_host_contract.sh` proves it with a -restricted `PATH`. +restricted `PATH`, and proves the lane executor there too: that the lanes +really overlap, that a `2` in one beats a `1` in another whichever finishes +first, that a failure in one lane does not stop the other, and that no two +lanes' output interleaves. ## Licence diff --git a/__pycache__/sdlccpython-314.pyc b/__pycache__/sdlccpython-314.pyc deleted file mode 100644 index 5730412..0000000 Binary files a/__pycache__/sdlccpython-314.pyc and /dev/null differ diff --git a/bin/sdlc b/bin/sdlc index c6ebdc4..5335775 100755 --- a/bin/sdlc +++ b/bin/sdlc @@ -134,8 +134,12 @@ esac CACHE_VOLUME=${SDLC_CACHE_VOLUME:-crunchtools-hook-cache} need_volume() { - podman volume exists "$1" 2>/dev/null || podman volume create "$1" >/dev/null \ - || die "cannot create the podman volume $1" + podman volume exists "$1" 2>/dev/null && return 0 + podman volume create "$1" >/dev/null 2>&1 && return 0 + # Two lanes reach this at the same moment on a cold machine and one of them + # loses the create. Losing that race is not a failure; not having the volume + # afterwards is. + podman volume exists "$1" 2>/dev/null || die "cannot create the podman volume $1" } # ------------------------------------------------------------------ podman run @@ -249,8 +253,8 @@ t_gatehouse() { # # Which checks run is decided by the repo's [tool.sdlc.commit], [tool.sdlc.push] # and [tool.sdlc.ci] blocks, and this shell cannot read TOML. So it asks the -# image for a plan -- one line per step, `check TAB mode TAB payload` -- and -# carries it out. The modes: +# image for a plan -- one line per step, `check TAB lane TAB mode TAB payload` -- +# and carries it out. The lane is handled under "the lanes" below. The modes: # # image-ro | image-rw `sdlc ` in the image, the tree ro or rw # local a check this file answers itself @@ -310,21 +314,18 @@ run_step() { # mode payload -> the step's exit status esac } -run_plan() { # commit | push | ci | - _plan=$(run_in_image ro nostdin "$IMAGE" plan "$1") \ - || die "the image could not plan '$1'; nothing was checked" - [ -n "$_plan" ] || die "the image planned nothing for '$1'" - +run_serial() { # the plan -> its worst step status _worst=0 _failed= _ifs=$IFS set -f IFS=' ' - for _line in $_plan; do + for _line in $1; do IFS=$_ifs _check=${_line%%"$TAB"*} _rest=${_line#*"$TAB"} + _rest=${_rest#*"$TAB"} # the lane; one lane, so nothing to decide _mode=${_rest%%"$TAB"*} _payload=${_rest#*"$TAB"} # One failed step ends that check; the next check still runs. @@ -344,6 +345,219 @@ run_plan() { # commit | push | ci | return "$_worst" } +# ------------------------------------------------------------------- the lanes +# +# The plan's second field is a lane. Steps of one lane run in sequence, in plan +# order; lanes run at the same time. Which steps share a lane is the image's +# decision (see "the lanes" in `sdlc`), so no check name appears in this file -- +# a repo with one lane never reaches any of this and runs through run_serial +# exactly as it did before lanes existed. +# +# Two things change when there is more than one lane: +# +# - The CPU budget is DIVIDED between them. The budget is for the whole gate, +# not for each container: two lanes at half the budget take the same share +# of the machine that one lane at the whole budget took, so a laptop running +# `sdlc ci` is no worse off than before. +# - Every lane's output is buffered to a file per CHECK and printed whole, in +# plan order, as each check finishes. Two lanes writing to one terminal +# produce a log nobody can read. The cost is that the `== ` markers +# inside a block all arrive at once, so each block ends with how long its +# check took -- which is the number people were reading off those markers. + +OUT= # the lane scratch directory, while lanes are running + +# How long the printer waits with NO CHECK FINISHING before it stops waiting on +# the order. What it measures is silence, not death, and two things cause it: +# a lane killed outright -- SIGKILL, the OOM killer, a cancelled CI job -- which +# runs no EXIT trap and so never reports at all; or a single check that honestly +# takes longer than this. The first is the one worth guarding, because without +# the guard the printer polls for a marker that is never coming. The second is +# why the default is an hour and why tripping it is cosmetic: the order is +# abandoned, the exit status is not, because the reap below is the authority. +# Settable so the test can reach the boundary without waiting an hour for it. +STALL_MAX=${SDLC_LANE_STALL:-3600} +case "$STALL_MAX" in ''|*[!0-9]*) die "SDLC_LANE_STALL must be a whole number" ;; esac + +now_s() { date +%s 2>/dev/null || printf 0; } + +_lanes_running() { # 0 while any lane has not yet written its done marker + _j=0 + while [ "$_j" -lt "$NLANES" ]; do + _j=$((_j + 1)) + if [ ! -f "$OUT/lane-done.$_j" ]; then return 0; fi + done + return 1 +} + +_close_check() { # the open check's block is complete + if [ -n "${_cur:-}" ]; then + printf '%s\n' "$_cur_rc" > "$OUT/rc.$_cur" + _cur= + fi +} + +run_lane() { # lane-index steps-file lane-cpus -> the lane's worst + _k=$1 + GATE_CPUS=$3 + _worst=0 + _failed= + _cur= + _cur_rc=0 + # However this subshell ends -- a step's `die` exits it where it stands -- the + # check it was on must get a verdict file and the lane must get its done + # marker, or the printer waits for a block that is never coming. + trap 'printf "2\n" > "$OUT/rc.${_cur:-0}"; printf "2\n" > "$OUT/lane-done.$_k"' EXIT + + # `index TAB check TAB mode TAB payload`: the lane is gone, this file is one, + # and the index into the print order was resolved when the file was written + # rather than by an awk per step. + while IFS= read -r _line; do + _i=${_line%%"$TAB"*} + _rest=${_line#*"$TAB"} + _check=${_rest%%"$TAB"*} + _rest=${_rest#*"$TAB"} + _mode=${_rest%%"$TAB"*} + _payload=${_rest#*"$TAB"} + case " $_failed " in *" $_check "*) continue ;; esac + # A check's steps are contiguous within its lane, so a different index means + # the previous block is finished and can be printed. + if [ "$_i" != "${_cur:-}" ]; then _close_check; _cur=$_i; _cur_rc=0; fi + _t0=$(now_s) + _rc=0 + run_step "$_mode" "$_payload" >> "$OUT/out.$_i" 2>&1 || _rc=$? + _took=$(( $(now_s) - _t0 )) + if [ "$_rc" -ne 0 ]; then + _failed="$_failed $_check" + _cur_rc=$_rc + printf '\n%s: %s %s after %ss\n' "$ME" "$_check" \ + "$([ "$_rc" -eq 2 ] && echo 'COULD NOT RUN' || echo 'FAILED')" "$_took" \ + >> "$OUT/out.$_i" + # A later 1 must not mask an earlier 2, in this lane or any other. + if [ "$_rc" -eq 2 ] || [ "$_worst" -eq 2 ]; then _worst=2; else _worst=1; fi + else + printf '\n== %s passed in %ss\n' "$_check" "$_took" >> "$OUT/out.$_i" + fi + done < "$2" + + _close_check + # The trap stays armed through this: it writes the same marker, and its only + # reader asks whether the file exists. Clearing it first would leave a window + # where a failed write means no marker at all, and the printer waiting on one. + printf '%s\n' "$_worst" > "$OUT/lane-done.$_k" + return "$_worst" +} + +run_lanes() { # the plan -> the worst status of every lane + OUT=$(mktemp -d) || die "cannot create a scratch directory for the lane logs" + trap 'rm -rf "$OUT"' EXIT HUP INT TERM + + # The checks in plan order, first appearance: the order blocks are printed in. + printf '%s\n' "$1" | cut -f1 | awk '!seen[$0]++' > "$OUT/order" + printf '%s\n' "$1" | cut -f2 | awk '!seen[$0]++' > "$OUT/lanes" + NCHECKS=$(awk 'END {print NR}' "$OUT/order") + NLANES=$(awk 'END {print NR}' "$OUT/lanes") + + # Floor of one, so a budget smaller than the lane count does overshoot it by + # a CPU or two. The alternative is running the lanes one after another on a + # single-CPU budget, which is the thing this exists to stop doing. + _lane_cpus=$(( GATE_CPUS / NLANES )) + [ "$_lane_cpus" -lt 1 ] && _lane_cpus=1 + + # Once, here, rather than from whichever lane gets there first. + need_volume "$CACHE_VOLUME" + + printf '%s: %s lanes at the same time, %s CPU(s) each\n' "$ME" "$NLANES" "$_lane_cpus" + _k=0 + _pids= + while IFS= read -r _lane; do + _k=$((_k + 1)) + # Number each check by first appearance in the WHOLE plan -- the order the + # blocks print in -- and keep this lane's steps, index first. + printf '%s\n' "$1" | awk -F"$TAB" -v OFS="$TAB" -v l="$_lane" ' + !($1 in idx) { idx[$1] = ++n } + $2 == l { print idx[$1], $1, $3, $4 }' > "$OUT/steps.$_k" + printf '%s: lane %s: %s\n' "$ME" "$_lane" \ + "$(cut -f2 "$OUT/steps.$_k" | awk '!seen[$0]++' | tr '\n' ' ')" + run_lane "$_k" "$OUT/steps.$_k" "$_lane_cpus" & + _pids="$_pids $!" + done < "$OUT/lanes" + + # Print each block whole, in plan order, as soon as it is finished. A lane + # that finishes early is HELD BACK, never dropped and never cut into another + # lane's output; the lanes themselves keep running while this waits. + _i=1 + _stall=0 + while [ "$_i" -le "$NCHECKS" ]; do + if [ -f "$OUT/rc.$_i" ]; then + if [ -f "$OUT/out.$_i" ]; then cat "$OUT/out.$_i"; fi + _i=$((_i + 1)) + _stall=0 + continue + fi + _lanes_running || break + # Nothing has finished for STALL_MAX seconds. Either a lane was killed and + # will never report, or a check is honestly taking that long. Either way, + # stop waiting on the order and fall through to the reap. + if [ "$_stall" -ge "$STALL_MAX" ]; then + printf '%s: no check has finished for %ss.\n' "$ME" "$STALL_MAX" + printf '%s: waiting for every lane to stop, then printing what is left,\n' "$ME" + printf '%s: which may not be in plan order. Raise SDLC_LANE_STALL if a\n' "$ME" + printf '%s: check here really does take that long.\n' "$ME" + break + fi + _stall=$((_stall + 1)) + sleep 1 + done + + # Reaped BEFORE the remaining blocks are printed, for two reasons: the status + # is the answer and is never guessed from what a file happened to contain, + # and once every lane has stopped nothing is still writing, so the blocks + # below are whole and still in plan order. + _worst=0 + for _p in $_pids; do + _rc=0 + wait "$_p" || _rc=$? + case "$_rc" in + 0) ;; + 2) _worst=2 ;; + *) [ "$_worst" -eq 2 ] || _worst=1 ;; + esac + done + + # Whatever the loop above did not reach: a lane that stopped where it stood + # leaves the checks behind it with no block at all. Say so rather than print + # nothing, which reads exactly like a check that passed. + while [ "$_i" -le "$NCHECKS" ]; do + if [ -f "$OUT/out.$_i" ]; then + cat "$OUT/out.$_i" + else + printf '\n%s: %s NEVER RAN: its lane stopped before reaching it\n' "$ME" \ + "$(awk -v n="$_i" 'NR == n {print; exit}' "$OUT/order")" >&2 + fi + _i=$((_i + 1)) + done + + rm -rf "$OUT" + trap - EXIT HUP INT TERM + return "$_worst" +} + +run_plan() { # commit | push | ci | + _plan=$(run_in_image ro nostdin "$IMAGE" plan "$1") \ + || die "the image could not plan '$1'; nothing was checked" + [ -n "$_plan" ] || die "the image planned nothing for '$1'" + + _n=$(printf '%s\n' "$_plan" | awk -F"$TAB" '!seen[$2]++ {n++} END {print n + 0}') + _rc=0 + if [ "${_n:-1}" -le 1 ]; then + run_serial "$_plan" || _rc=$? + else + run_lanes "$_plan" || _rc=$? + fi + return "$_rc" +} + # ---------------------------------------------------------------------- dispatch case "${1:---help}" in diff --git a/pyproject.toml b/pyproject.toml index 582a207..6613584 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ [project] name = "python-sdlc" -version = "0.2.2" +version = "0.3.0" description = "One container image and one script for every CrunchTools Python SDLC check" requires-python = ">=3.14" license = "AGPL-3.0-or-later" diff --git a/sdlc b/sdlc index 5f07af4..9d37060 100755 --- a/sdlc +++ b/sdlc @@ -280,12 +280,13 @@ def kind_of(cfg: dict, name: str) -> str | None: # -------------------------------------------------------------------- the plan # # The host half (bin/sdlc) is POSIX shell and cannot read TOML, so it asks this -# script what to do and then does it. One line per step, three fields split by a +# script what to do and then does it. One line per step, four fields split by a # tab: # -# TAB TAB +# TAB TAB TAB # -# where mode is: +# where lane is the group the step runs in -- see "the lanes" below -- and mode +# is: # image-ro / image-rw run `sdlc ` in this image, tree ro or rw # local bin/sdlc's own check named # host is a podman argv, its words joined by @@ -321,27 +322,102 @@ def plan_lines(cfg: dict, check: str) -> list[tuple[str, str, str]]: return [(check, "image-rw" if check in WRITES_TREE else "image-ro", check)] +# ------------------------------------------------------------------- the lanes +# +# `sdlc ci` used to run every check one after another, so trentina's image build +# -- which depends on none of mypy, pytest, semgrep or pip-audit -- waited for +# all four. The job took the sum of its checks instead of the length of its +# longest chain: 5m09s on a self-hosted runner, 16m05s on a GitHub-hosted one. +# +# So each plan line carries a lane. Steps of one lane run in sequence, in plan +# order; lanes run at the same time. The decision is made HERE, in the image, +# because this is the half that knows what each check does -- bin/sdlc only +# carries it out, and no check name appears in that file. +# +# Two rules, and neither names a repo: +# +# - An `image-build` check and every `pytest-in-image` check that runs the +# image it tags are ONE lane. The build has to finish before the tests +# start, and two builds writing one tag must not race, so a shared tag is a +# shared lane. +# - Everything else goes in MAIN_LANE, in plan order. That is where `uv sync` +# happens: mypy, pytest and pip-audit each sync the project venv in the +# shared cache volume, and one lane makes those sequential. uv does lock the +# target environment (uv 0.13, "Cache safety"), but a gate should not bet +# its venv on a lock it did not take. +# +# And one refusal: a command that REWRITES THE TREE gets no lanes at all. ruff +# rewrites your source and gourmand writes its cache, and a build context or an +# in-image mount reading those files while they change is a check judging a tree +# that no longer exists. `sdlc commit` is ruff, so commit is always sequential, +# which is what "seconds, offline, free" wanted anyway. +# +# A repo with no image-build check therefore gets one lane, and one lane is the +# launcher's sequential path -- the same run, the same output, as before lanes +# existed. `sdlc push` has no image chain either, so push is unchanged too. + +MAIN_LANE = "main" + + +def lane_of(cfg: dict, checks: list[str]) -> dict[str, str]: + """check -> lane, for one command's resolved check list. + + An image chain is keyed by the image reference, not by position: a + `pytest-in-image` may be listed before the `image-build` that makes what it + runs, and two builds may legitimately produce two different tags. + """ + # Nothing runs beside a check that rewrites the tree: one lane, in order. + if any(check in WRITES_TREE for check in checks): + return dict.fromkeys(checks, MAIN_LANE) + lanes: dict[str, str] = {} + by_tag: dict[str, str] = {} + for check in checks: + if kind_of(cfg, check) != "image-build": + continue + tag = str(check_settings(cfg, check).get("tag", default_image_tag())) + lanes[check] = by_tag.setdefault(tag, check) + for check in checks: + if kind_of(cfg, check) != "pytest-in-image": + continue + image = str(check_settings(cfg, check).get("image", default_image_tag())) + # No build of that image in this command: the repo is testing something + # already built, so it owes nothing to another lane and gets its own. + lanes[check] = by_tag.get(image, check) + return {check: lanes.get(check, MAIN_LANE) for check in checks} + + def t_plan(target: str) -> int: cfg = sdlc_config() checks = resolve(cfg, target) if target in COMMANDS else [target] for check in checks: if check not in BUILTINS and kind_of(cfg, check) is None: cannot(f"unknown check: {check}") + lanes = lane_of(cfg, checks) + for check in checks: + lane = plain(lanes[check], "a lane name") for name, mode, payload in plan_lines(cfg, check): plain(name, "a check name") if mode != "host": plain(payload, name) - print(f"{name}\t{mode}\t{payload}") + print(f"{name}\t{lane}\t{mode}\t{payload}") return CLEAN def t_tiers() -> int: cfg = sdlc_config() for command in COMMANDS: + checks = resolve(cfg, command) + lanes = lane_of(cfg, checks) + parallel = len(set(lanes.values())) > 1 print(f"sdlc {command}:") - for check in resolve(cfg, command): + for check in checks: kind = kind_of(cfg, check) - print(f" {check}" + (f" ({kind})" if kind else "")) + note = f" ({kind})" if kind else "" + if parallel: + note += f" [lane {lanes[check]}]" + print(f" {check}{note}") + if parallel: + print(f" {len(set(lanes.values()))} lanes, run at the same time") if skipped(): print(f"\nSDLC_SKIP is skipping: {', '.join(sorted(skipped()))}") return CLEAN diff --git a/tests/contract.sh b/tests/contract.sh index 8806ccf..e487e95 100755 --- a/tests/contract.sh +++ b/tests/contract.sh @@ -255,6 +255,59 @@ case "$plan" in *) ok "sdlc plan ci leaves out the two reviewers" ;; esac +# A repo with no image chain gets ONE lane, which is the launcher's sequential +# path: the same run it had before lanes existed. +lanes=$(podman run --rm -v "$FIXTURE:/repo:ro,z" -w /repo --entrypoint="" \ + "$IMAGE" sdlc plan ci 2>&1 | cut -f2 | sort -u | tr '\n' ' ') +if [ "$lanes" = "main " ]; then + ok "a repo with no image build plans one lane" +else + bad "a repo with no image build planned lanes '$lanes', want 'main '" +fi + +# And a repo with one gets two: the build and the tests that run what it built +# in their own lane, every uv sync in the other. This is the seam between the +# TOML the image reads and the parallel executor in the shell, so it is checked +# on the real image rather than only in the unit test. +CHAIN=$(mktemp -d) +mkdir -p "$CHAIN/tests" +printf 'FROM scratch\n' > "$CHAIN/Containerfile" +cat > "$CHAIN/pyproject.toml" <<'TOML' +[project] +name = "chain" +version = "0.0.0" + +[tool.sdlc.ci] +checks = ["image", "image-tests"] + +[tool.sdlc.ci.image] +kind = "image-build" + +[tool.sdlc.ci.image-tests] +kind = "pytest-in-image" +tests = ["tests/"] +TOML + +chain_plan=$(podman run --rm -v "$CHAIN:/repo:ro,z" -w /repo --entrypoint="" \ + -e SDLC_HOST_REPO="$CHAIN" -e SDLC_CACHE_VOLUME=contract-test \ + "$IMAGE" sdlc plan ci 2>&1) +rm -rf "$CHAIN" +nlanes=$(printf '%s\n' "$chain_plan" | cut -f2 | sort -u | grep -c .) +off_main=$(printf '%s\n' "$chain_plan" | awk -F'\t' '$2 != "main" {print $1}' | sort -u | tr '\n' ' ') +syncers=$(printf '%s\n' "$chain_plan" | awk -F'\t' ' + $1 == "mypy" || $1 == "pytest" || $1 == "pip-audit" {print $2}' | sort -u | grep -c .) +if [ "$nlanes" = 2 ] && [ "$off_main" = "image image-tests " ]; then + ok "an image chain plans a second lane, and only the chain is in it" +else + bad "an image chain planned $nlanes lane(s), off-main: '$off_main'" + printf '%s\n' "$chain_plan" | sed 's/^/ /' +fi +if [ "$syncers" = 1 ]; then + ok "mypy, pytest and pip-audit share a lane, so two uv syncs cannot race" +else + bad "the checks that run uv sync were spread across $syncers lanes" +fi + # And it must still FAIL on something broken -- a gate that passes everything is # the other way to be useless. cat > "$FIXTURE/src/broken.py" <<'PY' diff --git a/tests/test_host_contract.sh b/tests/test_host_contract.sh index 2707aa4..f22d62c 100755 --- a/tests/test_host_contract.sh +++ b/tests/test_host_contract.sh @@ -46,30 +46,75 @@ done # POSIX utilities. The rule is about language toolchains and linters, not about # rebuilding coreutils -- every machine with a shell has these. -for prog in sh awk sed tr grep basename dirname nproc cat mktemp rm ln head id; do +POSIX_UTILS="sh awk sed tr grep basename dirname nproc cat mktemp rm ln head id + cut sleep date wc" +for prog in $POSIX_UTILS; do if full=$(command -v "$prog" 2>/dev/null); then ln -sf "$full" "$SHIM/$prog"; fi done # A fake podman that records what it was asked to do and succeeds. Every target # that needs a container must reach THIS and nothing else. Asked for a plan, it # answers with one step of each mode, so the launcher's whole executor runs. +# +# A plan line is `check TAB lane TAB mode TAB payload`. The plans whose names +# begin `lanes-` exercise the parallel executor: their steps print markers, +# sleep and exit on command, so a test can pin what the launcher does with a +# lane that fails, a lane that is slow, and two lanes writing one terminal. TAB=$(printf '\t') US=$(printf '\037') cat > "$SHIM/podman" <> "$LOG" +for _last in "\$@"; do :; done case " \$* " in *" plan gatehouse-direct "*) - printf 'gatehouse${TAB}local${TAB}gatehouse\n' ;; + printf 'gatehouse${TAB}main${TAB}local${TAB}gatehouse\n' ;; *" plan evil "*) - printf 'evil${TAB}host${TAB}sh${US}-c${US}touch $SHIM/pwned\n' ;; + printf 'evil${TAB}main${TAB}host${TAB}sh${US}-c${US}touch $SHIM/pwned\n' ;; *" plan subst "*) - printf 'subst${TAB}host${TAB}podman${US}run${US}\\\$(touch $SHIM/pwned)\n' ;; + printf 'subst${TAB}main${TAB}host${TAB}podman${US}run${US}\\\$(touch $SHIM/pwned)\n' ;; + *" plan lanes-interleave "*) + printf 'a1${TAB}one${TAB}image-ro${TAB}tick-a1\n' + printf 'a2${TAB}one${TAB}image-ro${TAB}tick-a2\n' + printf 'b1${TAB}two${TAB}image-ro${TAB}tick-b1\n' + printf 'b2${TAB}two${TAB}image-ro${TAB}tick-b2\n' ;; + *" plan lanes-two-first "*) + printf 'x${TAB}one${TAB}image-ro${TAB}rc-two\n' + printf 'y${TAB}two${TAB}image-ro${TAB}slow-rc-one\n' ;; + *" plan lanes-one-first "*) + printf 'x${TAB}one${TAB}image-ro${TAB}rc-one\n' + printf 'y${TAB}two${TAB}image-ro${TAB}slow-rc-two\n' ;; + *" plan lanes-keeps-going "*) + printf 'x${TAB}one${TAB}image-ro${TAB}rc-one\n' + printf 'x${TAB}one${TAB}image-ro${TAB}must-not-run\n' + printf 'w${TAB}one${TAB}image-ro${TAB}after-the-failure\n' + printf 'y${TAB}two${TAB}image-ro${TAB}other-lane-first\n' + printf 'z${TAB}two${TAB}image-ro${TAB}other-lane-second\n' ;; + *" plan lanes-stall "*) + printf 'f${TAB}one${TAB}image-ro${TAB}quiet-ok\n' + printf 's${TAB}two${TAB}image-ro${TAB}very-slow-rc-one\n' ;; + *" plan lanes-budget "*) + printf 'p${TAB}one${TAB}host${TAB}podman${US}build${US}-t${US}localhost/x:sdlc${US}.\n' + printf 'q${TAB}two${TAB}image-ro${TAB}quiet-ok\n' ;; *" plan "*) - printf 'mypy${TAB}image-ro${TAB}mypy\n' - printf 'ruff-check${TAB}image-rw${TAB}ruff-check\n' - printf 'image${TAB}host${TAB}podman${US}build${US}-t${US}localhost/x:sdlc${US}.\n' - printf 'image-tests${TAB}host${TAB}podman${US}run${US}--rm${US}localhost/x:sdlc${US}-m${US}pytest\n' ;; + printf 'mypy${TAB}main${TAB}image-ro${TAB}mypy\n' + printf 'ruff-check${TAB}main${TAB}image-rw${TAB}ruff-check\n' + printf 'image${TAB}main${TAB}host${TAB}podman${US}build${US}-t${US}localhost/x:sdlc${US}.\n' + printf 'image-tests${TAB}main${TAB}host${TAB}podman${US}run${US}--rm${US}localhost/x:sdlc${US}-m${US}pytest\n' ;; +esac +case "\$_last" in + tick-*) + printf 'START %s\n' "\$_last" >> "$SHIM/race.log" + printf 'BEGIN %s\n' "\$_last"; sleep 1; printf 'END %s\n' "\$_last" + printf 'DONE %s\n' "\$_last" >> "$SHIM/race.log" ;; + rc-one) printf 'RAN rc-one\n'; exit 1 ;; + rc-two) printf 'RAN rc-two\n'; exit 2 ;; + slow-rc-one) printf 'RAN slow-rc-one\n'; sleep 2; exit 1 ;; + very-slow-rc-one) printf 'RAN very-slow-rc-one\n'; sleep 6; printf 'LAST LINE\n'; exit 1 ;; + slow-rc-two) printf 'RAN slow-rc-two\n'; sleep 2; exit 2 ;; + must-not-run) printf 'RAN must-not-run\n'; touch "$SHIM/must-not-run" ;; + after-the-failure|other-lane-first|other-lane-second) + printf 'RAN %s\n' "\$_last" ;; esac exit 0 EOF @@ -113,6 +158,155 @@ else bad "a host step's word was evaluated as shell" fi +printf '\n== the lanes\n\n' + +# Steps of one lane run in sequence; lanes run at the same time. What has to +# hold is not the speed -- it is that the answer and the log do not depend on +# which lane happened to finish first. + +RUN=$(mktemp -d) +trap 'rm -rf "$SHIM" "$RUN"' EXIT 0 + +lanes() { # target [stall-seconds] -> exit status, output in $RUN/out + rm -f "$SHIM/must-not-run" + _rc=0 + PATH="$SHIM" SDLC_CPUS=4 SDLC_LANE_STALL=${2:-3600} \ + sh bin/sdlc "$1" > "$RUN/out" 2>&1 || _rc=$? + return "$_rc" +} + +# A single-lane plan takes the sequential path: no lane banner, and the log is +# the one a repo without an image chain has always had. +lanes ci && rc=0 || rc=$? +if ! grep -q 'lanes at the same time' "$RUN/out"; then + ok "a single-lane plan does not go through the lane executor" +else + bad "a single-lane plan was split into lanes" +fi + +# Two lanes, four checks, each step printing BEGIN and END a second apart. Run +# unbuffered the two lanes would alternate; what must come out is each check's +# block whole, in PLAN order -- a1 a2 b1 b2 -- however the lanes ran. +rm -f "$SHIM/race.log" +lanes lanes-interleave && rc=0 || rc=$? + +# First: the lanes really did overlap, so the ordering assertion below is +# testing the buffering and not a sequential run that could not interleave if +# it tried. Each step records when it started and finished in one shared file; +# both lanes' first steps must have started before either had finished. +if [ "$(head -2 "$SHIM/race.log" | grep -c '^START ')" -eq 2 ]; then + ok "both lanes were running at the same time" +else + bad "the lanes ran one after another: $(tr '\n' ' ' < "$SHIM/race.log")" +fi + +got=$(grep -E '^(BEGIN|END) ' "$RUN/out" | tr '\n' ' ') +want="BEGIN tick-a1 END tick-a1 BEGIN tick-a2 END tick-a2 BEGIN tick-b1 END tick-b1 BEGIN tick-b2 END tick-b2 " +if [ "$got" = "$want" ]; then + ok "each check's block is printed whole, in plan order, never interleaved" +else + bad "lane output interleaved or reordered: $got" +fi +if [ "$rc" -eq 0 ]; then + ok "two passing lanes exit 0" +else + bad "two passing lanes exited $rc" +fi +# And the blocks must carry a duration, because buffering collapses the +# timestamps people were reading the `== ` markers for. +if grep -q '^== a1 passed in [0-9]*s' "$RUN/out"; then + ok "a block ends with how long its check took" +else + bad "a block does not report its check's duration" +fi +if ! grep -q 'no check has finished for' "$RUN/out"; then + ok "a healthy run never claims nothing is finishing" +else + bad "a healthy run claimed no check had finished" +fi + +# Nothing finishing for STALL_MAX seconds. The case worth guarding is a lane +# killed outright, which runs no EXIT trap and so never reports at all, leaving +# the printer polling for a marker that is never coming. At the threshold it +# gives up the ORDER -- and must still reap every lane for its real status, and +# still print the late block WHOLE rather than reading a file another process +# is writing. Reached here with a one-second threshold and a check that takes +# six, which is the OTHER way in and the cheaper one to arrange. +lanes lanes-stall 1 && rc=0 || rc=$? +if grep -q 'no check has finished for 1s' "$RUN/out"; then + ok "the printer gives up the order and says what it measured" +else + bad "the stall threshold was never reached" +fi +if [ "$rc" -eq 1 ]; then + ok "a lane's status is still collected after the order is given up" +else + bad "giving up on the order lost the lane's status (exit $rc)" +fi +if grep -q '^== f passed in [0-9]*s' "$RUN/out"; then + ok "the block that did finish in time is still printed" +else + bad "the finished block was dropped when the order was given up" +fi +# LAST LINE is written six seconds in, after the printer gave up at one. It can +# only be here if the block was printed after the lane was reaped. +if grep -q '^LAST LINE$' "$RUN/out" && grep -q 'sdlc: s FAILED after' "$RUN/out"; then + ok "the late block is printed whole, after its lane was reaped" +else + bad "the late block was read while its lane was still writing it" + sed 's/^/ /' "$RUN/out" +fi + +# A 2 anywhere wins over a 1, in either finishing order. Fast-2/slow-1 and +# fast-1/slow-2 are two different code paths through the aggregation, and the +# second is the one a naive "last writer wins" gets wrong. +lanes lanes-two-first && rc=0 || rc=$? +if [ "$rc" -eq 2 ]; then + ok "a 2 that finishes first beats a 1 that finishes last" +else + bad "fast 2 + slow 1 exited $rc, should be 2" +fi +lanes lanes-one-first && rc=0 || rc=$? +if [ "$rc" -eq 2 ]; then + ok "a 2 that finishes last beats a 1 that finished first" +else + bad "fast 1 + slow 2 exited $rc, should be 2" +fi + +# A failed step ends its OWN check. The next check in that lane still runs, and +# so does everything in every other lane -- that rule is older than the lanes +# and the lanes must not quietly narrow it. +lanes lanes-keeps-going && rc=0 || rc=$? +if [ "$rc" -eq 1 ]; then + ok "a lane that finds something exits 1 when nothing could not run" +else + bad "a found-something run exited $rc, should be 1" +fi +if [ -e "$SHIM/must-not-run" ]; then + bad "a failed check's remaining steps still ran" +else + ok "a failed step ends its own check" +fi +for marker in after-the-failure other-lane-first other-lane-second; do + if grep -q "RAN $marker" "$RUN/out"; then + ok "$marker ran despite the failure in lane one" + else + bad "$marker was skipped because another check failed" + fi +done + +# The CPU budget is for the whole gate, not for each container: two lanes out +# of a budget of 4 get 2 each, so a laptop running `sdlc ci` is no worse off +# than it was when the checks were sequential. +: > "$LOG" +lanes lanes-budget || true +if grep -q -- '--cpus 2 ' "$LOG" && grep -q -- '--cpu-quota 200000' "$LOG"; then + ok "a budget of 4 is split 2 and 2 between two lanes" +else + bad "the lanes did not divide the CPU budget" + grep -o -- '--cpus [0-9]*\|--cpu-quota [0-9]*' "$LOG" | sort -u | sed 's/^/ /' +fi + # A container that succeeds must report success, whatever the caller's last # exit status was. Gatehouse runs right after `git diff --quiet`, which exits 1 # whenever there IS a diff to review; that 1 leaked through a shared variable @@ -122,7 +316,7 @@ GITSHIM=$(mktemp -d) for prog in pre-commit gh; do [ -e "$SHIM/$prog" ] && ln -s "$SHIM/$prog" "$GITSHIM/$prog" done -for prog in sh awk sed tr grep basename dirname nproc cat mktemp rm ln head id podman; do +for prog in $POSIX_UTILS podman; do [ -e "$SHIM/$prog" ] && ln -s "$(readlink -f "$SHIM/$prog")" "$GITSHIM/$prog" done cat > "$GITSHIM/git" < dict: S.host_step("x", ["podman", "run", "a b"]) == ("x", "host", "podman\x1frun\x1fa b"), ) +# ------------------------------------------------------------------- the lanes +# +# Steps of one lane run in sequence; lanes run at the same time. Getting this +# wrong is not a slow job, it is a check that ran against an image that had not +# been built yet, or two `uv sync`es in one venv. So every rule is pinned. + +no_image = S.lane_of(empty, S.resolve(empty, "ci")) +check( + "a repo with no image-build gets one lane", + set(no_image.values()) == {S.MAIN_LANE}, + str(no_image), +) + +chain = { + "ci": { + "checks": ["image", "image-tests"], + "image": {"kind": "image-build"}, + "image-tests": {"kind": "pytest-in-image", "tests": ["tests/"]}, + } +} +lanes = S.lane_of(chain, S.resolve(chain, "ci")) +check( + "an image chain is one lane, so the tests follow the build", + lanes["image"] == lanes["image-tests"], + str(lanes), +) +check("the image chain is not the main lane", lanes["image"] != S.MAIN_LANE, str(lanes)) +check( + "the other checks stay in the main lane", + all(lanes[c] == S.MAIN_LANE for c in S.resolve(chain, "ci") if not c.startswith("image")), + str(lanes), +) +check( + "every uv sync is in one lane, so two cannot race in one venv", + len({lanes[c] for c in ("mypy", "pytest", "pip-audit")}) == 1, + str(lanes), +) + +# A pytest-in-image listed BEFORE its build still joins that build's lane: the +# chain is keyed by the image reference, not by the order the repo wrote. +reordered = { + "ci": { + "checks": ["image-tests", "image"], + "image": {"kind": "image-build"}, + "image-tests": {"kind": "pytest-in-image", "tests": ["tests/"]}, + } +} +lanes = S.lane_of(reordered, S.resolve(reordered, "ci")) +check("a chain written backwards is still one lane", lanes["image"] == lanes["image-tests"]) + +# Two builds writing ONE tag share a lane: concurrent builds of the same tag +# would race to leave the last one standing. +same_tag = { + "ci": { + "checks": ["a", "b"], + "a": {"kind": "image-build"}, + "b": {"kind": "image-build"}, + } +} +lanes = S.lane_of(same_tag, S.resolve(same_tag, "ci")) +check("two builds of one tag share a lane", lanes["a"] == lanes["b"], str(lanes)) + +two_tags = { + "ci": { + "checks": ["a", "b"], + "a": {"kind": "image-build", "tag": "localhost/a:sdlc"}, + "b": {"kind": "image-build", "tag": "localhost/b:sdlc"}, + } +} +lanes = S.lane_of(two_tags, S.resolve(two_tags, "ci")) +check("two builds of two tags get two lanes", lanes["a"] != lanes["b"], str(lanes)) + +# Tests against an image nothing in this command builds owe no lane anything. +prebuilt = { + "ci": { + "checks": ["image-tests"], + "image-tests": { + "kind": "pytest-in-image", + "tests": ["tests/"], + "image": "quay.io/x/y:1", + }, + } +} +lanes = S.lane_of(prebuilt, S.resolve(prebuilt, "ci")) +check( + "tests on a prebuilt image get their own lane", + lanes["image-tests"] not in (S.MAIN_LANE, None) and lanes["image-tests"] == "image-tests", + str(lanes), +) + +# A check that rewrites the tree takes the whole command sequential. A build +# context or an in-image mount reading those files while ruff rewrites them is +# a check judging a tree that no longer exists. +rewrites = { + "ci": { + "checks": ["ruff-format", "image", "image-tests"], + "image": {"kind": "image-build"}, + "image-tests": {"kind": "pytest-in-image", "tests": ["tests/"]}, + } +} +lanes = S.lane_of(rewrites, S.resolve(rewrites, "ci")) +check( + "a check that rewrites the tree gets no lanes beside it", + set(lanes.values()) == {S.MAIN_LANE}, + str(lanes), +) +commit = S.lane_of(empty, S.resolve(empty, "commit")) +check("commit is always one lane", set(commit.values()) == {S.MAIN_LANE}, str(commit)) + +# ------------------------------------------------- the plan the launcher reads +# +# Four tab-separated fields now, and the launcher splits on exactly that. A +# plan line that does not have them is a step the shell mis-reads. + + +def plan_of(target: str, pyproject: str, **env) -> list[list[str]]: + with tempfile.TemporaryDirectory() as root: + Path(root, "pyproject.toml").write_text(pyproject) + Path(root, "tests").mkdir() + Path(root, "Containerfile").write_text("FROM scratch\n") + S.REPO = Path(root) + saved = dict(os.environ) + os.environ.update( + {"SDLC_HOST_REPO": root, "SDLC_CACHE_VOLUME": "testvol", "SDLC_PROJECT": "fx", **env} + ) + buf = io.StringIO() + try: + with contextlib.redirect_stdout(buf): + S.t_plan(target) + finally: + os.environ.clear() + os.environ.update(saved) + return [line.split("\t") for line in buf.getvalue().splitlines()] + + +TRENTINA = """ +[project] +name = "fx" +version = "0.0.0" + +[tool.sdlc.ci] +checks = ["image", "image-tests"] +skip = ["codeql"] + +[tool.sdlc.ci.image] +kind = "image-build" + +[tool.sdlc.ci.image-tests] +kind = "pytest-in-image" +tests = ["tests/"] +""" + +PLAN_FIELDS = 4 # check, lane, mode, payload -- what bin/sdlc splits on +CHAIN_LANES = 2 # the main lane, and the image chain's + +rows = plan_of("ci", TRENTINA) +check( + "every plan line has four fields", + all(len(r) == PLAN_FIELDS for r in rows), + str(rows), +) +check( + "the modes are still the third field", + {r[2] for r in rows} <= {"image-ro", "image-rw", "local", "host"}, + str({r[2] for r in rows}), +) +lanes_seen = [r[1] for r in rows] +check( + "the plan has exactly two lanes", + len(set(lanes_seen)) == CHAIN_LANES, + str(set(lanes_seen)), +) +check( + "the build and its tests are the only steps off the main lane", + {r[0] for r in rows if r[1] != S.MAIN_LANE} == {"image", "image-tests"}, + str(lanes_seen), +) +# The launcher groups a lane's steps by check and prints blocks in plan order, +# which both assume a check's steps are contiguous within its lane. +for lane in set(lanes_seen): + seq = [r[0] for r in rows if r[1] == lane] + runs = [c for i, c in enumerate(seq) if i == 0 or seq[i - 1] != c] + check(f"lane {lane}: each check's steps are contiguous", len(runs) == len(set(runs)), str(seq)) + +single = plan_of("push", '[project]\nname = "fx"\nversion = "0.0.0"\n') +check( + "push is one lane, so it runs exactly as it did before", + {r[1] for r in single} == {S.MAIN_LANE}, + str({r[1] for r in single}), +) + def main() -> int: print()