diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 1131956..81f0982 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -24,11 +24,6 @@ on: schedule: - cron: "0 6 * * 1" workflow_dispatch: - # Gourmand and Gatehouse dispatch this when they release, because this image - # bakes them. That is what keeps a consuming repo's one pin honest without - # anyone remembering to bump something. - repository_dispatch: - types: [parent-image-updated] concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} @@ -58,8 +53,7 @@ jobs: set -eu podman run --rm -v "$PWD:/mnt:ro" -w /mnt \ docker.io/koalaman/shellcheck:v0.11.0 bin/sdlc tests/contract.sh \ - tests/test_host_contract.sh tests/test_tiers.sh - tests/test_tiers.sh + tests/test_host_contract.sh tests/test_host_contract.sh - name: Build @@ -84,24 +78,16 @@ jobs: - name: Contract test run: tests/contract.sh "localhost/python-sdlc:$GITHUB_SHA" - # The Container Image profile recommends a CVE scan, and this is the one - # place Trivy belongs: it reads the OS packages in the image layers - # (glibc, openssl, zlib), which pip-audit never sees because pip-audit - # reads a Python lockfile. Scott's call is that Trivy is CI-only, and a - # container-image repo's CI is exactly where the image exists. - # - # `continue-on-error` per the profile: a base-image CVE with no fix - # available must not block a tooling release, but it must be visible. - - name: Trivy - continue-on-error: true + # The planner and the CodeQL path matching, run in the image because + # they need its interpreter and PyYAML. + - name: Unit tests run: | set -eu - podman save --format docker-archive \ - "localhost/python-sdlc:$GITHUB_SHA" -o /tmp/image.tar - podman run --rm -v /tmp:/tmp:z \ - docker.io/aquasec/trivy:0.58.1 image \ - --input /tmp/image.tar --scanners vuln \ - --severity CRITICAL,HIGH --ignore-unfixed --exit-code 1 + for t in tests/test_plan.py tests/test_codeql_globs.py; do + podman run --rm -v "$PWD:/src:ro,z" -w /src \ + --entrypoint /opt/constitution-venv/bin/python3 \ + "localhost/python-sdlc:$GITHUB_SHA" "$t" + done - name: Log in to Quay if: github.event_name != 'pull_request' @@ -201,3 +187,34 @@ jobs: echo "promoted $version" ;; esac + + # Step 3 of a release (update-tools.yml describes all three): once a version + # tag's image is promoted -- so the tag a consumer will pull exists -- open a + # bump PR in every repo that pins python-sdlc. Each repo's own CI decides + # whether its bump lands. + bump-consumers: + name: Bump every repo that pins python-sdlc + needs: promote + if: startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Mint CI token + id: ci-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ vars.CRUNCHTOOLS_CI_APP_ID || secrets.CRUNCHTOOLS_CI_APP_ID }} + private-key: ${{ secrets.CRUNCHTOOLS_CI_APP_KEY }} + owner: crunchtools + + - name: Open the bump pull requests + env: + GH_TOKEN: ${{ steps.ci-token.outputs.token }} + run: | + gh auth setup-git + python3 scripts/bump_consumers.py "$GITHUB_REF_NAME" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..0c015bc --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,68 @@ +# A merge that changes the version IS the release. When main's pyproject.toml +# names a version with a CHANGELOG heading and no tag yet, this tags it, and the +# tag push runs container.yml, which promotes the image to that version and then +# opens a bump PR in every repo that pins python-sdlc. +# +# Without this, a tool update that merged itself would sit untagged until a +# person noticed, and no repo would receive it. + +name: Release + +on: + push: + branches: [main] + +concurrency: + group: release + cancel-in-progress: false + +permissions: {} + +jobs: + release: + name: Tag a new version + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + + - name: Is there an untagged version? + id: version + env: + GH_TOKEN: ${{ github.token }} + run: | + set -eu + v=$(sed -n 's/^version = "\(.*\)"$/\1/p' pyproject.toml | head -1) + [ -n "$v" ] || { echo "pyproject.toml names no version" >&2; exit 1; } + if gh release view "v$v" >/dev/null 2>&1; then + echo "v$v is already released" + echo "tag=" >> "$GITHUB_OUTPUT" + elif grep -q "^## \[$v\]" CHANGELOG.md; then + echo "tag=v$v" >> "$GITHUB_OUTPUT" + else + echo "v$v has no CHANGELOG heading; not releasing" >&2 + exit 1 + fi + + # The App's token, because a tag created with GITHUB_TOKEN does not start + # container.yml, and the release would never be promoted. + - name: Mint CI token + if: steps.version.outputs.tag != '' + id: ci-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ vars.CRUNCHTOOLS_CI_APP_ID || secrets.CRUNCHTOOLS_CI_APP_ID }} + private-key: ${{ secrets.CRUNCHTOOLS_CI_APP_KEY }} + owner: crunchtools + + - name: Release + if: steps.version.outputs.tag != '' + env: + GH_TOKEN: ${{ steps.ci-token.outputs.token }} + TAG: ${{ steps.version.outputs.tag }} + run: | + set -eu + v=${TAG#v} + notes=$(awk -v h="## [$v]" 'index($0, h) == 1 {f=1; next} f && /^## \[/ {exit} f' CHANGELOG.md) + gh release create "$TAG" --target "$GITHUB_SHA" --title "$TAG" --notes "$notes" diff --git a/.github/workflows/update-tools.yml b/.github/workflows/update-tools.yml new file mode 100644 index 0000000..aead4d1 --- /dev/null +++ b/.github/workflows/update-tools.yml @@ -0,0 +1,71 @@ +# A Gourmand or Gatehouse release reaches every Python repo in three steps, and +# this is the first: move the pin and bump this image's patch version, as a pull +# request that merges itself when CI is green. +# +# 1. this workflow new tool release -> PR moving the pin here +# 2. release.yml that PR merges -> python-sdlc vX.Y.Z is released +# 3. container.yml the release is promoted -> a bump PR in every repo +# that pins python-sdlc (scripts/bump_consumers.py) +# +# Gourmand and Gatehouse send `parent-image-updated` when they release. The +# weekly run catches a dispatch that was lost. + +name: Update baked tools + +on: + repository_dispatch: + types: [parent-image-updated] + schedule: + - cron: "0 7 * * 1" + workflow_dispatch: + +concurrency: + group: update-tools + cancel-in-progress: false + +permissions: {} + +jobs: + update: + name: Move the Gourmand and Gatehouse pins + runs-on: ubuntu-latest + permissions: + contents: read + steps: + # The crunchtools-ci App's token, not GITHUB_TOKEN: a pull request opened + # with GITHUB_TOKEN does not trigger CI, so it could never turn green and + # merge itself. + - name: Mint CI token + id: ci-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ vars.CRUNCHTOOLS_CI_APP_ID || secrets.CRUNCHTOOLS_CI_APP_ID }} + private-key: ${{ secrets.CRUNCHTOOLS_CI_APP_KEY }} + owner: crunchtools + + - uses: actions/checkout@v7 + with: + token: ${{ steps.ci-token.outputs.token }} + + - name: Compare the pins to the latest releases + id: update + env: + GH_TOKEN: ${{ steps.ci-token.outputs.token }} + run: python3 scripts/update_tools.py + + - name: Open the pull request and let it merge itself + if: steps.update.outputs.changed == 'true' + env: + GH_TOKEN: ${{ steps.ci-token.outputs.token }} + VERSION: ${{ steps.update.outputs.version }} + SUMMARY: ${{ steps.update.outputs.summary }} + run: | + set -eu + branch="chore/tools-$VERSION" + git switch -c "$branch" + git -c user.name=crunchtools-ci -c user.email=ci@crunchtools.com \ + commit -qam "chore: python-sdlc $VERSION ($SUMMARY)" + git push -q -u origin "$branch" + url=$(gh pr create --head "$branch" --title "chore: python-sdlc $VERSION" \ + --body "Moves the baked tool pins: $SUMMARY. Merging releases python-sdlc $VERSION, which opens a bump PR in every repo that pins it.") + gh pr merge "$url" --auto --merge diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d7f4766..acd671f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,18 +1,12 @@ # This repo's own gates. # -# Deliberately the CLASSIC fleet shape, not python-sdlc's own hooks, for two -# reasons: +# The classic fleet shape, not python-sdlc's own two hooks: this repo builds the +# image, so gating it on that image is circular on a first clone. Constitution +# XII (v1.30.0) exempts it by name (`sdlc_exempt`), with gourmand and gatehouse. # -# 1. Constitution XII currently REQUIRES a hook with `id: gourmand` running -# `quay.io/crunchtools/gourmand` and one with `id: gatehouse` running -# `quay.io/crunchtools/gatehouse`, and `validate-constitution.py` greps -# this file for those strings. Until the XII amendment lands, a repo using -# python-sdlc's hooks fails `Constitution / Validate constitution`. -# 2. Bootstrapping. This repo builds the image, so gating it on the image it -# has not built yet is circular on a first clone. -# -# This file switches to the one-block form once the amendment is tagged. That -# is the point at which python-sdlc gates itself. +# No container build at pre-push: that is minutes, and Scott's rule is that a +# push never waits on one. CI builds the image and runs the contract test +# against the pushed digest. default_install_hook_types: [pre-commit, pre-merge-commit, pre-push] @@ -27,12 +21,12 @@ repos: rev: v0.17.1 hooks: - id: ruff-check - args: [--fix, sdlc, validate, tests] + args: [--fix, sdlc, validate, tests, scripts] pass_filenames: false always_run: true stages: [pre-commit, pre-merge-commit] - id: ruff-format - args: [sdlc, validate, tests] + args: [sdlc, validate, tests, scripts] pass_filenames: false always_run: true stages: [pre-commit, pre-merge-commit] @@ -41,22 +35,13 @@ repos: hooks: - id: shellcheck name: shellcheck (bin/sdlc and the tests) - entry: podman --events-backend=none run --rm --log-driver=none -v .:/mnt:ro,Z -w /mnt docker.io/koalaman/shellcheck:v0.11.0 bin/sdlc tests/contract.sh tests/test_host_contract.sh tests/test_tiers.sh + entry: podman --events-backend=none run --rm --log-driver=none -v .:/mnt:ro,Z -w /mnt docker.io/koalaman/shellcheck:v0.11.0 bin/sdlc tests/contract.sh tests/test_host_contract.sh language: system pass_filenames: false always_run: true stages: [pre-commit, pre-merge-commit] - # The two tests that keep this repo's own promises. Both are pure host - # work -- no image needed -- so they belong at commit, where they are free. - - id: tiers - name: tier membership agrees with .pre-commit-hooks.yaml - entry: tests/test_tiers.sh - language: script - pass_filenames: false - always_run: true - stages: [pre-commit, pre-merge-commit] - + # Pure host work, no image needed, so it belongs at commit. - id: host-contract name: bin/sdlc needs only git, podman, pre-commit and gh entry: tests/test_host_contract.sh @@ -95,15 +80,3 @@ repos: pass_filenames: false always_run: true stages: [pre-commit, pre-merge-commit] - - # The contract test needs a built image, so it runs at pre-push rather than - # on every commit. - - repo: local - hooks: - - id: contract - name: contract test (builds the image) - entry: sh -c 'podman build -q -f Containerfile -t localhost/python-sdlc:precommit . && tests/contract.sh localhost/python-sdlc:precommit' - language: system - pass_filenames: false - always_run: true - stages: [pre-push] diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 9ed7547..19b0451 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -1,134 +1,33 @@ -# The hooks a consuming repo gets from ONE `repo:` block. +# Two hooks, one per git moment. Which checks each one runs is decided by the +# consuming repo's pyproject.toml -- [tool.sdlc.commit] and [tool.sdlc.push] -- +# and by the defaults in the image, never by this file. One list, in one place. # -# This file is the single authority for tier membership. `bin/sdlc` parses the -# `stages:` lines below to answer `sdlc commit` and `sdlc push`, and -# tests/test_tiers.sh asserts that parse still agrees with this file. Two copies -# of the tier table is how trentina's mypy hook drifted from its own script's -# resource budget the moment that budget stopped being a constant. -# -# THE TIER RULE (constitution XII): -# -# pre-commit what REWRITES your code in under a second, offline, free -# pre-push what renders a VERDICT, once, over origin/main...HEAD -- -# the same set and the same diff CI reviews -# -# That last clause is the one with teeth. If the push tier is green, CI is -# green, because they are the same checks on the same bytes. +# sdlc-commit `sdlc commit`: seconds, offline, free -- formatting and lint +# fixes, and the check that the push hook is installed +# sdlc-push `sdlc push`: every verdict, once, over origin/...HEAD; +# never a container build # # `language: script` because pre-commit resolves a script entry against this -# cloned repo (`unsupported_script.py` -> `prefix.path(cmd[0])`) and chdirs to -# the consumer's toplevel first, so `bin/sdlc` runs with the repo as $PWD. -# NOT `language: docker_image`: that shells out to `docker`, and the host -# contract gives us podman. - -# ---------------------------------------------------------------- commit tier - -- id: sdlc-ruff-check - name: ruff check (--fix) - description: Lint and auto-fix. Sub-second, offline, and it fixes rather than complains. - entry: bin/sdlc ruff-check - language: script - pass_filenames: false - always_run: true - stages: [pre-commit, pre-merge-commit] - -- id: sdlc-ruff-format - name: ruff format - description: Format. Same reasoning as ruff check -- free at commit, annoying at push. - entry: bin/sdlc ruff-format - language: script - pass_filenames: false - always_run: true - stages: [pre-commit, pre-merge-commit] +# cloned repo and chdirs to the consumer's toplevel first, so `bin/sdlc` runs +# with the repo as $PWD. Not `language: docker_image`: that shells out to +# `docker`, and the host contract gives us podman. +# +# To skip one check for a false positive: SDLC_SKIP=gatehouse git push. The +# finding still has to be answered on the pull request. -# Cheap, offline, and the one thing that proves the push tier will actually run. -# `pre-commit install` in an existing clone does not add stages it did not -# already wire, so `default_install_hook_types` can silently do nothing -- and -# with every verdict now at pre-push, that failure costs you EVERY gate rather -# than one. This is the hook that notices. -- id: sdlc-hooks-installed - name: pre-push hooks are installed - description: Fails if the pre-push hook is missing, which would make the whole push tier a no-op. - entry: bin/sdlc hooks-installed +- id: sdlc-commit + name: sdlc commit + description: Formatting and lint fixes, and the check that the push hook is installed. + entry: bin/sdlc commit language: script pass_filenames: false always_run: true stages: [pre-commit, pre-merge-commit] -# ------------------------------------------------------------------ push tier - -- id: sdlc-mypy - name: mypy - description: Type check. Needs the project's locked deps, so it cannot be sub-second. - entry: bin/sdlc mypy - language: script - pass_filenames: false - always_run: true - stages: [pre-push] - -- id: sdlc-pytest - name: pytest (full suite, containerised) - entry: bin/sdlc pytest - language: script - pass_filenames: false - always_run: true - stages: [pre-push] - -- id: sdlc-gourmand - name: Gourmand (AI slop detection) - entry: bin/sdlc gourmand - language: script - pass_filenames: false - always_run: true - stages: [pre-push] - -# One Gatehouse hook, not two. The old pair existed because of a spelling -# problem: at pre-commit time `origin/main...HEAD` is the already-committed work -# and EXCLUDES the change being made, so a staged-diff hook was the workaround. -# At pre-push HEAD already holds every commit, so the range is exactly the diff -# the PR will show and exactly what CI reviews. -- id: sdlc-gatehouse - name: Gatehouse (AI code review, branch diff) - description: Reviews origin/main...HEAD -- the same diff CI's Gatehouse reviews. - entry: bin/sdlc gatehouse - language: script - pass_filenames: false - always_run: true - stages: [pre-push] - -- id: sdlc-semgrep - name: Semgrep (rule tests + scan) - entry: bin/sdlc semgrep - language: script - pass_filenames: false - always_run: true - stages: [pre-push] - -# Gates, unlike the 17 repos in the fleet running `pip-audit … || true`, where -# the scan cannot fail and never has. -- id: sdlc-pip-audit - name: Dependency CVE scan (pip-audit) - entry: bin/sdlc pip-audit - language: script - pass_filenames: false - always_run: true - stages: [pre-push] - -- id: sdlc-constitution - name: Constitution validator - description: Validates the manifest against the constitution version the repo itself pins. - entry: bin/sdlc constitution - language: script - pass_filenames: false - always_run: true - stages: [pre-push] - -# Minutes, and a database build. Local because Scott asked for it, at pre-push -# because it is not a linter. Affordable at all only because the Python-only -# CodeQL bundle is 110 MB against the full bundle's 1.75 GB. -- id: sdlc-codeql - name: CodeQL (python-security-extended + repo queries) - entry: bin/sdlc codeql +- id: sdlc-push + name: sdlc push + description: Every verdict, once, over the branch diff -- the checks CI runs. + entry: bin/sdlc push language: script pass_filenames: false always_run: true diff --git a/CHANGELOG.md b/CHANGELOG.md index a564175..b2852bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,65 @@ All notable changes to this project are documented here, following ## [Unreleased] +## [0.2.0] - 2026-11-04 + +### Changed + +- **Three commands, three config blocks.** `sdlc commit` (the pre-commit hook), + `sdlc push` (the pre-push hook) and `sdlc ci` (the CI job) are configured by + `[tool.sdlc.commit]`, `[tool.sdlc.push]` and `[tool.sdlc.ci]`. Each runs its + defaults; a block's `checks` adds and `skip` removes; a check's settings live + in its own `[tool.sdlc..]` table. `sdlc tiers` prints what + each command runs. Constitution XII v1.30.0 requires this shape. +- **ci = push minus the two reviewers, plus the repo's own checks.** CI runs + Gourmand and Gatehouse through their own workflows; running them again here + would pay twice and make the reviewer blocking. +- **The hooks file exports two hooks**, `sdlc-commit` and `sdlc-push`, instead + of eleven. Which checks run is decided by the config alone, so there is no + second list to keep in sync. +- **Paths come from each tool's own config.** `ruff check .`, `ruff format .` + and bare `mypy` read `[tool.ruff]` and `[tool.mypy] files`. `lint_paths`, + `type_paths`, `scan_paths`, `test_env`, `pytest_args`, `semgrep_config`, + `codeql_config` and `codeql_gate` are gone; a stray key in `[tool.sdlc]` is + refused rather than ignored. +- **CodeQL fails on new findings.** Any finding not in + `.sdlc/codeql-baseline.json` fails; `sdlc codeql-baseline` writes the file. + It used to report and pass, which made it a push-tier check that could not + fail. + +### Added + +- **Kinds for a repo's own checks:** `image-build` and `pytest-in-image`. The + second runs tests inside the built image with no network, a read-only root + and no capabilities, with pytest installed into the cache volume at the + versions the repo's `uv.lock` pins. There is no shell kind. +- `SDLC_SKIP=[,]` skips a check once. pre-commit's own `SKIP=` + cannot reach inside a single hook. +- `bin/sdlc` refuses any planned host step that is not `podman build` or + `podman run`, so the host contract is held by the launcher, not trusted to a + repo's config. A host step is an argv split on the ASCII unit separator and + never `eval`ed, and the plan refuses any value with a tab, a newline or that + separator in it, so a config value cannot forge a second plan line or be read + as shell. `tests/test_host_contract.sh` proves both. +- `tests/test_plan.py`; it and `tests/test_codeql_globs.py`, which CI never ran + before, now run in the built image on every build. +- **A new Gourmand or Gatehouse reaches every Python repo on its own.** Their + releases trigger `update-tools.yml`, which moves the pin and bumps this + image's patch version in a pull request that merges itself when green; + `release.yml` tags the merged version; and after the tag's image is promoted, + `container.yml` opens an auto-merging bump PR in every repo that pins + python-sdlc (`scripts/bump_consumers.py`). Rebuilding on a tool release, which + is what v0.1.0 did, changed nothing: the Containerfile pins exact versions. + +### Removed + +- Trivy, from this image's own build. pip-audit covers dependencies; fleet + image scanning moves to a scheduled job (RT #1551). +- `tests/test_tiers.sh` and `examples/pre-commit.yaml`; the README holds the + one block a repo copies. +- The contract test at this repo's own pre-push, which built the image: a push + never waits on a container build. CI runs it against the pushed digest. + ## [0.1.0] - 2026-11-04 First release. One container image and one script holding every CrunchTools @@ -94,5 +153,6 @@ cached tag was never refreshed — that gate had been running a two-week-old reviewer while CI pulled fresh. Local and CI were different reviewers on the same diff. -[Unreleased]: https://github.com/crunchtools/python-sdlc/compare/v0.1.0...HEAD +[Unreleased]: https://github.com/crunchtools/python-sdlc/compare/v0.2.0...HEAD +[0.2.0]: https://github.com/crunchtools/python-sdlc/releases/tag/v0.2.0 [0.1.0]: https://github.com/crunchtools/python-sdlc/releases/tag/v0.1.0 diff --git a/Containerfile b/Containerfile index 5da2496..f882c22 100644 --- a/Containerfile +++ b/Containerfile @@ -44,7 +44,7 @@ LABEL description="Every CrunchTools Python SDLC check in one image: ruff, mypy, LABEL org.opencontainers.image.source="https://github.com/crunchtools/python-sdlc" LABEL org.opencontainers.image.description="Every CrunchTools Python SDLC check in one image." LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later" -LABEL version="0.1.0" +LABEL version="0.2.0" # What the image carries, readable without starting anything. `sdlc versions` # prints these and a consuming repo's CI pins them. diff --git a/IMAGE_TAG b/IMAGE_TAG index 6e8bf73..0ea3a94 100644 --- a/IMAGE_TAG +++ b/IMAGE_TAG @@ -1 +1 @@ -0.1.0 +0.2.0 diff --git a/README.md b/README.md index f8f11b1..bf6df20 100644 --- a/README.md +++ b/README.md @@ -1,85 +1,155 @@ # python-sdlc -One container image and one script for every CrunchTools Python SDLC check. +Every CrunchTools Python check in one container image, run the same way on a +laptop and in CI. -A consuming repo's whole gate configuration is one block: +Constitution XII (v1.30.0) requires it for every Python repo — any repo with a +`pyproject.toml` — except `gourmand`, `gatehouse` and this one, which build what +the image contains. + +## Three commands, three config blocks + +| Command | Runs when | Config block in `pyproject.toml` | +|---|---|---| +| `sdlc commit` | the pre-commit hook | `[tool.sdlc.commit]` | +| `sdlc push` | the pre-push hook | `[tool.sdlc.push]` | +| `sdlc ci` | the CI job named `sdlc ci` | `[tool.sdlc.ci]` | + +- **commit** takes seconds, works offline and costs nothing: formatting and + lint fixes. +- **push** renders every verdict, once, over `origin/...HEAD`. + It never builds a container — that is minutes, and a push you wait minutes + for is a push people stop making. +- **ci** runs push's checks except Gourmand and Gatehouse, which CI already runs + through their own workflows, plus the repo's own checks, such as building its + image. + +Because the laptop and CI run the same image and the same commands, a green +push predicts a green CI. + +## Wiring a repo + +`.pre-commit-config.yaml`, the whole file: ```yaml -# .pre-commit-config.yaml -default_install_hook_types: [pre-commit, pre-merge-commit, pre-push] +default_install_hook_types: [pre-commit, pre-push] repos: - repo: https://github.com/crunchtools/python-sdlc - rev: v0.1.0 + rev: v0.2.0 hooks: - - id: sdlc-ruff-check - - id: sdlc-ruff-format - - id: sdlc-hooks-installed - - id: sdlc-mypy - - id: sdlc-pytest - - id: sdlc-gourmand - - id: sdlc-gatehouse - - id: sdlc-semgrep - - id: sdlc-pip-audit - - id: sdlc-constitution - - id: sdlc-codeql + - id: sdlc-commit + - id: sdlc-push ``` -That `rev:` pins the launcher and the image together, and the image pins every -tool inside it. One pin per repo. +CI, one job, which reads the release from that same `rev:`: -## Why +```yaml + sdlc: + name: sdlc ci + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: sdlc ci + run: | + rev=$(awk '/repo: https:\/\/github.com\/crunchtools\/python-sdlc/ {f=1; next} + f && /rev:/ {print $2; exit}' .pre-commit-config.yaml) + git clone -q --depth 1 --branch "$rev" https://github.com/crunchtools/python-sdlc "$RUNNER_TEMP/sdlc" + $RUNNER_TEMP/sdlc/bin/sdlc ci +``` -Across 69 CrunchTools checkouts (35 Python), the tools were right and the -versions were not: +That `rev:` pins the launcher, the image and every tool inside it, for the +laptop and CI both. One pin. -| | | +## What a repo gets with no config at all + +| Command | Runs | |---|---| -| ruff `rev:` in pre-commit | **9 distinct versions**, v0.8.6 → v0.17.1 | -| ruff `line-length` | 5 values: 88, 100, 110, 120, unset | -| mypy | **15 of 35 Python repos had none at all** | -| `pip-audit` | 17 repos ran it as `… \|\| true`, so the scan could not fail | -| Trivy action | `@v0.36.0`, a SHA, and `@master` in 6 places | +| `commit` | ruff-check, ruff-format, hooks-installed | +| `push` | mypy, pytest, gourmand, gatehouse, semgrep, pip-audit, constitution, codeql | +| `ci` | mypy, pytest, semgrep, pip-audit, constitution, codeql | -And a concrete one, found while building this: `trentina/scripts/gates.sh` pins -`gatehouse:latest`, podman's default pull policy is `missing`, so a cached tag -is never refreshed. That gate had been running a **two-week-old reviewer** while -CI pulled fresh. Local and CI were different reviewers on the same diff, which -is the failure mode this image exists to remove. +`sdlc tiers` prints what each command runs in the repo you are in. -## The tier rule +## Configuring it -Constitution XII. Not lint-versus-tests — **what rewrites your code in under a -second** versus **what renders a verdict**. +One rule for all three blocks: -| Tier | Targets | Properties | -|---|---|---| -| `pre-commit` | `ruff-check --fix`, `ruff-format`, `hooks-installed` | sub-second, offline, free, auto-fixing | -| `pre-push` | `mypy`, `pytest`, `gourmand`, `gatehouse`, `semgrep`, `pip-audit`, `constitution`, `codeql` | one run per push, over `origin/main...HEAD` | +- `checks` **adds** to that command's defaults; `skip` removes from them. + What push skips, ci skips too. +- A check that needs settings gets its own table: + `[tool.sdlc..]`. +- A check that is not built in must say what `kind` it is. +- Check names are unique, and `sdlc ` runs any one of them — the inner + loop. -The push tier is **the same set and the same diff CI reviews**. That is the -whole mechanism behind things sailing through CI: if the push tier is green, CI -is green, because they are the same checks on the same bytes. +Paths are not set here. Each tool reads its own config: `[tool.ruff]` excludes, +`[tool.mypy] files`, `[tool.pytest.ini_options] testpaths`. -Commits are therefore instant and work offline. `mypy` is on the push side -despite feeling like linting: it needs the project's locked dependencies -installed to see type stubs, so it cannot be sub-second. +A typical repo writes nothing. Trentina writes this: -## Inner loop +```toml +[tool.sdlc.ci] +checks = ["image", "image-tests"] -Run one target, as often as you like, while fixing it: +[tool.sdlc.ci.image] +kind = "image-build" +build_args = { VERSION = "@project.version" } +labels = { "org.opencontainers.image.revision" = "@git.sha" } -```bash -sdlc mypy -sdlc semgrep -sdlc pytest -sdlc list # what targets exist -sdlc tiers # which tier each one runs in -sdlc versions # the exact tool versions this image carries +[tool.sdlc.ci.image-tests] +kind = "pytest-in-image" +tests = ["tests/image/"] ``` -`sdlc` is `bin/sdlc` from this repo; pre-commit clones it for you, or add it to -`PATH` for direct use. +### Built-in checks and their settings + +| Check | Settings | +|---|---| +| `pytest` | `env` (table), `args` (default `["-q", "-rfE"]`) | +| `semgrep` | `config` (default `.semgrep`), `tests` (default `.semgrep/run-tests.sh`), `paths` (default `["."]`) | +| `codeql` | `config` (default `.github/codeql/codeql-config.yml`), `baseline` (default `.sdlc/codeql-baseline.json`) | +| `constitution` | `manifest` (default `.specify/memory/constitution.md`) | +| `ruff-check`, `ruff-format`, `mypy`, `gourmand`, `gatehouse`, `pip-audit`, `hooks-installed` | none | + +### Kinds, for a repo's own checks + +**`image-build`** — `podman build`, leaving the image at a known tag for the +next check and for a CI publish step. + +| Setting | Type | Default | +|---|---|---| +| `containerfile` | path in the repo | `"Containerfile"` | +| `context` | path in the repo | `"."` | +| `build_args` | table | none | +| `labels` | table | none | +| `tag` | string | `"localhost/:sdlc"` | + +**`pytest-in-image`** — runs tests inside a built image, locked down the way +production runs it: no network, a read-only root, no capabilities. pytest is +installed into the cache volume at the versions the repo's `uv.lock` pins, not +added to the image. + +| Setting | Type | Default | +|---|---|---| +| `tests` | list of paths | **required** | +| `mounts` | list of paths in the repo, mounted read-only | `["tests"]` | +| `env` | table | none | +| `args` | list | `["-q", "-rfE"]` | +| `image` | string | `"localhost/:sdlc"` | +| `network` | boolean | `false` | + +Paths must stay inside the repository; `..` or an absolute path is refused. +Values in `build_args`, `labels` and `env` may be `@project.version`, +`@git.sha` or `@now`. + +There is no shell kind. A repo that needs something these cannot express gets a +new kind here, so the config stays something a person can read. + +## Skipping a check + +`SDLC_SKIP=gatehouse git push` skips one check, once, for a false positive. The +finding still has to be answered on the pull request. ## Exit codes @@ -89,104 +159,42 @@ sdlc versions # the exact tool versions this image carries 2 COULD NOT RUN ``` -**2 is never to be worked around.** A gate that cannot run and reports success -is worse than no gate, because the pipeline goes green and nobody looks again. -Constitution XII was written after a repo's test runner "skipped three gates -with a warning and printed the other results as if that were a pass". - -This is also where the script deliberately differs from the `gates.sh` it -replaces: that mapped podman's own 125–127 onto exit 1, reporting "found -something" for "could not invoke the container". - -## Per-repo configuration +`2` is never treated as a pass. A gate that cannot run and reports success is +worse than no gate. -The image carries tools; the repo carries policy. `pyproject.toml`'s -`[tool.ruff]`, `[tool.mypy]`, `[tool.pytest]`, plus `.semgrep/`, -`.github/codeql/`, `gourmand*.toml` and `.gatehouse-ignore` are read from the -repo as they always were. Baking a ruleset into the image would mean one repo's -rule change breaking 35. +## CodeQL fails only on new findings -One optional table covers what a fleet-wide script cannot guess: - -```toml -[tool.sdlc] -lint_paths = ["src", "tests", "corpora"] # default: ["src", "tests"] -type_paths = ["src"] -scan_paths = ["src"] -semgrep_config = ".semgrep" -semgrep_tests = ".semgrep/run-tests.sh" -codeql_config = ".github/codeql/codeql-config.yml" -pytest_args = ["-q", "-rfE"] - -[tool.sdlc.test_env] -GEMINI_API_KEY = "test_key_for_ci" -``` - -Set each value to what your CI already does, so a local verdict is the CI -verdict. +`codeql` fails on any finding not in `.sdlc/codeql-baseline.json`. The baseline +is the findings someone has looked at and accepted, committed where a reviewer +sees it. `sdlc codeql-baseline` writes it from the current findings. See +`docs/codeql.md`. ## What is in the image | Tool | Pinned by | |---|---| -| ruff, semgrep, pip-audit | this image (`uv tool install`, one venv each) | -| Gourmand, Gatehouse | this image (`COPY --from` their official images) | -| CodeQL | this image (the **Python-only** bundle, 110 MB; the full one is 1.75 GB) | -| constitution validator | the repo being checked — cloned at the tag its own manifest names | -| **mypy, pytest** | **the repo's `uv.lock`** | - -That last row is the one honest seam. mypy runs plugins and pytest runs the -project's plugins *inside their own interpreter*, so they have to come from the -project's environment, not from here. That is still one pin per repo — the -lockfile, which is where constitution XV already says a Python dependency's -version belongs — but it is not this image's pin, and `sdlc` exits **2** rather -than reporting a pass if either is missing from the lock. - -The constitution validator is cloned rather than baked for a different reason: -the fleet spans `Inherits:` v1.0.0 to v1.29.0, so a baked copy would judge most -repos by rules they have not adopted. - -`sdlc versions` prints what is installed next to what was pinned, because the -two answer different questions. They need not match: `crunchtools/gourmand:1.0.0` -is that packaging repo's own first release and the binary inside it is upstream -`mattdm/gourmand` v0.16.5. Both are correct. The tag is what CI pins; the -self-reported version is which upstream you are actually running, and a tag -alone does not tell you that. - -## Trivy is not here - -Deliberately, and it is CI-only. It is **not** redundant with `pip-audit`: -pip-audit reads the Python lock, Trivy reads the OS packages in the image layers -(glibc, openssl, zlib), and nothing else in the pipeline looks at those. CI is -where the built image exists, so that is where it runs. - -## How the repo gets into the container - -Gourmand and Gatehouse have been doing this across the fleet for a year, so -this copies their convention rather than inventing one: - -```bash -podman run --rm -v "$PWD:/repo:ro,Z" -w /repo quay.io/crunchtools/python-sdlc:v0.1.0 mypy -``` +| ruff, semgrep, pip-audit | this image | +| Gourmand, Gatehouse | this image, copied from their official images | +| CodeQL | this image (the Python-only bundle, 110 MB rather than 1.75 GB) | +| constitution validator | the repo — cloned at the release its own manifest names | +| mypy, pytest | the repo's `uv.lock`, because they load the project's own plugins | -The container runs as its own root. Under rootless podman that maps to the -invoking user, so a file `ruff --fix` writes is owned by the developer and the -shared cache volume is writable — with no `--user` and no `--userns=keep-id`. +`sdlc versions` prints what the image carries. -One deliberate difference from those two: `z`, not `Z`. A tier runs several -containers over one tree and `Z` relabels it *exclusively*, taking it out from -under whichever started first. +## How a new Gourmand or Gatehouse arrives + +Their releases trigger a rebuild here, which is released as a new python-sdlc +version, which opens a version-bump pull request in each Python repo. The laptop +then runs the reviewer CI runs. ## Host contract -Constitution XII: a developer machine has `git`, `podman`, `pre-commit` and -`gh`, plus `~/.config/mcp-env/gatehouse.env`, and nothing else may be assumed. -`bin/sdlc` is POSIX shell and runs only `git` and `podman`; everything else is -inside the image. `tests/test_host_contract.sh` asserts that, rather than -relying on the validator not looking. +A developer machine needs `git`, `podman`, `pre-commit` and `gh`. `bin/sdlc` is +POSIX shell, runs only `git` and `podman`, and refuses any step that is not a +`podman build` or `podman run`. `tests/test_host_contract.sh` proves it with a +restricted `PATH`. ## Licence -AGPL-3.0-or-later. The CodeQL bundle is redistributed under GitHub's own terms -for the CodeQL CLI, which permit use for analysing repositories — see -`docs/codeql.md`. +AGPL-3.0-or-later. The CodeQL bundle is redistributed under GitHub's CodeQL CLI +terms — see `docs/codeql.md`. diff --git a/bin/sdlc b/bin/sdlc index e9f8ee1..065d3e1 100755 --- a/bin/sdlc +++ b/bin/sdlc @@ -158,6 +158,9 @@ run_in_image() { -e SEMGREP_SETTINGS_FILE=/tmp/semgrep.yml \ -e "SDLC_PROJECT=$PROJECT" \ -e "SDLC_JOBS=$GATE_CPUS" \ + -e "SDLC_HOST_REPO=$REPO" \ + -e "SDLC_CACHE_VOLUME=$CACHE_VOLUME" \ + ${SDLC_SKIP:+-e SDLC_SKIP} \ ${SDLC_GATEHOUSE_SAMPLES:+-e SDLC_GATEHOUSE_SAMPLES} \ "$@" @@ -225,81 +228,129 @@ t_gatehouse() { --env-file "$_env" -i "$IMAGE" gatehouse } -# -------------------------------------------------------------- tier membership +# --------------------------------------------------------------------- the plan # -# Parsed from .pre-commit-hooks.yaml, which is the single authority. -# tests/test_tiers.sh asserts this parse still agrees with that file. - -targets_in_stage() { - awk -v want="$1" ' - /^- id:/ { entry=""; stages="" } - /^ entry: bin\/sdlc/ { entry=$NF } - /^ stages:/ { stages=$0 } - stages != "" && entry != "" { - if (index(stages, want)) print entry - entry=""; stages="" - } - ' "$HOOKS_DIR/.pre-commit-hooks.yaml" +# Which checks run is decided by the repo's [tool.sdlc.commit], [tool.sdlc.push] +# and [tool.sdlc.ci] blocks, and this shell cannot read TOML. So it asks the +# image for a plan -- one line per step, `check TAB mode TAB payload` -- and +# carries it out. The modes: +# +# image-ro | image-rw `sdlc ` in the image, the tree ro or rw +# local a check this file answers itself +# host a podman argv, words joined by the unit separator +# +# A host line that does not start with `podman build` or `podman run` is +# refused: the host contract is held here, by the launcher, not trusted to +# anything a repo's config says. The argv is SPLIT on the separator, never +# `eval`ed, so nothing in it is ever read as shell -- no expansion, no command +# substitution -- before or after that check. + +TAB=$(printf '\t') +US=$(printf '\037') + +run_host() { + _ifs=$IFS + IFS=$US + # shellcheck disable=SC2086 # splitting on the separator is the point + set -- $1 + IFS=$_ifs + [ "${1:-}" = podman ] || die "a host step may only run podman; refused: ${1:-}" + _sub=${2:-} + shift 2 + # The budget is applied here so it is defined once, whatever the step. + case "$_sub" in + build) set -- build --cpu-period 100000 --cpu-quota "$(( GATE_CPUS * 100000 ))" \ + --memory "$GATE_MEMORY" "$@" ;; + run) set -- --events-backend=none run --log-driver=none \ + --cpus "$GATE_CPUS" --memory "$GATE_MEMORY" --cpu-shares 256 "$@" ;; + *) die "a host step may only build or run; refused: podman $_sub" ;; + esac + _rc=0 + podman "$@" &2; return 2 ;; + esac +} + +run_step() { # mode payload -> the step's exit status + case "$1" in + image-ro) run_in_image ro nostdin "$IMAGE" "$2" ;; + image-rw) run_in_image rw nostdin "$IMAGE" "$2" ;; + local) + case "$2" in + hooks-installed) ( t_hooks_installed ) ;; + gatehouse) ( t_gatehouse ) ;; + *) die "the image asked for an unknown local check: $2" ;; + esac ;; + host) run_host "$2" ;; + *) die "the image asked for an unknown step mode: $1" ;; + esac } -run_tier() { - _stage=$1 - _targets=$(targets_in_stage "$_stage") - [ -n "$_targets" ] || die "no targets found for stage $_stage in .pre-commit-hooks.yaml" +run_plan() { # commit | push | ci | + _plan=$(run_in_image ro nostdin "$IMAGE" plan "$1") \ + || die "the image could not plan '$1'; nothing was checked" + [ -n "$_plan" ] || die "the image planned nothing for '$1'" + _worst=0 - for _t in $_targets; do + _failed= + _ifs=$IFS + set -f + IFS=' +' + for _line in $_plan; do + IFS=$_ifs + _check=${_line%%"$TAB"*} + _rest=${_line#*"$TAB"} + _mode=${_rest%%"$TAB"*} + _payload=${_rest#*"$TAB"} + # One failed step ends that check; the next check still runs. + case " $_failed " in *" $_check "*) continue ;; esac _rc=0 - "$0" "$_t" || _rc=$? - # A later 1 must not mask an earlier 2. - if [ "$_rc" -eq 2 ] || [ "$_worst" -eq 2 ]; then _worst=2 - elif [ "$_rc" -ne 0 ]; then _worst=1 + run_step "$_mode" "$_payload" || _rc=$? + if [ "$_rc" -ne 0 ]; then + _failed="$_failed $_check" + printf '\n%s: %s %s\n' "$ME" "$_check" \ + "$([ "$_rc" -eq 2 ] && echo 'COULD NOT RUN' || echo 'FAILED')" >&2 + # A later 1 must not mask an earlier 2. + if [ "$_rc" -eq 2 ] || [ "$_worst" -eq 2 ]; then _worst=2; else _worst=1; fi fi done + IFS=$_ifs + set +f return "$_worst" } # ---------------------------------------------------------------------- dispatch -case "${1:-list}" in - # Host-only. - hooks-installed) t_hooks_installed ;; - gatehouse) SDLC_DEPTH=0 t_gatehouse ;; - - # Batches. Named by stage so there is one vocabulary, not two. - commit) SDLC_DEPTH=0 run_tier pre-commit ;; - push) SDLC_DEPTH=0 run_tier pre-push ;; - - # These write the tree, so they are the only rw mounts. - # ruff-check --fix / ruff-format rewrite your source, which is the point - # gourmand writes .gourmand-cache/ (which is why the - # fleet's own hook mounts `-v .:/src:Z` - # read-write while gatehouse's is ro) - # - # Gourmand was on `ro` here at first and appeared to pass, because the repo it - # was tested against already had a .gourmand-cache/ directory from an earlier - # run. A clean checkout gave "Read-only file system (os error 30)" and exit 3. - ruff-check|ruff-format|gourmand) - run_in_image rw nostdin "$IMAGE" "$1" ;; - - list|versions|mypy|pytest|semgrep|pip-audit|constitution|codeql) - run_in_image ro nostdin "$IMAGE" "$1" ;; - - tiers) printf 'pre-commit:\n'; targets_in_stage pre-commit | sed 's/^/ /' - printf 'pre-push:\n'; targets_in_stage pre-push | sed 's/^/ /' ;; - +case "${1:---help}" in -h|--help|help) cat <... | commit | push | list | versions | tiers - - commit | push run every target in that tier (the hooks do this) - run one target -- the inner loop - list targets the image provides +usage: $ME commit | push | ci | | tiers | list | versions | codeql-baseline + + commit the pre-commit hook: seconds, offline, free + push the pre-push hook: every verdict, no container builds + ci what CI runs: push's checks except the two reviewers, + plus the repo's own + one check by name -- the inner loop + tiers what each command runs in this repo + list the built-in checks and the kinds versions the exact tool versions the image carries - tiers which targets run in which tier + codeql-baseline accept every current CodeQL finding into the baseline +Configure each command in pyproject.toml: [tool.sdlc.commit], +[tool.sdlc.push], [tool.sdlc.ci]. Skip one check once: SDLC_SKIP=name. Exit 2 means a check COULD NOT RUN and is never to be worked around. Image: $IMAGE EOF ;; - *) printf '%s: unknown target: %s (try --help)\n' "$ME" "$1" >&2; exit 2 ;; + tiers|list|versions) run_in_image ro nostdin "$IMAGE" "$1" ;; + codeql-baseline) run_in_image rw nostdin "$IMAGE" "$1" ;; + *) SDLC_DEPTH=0 run_plan "$1" ;; esac diff --git a/docs/codeql.md b/docs/codeql.md index 42f6643..51c1894 100644 --- a/docs/codeql.md +++ b/docs/codeql.md @@ -13,31 +13,24 @@ It builds a database. On trentina that is 747 modules in ~42 s, then ~55 query evaluations, for a few minutes end to end. That is not a linter and cannot run on every commit. -## Why findings do not block by default - -`[tool.sdlc] codeql_gate` defaults to **false**, and this is parity with CI -rather than laxness. - -GitHub's CodeQL Action uploads SARIF to code scanning. The `CodeQL Analysis` -check **passes whether or not there are findings** — each finding becomes an -alert in the Security tab, where it is triaged and often dismissed. Measured on -trentina's `main` at `c32a1dc9`, a commit whose CodeQL check is green: **19 -findings**, including `py/clear-text-logging-sensitive-data` and trentina's own -`trentina/tool-arg-to-log`. - -So a local gate that blocked on any finding would be strictly stricter than CI. -On trentina it would block every push until all 19 were fixed or suppressed, -which inverts the promise this whole image is built on — that a green push tier -means a green CI. A gate nobody can pass is a gate everybody skips, and -`SKIP=sdlc-codeql` becomes muscle memory. - -**The honest gap:** without GitHub's dismissal state, the local run cannot tell -a known, accepted finding from a new one. So it reports rather than judges. - -**The fix, which is the next piece of work here:** a committed baseline of -accepted findings, so only *new* ones block. That is how a repo gets a CodeQL -gate with teeth without first clearing a backlog. Until then, -`codeql_gate = true` is available for a repo whose findings are already at zero. +## It fails only on new findings + +A check in the push tier that cannot fail is a check that lies about what it +did, so `codeql` fails. But on trentina's `main` it reports 19 findings that +were triaged and accepted in GitHub's Security tab, which a local run cannot +see. Failing on all of them would block every push until a backlog was cleared, +and people would learn `SDLC_SKIP=codeql` instead. + +So the acceptance is committed: `.sdlc/codeql-baseline.json` lists the findings +someone has looked at, and only a finding NOT in it fails. Each entry is the +rule, the file and CodeQL's own line hash — the hash, not the line number, so a +finding that moves because code was added above it is still the same finding. + + sdlc codeql-baseline # accept every current finding + +That command is the one way to make a finding stop failing without fixing it, +and it lands in a commit a reviewer reads. With no baseline file, every finding +fails. ## The config file diff --git a/examples/pre-commit.yaml b/examples/pre-commit.yaml deleted file mode 100644 index 10ba9b2..0000000 --- a/examples/pre-commit.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Drop in as .pre-commit-config.yaml. This is the whole file. -# -# `default_install_hook_types` must stay in the CONSUMER: a hooks repo cannot -# set it, and without it `pre-commit install` wires only pre-commit -- so the -# pre-push tier silently never runs. With every verdict now at pre-push, that -# costs you EVERY gate rather than one, which is why `sdlc-hooks-installed` is -# in the commit tier to notice. - -default_install_hook_types: [pre-commit, pre-merge-commit, pre-push] - -repos: - - repo: https://github.com/crunchtools/python-sdlc - rev: v0.1.0 - hooks: - # Commit tier: sub-second, offline, free, and auto-fixing. - - id: sdlc-ruff-check - - id: sdlc-ruff-format - - id: sdlc-hooks-installed - - # Push tier: everything that renders a verdict, once, over - # origin/main...HEAD -- the same set and the same diff CI reviews. - - id: sdlc-mypy - - id: sdlc-pytest - - id: sdlc-gourmand - - id: sdlc-gatehouse - - id: sdlc-semgrep - - id: sdlc-pip-audit - - id: sdlc-constitution - - id: sdlc-codeql diff --git a/pyproject.toml b/pyproject.toml index c86f045..95d4e61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ [project] name = "python-sdlc" -version = "0.1.0" +version = "0.2.0" description = "One container image and one script for every CrunchTools Python SDLC check" requires-python = ">=3.14" license = "AGPL-3.0-or-later" @@ -50,9 +50,3 @@ max-complexity = 10 "validate" = ["S404"] # A test asserts. That is what it is for. "tests/*" = ["S101", "PT018"] - -# Dogfooding: once the XII amendment lands and this repo's hooks switch to -# python-sdlc's own, these are the seams it will read. -[tool.sdlc] -lint_paths = ["sdlc", "validate", "tests"] -type_paths = ["sdlc", "validate"] diff --git a/scripts/bump_consumers.py b/scripts/bump_consumers.py new file mode 100755 index 0000000..d0aeb4b --- /dev/null +++ b/scripts/bump_consumers.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Open a version-bump pull request in every repo that pins python-sdlc. + +Run after a release is promoted, so the image a bump points at exists. Each +consuming repo gets one PR moving its `rev:`, queued for auto-merge, and its own +CI decides whether it lands -- a bad release fails per repo, not fleet-wide. +The same shape as crunchtools/constitution's scripts/fleet-bump.py: clone, +rewrite, push, open, queue, and name the repos that failed so they can be rerun. + +Usage: scripts/bump_consumers.py vX.Y.Z [--only repo,repo] [--dry-run] +Needs GH_TOKEN able to push branches and open PRs across the org. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +ORG = "crunchtools" +SELF = "python-sdlc" +PIN = re.compile( + r"(-\s*repo:\s*https://github\.com/crunchtools/python-sdlc(?:\.git)?\s*\n\s*rev:\s*)(v\d+\.\d+\.\d+)" +) + + +def gh(*args: str) -> str: + return subprocess.run(["gh", *args], capture_output=True, text=True, check=True).stdout + + +def org_repos() -> list[str]: + out = gh( + "api", + "--paginate", + f"orgs/{ORG}/repos?per_page=100", + "--jq", + ".[] | select(.archived | not) | .name", + ) + return sorted(name for name in out.split() if name != SELF) + + +def pinned_rev(repo: str) -> str | None: + """The python-sdlc rev a repo pins, or None if it does not pin one.""" + proc = subprocess.run( + ["gh", "api", f"repos/{ORG}/{repo}/contents/.pre-commit-config.yaml"], + capture_output=True, + text=True, + check=False, + ) + # Only a missing file means "does not pin". A rate limit, an auth failure + # or a timeout must fail the run, or the repo is silently skipped and the + # run reports success without having bumped it. + if proc.returncode != 0: + if "HTTP 404" in proc.stderr: + return None + raise RuntimeError(f"cannot read {repo}'s .pre-commit-config.yaml: {proc.stderr.strip()}") + raw = proc.stdout + text = base64.b64decode(json.loads(raw)["content"]).decode("utf-8", "replace") + match = PIN.search(text) + return match.group(2) if match else None + + +def bump(repo: str, version: str) -> str: + branch = f"chore/python-sdlc-{version}" + with tempfile.TemporaryDirectory() as workdir: + root = Path(workdir) / repo + gh("repo", "clone", f"{ORG}/{repo}", str(root), "--", "--depth", "1", "-q") + config = root / ".pre-commit-config.yaml" + text, n = PIN.subn(rf"\g<1>{version}", config.read_text(encoding="utf-8")) + if n != 1: + raise RuntimeError(f"expected one python-sdlc pin, found {n}") + config.write_text(text, encoding="utf-8") + + def git(*a: str) -> None: + subprocess.run(["git", "-C", str(root), *a], check=True, capture_output=True) + + git("checkout", "-q", "-b", branch) + git( + "-c", + "user.name=crunchtools-ci", + "-c", + "user.email=ci@crunchtools.com", + "commit", + "-qam", + f"chore: python-sdlc {version}", + ) + git("push", "-q", "-u", "origin", branch) + url = gh( + "pr", + "create", + "-R", + f"{ORG}/{repo}", + "--head", + branch, + "--title", + f"chore: python-sdlc {version}", + "--body", + f"Moves the python-sdlc pin to {version}. Its own CI " + f"decides whether it lands; it merges itself when green.", + ).strip() + try: + gh("pr", "merge", url, "--auto", "--merge") + except subprocess.CalledProcessError as exc: + reason = exc.stderr.strip() + print(f" warning: {repo}: auto-merge not queued: {reason}", file=sys.stderr) + return url + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("version") + ap.add_argument("--only", help="comma-separated repo names") + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + if not re.fullmatch(r"v\d+\.\d+\.\d+", args.version): + print(f"not a release tag: {args.version}", file=sys.stderr) + return 2 + + repos = args.only.split(",") if args.only else org_repos() + failed = [] + for repo in repos: + try: + rev = pinned_rev(repo) + except RuntimeError as read_error: + print(f"{repo}: {read_error}", file=sys.stderr) + failed.append(repo) + continue + if rev is None or rev == args.version: + continue + print(f"{repo}: {rev} -> {args.version}") + if args.dry_run: + continue + try: + print(f" {bump(repo, args.version)}") + except subprocess.CalledProcessError as command_error: + print(f" FAILED: {command_error}: {command_error.stderr}", file=sys.stderr) + failed.append(repo) + except RuntimeError as bump_error: + print(f" FAILED: {bump_error}", file=sys.stderr) + failed.append(repo) + if failed: + print(f"rerun with --only {','.join(failed)}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_tools.py b/scripts/update_tools.py new file mode 100755 index 0000000..476d590 --- /dev/null +++ b/scripts/update_tools.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Move the baked Gourmand and Gatehouse pins to their latest releases. + +The Containerfile pins both exactly, so rebuilding when one of them releases +changes nothing: the build pulls the same version it pulled before. What a +release has to do is MOVE THE PIN. This script does that, and when anything +moved it also bumps this image's own PATCH version, so the change ships as a +python-sdlc release that every Python repo then receives as a bump PR. + +Run by .github/workflows/update-tools.yml. Writes `changed=true|false` and a +one-line `summary` to $GITHUB_OUTPUT when that is set. + +Usage: scripts/update_tools.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys +from datetime import UTC, datetime +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +TOOLS = {"GOURMAND_VERSION": "crunchtools/gourmand", "GATEHOUSE_VERSION": "crunchtools/gatehouse"} +SEMVER = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$") + + +def semver(text: str) -> tuple[int, int, int]: + match = SEMVER.match(text.strip()) + if not match: + raise ValueError(f"not a release version: {text!r}") + major, minor, patch = (int(g) for g in match.groups()) + return major, minor, patch + + +def latest_release(repo: str) -> str: + """The highest release TAG, not the "latest release" object. + + crunchtools/gourmand tags its releases and creates no GitHub Release, so + `releases/latest` answers 404 for it. Tags are what both repos have. + """ + out = subprocess.run( + ["gh", "api", "--paginate", f"repos/{repo}/tags", "--jq", ".[].name"], + capture_output=True, + text=True, + check=True, + ).stdout.split() + versions = sorted(semver(tag) for tag in out if SEMVER.match(tag)) + if not versions: + raise ValueError(f"{repo} has no release tags") + return ".".join(str(n) for n in versions[-1]) + + +def pinned(containerfile: str, arg: str) -> str: + values = set(re.findall(rf"^ARG {arg}=(\S+)$", containerfile, re.M)) + if len(values) != 1: + raise ValueError(f"{arg} is pinned {len(values)} different ways: {sorted(values)}") + return values.pop() + + +def bump_patch(version: str) -> str: + major, minor, patch = semver(version) + return f"{major}.{minor}.{patch + 1}" + + +def rewrite(path: Path, old: str, new: str, count: int | None = None) -> None: + text = path.read_text(encoding="utf-8") + found = text.count(old) + if found == 0 or (count is not None and found != count): + raise ValueError(f"{path.name}: expected {count or 'some'} of {old!r}, found {found}") + path.write_text(text.replace(old, new), encoding="utf-8") + + +def changelog_entry(version: str, moves: list[tuple[str, str, str]]) -> str: + today = datetime.now(UTC).strftime("%Y-%m-%d") + lines = [f"## [{version}] - {today}", "", "### Changed", ""] + lines += [f"- {repo.split('/')[1]} {old} -> {new}." for repo, old, new in moves] + return "\n".join(lines) + "\n\n" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + containerfile = ROOT / "Containerfile" + text = containerfile.read_text(encoding="utf-8") + moves = [] + for arg, repo in TOOLS.items(): + current = pinned(text, arg) + latest = latest_release(repo) + if semver(latest) > semver(current): + moves.append((repo, current, latest, arg)) + + output = os.environ.get("GITHUB_OUTPUT") + if not moves: + print("every baked tool is at its latest release") + if output: + with Path(output).open("a", encoding="utf-8") as fh: + fh.write("changed=false\n") + return 0 + + image_tag = ROOT / "IMAGE_TAG" + current = image_tag.read_text(encoding="utf-8").strip() + version = bump_patch(current) + summary = ", ".join(f"{repo.split('/')[1]} {old} -> {new}" for repo, old, new, _ in moves) + print(f"python-sdlc {current} -> {version}: {summary}") + if args.dry_run: + return 0 + + for _, old, new, arg in moves: + # Both copies: one before the first FROM, one after it (an ARG declared + # before FROM is out of scope inside the stage). + rewrite(containerfile, f"ARG {arg}={old}\n", f"ARG {arg}={new}\n", count=2) + rewrite(containerfile, f'LABEL version="{current}"', f'LABEL version="{version}"', count=1) + rewrite(ROOT / "pyproject.toml", f'version = "{current}"', f'version = "{version}"', count=1) + image_tag.write_text(version + "\n", encoding="utf-8") + entry = changelog_entry(version, [(r, o, n) for r, o, n, _ in moves]) + rewrite(ROOT / "CHANGELOG.md", "## [Unreleased]\n\n", "## [Unreleased]\n\n" + entry, count=1) + + if output: + with Path(output).open("a", encoding="utf-8") as fh: + fh.write(f"changed=true\nversion={version}\nsummary={summary}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/sdlc b/sdlc index 77a4a01..5f07af4 100755 --- a/sdlc +++ b/sdlc @@ -33,7 +33,9 @@ import shutil import subprocess import sys import tomllib +from datetime import UTC, datetime from pathlib import Path +from typing import NoReturn import yaml @@ -100,7 +102,7 @@ class CannotRunError(Exception): """Raised for anything that is not the tool's own verdict. Becomes exit CANNOT_RUN.""" -def cannot(msg: str) -> None: +def cannot(msg: str) -> NoReturn: raise CannotRunError(msg) @@ -111,32 +113,405 @@ def step(msg: str) -> None: # --------------------------------------------------------------- repo settings -def settings() -> dict: - """The `[tool.sdlc]` table, which is the ONE declared seam per repo. +def pyproject() -> dict: + """The repo's pyproject.toml, or {} when it has none. - gates.sh hard-codes trentina's policy in three places -- `GEMINI_API_KEY` - for the suite, `src/` as the scan path, `.semgrep/run-tests.sh` as the rule - tests. None of that is Python-generic, so a fleet-wide script has to read it - rather than assume it. Everything here has a default that is right for a - repo laid out the way the fleet lays repos out. - - A repo with NO pyproject.toml is not an error here. Three targets -- - `constitution`, `gourmand` and `gatehouse` -- are language-agnostic, and - this image's own repo is a Container Image profile repo with no - pyproject.toml that still wants all three. Demanding one made the tool - unable to check itself, which is how this was found. - - The Python-specific targets assert the project separately, through - `need_project()`. + A repo with NO pyproject.toml is not an error here. `constitution`, + `gourmand` and `gatehouse` are language-agnostic, and this image's own repo + has none of its own policy to state. The Python-specific checks assert the + project separately, through `need_project()`. """ path = REPO / "pyproject.toml" if not path.is_file(): return {} try: - data = tomllib.loads(path.read_text(encoding="utf-8")) + return tomllib.loads(path.read_text(encoding="utf-8")) except (OSError, tomllib.TOMLDecodeError) as exc: cannot(f"cannot read {path}: {exc}") - return data.get("tool", {}).get("sdlc", {}) or {} + + +# ------------------------------------------------------------ the three commands +# +# One rule for all three config blocks, so what you read is what runs: +# +# [tool.sdlc.commit] `sdlc commit`, the pre-commit hook +# [tool.sdlc.push] `sdlc push`, the pre-push hook +# [tool.sdlc.ci] `sdlc ci`, the CI job named `sdlc ci` +# +# Each command runs its defaults. A block's `checks` ADDS to them and its `skip` +# removes from them. A check that needs settings has its own table, +# `[tool.sdlc..]`, and a check the image does not have built in +# must say what `kind` it is. `sdlc tiers` prints exactly what will run. + +COMMANDS = ("commit", "push", "ci") + +DEFAULTS = { + # Seconds, offline, free: what rewrites your code, plus the one check that + # notices the push hook was never installed. + "commit": ["ruff-check", "ruff-format", "hooks-installed"], + # Everything that renders a verdict, once, over origin/...HEAD. + # Never a container build: that is minutes, and a push you wait minutes for + # is a push people stop making. + "push": [ + "mypy", + "pytest", + "gourmand", + "gatehouse", + "semgrep", + "pip-audit", + "constitution", + "codeql", + ], +} + +# CI runs push's checks except these two, which it gets from their own +# workflows: Gatehouse's posts PR comments and must never be a blocking check, +# and the constitution requires Gourmand's CI job to be the shared reusable one. +# Running them again here would pay twice and make the reviewer blocking. +CI_EXCLUDES = ("gourmand", "gatehouse") + +# Built-in checks that are a question about the developer's clone or need its +# git credentials, so bin/sdlc answers them on the host. +LOCAL_CHECKS = ("hooks-installed", "gatehouse") + +# Built-in checks that write the tree. ruff rewrites your source, which is the +# point; gourmand writes .gourmand-cache/ and fails on a read-only mount. +WRITES_TREE = ("ruff-check", "ruff-format", "gourmand") + +# Words that are commands of this script, so no check may be named one. +RESERVED = (*COMMANDS, "plan", "tiers", "list", "versions", "pytest-deps", "codeql-baseline") + + +def sdlc_config() -> dict: + """`[tool.sdlc]`, checked for shape before anything trusts it.""" + cfg = pyproject().get("tool", {}).get("sdlc", {}) or {} + stray = sorted(set(cfg) - set(COMMANDS)) + if stray: + cannot( + f"[tool.sdlc] has {', '.join(stray)}; the only blocks are " + f"[tool.sdlc.commit], [tool.sdlc.push] and [tool.sdlc.ci]" + ) + # The WHOLE config is checked whichever command runs, so a typo in the ci + # block is found on the next commit rather than in CI. + for command in COMMANDS: + for check in resolve(cfg, command): + kind_of(cfg, check) + return cfg + + +def block(cfg: dict, command: str) -> dict: + value = cfg.get(command, {}) + if not isinstance(value, dict): + cannot(f"[tool.sdlc.{command}] must be a table") + return value + + +def names(value: object, where: str) -> list[str]: + if value is None: + return [] + if not isinstance(value, list) or not all(isinstance(v, str) for v in value): + cannot(f"{where} must be a list of check names") + return list(value) + + +def skipped() -> set[str]: + """`SDLC_SKIP=gatehouse,codeql` -- the escape hatch for one false positive. + + The push tier is one pre-commit hook now, so pre-commit's own `SKIP=` would + skip all of it or none of it. A finding skipped here still has to be + answered on the pull request. + """ + return {s.strip() for s in os.environ.get("SDLC_SKIP", "").split(",") if s.strip()} + + +def resolve(cfg: dict, command: str) -> list[str]: + """The checks `command` runs in this repo, in order.""" + if command == "ci": + base = [c for c in resolve(cfg, "push") if c not in CI_EXCLUDES] + else: + base = list(DEFAULTS[command]) + own = block(cfg, command) + drop = set(names(own.get("skip"), f"[tool.sdlc.{command}] skip")) + if command != "ci": + drop |= skipped() + out = [c for c in base if c not in drop] + for name in names(own.get("checks"), f"[tool.sdlc.{command}] checks"): + if name not in out: + out.append(name) + return out + + +def check_settings(cfg: dict, name: str) -> dict: + """A check's own table, wherever it is written. Names are unique.""" + found = [ + (command, tbl) + for command in COMMANDS + for key, tbl in block(cfg, command).items() + if key == name and isinstance(tbl, dict) + ] + if len(found) > 1: + cannot(f"{name} has a settings table in more than one block; check names are unique") + return found[0][1] if found else {} + + +def kind_of(cfg: dict, name: str) -> str | None: + """None for a built-in check; the kind for a repo's own.""" + settings = check_settings(cfg, name) + if name in BUILTINS: + if "kind" in settings: + cannot(f"{name} is built in; its table cannot set a kind") + return None + if name in RESERVED: + cannot(f"{name} is a command of this tool and cannot be a check name") + kind = settings.get("kind") + if kind is not None and not isinstance(kind, str): + cannot(f"{name}: kind must be a name, one of {', '.join(KINDS)}") + if kind is None: + cannot( + f"{name} is not a built-in check, so it needs a table " + f"[tool.sdlc..{name}] that says what kind it is " + f"(one of: {', '.join(KINDS)})" + ) + if kind not in KINDS: + cannot(f"{name} has kind {kind!r}; the kinds are {', '.join(KINDS)}") + return kind + + +# -------------------------------------------------------------------- the plan +# +# The host half (bin/sdlc) is POSIX shell and cannot read TOML, so it asks this +# script what to do and then does it. One line per step, three fields split by a +# tab: +# +# TAB TAB +# +# where mode is: +# image-ro / image-rw run `sdlc ` in this image, tree ro or rw +# local bin/sdlc's own check named +# host is a podman argv, its words joined by +# the ASCII unit separator (0x1f) +# +# bin/sdlc refuses a host line that does not start with `podman`, so the host +# contract is held by the launcher rather than trusted to the config. It splits +# the argv on the separator and never `eval`s it, and no field may contain a +# tab, a newline or the separator: a config value carrying one could otherwise +# forge a second plan line, which is how a repo's TOML would reach the host's +# shell. + +UNIT_SEPARATOR = "\x1f" +FORBIDDEN_IN_PLAN = ("\t", "\n", "\r", UNIT_SEPARATOR) + + +def plain(value: str, where: str) -> str: + if any(c in value for c in FORBIDDEN_IN_PLAN): + cannot(f"{where} contains a tab, a newline or a control separator, which is refused") + return value + + +def host_step(name: str, argv: list[str]) -> tuple[str, str, str]: + return (name, "host", UNIT_SEPARATOR.join(plain(a, name) for a in argv)) + + +def plan_lines(cfg: dict, check: str) -> list[tuple[str, str, str]]: + kind = kind_of(cfg, check) + if kind is not None: + return KINDS[kind](check, check_settings(cfg, check)) + if check in LOCAL_CHECKS: + return [(check, "local", check)] + return [(check, "image-rw" if check in WRITES_TREE else "image-ro", check)] + + +def t_plan(target: str) -> int: + cfg = sdlc_config() + checks = resolve(cfg, target) if target in COMMANDS else [target] + for check in checks: + if check not in BUILTINS and kind_of(cfg, check) is None: + cannot(f"unknown check: {check}") + for name, mode, payload in plan_lines(cfg, check): + plain(name, "a check name") + if mode != "host": + plain(payload, name) + print(f"{name}\t{mode}\t{payload}") + return CLEAN + + +def t_tiers() -> int: + cfg = sdlc_config() + for command in COMMANDS: + print(f"sdlc {command}:") + for check in resolve(cfg, command): + kind = kind_of(cfg, check) + print(f" {check}" + (f" ({kind})" if kind else "")) + if skipped(): + print(f"\nSDLC_SKIP is skipping: {', '.join(sorted(skipped()))}") + return CLEAN + + +# ------------------------------------------------------------------- the kinds +# +# A repo's own checks are named kinds with parameters, never shell. A repo that +# needs something these cannot express gets a new kind HERE, so the config +# stays something a person can read. + + +def project_name() -> str: + return os.environ.get("SDLC_PROJECT", REPO.name) + + +def default_image_tag() -> str: + return f"localhost/{project_name()}:sdlc" + + +def token(value: object) -> str: + """Resolve `@project.version`, `@git.sha` and `@now`; pass anything else.""" + text = str(value) + if not text.startswith("@"): + return text + if text == "@project.version": + version = pyproject().get("project", {}).get("version") + if not version: + cannot("@project.version: pyproject.toml has no [project] version") + return str(version) + if text == "@git.sha": + try: + proc = subprocess.run( + ["git", "-c", "safe.directory=*", "rev-parse", "HEAD"], + cwd=REPO, + capture_output=True, + text=True, + check=False, + ) + except OSError as exc: + cannot(f"@git.sha: cannot run git: {exc}") + if proc.returncode != 0 or not proc.stdout.strip(): + cannot("@git.sha: git could not read HEAD") + return proc.stdout.strip() + if text == "@now": + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + cannot(f"unknown value {text}; known: @project.version, @git.sha, @now") + + +def in_repo(name: str, key: str, value: object) -> str: + """A repo-relative path that stays inside the repo, or a refusal. + + A setting that names a path is repo-controlled, and these paths reach a + host bind mount or a build context. `../../.config` or `/` would hand the + host's files to the repo's own code, so anything absolute or outside the + tree is refused. + """ + text = str(value) + target = (REPO / text).resolve() + if Path(text).is_absolute() or not target.is_relative_to(REPO.resolve()): + cannot(f"{name}: {key} {text!r} is outside the repository") + if not target.exists(): + cannot(f"{name}: {key} {text!r} does not exist") + return str(target.relative_to(REPO.resolve())) or "." + + +def table(settings: dict, key: str) -> dict: + value = settings.get(key, {}) + if not isinstance(value, dict): + cannot(f"{key} must be a table") + return value + + +def kind_image_build(name: str, settings: dict) -> list[tuple[str, str, str]]: + """Build the repo's image and leave it at a known tag. + + The tag is what `pytest-in-image` runs and what a CI publish step picks up, + so the image that is published is the image that was tested. + """ + containerfile = in_repo(name, "containerfile", settings.get("containerfile", "Containerfile")) + context = in_repo(name, "context", settings.get("context", ".")) + argv = ["podman", "build", "-f", containerfile] + for key, value in table(settings, "build_args").items(): + argv += ["--build-arg", f"{key}={token(value)}"] + for key, value in table(settings, "labels").items(): + argv += ["--label", f"{key}={token(value)}"] + argv += ["-t", str(settings.get("tag", default_image_tag())), context] + return [host_step(name, argv)] + + +def kind_pytest_in_image(name: str, settings: dict) -> list[tuple[str, str, str]]: + """Run tests INSIDE the built image, locked down the way production runs it. + + For tests that only mean something there: that a native library loads in a + slim runtime, that the server starts and answers. pytest is not added to the + image -- it is installed once into the cache volume, at the versions the + repo's own uv.lock pins, and put on PYTHONPATH. + """ + tests = names(settings.get("tests"), f"{name}.tests") + if not tests: + cannot(f"{name}: tests is required -- the paths to run inside the image") + host = os.environ.get("SDLC_HOST_REPO") + volume = os.environ.get("SDLC_CACHE_VOLUME") + if not host or not volume: + cannot(f"{name}: run through bin/sdlc, which supplies the host paths") + argv = [ + "podman", + "run", + "--rm", + "--read-only", + "--tmpfs", + "/tmp:rw,exec,size=2g,mode=1777", + "--cap-drop=all", + "--security-opt", + "no-new-privileges", + "-v", + f"{volume}:/sdlc-cache:ro", + "-e", + f"PYTHONPATH=/sdlc-cache/pytest/{project_name()}", + "-e", + "HOME=/tmp", + "-w", + "/repo", + "--entrypoint", + "python", + ] + if not settings.get("network", False): + argv += ["--network", "none"] + for mount in names(settings.get("mounts", ["tests"]), f"{name}.mounts"): + rel = in_repo(name, "mount", mount) + argv += ["-v", f"{host}/{rel}:/repo/{rel}:ro,z"] + for key, value in table(settings, "env").items(): + argv += ["-e", f"{key}={token(value)}"] + argv += [str(settings.get("image", default_image_tag()))] + argv += ["-m", "pytest", "-p", "no:cacheprovider"] + argv += names(settings.get("args", ["-q", "-rfE"]), f"{name}.args") + argv += tests + return [(name, "image-ro", "pytest-deps"), host_step(name, argv)] + + +KINDS = { + "image-build": kind_image_build, + "pytest-in-image": kind_pytest_in_image, +} + +# pytest and the plugins an in-image run needs. Pure Python, so one install +# serves a target image on any Python the lock admits. +PYTEST_PACKAGES = ("pytest", "pytest-asyncio", "pytest-xdist", "pytest-timeout") + + +def t_pytest_deps() -> int: + """Install the lock's pytest into the cache volume for `pytest-in-image`.""" + need_lock_dep("pytest") + lock = tomllib.loads((REPO / "uv.lock").read_text(encoding="utf-8")) + pins = sorted( + f"{pkg['name']}=={pkg['version']}" + for pkg in lock.get("package", []) + if pkg.get("name") in PYTEST_PACKAGES and pkg.get("version") + ) + target = CACHE / "pytest" / project_name() + stamp = target / ".sdlc-pins" + if stamp.is_file() and stamp.read_text(encoding="utf-8") == "\n".join(pins): + return CLEAN + shutil.rmtree(target, ignore_errors=True) + step(f"install {', '.join(pins)} for the in-image run") + rc, _ = run(["uv", "pip", "install", "--target", str(target), *pins], env=project_env()) + if rc != 0: + cannot("could not install pytest for the in-image run") + stamp.write_text("\n".join(pins), encoding="utf-8") + return CLEAN def need_project() -> None: @@ -149,21 +524,6 @@ def need_project() -> None: ) -def paths_for(cfg: dict, key: str, default: list[str]) -> list[str]: - """A path list from `[tool.sdlc]`, keeping only what exists. - - Filtering matters: ruff and semgrep both exit non-zero on a path that is not - there, which would read as "the check found something". - """ - wanted = cfg.get(key, default) - if isinstance(wanted, str): - wanted = [wanted] - present = [p for p in wanted if (REPO / p).exists()] - if not present: - cannot(f"none of the {key} paths exist in the repo: {wanted}") - return present - - def project_env() -> dict[str, str]: """Environment for the uv-managed project venv. @@ -278,28 +638,31 @@ def sync_project(env: dict[str, str]) -> None: # ---------------------------------------------------------------------- targets -def t_ruff_check(cfg: dict) -> int: +# Every check takes its own settings table, `[tool.sdlc..]`, +# empty when the repo wrote none. Paths are NOT settings here: each tool reads +# its own config (`[tool.ruff]` excludes, `[tool.mypy] files`, +# `[tool.pytest] testpaths`), so there is one place to say what gets checked. + + +def t_ruff_check(s: dict) -> int: need_project() - paths = paths_for(cfg, "lint_paths", ["src", "tests"]) - step(f"ruff check --fix {' '.join(paths)}") - return run(["ruff", "check", "--fix", *paths])[0] + step("ruff check --fix .") + return run(["ruff", "check", "--fix", "."])[0] -def t_ruff_format(cfg: dict) -> int: +def t_ruff_format(s: dict) -> int: need_project() - paths = paths_for(cfg, "lint_paths", ["src", "tests"]) - step(f"ruff format {' '.join(paths)}") - return run(["ruff", "format", *paths])[0] + step("ruff format .") + return run(["ruff", "format", "."])[0] -def t_mypy(cfg: dict) -> int: +def t_mypy(s: dict) -> int: + """Type check what `[tool.mypy] files` names. mypy's own exit 2 is ours too.""" need_project() need_lock_dep("mypy") env = project_env() sync_project(env) - paths = paths_for(cfg, "type_paths", ["src"]) - name = os.environ.get("SDLC_PROJECT", REPO.name) - step(f"mypy {' '.join(paths)}") + step("mypy") return run( [ "uv", @@ -308,23 +671,21 @@ def t_mypy(cfg: dict) -> int: "--all-extras", "mypy", "--cache-dir", - str(CACHE / "mypy" / name), - *paths, + str(CACHE / "mypy" / project_name()), ], env=env, )[0] -def t_pytest(cfg: dict) -> int: +def t_pytest(s: dict) -> int: need_project() need_lock_dep("pytest") env = project_env() - # Per-repo test environment: the one trentina seam that cannot be guessed. - for key, value in (cfg.get("test_env") or {}).items(): - env[str(key)] = str(value) + for key, value in table(s, "env").items(): + env[str(key)] = token(value) sync_project(env) jobs = os.environ.get("SDLC_JOBS", "0") - args = cfg.get("pytest_args", ["-q", "-rfE"]) + args = names(s.get("args", ["-q", "-rfE"]), "pytest.args") step("pytest") cmd = [ "uv", @@ -342,12 +703,12 @@ def t_pytest(cfg: dict) -> int: return run(cmd, env=env)[0] -def t_gourmand(cfg: dict) -> int: +def t_gourmand(s: dict) -> int: step("gourmand check --full") return run(["gourmand", "check", "--full", "."])[0] -def t_gatehouse(cfg: dict) -> int: +def t_gatehouse(s: dict) -> int: """Review the branch. The diff arrives on stdin, computed by the host. It is the host's job because the diff needs origin fetched, which needs the @@ -366,18 +727,23 @@ def t_gatehouse(cfg: dict) -> int: return run(cmd, stdin=diff)[0] -def t_semgrep(cfg: dict) -> int: +def t_semgrep(s: dict) -> int: + """The repo's own rules. Semgrep has no file-based scan list, so `paths` + is the one path setting here; `.semgrepignore` narrows it further.""" need_project() - config = cfg.get("semgrep_config", ".semgrep") + config = str(s.get("config", ".semgrep")) if not (REPO / config).exists(): - cannot(f"semgrep config {config} does not exist; set tool.sdlc.semgrep_config") - tests = cfg.get("semgrep_tests", ".semgrep/run-tests.sh") + cannot(f"semgrep config {config} does not exist; set config in its table, or skip it") + tests = str(s.get("tests", ".semgrep/run-tests.sh")) if (REPO / tests).is_file(): step(f"semgrep rule tests ({tests})") rc, _ = run(["sh", tests], env=project_env()) if rc != 0: return rc - paths = paths_for(cfg, "scan_paths", ["src"]) + paths = names(s.get("paths", ["."]), "semgrep.paths") + missing = [p for p in paths if not (REPO / p).exists()] + if missing: + cannot(f"semgrep paths do not exist: {', '.join(missing)}") step(f"semgrep scan --config {config}") return run( ["semgrep", "scan", "--config", config, "--error", "--metrics=off", *paths], @@ -385,7 +751,7 @@ def t_semgrep(cfg: dict) -> int: )[0] -def t_pip_audit(cfg: dict) -> int: +def t_pip_audit(s: dict) -> int: """Audit the project's locked dependencies, and GATE on it. 17 repos in the fleet run this as `pip-audit … || true`, which means the @@ -425,7 +791,7 @@ def t_pip_audit(cfg: dict) -> int: return rc -def t_constitution(cfg: dict) -> int: +def t_constitution(s: dict) -> int: """Validate the repo's manifest against the constitution IT pins. The constitution is cloned at the tag the repo's own manifest names, never @@ -435,23 +801,23 @@ def t_constitution(cfg: dict) -> int: its own directory, so a partial copy makes its pin, file and version checks silently vanish rather than fail. """ - manifest = Path(cfg.get("constitution", ".specify/memory/constitution.md")) + manifest = Path(str(s.get("manifest", ".specify/memory/constitution.md"))) if not (REPO / manifest).is_file(): cannot(f"{manifest} does not exist") return run(["sdlc-validate", str(manifest)], env=project_env())[0] -def t_codeql(cfg: dict) -> int: +def t_codeql(s: dict) -> int: default_config = ".github/codeql/codeql-config.yml" - config = cfg.get("codeql_config", default_config) + config = str(s.get("config", default_config)) # A config the repo NAMED and that is not there is a missing config path, # which the contract says is a 2 -- silently falling back to a default query # suite would run a different analysis than the repo asked for and call it a # pass. A repo that simply has no config, and named none, gets the default # suite; that is not a missing path. if config != default_config and not (REPO / config).is_file(): - cannot(f"tool.sdlc.codeql_config names {config}, which does not exist") - db = CACHE / "codeql" / os.environ.get("SDLC_PROJECT", REPO.name) + cannot(f"codeql config {config} does not exist") + db = CACHE / "codeql" / project_name() sarif = Path("/tmp/codeql-results.sarif") shutil.rmtree(db, ignore_errors=True) # An unwritable cache is a 2, not a traceback and not a 1. The same bug in @@ -496,7 +862,8 @@ def t_codeql(cfg: dict) -> int: # pile of findings CI does not, which teaches people to ignore the gate. queries, ignores = read_codeql_config(REPO / config) analyze += queries or ["python-security-extended.qls"] - return run_codeql(create, analyze, sarif, ignores, gate=bool(cfg.get("codeql_gate", False))) + run_codeql(create, analyze, sarif) + return report_sarif(sarif, ignores, baseline_path(s)) def path_ignored(artifact: str, ignores: list[str]) -> bool: @@ -506,8 +873,8 @@ def path_ignored(artifact: str, ignores: list[str]) -> bool: it has to match CodeQL's own semantics, and those include GLOBS -- `tests/**` and `**/vendor/*` are both legal in a code-scanning config. Matching only literal paths and directory prefixes would silently fail to suppress them, - and the local gate would then report findings CI ignores. With - `codeql_gate = true` it would fail on them. + and the local gate would then report -- and fail on -- findings CI + ignores. Three forms, because CodeQL accepts all three: - an exact path @@ -576,14 +943,8 @@ def read_codeql_config(path: Path) -> tuple[list[str], list[str]]: return queries, ignores -def run_codeql( - create: list[str], - analyze: list[str], - sarif: Path, - ignores: list[str], - gate: bool = False, -) -> int: - +def run_codeql(create: list[str], analyze: list[str], sarif: Path) -> None: + """Build the database and analyse it, or fail as 'could not run'.""" step("codeql database create") rc, _ = run(create, env=project_env()) if rc != 0: @@ -592,72 +953,99 @@ def run_codeql( rc, _ = run(analyze, env=project_env()) if rc != 0: # Analysis failing is not the tool's verdict ON the code -- it means no - # verdict was reached. Returning its status would report "found - # something" for "could not look", which is the one thing the exit-code - # contract forbids. + # verdict was reached, which the exit-code contract makes a 2. cannot(f"codeql database analyze failed (exit {rc}); nothing was judged") - return report_sarif(sarif, ignores, gate) - - -def report_sarif(sarif: Path, ignores: list[str], gate: bool = False) -> int: - """Turn a SARIF file into the exit-code contract. - `database analyze` exits 0 whether or not it found anything, so the verdict - has to be computed from the results. - WHETHER findings block is `[tool.sdlc] codeql_gate`, and it defaults to - FALSE, because that is what CI does. GitHub's CodeQL Action uploads SARIF to - code scanning: the `CodeQL Analysis` check passes regardless of findings, - and each finding becomes an alert that is triaged -- and often dismissed -- - in the Security tab. Measured on trentina's main: 19 findings on a commit - whose CodeQL check is green. +def baseline_path(s: dict) -> Path: + return REPO / str(s.get("baseline", ".sdlc/codeql-baseline.json")) - So blocking locally on any finding would make this hook STRICTER than CI and - block every push until all 19 were fixed, which inverts the promise that a - green push tier means a green CI. A gate nobody can pass is a gate everybody - skips. - The honest gap: without GitHub's dismissal state this cannot tell a known, - accepted finding from a new one, so it reports rather than judges. The fix - is a committed baseline so only NEW findings block; that is the next piece - of work on this target, not something today's default pretends to do. - """ +def sarif_findings(sarif: Path, ignores: list[str]) -> list[dict]: + """Every result outside `paths-ignore`, with the key a baseline matches on.""" try: data = json.loads(sarif.read_text(encoding="utf-8")) except (OSError, ValueError) as exc: cannot(f"codeql wrote no readable SARIF: {exc}") - found = 0 - skipped = 0 + out = [] for run_ in data.get("runs", []): for result in run_.get("results", []): - rule = result.get("ruleId", "?") - msg = (result.get("message") or {}).get("text", "") - locs = result.get("locations") or [{}] - phys = locs[0].get("physicalLocation") or {} - art = (phys.get("artifactLocation") or {}).get("uri", "?") - line = (phys.get("region") or {}).get("startLine", "?") - if path_ignored(art, ignores): - skipped += 1 + phys = ((result.get("locations") or [{}])[0]).get("physicalLocation") or {} + uri = (phys.get("artifactLocation") or {}).get("uri", "?") + if path_ignored(uri, ignores): continue - print(f" {art}:{line} [{rule}] {msg}") - found += 1 - if skipped: - print(f" ({skipped} result(s) in paths-ignore entries, not counted)") - if not found: - print("codeql: no results") - return 0 - print(f"\ncodeql: {found} result(s)") - if gate: - return 1 - print( - "NOT blocking: CI's CodeQL check passes with findings too -- they become\n" - "code-scanning alerts to triage, not a failed build. Set\n" - "[tool.sdlc] codeql_gate = true to block on them here." - ) - return 0 + line = (phys.get("region") or {}).get("startLine", "?") + # The line HASH, not the line number, so a finding that moves + # because code was added above it is still the same finding. + fingerprint = (result.get("partialFingerprints") or {}).get( + "primaryLocationLineHash", f"line:{line}" + ) + out.append( + { + "rule": result.get("ruleId", "?"), + "path": uri, + "fingerprint": fingerprint, + "line": line, + "message": (result.get("message") or {}).get("text", ""), + } + ) + return out -TARGETS = { +def finding_key(f: dict) -> tuple[str, str, str]: + return (f["rule"], f["path"], f["fingerprint"]) + + +def report_sarif(sarif: Path, ignores: list[str], baseline: Path) -> int: + """Fail on any finding that is not in the committed baseline. + + CodeQL in the push tier has to be able to fail, or it is a check that cannot + fail sitting next to ones that can. But trentina's green main carries 19 + findings that were triaged and accepted in GitHub's Security tab, which this + cannot see. The baseline is that acceptance, committed: the findings a + person has looked at. Anything not in it is new, and new blocks. + + `sdlc codeql-baseline` writes it. With no baseline file, every finding is + new. + """ + accepted: set[tuple[str, str, str]] = set() + if baseline.is_file(): + try: + for entry in json.loads(baseline.read_text(encoding="utf-8")): + accepted.add((entry["rule"], entry["path"], entry["fingerprint"])) + except (OSError, ValueError, KeyError, TypeError) as exc: + cannot(f"cannot read the CodeQL baseline {baseline.relative_to(REPO)}: {exc}") + findings = sarif_findings(sarif, ignores) + new = [f for f in findings if finding_key(f) not in accepted] + known = len(findings) - len(new) + for f in new: + print(f" NEW {f['path']}:{f['line']} [{f['rule']}] {f['message']}") + print(f"codeql: {len(new)} new, {known} in the baseline") + return FOUND if new else CLEAN + + +def t_codeql_baseline() -> int: + """Accept every current finding: write them to the baseline file. + + The one deliberate way to make a finding stop failing besides fixing it, and + it lands in a commit someone reviews. + """ + settings = check_settings(sdlc_config(), "codeql") + config = str(settings.get("config", ".github/codeql/codeql-config.yml")) + _, ignores = read_codeql_config(REPO / config) + # Its verdict does not matter here: findings are what is being accepted. + t_codeql(settings) + sarif = Path("/tmp/codeql-results.sarif") + findings = sorted(sarif_findings(sarif, ignores), key=finding_key) + path = baseline_path(settings) + path.parent.mkdir(parents=True, exist_ok=True) + entries = [{k: f[k] for k in ("rule", "path", "fingerprint")} for f in findings] + path.write_text(json.dumps(entries, indent=2) + "\n", encoding="utf-8") + print(f"wrote {len(entries)} finding(s) to {path.relative_to(REPO)}") + return CLEAN + + +BUILTINS = { "ruff-check": t_ruff_check, "ruff-format": t_ruff_format, "mypy": t_mypy, @@ -668,60 +1056,54 @@ TARGETS = { "pip-audit": t_pip_audit, "constitution": t_constitution, "codeql": t_codeql, + # Answered by bin/sdlc on the host; listed so the name is known here. + "hooks-installed": None, } -# Tier membership is NOT stated here. `.pre-commit-hooks.yaml` in the hooks repo -# is the single authority, and `bin/sdlc` reads it. Two copies of the tier table -# is how gates.sh's mypy hook drifted from the script's own resource budget the -# moment that budget stopped being a constant. + +def t_list() -> int: + print("built-in checks:\n " + "\n ".join(BUILTINS)) + print("kinds for a repo's own checks:\n " + "\n ".join(KINDS)) + return CLEAN + + +def t_versions() -> int: + print(f"{'tool':12} {'pinned':24} self-reported") + for name, pinned in BAKED.items(): + print(f"{name:12} {pinned:24} {self_reported(name)}") + return CLEAN + + +TOOL_COMMANDS = { + "tiers": t_tiers, + "list": t_list, + "versions": t_versions, + "pytest-deps": t_pytest_deps, + "codeql-baseline": t_codeql_baseline, +} def main() -> int: ap = argparse.ArgumentParser( prog="sdlc", - description="Run CrunchTools Python SDLC checks. Exit 2 means a check " - "COULD NOT RUN and is never to be worked around.", + description="The in-image half of `sdlc`. Exit 2 means a check COULD NOT RUN " + "and is never to be worked around.", ) - ap.add_argument("targets", nargs="*", help="one or more target names, or 'list'/'versions'") + ap.add_argument("what", help="a check name, or plan/tiers/list/versions") + ap.add_argument("arg", nargs="?", help="for plan: commit, push, ci or a check name") args = ap.parse_args() - if not args.targets or args.targets == ["list"]: - print("targets:") - for name in TARGETS: - print(f" {name}") - print("\nTiers come from .pre-commit-hooks.yaml, not from this script.") - return 0 - - if args.targets == ["versions"]: - print(f"{'tool':12} {'pinned':24} self-reported") - for name, pinned in BAKED.items(): - print(f"{name:12} {pinned:24} {self_reported(name)}") - print( - "\nPin the 'pinned' column in CI so a local finding is the CI finding.\n" - "The two columns answer different questions and need not match: a\n" - "wrapper image carries its own version (gourmand 1.0.0 is the\n" - "packaging release) around an upstream build (0.16.5). Both are\n" - "shown because the tag is what you pin and the binary is what runs." - ) - return 0 - - unknown = [t for t in args.targets if t not in TARGETS] - if unknown: - print(f"sdlc: unknown target(s): {', '.join(unknown)}", file=sys.stderr) - print(f"known: {', '.join(TARGETS)}", file=sys.stderr) - return CANNOT_RUN - - cfg = settings() - worst = 0 - for name in args.targets: - rc = TARGETS[name](cfg) - if rc == 0: - continue - print(f"\nGATE FAILED: {name}", file=sys.stderr) - # A later 1 must not mask an earlier 2: "could not run" outranks - # "found something", because it is the one a person has to act on. - worst = CANNOT_RUN if CANNOT_RUN in (rc, worst) else FOUND - return worst + if args.what == "plan": + if not args.arg: + cannot("plan needs commit, push, ci or a check name") + return t_plan(args.arg) + tool = TOOL_COMMANDS.get(args.what) + if tool is not None: + return tool() + fn = BUILTINS.get(args.what) + if fn is None: + cannot(f"{args.what} is not a check this image runs itself") + return fn(check_settings(sdlc_config(), args.what)) if __name__ == "__main__": diff --git a/tests/contract.sh b/tests/contract.sh index 376647e..f99a058 100755 --- a/tests/contract.sh +++ b/tests/contract.sh @@ -196,10 +196,8 @@ version = "0.0.0" [tool.ruff] line-length = 100 -[tool.sdlc] -lint_paths = ["src"] -scan_paths = ["src"] -semgrep_config = ".semgrep" +[tool.sdlc.push.semgrep] +paths = ["src"] TOML # Clean, formatted, lint-free, and not slop: every offline target must pass it. @@ -239,6 +237,24 @@ for target in ruff-check ruff-format semgrep; do fi done +# The plan the launcher reads must name every default check, in order, and +# resolve a repo's own setting. This is the seam between the TOML the image +# reads and the shell that acts on it, so it is checked on the real image. +plan=$(podman run --rm -v "$FIXTURE:/repo:ro,z" -w /repo --entrypoint="" \ + "$IMAGE" sdlc plan push 2>&1 | cut -f1 | tr '\n' ' ') +want="mypy pytest gourmand gatehouse semgrep pip-audit constitution codeql " +if [ "$plan" = "$want" ]; then + ok "sdlc plan push lists the push defaults in order" +else + bad "sdlc plan push gave '$plan', want '$want'" +fi +plan=$(podman run --rm -v "$FIXTURE:/repo:ro,z" -w /repo --entrypoint="" \ + "$IMAGE" sdlc plan ci 2>&1 | cut -f1 | tr '\n' ' ') +case "$plan" in + *gourmand*|*gatehouse*) bad "sdlc plan ci runs a reviewer CI already runs: $plan" ;; + *) ok "sdlc plan ci leaves out the two reviewers" ;; +esac + # And it must still FAIL on something broken -- a gate that passes everything is # the other way to be useless. cat > "$FIXTURE/src/broken.py" <<'PY' diff --git a/tests/test_codeql_globs.py b/tests/test_codeql_globs.py index c1ae8a5..78d5e5e 100755 --- a/tests/test_codeql_globs.py +++ b/tests/test_codeql_globs.py @@ -3,7 +3,7 @@ This is the local stand-in for a config the CodeQL CLI will not read (see docs/codeql.md), so an entry CodeQL honours and this does not means the local -gate reports findings CI ignores -- and with `codeql_gate = true`, fails on +gate reports findings CI ignores -- and, since CodeQL fails on new findings, fails on them. trentina's config excludes `tests/fixtures/boundary-review`, which holds deliberately vulnerable code used to grade a reviewer, so getting this wrong is not hypothetical. diff --git a/tests/test_host_contract.sh b/tests/test_host_contract.sh index 8e26751..9cda1b4 100755 --- a/tests/test_host_contract.sh +++ b/tests/test_host_contract.sh @@ -51,17 +51,31 @@ for prog in sh awk sed tr grep basename dirname nproc cat mktemp rm ln head id; done # A fake podman that records what it was asked to do and succeeds. Every target -# that needs a container must reach THIS and nothing else. +# that needs a container must reach THIS and nothing else. Asked for a plan, it +# answers with one step of each mode, so the launcher's whole executor runs. +TAB=$(printf '\t') +US=$(printf '\037') cat > "$SHIM/podman" <> "$LOG" +case " \$* " in + *" plan evil "*) + printf 'evil${TAB}host${TAB}sh${US}-c${US}touch $SHIM/pwned\n' ;; + *" plan subst "*) + printf 'subst${TAB}host${TAB}podman${US}run${US}\\\$(touch $SHIM/pwned)\n' ;; + *" plan "*) + printf 'mypy${TAB}image-ro${TAB}mypy\n' + printf 'ruff-check${TAB}image-rw${TAB}ruff-check\n' + printf 'image${TAB}host${TAB}podman${US}build${US}-t${US}localhost/x:sdlc${US}.\n' + printf 'image-tests${TAB}host${TAB}podman${US}run${US}--rm${US}localhost/x:sdlc${US}-m${US}pytest\n' ;; +esac exit 0 EOF chmod +x "$SHIM/podman" # Each target run with only the above on PATH. If bin/sdlc reaches for python3, # uv, jq or a real tool binary, the shell cannot find it and the target fails. -for target in list versions tiers mypy pytest gourmand semgrep pip-audit constitution codeql ruff-check ruff-format; do +for target in list versions tiers commit push ci mypy ruff-check image; do if PATH="$SHIM" sh bin/sdlc "$target" >/dev/null 2>&1; then ok "$target needs only the contract's tools" else @@ -79,6 +93,24 @@ else bad "hooks-installed could not run with a restricted PATH" fi +# The launcher, not the config, holds the host contract: a plan step that is +# not a podman command must be refused, whatever a repo's config asked for. +PATH="$SHIM" sh bin/sdlc evil >/dev/null 2>&1 && rc=0 || rc=$? +if [ "$rc" -eq 2 ] && [ ! -e "$SHIM/pwned" ]; then + ok "a host step that is not podman is refused (exit 2) and never runs" +else + bad "a non-podman host step was not refused (exit $rc)" +fi + +# And a podman argument that LOOKS like a command substitution is passed to +# podman as a literal word, never run by a shell. +PATH="$SHIM" sh bin/sdlc subst >/dev/null 2>&1 || true +if [ ! -e "$SHIM/pwned" ]; then + ok "a host step's words are never evaluated as shell" +else + bad "a host step's word was evaluated as shell" +fi + # The targets that run a container must actually have tried to. Without this the # loop above would also pass if every target silently did nothing. if [ -s "$LOG" ]; then diff --git a/tests/test_plan.py b/tests/test_plan.py new file mode 100644 index 0000000..ea48076 --- /dev/null +++ b/tests/test_plan.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""What each command runs, decided from a repo's pyproject.toml. + +The launcher is shell and cannot read TOML, so it does exactly what `sdlc plan` +tells it. A wrong answer here is a check that silently does not run, so the +rules are pinned case by case: + + - each command runs its defaults; a block's `checks` adds, `skip` removes + - ci runs push's checks except the two reviewers CI already runs elsewhere + - a check that is not built in must name a kind, and only real kinds exist + - the whole config is checked whichever command runs + - CodeQL fails only on findings that are not in the committed baseline + +Run inside the image, which has the interpreter and PyYAML the runner needs: + podman run --rm -v .:/src:ro,z -w /src --entrypoint /opt/constitution-venv/bin/python3 \ + quay.io/crunchtools/python-sdlc: tests/test_plan.py +""" + +from __future__ import annotations + +import importlib.machinery +import importlib.util +import json +import sys +import tempfile +from pathlib import Path + +_SPEC = importlib.util.spec_from_loader( + "sdlc_under_test", + importlib.machinery.SourceFileLoader( + "sdlc_under_test", str(Path(__file__).resolve().parent.parent / "sdlc") + ), +) +assert _SPEC is not None and _SPEC.loader is not None +S = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(S) + +FAILED: list[str] = [] + + +def check(name: str, cond: bool, detail: str = "") -> None: + print(f"{'ok ' if cond else 'FAIL'} {name}" + (f" -- {detail}" if detail and not cond else "")) + if not cond: + FAILED.append(name) + + +def cannot_run(fn, *args) -> str | None: + """The CannotRun message `fn` raised, or None if it did not raise.""" + try: + fn(*args) + except S.CannotRunError as exc: + return str(exc) + return None + + +# --------------------------------------------------------------- resolution + +empty: dict = {} +check( + "commit defaults", + S.resolve(empty, "commit") == ["ruff-check", "ruff-format", "hooks-installed"], +) +check("push defaults", S.resolve(empty, "push") == S.DEFAULTS["push"]) +ci = S.resolve(empty, "ci") +check("ci leaves out the reviewers", "gourmand" not in ci and "gatehouse" not in ci, str(ci)) +check( + "ci keeps the other push checks", + ci == [c for c in S.DEFAULTS["push"] if c not in S.CI_EXCLUDES], +) + +cfg = {"push": {"skip": ["codeql"]}, "ci": {"checks": ["image"], "image": {"kind": "image-build"}}} +check("push skip removes a default", "codeql" not in S.resolve(cfg, "push")) +check("a push skip carries into ci", "codeql" not in S.resolve(cfg, "ci")) +check("ci checks adds after the defaults", S.resolve(cfg, "ci")[-1] == "image") +check("a repo check has its kind", S.kind_of(cfg, "image") == "image-build") +check("a built-in has no kind", S.kind_of(cfg, "mypy") is None) + +# ----------------------------------------------------------------- refusals + +msg = cannot_run(S.kind_of, {"commit": {"checks": ["mystery"]}}, "mystery") +check("an unknown check needs a kind", msg is not None and "needs a table" in msg, str(msg)) +msg = cannot_run(S.kind_of, {"commit": {"x": {"kind": "shell"}}}, "x") +check("there is no shell kind", msg is not None and "the kinds are" in msg, str(msg)) +msg = cannot_run(S.kind_of, {"commit": {"mypy": {"kind": "image-build"}}}, "mypy") +check("a built-in cannot be given a kind", msg is not None and "built in" in msg, str(msg)) +msg = cannot_run(S.kind_of, {"commit": {"plan": {"kind": "image-build"}}}, "plan") +check("a command word cannot be a check", msg is not None and "command of this tool" in msg) +msg = cannot_run(S.check_settings, {"push": {"x": {"kind": "image-build"}}, "ci": {"x": {}}}, "x") +check("check names are unique across blocks", msg is not None and "more than one block" in msg) +msg = cannot_run(S.resolve, {"push": {"checks": "mypy"}}, "push") +check("checks must be a list", msg is not None and "list of check names" in msg) + + +def config_error(pyproject: str) -> str | None: + with tempfile.TemporaryDirectory() as root: + Path(root, "pyproject.toml").write_text(pyproject) + S.REPO = Path(root) + return cannot_run(S.sdlc_config) + + +msg = config_error("[tool.sdlc.post]\n") +check("a block that is not a command is refused", msg is not None and "only blocks are" in msg) +msg = config_error('[tool.sdlc.ci]\nchecks = ["typo"]\n') +check("a typo in ci is found on commit", msg is not None and "typo" in msg, str(msg)) + +# ----------------------------------------------------------- codeql baseline + + +def sarif(results: list[tuple[str, str, str]]) -> dict: + return { + "runs": [ + { + "results": [ + { + "ruleId": rule, + "message": {"text": "x"}, + "partialFingerprints": {"primaryLocationLineHash": fp}, + "locations": [ + { + "physicalLocation": { + "artifactLocation": {"uri": path}, + "region": {"startLine": 1}, + } + } + ], + } + for rule, path, fp in results + ] + } + ] + } + + +with tempfile.TemporaryDirectory() as root: + S.REPO = Path(root) + out = Path(root, "r.sarif") + base = Path(root, ".sdlc", "codeql-baseline.json") + known = ("py/a", "src/a.py", "h1") + out.write_text(json.dumps(sarif([known]))) + check("no baseline: every finding fails", S.report_sarif(out, [], base) == S.FOUND) + base.parent.mkdir() + base.write_text(json.dumps([{"rule": known[0], "path": known[1], "fingerprint": known[2]}])) + check("a baselined finding passes", S.report_sarif(out, [], base) == S.CLEAN) + out.write_text(json.dumps(sarif([known, ("py/b", "src/b.py", "h2")]))) + check("a new finding fails", S.report_sarif(out, [], base) == S.FOUND) + out.write_text(json.dumps(sarif([known, ("py/b", "tests/fx/b.py", "h2")]))) + check("paths-ignore still applies", S.report_sarif(out, ["tests/fx"], base) == S.CLEAN) + base.write_text("not json") + check( + "an unreadable baseline cannot run", + cannot_run(S.report_sarif, out, [], base) is not None, + ) + +# ------------------------------------------------------- path confinement + +with tempfile.TemporaryDirectory() as root: + S.REPO = Path(root) + Path(root, "tests").mkdir() + check("a repo path is accepted", S.in_repo("x", "mount", "tests") == "tests") + for bad in ("../..", "/etc", "tests/../../.."): + msg = cannot_run(S.in_repo, "x", "mount", bad) + check(f"{bad!r} is refused", msg is not None and "outside the repository" in msg, str(msg)) + msg = cannot_run(S.kind_of, {"commit": {"x": {"kind": ["image-build"]}}}, "x") + check("a kind that is not a name is refused", msg is not None and "must be a name" in msg) + + +# ------------------------------------------------- the plan cannot be forged +# +# A config value with a newline or a tab in it would split into a second plan +# line the launcher then obeys, so the plan refuses them at the source. + +for bad in ("a\nevil\thost\tsh", "a\tb", "a\x1fb"): + msg = cannot_run(S.host_step, "image", ["podman", "build", "--build-arg", f"V={bad}"]) + check(f"{bad!r} cannot reach the plan", msg is not None and "refused" in msg, str(msg)) +check( + "a host step is its argv joined by the unit separator", + S.host_step("x", ["podman", "run", "a b"]) == ("x", "host", "podman\x1frun\x1fa b"), +) + + +def main() -> int: + print() + if FAILED: + print(f"{len(FAILED)} FAILED") + return 1 + print("all passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_tiers.sh b/tests/test_tiers.sh deleted file mode 100755 index a633aa2..0000000 --- a/tests/test_tiers.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/sh -# `.pre-commit-hooks.yaml` is the single authority for tier membership, and -# `bin/sdlc` parses it with awk to answer `sdlc commit` and `sdlc push`. -# -# This test is what makes that one authority rather than two. The parse is a -# small awk program over YAML, which is exactly the kind of thing that keeps -# working while quietly reading the wrong thing -- and if it ever returns an -# empty or partial list, `sdlc push` runs fewer gates than the hooks do and -# nothing says so. gates.sh's mypy hook drifted from its own script's resource -# budget the moment that budget stopped being a constant; this is the guard -# against the same shape of bug here. -# -# Usage: tests/test_tiers.sh - -set -eu - -cd "$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)" -FAILED=0 -ok() { printf 'ok %s\n' "$*"; } -bad() { printf 'FAIL %s\n' "$*"; FAILED=$((FAILED + 1)); } - -printf '== tier membership\n\n' - -HOOKS=.pre-commit-hooks.yaml - -# The expected answer, derived from the file a DIFFERENT way than bin/sdlc -# derives it. Two independent readings that agree is the point; reusing the same -# awk would only prove it is self-consistent. -expected_for() { - python3 - "$1" <<'PY' 2>/dev/null || return 1 -import re, sys -want = sys.argv[1] -text = open(".pre-commit-hooks.yaml", encoding="utf-8").read() -for block in re.split(r"^- id:", text, flags=re.M)[1:]: - m = re.search(r"^ entry: bin/sdlc (\S+)", block, re.M) - s = re.search(r"^ stages: \[(.*)\]", block, re.M) - if m and s and want in [x.strip() for x in s.group(1).split(",")]: - print(m.group(1)) -PY -} - -# What bin/sdlc itself will act on. -actual_for() { sh bin/sdlc tiers | awk -v w="$1:" '$0==w {f=1; next} /^[a-z-]+:$/ {f=0} f {print $1}'; } - -for stage in pre-commit pre-push; do - if ! exp=$(expected_for "$stage"); then - bad "cannot compute the expected $stage list (python3 unavailable in CI?)" - continue - fi - act=$(actual_for "$stage") - if [ "$(printf '%s' "$exp" | sort)" = "$(printf '%s' "$act" | sort)" ]; then - ok "$stage: $(printf '%s' "$act" | tr '\n' ' ')" - else - bad "$stage disagrees" - printf ' hooks file says: %s\n' "$(printf '%s' "$exp" | tr '\n' ' ')" - printf ' bin/sdlc says: %s\n' "$(printf '%s' "$act" | tr '\n' ' ')" - fi - [ -n "$act" ] || bad "$stage parsed to an EMPTY list; the tier would run nothing" -done - -# Every hook must name a target the dispatcher knows, or the hook fails at the -# worst possible moment -- on somebody else's commit. Compared as sets, by -# asking bin/sdlc itself rather than pattern-matching its source: it exits 2 on -# an unknown target, and `tiers` needs no container. -DISPATCHABLE=$(sh bin/sdlc tiers | sed -n 's/^ \([a-z-]*\)$/\1/p') -grep -oE '^ entry: bin/sdlc [a-z-]+' "$HOOKS" | awk '{print $NF}' | while read -r target; do - if printf '%s\n' "$DISPATCHABLE" | grep -qx "$target"; then - continue - fi - printf 'FAIL hook target %s is not in any tier bin/sdlc reports\n' "$target" - exit 1 -done || FAILED=$((FAILED + 1)) -ok "every hook target appears in a tier bin/sdlc reports" - -# The tier rule itself, asserted rather than left to prose: nothing that needs -# the network or costs money may sit in the commit tier, because the promise is -# that commits are instant and work offline. -for target in $(expected_for pre-commit); do - case "$target" in - ruff-check|ruff-format|hooks-installed) ;; - *) bad "'$target' is in the commit tier; only sub-second offline targets belong there" ;; - esac -done -ok "the commit tier holds only sub-second offline targets" - -printf '\n' -if [ "$FAILED" -eq 0 ]; then - printf 'tier membership PASSED\n'; exit 0 -fi -printf 'tier membership FAILED: %s check(s)\n' "$FAILED"; exit 1