From d63fabf101953ddf4e202bfa1595bdc3cefc422f Mon Sep 17 00:00:00 2001 From: leynos Date: Mon, 14 Sep 2026 17:38:03 +0200 Subject: [PATCH 1/4] Move ci.yml's inline shell into Cyclopts scripts under scripts/ci The five substantive `run:` blocks in .github/workflows/ci.yml (staging gawk for the sandbox, installing actionlint, reporting sccache statistics, discarding the instrumented build trees, and installing Kani) become Python scripts under scripts/ci, written to docs/scripting-standards.md: each is a Cyclopts application reading its parameters from INPUT_* and ambient GitHub Actions environment variables, running external programs through cuprum's allowlisted catalogue rather than a shell, and using pathlib for the filesystem. The workflow invokes each as `uv run --script scripts/ci/.py`, so every pin sits beside its step as an `env:` entry the contract tests can read, and ci.yml drops from 400 lines to 329. Setup uv moves ahead of the first script in build-test and is added to kani-smoke. The actionlint installer no longer runs the upstream downloader script behind a curl override: the checksum-verified archive is the whole artefact, so the single `actionlint` member is extracted directly. Both installers refuse links and directories in archives and accept only https:// and file:// URLs. scripts/ci/ci_support.py holds the shared catalogue, runner, GITHUB_PATH publication, and verified download and extraction. scripts/tests/test_ci_* cover every script's happy path and failure modes with pytest and cmd-mox shims for sudo, sccache, df, and cargo, and serve release fixtures over file:// so the real download, verification, and extraction path runs without a network. `make test-ci-scripts` runs them; CI runs it before Format. The scripts pin cyclopts and cuprum in their PEP 723 blocks at the Makefile's CYCLOPTS_VERSION and CUPRUM_VERSION, held equal by a test, and ty checks them alongside the other Python sources. The workflow contracts for the retired inline blocks now assert the script invocation and the pins each step passes; the behaviours they spelled out line by line live in the script tests. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 147 +++------- Makefile | 17 +- docs/developers-guide.md | 95 ++++--- pyproject.toml | 3 + scripts/ci/ci_support.py | 256 ++++++++++++++++++ scripts/ci/discard_instrumented_tree.py | 91 +++++++ scripts/ci/install_actionlint.py | 151 +++++++++++ scripts/ci/install_kani.py | 197 ++++++++++++++ scripts/ci/report_sccache_stats.py | 86 ++++++ scripts/ci/stage_test_shell.py | 141 ++++++++++ scripts/tests/ci_script_support.py | 84 ++++++ .../test_ci_discard_instrumented_tree.py | 116 ++++++++ scripts/tests/test_ci_install_actionlint.py | 208 ++++++++++++++ scripts/tests/test_ci_install_kani.py | 256 ++++++++++++++++++ scripts/tests/test_ci_report_sccache_stats.py | 100 +++++++ scripts/tests/test_ci_scripts_metadata.py | 64 +++++ scripts/tests/test_ci_stage_test_shell.py | 166 ++++++++++++ scripts/tests/test_ci_support.py | 215 +++++++++++++++ .../actionlint_installer_contract.py | 117 ++------ tests/workflow_contracts/ci_lint_test.py | 78 ++---- .../github_actions_validation_test.py | 38 +-- tests/workflow_contracts/kani_cache_test.py | 124 +++------ .../sccache_contract_test.py | 12 +- 23 files changed, 2337 insertions(+), 425 deletions(-) create mode 100755 scripts/ci/ci_support.py create mode 100755 scripts/ci/discard_instrumented_tree.py create mode 100755 scripts/ci/install_actionlint.py create mode 100755 scripts/ci/install_kani.py create mode 100755 scripts/ci/report_sccache_stats.py create mode 100755 scripts/ci/stage_test_shell.py create mode 100644 scripts/tests/ci_script_support.py create mode 100644 scripts/tests/test_ci_discard_instrumented_tree.py create mode 100644 scripts/tests/test_ci_install_actionlint.py create mode 100644 scripts/tests/test_ci_install_kani.py create mode 100644 scripts/tests/test_ci_report_sccache_stats.py create mode 100644 scripts/tests/test_ci_scripts_metadata.py create mode 100644 scripts/tests/test_ci_stage_test_shell.py create mode 100644 scripts/tests/test_ci_support.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04ca53b5f..f2cb72a75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,20 +96,20 @@ jobs: with: mode: restore sccache-local: ${{ vars.NETSUKE_SCCACHE_LOCAL_DIR }} + - name: Setup uv + # First, because the CI helper scripts under scripts/ci run through + # `uv run --script` from the next step on, and the uv-driven Ruff, + # Pylint, and df12 gates need it before Format. + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 + with: + python-version: ${{ env.PYTHON_BASELINE }} + # The gate cache already owns the uv download and tool stores, and + # the shim directory the yamllint step writes into. + enable-cache: 'false' - name: Install test shell dependencies - run: | - sudo apt-get update - sudo apt-get install --yes --no-install-recommends gawk - # Ubuntu exposes awk through an alternatives symlink the sandbox - # probe cannot follow, so expose the packaged binary on PATH. - test_shell_bin="${RUNNER_TEMP}/netsuke-test-bin" - mkdir --parents "${test_shell_bin}" - install --mode=0755 "$(command -v gawk)" "${test_shell_bin}/awk" - echo "${test_shell_bin}" >> "${GITHUB_PATH}" - export PATH="${test_shell_bin}:${PATH}" - - command -v awk - awk --version + # Installs gawk and stages a regular-file `awk` the sandbox probe can + # follow; see the script's docstring and scripts/tests/test_ci_stage_test_shell.py. + run: uv run --script scripts/ci/stage_test_shell.py - name: Install sccache uses: taiki-e/install-action@18b1216eba7f8039b0f8d131d5473787f0edce68 # v2.85.3 with: @@ -144,58 +144,19 @@ jobs: # The gate cache owns `~/.local/bin`, so a warm run finds it present. with: version: ${{ env.MDTABLEFIX_VERSION }} - - name: Setup uv - # Before Format: the uv-driven Ruff, Pylint, and df12 gates need it. - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 - with: - python-version: ${{ env.PYTHON_BASELINE }} - # The gate cache already owns the uv download and tool stores, and - # the shim directory the next step writes into. - enable-cache: 'false' - name: Install yamllint run: | uv tool install "yamllint==${YAMLLINT_VERSION}" echo "${UV_TOOL_BIN_DIR}" >> "$GITHUB_PATH" - name: Download actionlint id: get_actionlint - # The gate cache owns `actionlint`; reuse it only at the pinned - # version, so a bump cannot reuse a stale executable. - run: | - set -euo pipefail - readonly ACTIONLINT_VERSION='1.7.12' - if [[ -x ./actionlint ]] \ - && [[ "$(./actionlint --version | head --lines=1)" == "${ACTIONLINT_VERSION}" ]]; then - echo "actionlint ${ACTIONLINT_VERSION} restored from the cache volume" - exit 0 - fi - readonly ACTIONLINT_SHA256='8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8' - readonly ACTIONLINT_INSTALLER_COMMIT='914e7df21a07ef503a81201c76d2b11c789d3fca' - readonly ACTIONLINT_ARCHIVE="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" - readonly ACTIONLINT_RAW_BASE='https://raw.githubusercontent.com/rhysd/actionlint' - readonly ACTIONLINT_SCRIPT='scripts/download-actionlint.bash' - readonly ACTIONLINT_INSTALLER_URL="${ACTIONLINT_RAW_BASE}/${ACTIONLINT_INSTALLER_COMMIT}/${ACTIONLINT_SCRIPT}" - readonly ACTIONLINT_RELEASE_ROOT='https://github.com/rhysd/actionlint/releases/download' - readonly ACTIONLINT_RELEASE_BASE="${ACTIONLINT_RELEASE_ROOT}/v${ACTIONLINT_VERSION}" - readonly ACTIONLINT_RELEASE_URL="${ACTIONLINT_RELEASE_BASE}/${ACTIONLINT_ARCHIVE}" - ACTIONLINT_INSTALLER_PATH="$(mktemp)" - ACTIONLINT_ARCHIVE_PATH="$(mktemp)" - trap 'rm -f "${ACTIONLINT_INSTALLER_PATH}" "${ACTIONLINT_ARCHIVE_PATH}"' EXIT - # Feed the verified archive to reviewed installer without second download. - curl() { - if [[ "${*: -1}" == "${ACTIONLINT_RELEASE_URL}" ]]; then - cat "${ACTIONLINT_ARCHIVE_PATH}" - else - command curl "$@" - fi - } - export -f curl - command curl --fail --location --show-error --output "${ACTIONLINT_INSTALLER_PATH}" \ - "${ACTIONLINT_INSTALLER_URL}" - command curl --fail --location --show-error --output "${ACTIONLINT_ARCHIVE_PATH}" \ - "${ACTIONLINT_RELEASE_URL}" - printf '%s %s\n' "${ACTIONLINT_SHA256}" "${ACTIONLINT_ARCHIVE_PATH}" | sha256sum --check -- - bash "${ACTIONLINT_INSTALLER_PATH}" "${ACTIONLINT_VERSION}" - shell: bash + # The gate cache owns `actionlint`; the script reuses it only at the + # pinned version, so a bump cannot reuse a stale executable, and + # otherwise installs the checksum-verified release archive. + env: + INPUT_ACTIONLINT_VERSION: '1.7.12' + INPUT_ACTIONLINT_SHA256: 8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 + run: uv run --script scripts/ci/install_actionlint.py - name: Install Nixie uses: >- leynos/shared-actions/.github/actions/install-nixie@e041cb75c35c3524201a32d5e57c87408fbd5874 @@ -203,6 +164,8 @@ jobs: python-version: '3.14' - name: Markdown formatter checker tests run: make test-markdown-format + - name: CI helper script tests + run: make test-ci-scripts - name: Format run: make check-fmt - name: Install Whitaker @@ -262,22 +225,13 @@ jobs: if-no-files-found: error - name: Show sccache statistics if: always() - run: | - sccache --show-stats | tee sccache-stats.txt - sccache --show-stats --stats-format=json > sccache-stats.json - { - printf '### sccache\n\n```text\n' - cat sccache-stats.txt - printf '```\n' - } >> "$GITHUB_STEP_SUMMARY" + # Text and JSON files plus a step-summary section, from one script. + run: uv run --script scripts/ci/report_sccache_stats.py - name: Discard the instrumented build tree # No consumer once the report exists, and the second tree on a volume # a sibling repository exhausted. Removed before any cache save. if: always() - run: | - df -h . - rm --recursive --force target/llvm-cov-target target/llvm-cov - df -h . + run: uv run --script scripts/ci/discard_instrumented_tree.py - name: Report peak resource use if: always() uses: ./.github/actions/memory-sampler @@ -336,6 +290,13 @@ jobs: with: mode: restore runner-image: ${{ env.NETSUKE_RUNNER_IMAGE }} + - name: Setup uv + # The installer below is a scripts/ci helper run through + # `uv run --script`; uv's own cache is not part of the Kani key. + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 + with: + python-version: ${{ env.PYTHON_BASELINE }} + enable-cache: 'false' - name: Setup Rust uses: leynos/shared-actions/.github/actions/setup-rust@e041cb75c35c3524201a32d5e57c87408fbd5874 with: @@ -348,44 +309,12 @@ jobs: - name: Install prebuilt Kani # Two payloads from pinned checksummed archives under # version-qualified directories, so a bump cannot reuse a stale one. - run: | - set -euo pipefail - kani_version="$(tr -d '\r\n' < tools/kani/VERSION)" - test "${kani_version}" = '0.67.0' - frontend_bin="${CARGO_HOME}/frontend/kani-${kani_version}" - kani_dir="${KANI_HOME}/kani-${kani_version}" - mkdir --parents "${frontend_bin}" "${KANI_HOME}" "${RUSTUP_HOME}" - export PATH="${frontend_bin}:${PATH}" - echo "${frontend_bin}" >> "$GITHUB_PATH" - - target='x86_64-unknown-linux-gnu' - quickinstall='https://github.com/cargo-bins/cargo-quickinstall' - upstream='https://github.com/model-checking/kani' - - if [[ ! -x "${frontend_bin}/cargo-kani" || ! -x "${frontend_bin}/kani" ]]; then - frontend_name="kani-verifier-${kani_version}-${target}.tar.gz" - frontend_archive="${RUNNER_TEMP}/${frontend_name}" - curl --fail --location --proto '=https' --tlsv1.2 \ - "${quickinstall}/releases/download/kani-verifier-${kani_version}/${frontend_name}" \ - --output "${frontend_archive}" - printf '%s %s\n' \ - 'ed2bafc239b834e14c6b66fc4838e342e3bc0b814e548e72ea30e84f83dc0974' \ - "${frontend_archive}" | sha256sum --check -- - tar --extract --gzip --file "${frontend_archive}" \ - --directory "${frontend_bin}" cargo-kani kani - fi - - if [[ ! -x "${kani_dir}/bin/kani-driver" ]]; then - bundle_name="kani-${kani_version}-${target}.tar.gz" - bundle="${RUNNER_TEMP}/${bundle_name}" - curl --fail --location --proto '=https' --tlsv1.2 \ - "${upstream}/releases/download/kani-${kani_version}/${bundle_name}" \ - --output "${bundle}" - printf '%s %s\n' \ - '3b5f7afd3b51603ee720db7bc1bc4fe46b5a4f5d36daad9939c4b4c658b51ac0' \ - "${bundle}" | sha256sum --check -- - cargo kani setup --use-local-bundle "${bundle}" - fi + # The version must equal tools/kani/VERSION, which the cache key reads. + env: + INPUT_KANI_VERSION: '0.67.0' + INPUT_FRONTEND_SHA256: ed2bafc239b834e14c6b66fc4838e342e3bc0b814e548e72ea30e84f83dc0974 + INPUT_BUNDLE_SHA256: 3b5f7afd3b51603ee720db7bc1bc4fe46b5a4f5d36daad9939c4b4c658b51ac0 + run: uv run --script scripts/ci/install_kani.py - name: Kani version check run: cargo kani --version | grep --fixed-strings "0.67.0" - name: Run Kani harnesses diff --git a/Makefile b/Makefile index a87b190e1..b07ef7db6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help all clean test test-nextest doctest test-workflow-contracts test-release-admission test-coverage-artifact test-markdown-format test-typos-config build release lint lint-clippy lint-whitaker lint-python github-actions-lint doc-coverage doc-coverage-test validate-coverage-artifact fmt check-fmt typecheck typecheck-python markdownlint spelling spelling-config spelling-helper-test nixie install-kani kani-check kani-full kani-ir install-verus verus formal-pr install-dev-fast dev-fast-check dev-build dev-test bench-build bench-config-load bench-glob-expansion +.PHONY: help all clean test test-nextest doctest test-workflow-contracts test-release-admission test-coverage-artifact test-markdown-format test-ci-scripts test-typos-config build release lint lint-clippy lint-whitaker lint-python github-actions-lint doc-coverage doc-coverage-test validate-coverage-artifact fmt check-fmt typecheck typecheck-python markdownlint spelling spelling-config spelling-helper-test nixie install-kani kani-check kani-full kani-ir install-verus verus formal-pr install-dev-fast dev-fast-check dev-build dev-test bench-build bench-config-load bench-glob-expansion RUST_TOOLCHAIN_FILE ?= rust-toolchain.toml # Export this path before shell probes expand it, so Make does not interpolate @@ -86,6 +86,12 @@ RUFF = $(UV_ENV) $(UV) tool run --from ruff==$(RUFF_VERSION) ruff # typecheck gate without any code change. Bump deliberately and fix new # diagnostics in the same commit. TY_VERSION ?= 0.0.74 +# The CI helper scripts under scripts/ci declare these in their PEP 723 +# blocks; scripts/tests/test_ci_scripts_metadata.py holds the two in step, so +# `uv run --script` in CI and the test and typecheck gates resolve one release. +CYCLOPTS_VERSION ?= 4.25.2 +CUPRUM_VERSION ?= 0.1.0 +CI_SCRIPT_DEPS = --with cyclopts==$(CYCLOPTS_VERSION) --with cuprum==$(CUPRUM_VERSION) # Every Python source the repository owns. Ruff and Pylint resolve their own # configuration and exclusions from pyproject.toml, so these paths only bound # the walk. @@ -183,6 +189,12 @@ test-markdown-format: ## Validate the Markdown formatter checker python -m pytest scripts/tests/test_check_markdown_format.py -c /dev/null \ --rootdir=. -p no:cacheprovider +test-ci-scripts: ## Test the CI helper scripts under scripts/ci with pytest and cmd-mox + @PYTHONPATH=scripts/ci:scripts $(UV_ENV) $(UV) run --no-project --python $(PYTHON_BASELINE) \ + --with pytest==9.0.2 --with cmd-mox==0.2.0 $(CI_SCRIPT_DEPS) \ + python -m pytest -p cmd_mox.pytest_plugin scripts/tests/test_ci_*.py \ + -c /dev/null --rootdir=. -p no:cacheprovider + test-typos-config: spelling-helper-test ## Verify the shared spelling-policy integration target/%/$(APP): ## Build binary in debug or release mode @@ -257,8 +269,9 @@ typecheck-python: ## Typecheck the Python sources with ty $(UV_ENV) $(UV) tool run --python $(PYTHON_BASELINE) \ --from ty==$(TY_VERSION) --with pytest==9.0.2 --with pytest-cov==7.0.0 \ --with 'pyyaml>=6' --with 'hypothesis>=6' --with 'cmd-mox==0.2.0' \ + $(CI_SCRIPT_DEPS) \ ty check --python-version $(PYTHON_BASELINE) \ - --extra-search-path scripts $(PYTHON_SOURCES) + --extra-search-path scripts/ci --extra-search-path scripts $(PYTHON_SOURCES) markdownlint: spelling ## Lint Markdown and enforce en-GB-oxendict spelling @unset FORCE_COLOR; $(MDLINT) "**/*.md" diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 03dcd6680..3b4eba3d3 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -1625,6 +1625,44 @@ backend. For a faster inner loop between gate runs, see For documentation changes, also run `make fmt`, `make markdownlint`, and `make nixie`. +### CI helper scripts + +The substantive steps of `.github/workflows/ci.yml` that are not `make` targets +or actions are Python scripts under [`scripts/ci`](../scripts/ci), written to +[`docs/scripting-standards.md`](scripting-standards.md): each is a +[Cyclopts](https://cyclopts.readthedocs.io/) application whose parameters come +from `INPUT_*` environment variables (and the ambient GitHub Actions variables +it needs, such as `RUNNER_TEMP` and `GITHUB_PATH`), runs external programs +through [cuprum](https://github.com/leynos/cuprum)'s allowlisted catalogue +rather than a shell, and uses `pathlib` for the filesystem. The workflow +invokes each one as `uv run --script scripts/ci/.py`, so the step body is +one line and every pin the script needs is an `env:` entry beside it, where the +workflow contract tests can read it. + +| Script | Workflow step | +| ------------------------------ | ------------------------------------ | +| `stage_test_shell.py` | Install test shell dependencies | +| `install_actionlint.py` | Download actionlint | +| `report_sccache_stats.py` | Show sccache statistics | +| `discard_instrumented_tree.py` | Discard the instrumented build tree | +| `install_kani.py` | Install prebuilt Kani (`kani-smoke`) | + +`scripts/ci/ci_support.py` holds what they share: the catalogue, a synchronous +runner, `GITHUB_PATH` publication, and checksum-verified download and +extraction of release archives. Downloads accept `https://` and `file://` URLs +only; the tests serve release fixtures over `file://` so the real download, +verification, and extraction path runs without a network. + +`make test-ci-scripts` runs their suite, `scripts/tests/test_ci_*.py`, with +pytest and [cmd-mox](https://github.com/leynos/cmd-mox), which shims the +external programs (`sudo`, `sccache`, `df`, `cargo`) so each script's happy +path and failure modes are exercised at the process boundary. CI runs the same +target before `make check-fmt`. The scripts pin `cyclopts` and `cuprum` in +their PEP 723 blocks at the Makefile's `CYCLOPTS_VERSION` and `CUPRUM_VERSION`, +held equal by `scripts/tests/test_ci_scripts_metadata.py`, and +`make typecheck-python` checks them with ty alongside the rest of the Python +sources. + ### GitHub Actions validation `make lint` includes `make github-actions-lint`, which runs `yamllint` against @@ -1637,40 +1675,19 @@ Install the pinned YAML linter locally with installs that exact version. Run the workflow checks with `make github-actions-lint` after installing both linters. -The following shell commands reproduce CI's actionlint v1.7.12 setup. They -download the installer at its pinned commit and the Linux `x86_64` release -archive, verify the archive's SHA-256, and feed that verified archive to the -installer so it cannot download a different artefact: +CI installs actionlint v1.7.12 through +[`scripts/ci/install_actionlint.py`](../scripts/ci/install_actionlint.py), +which reuses the gate-cached binary when it reports the pinned version and +otherwise downloads the Linux `x86_64` release archive, verifies its SHA-256 +against the pin, and extracts the single `actionlint` member. The same command +reproduces that setup locally; the pins are the two `INPUT_*` values beside the +step in `.github/workflows/ci.yml`: ```bash -ACTIONLINT_VERSION='1.7.12' -ACTIONLINT_SHA256='8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8' -ACTIONLINT_INSTALLER_COMMIT='914e7df21a07ef503a81201c76d2b11c789d3fca' -ACTIONLINT_ARCHIVE="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" -ACTIONLINT_RAW_BASE='https://raw.githubusercontent.com/rhysd/actionlint' -ACTIONLINT_RELEASE_ROOT='https://github.com/rhysd/actionlint/releases/download' -ACTIONLINT_INSTALLER_URL="${ACTIONLINT_RAW_BASE}/${ACTIONLINT_INSTALLER_COMMIT}/scripts" -ACTIONLINT_INSTALLER_URL+='/download-actionlint.bash' -ACTIONLINT_RELEASE_URL="${ACTIONLINT_RELEASE_ROOT}/v${ACTIONLINT_VERSION}/${ACTIONLINT_ARCHIVE}" -ACTIONLINT_INSTALLER_PATH="$(mktemp)" -ACTIONLINT_ARCHIVE_PATH="$(mktemp)" -trap 'rm -f "${ACTIONLINT_INSTALLER_PATH}" "${ACTIONLINT_ARCHIVE_PATH}"' EXIT -curl --fail --location --show-error --output "${ACTIONLINT_INSTALLER_PATH}" \ - "${ACTIONLINT_INSTALLER_URL}" -curl --fail --location --show-error --output "${ACTIONLINT_ARCHIVE_PATH}" \ - "${ACTIONLINT_RELEASE_URL}" -printf '%s %s\n' "${ACTIONLINT_SHA256}" "${ACTIONLINT_ARCHIVE_PATH}" \ - | sha256sum --check -- -curl() { - if [[ "${*: -1}" == "${ACTIONLINT_RELEASE_URL}" ]]; then - cat "${ACTIONLINT_ARCHIVE_PATH}" - else - command curl "$@" - fi -} -export -f curl -bash "${ACTIONLINT_INSTALLER_PATH}" "${ACTIONLINT_VERSION}" -make github-actions-lint +INPUT_ACTIONLINT_VERSION='1.7.12' \ +INPUT_ACTIONLINT_SHA256='8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8' \ + uv run --script scripts/ci/install_actionlint.py +make ACTIONLINT="$PWD/actionlint" github-actions-lint ``` [`tests/workflow_contracts/github_actions_validation_test.py`][github-actions-validation-test] @@ -1694,9 +1711,9 @@ sequenceDiagram CI->>Cache: Restore yamllint and actionlint alt actionlint cache miss - CI->>GitHub: Download pinned installer and v1.7.12 archive - CI->>CI: sha256sum --check archive - CI->>CI: Install actionlint + CI->>GitHub: Download the v1.7.12 release archive + CI->>CI: Verify the archive SHA-256 against the pin + CI->>CI: Extract actionlint CI->>Cache: Save actionlint end CI->>CI: uv tool install yamllint==1.38.0 @@ -2463,10 +2480,10 @@ The fixtures live in `test_support::dev_fast`: Add to `SANDBOX_UTILITIES` when a script gains a dependency; a missing entry surfaces as a test failure rather than as a silent fallback to the developer's own tools. Every allowlisted utility must also be provisioned on - CI's host `PATH`; `.github/workflows/ci.yml` installs GNU Awk through the - `gawk` package and exposes its binary directly as `awk` for the sandbox's - capability-backed executable probe. Its `write_fake` is the domain helper - described under + CI's host `PATH`; `scripts/ci/stage_test_shell.py`, run by + `.github/workflows/ci.yml`, installs GNU Awk through the `gawk` package and + stages its binary as a regular-file `awk` for the sandbox's capability-backed + executable probe. Its `write_fake` is the domain helper described under [temporary executable test helpers](#temporary-executable-test-helpers): it composes `write_exec_with_content`, supplying the shebang so call sites carry only the behaviour being faked. diff --git a/pyproject.toml b/pyproject.toml index 84d48a65e..819a44230 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,6 +99,9 @@ extend-ignore = [ # literal expectations; those idioms are the point of a test, not a smell. "scripts/tests/*.py" = ["S101", "PLR0913", "PLR0917", "PLR2004", "PLR6301"] "tests/workflow_contracts/*.py" = ["S101", "PLR0913", "PLR0917", "PLR2004", "PLR6301"] +# Cyclopts entry points declare one keyword-only parameter per environment +# input by design, so the argument-count ceilings do not apply to them. +"scripts/ci/*.py" = ["PLR0913", "PLR0917"] [tool.ruff.lint.flake8-import-conventions] # Declare the banned `from` imports. diff --git a/scripts/ci/ci_support.py b/scripts/ci/ci_support.py new file mode 100755 index 000000000..488af52d0 --- /dev/null +++ b/scripts/ci/ci_support.py @@ -0,0 +1,256 @@ +"""Shared helpers for the CI helper scripts under ``scripts/ci``. + +The scripts replace inline shell in ``.github/workflows/ci.yml``. Each one is +a Cyclopts application that reads its parameters from ``INPUT_*`` environment +variables (and the ambient GitHub Actions variables it needs), runs external +programs through cuprum's allowlisted catalogue, and uses ``pathlib`` for the +filesystem. This module holds the pieces they share: the catalogue, a thin +synchronous runner, ``GITHUB_PATH`` publication, and checksum-verified +download and extraction of release archives. + +Every function here is exercised by ``scripts/tests/test_ci_support.py``. +""" + +import hashlib +import os +import pathlib +import shutil +import tarfile +import urllib.parse +import urllib.request + +from cuprum import ( + CommandResult, + ExecutionContext, + Program, + ProgramCatalogue, + ProjectSettings, + sh, +) + +PROJECT_NAME = "netsuke-ci" +DOCUMENTATION = ("docs/developers-guide.md",) +#: Release archives come from GitHub over HTTPS; tests serve fixtures from +#: ``file://`` URLs so the real download path is exercised without a network. +ALLOWED_URL_SCHEMES = frozenset({"https", "file"}) +DOWNLOAD_TIMEOUT_SECONDS = 120.0 +EXECUTABLE_MODE = 0o755 + + +class CiScriptError(RuntimeError): + """An actionable failure whose message names the remediation.""" + + +def catalogue(*programs: str) -> ProgramCatalogue: + """Return a cuprum catalogue allowing exactly ``programs``. + + Parameters + ---------- + programs + Program names, or absolute paths for executables the script staged + itself, that the calling script is permitted to run. + + Returns + ------- + ProgramCatalogue + A catalogue whose single project lists ``programs``. + """ + return ProgramCatalogue( + projects=( + ProjectSettings( + name=PROJECT_NAME, + programs=tuple(Program(program) for program in programs), + documentation_locations=DOCUMENTATION, + noise_rules=(), + ), + ) + ) + + +def run( + program: str, + *args: str, + allowed: ProgramCatalogue, + env: dict[str, str] | None = None, + echo: bool = True, +) -> CommandResult: + """Run one allowlisted program synchronously and return its result. + + Parameters + ---------- + program + The program to run; it must be in ``allowed``. + args + Arguments passed verbatim, never through a shell. + allowed + The catalogue that permits ``program``. + env + Variables overlaid on the live environment for this run only. + echo + Whether to mirror the child's output to this process's streams as + well as capturing it, so the CI log shows what happened. + + Returns + ------- + CommandResult + The exit code and captured output; a non-zero exit is returned, not + raised, so callers decide what it means. + """ + command = sh.make(Program(program), catalogue=allowed)(*args) + return command.run_sync(echo=echo, context=ExecutionContext(env=env)) + + +def describe_failure(result: CommandResult) -> str: + """Return a one-line description of a failed command for an error message.""" + argv = " ".join((str(result.program), *result.argv)) + detail = (result.stderr or "").strip().splitlines() + suffix = f": {detail[-1]}" if detail else "" + return f"`{argv}` exited {result.exit_code}{suffix}" + + +def append_github_path(directory: pathlib.Path, github_path: pathlib.Path) -> None: + """Publish ``directory`` to later workflow steps through ``GITHUB_PATH``. + + Parameters + ---------- + directory + The directory to prepend to ``PATH`` for the rest of the job. + github_path + The file GitHub Actions names in ``GITHUB_PATH``. + """ + with github_path.open("a", encoding="utf-8") as handle: + handle.write(f"{directory}\n") + + +def prepend_path(directory: pathlib.Path) -> None: + """Prepend ``directory`` to this process's ``PATH`` for later commands.""" + current = os.environ.get("PATH", "") + os.environ["PATH"] = ( + f"{directory}{os.pathsep}{current}" if current else str(directory) + ) + + +def is_executable_file(path: pathlib.Path) -> bool: + """Return whether ``path`` is a regular file this process may execute.""" + return path.is_file() and os.access(path, os.X_OK) + + +def download(url: str, destination: pathlib.Path) -> None: + """Download ``url`` to ``destination``. + + Parameters + ---------- + url + An ``https://`` release URL, or a ``file://`` URL under test. + destination + The file to write; its parent must exist. + + Raises + ------ + CiScriptError + If the URL scheme is not allowed or the download fails. + """ + scheme = urllib.parse.urlsplit(url).scheme + if scheme not in ALLOWED_URL_SCHEMES: + message = ( + f"refusing to download {url!r}: only " + f"{sorted(ALLOWED_URL_SCHEMES)} URLs are permitted" + ) + raise CiScriptError(message) + try: + # The scheme is allowlisted just above; `file://` serves test fixtures. + with ( + urllib.request.urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECONDS) as response, # ruff: ignore[suspicious-url-open-usage] - scheme allowlisted above + destination.open("wb") as handle, + ): + shutil.copyfileobj(response, handle) + except OSError as error: + message = f"downloading {url} failed: {error}" + raise CiScriptError(message) from error + + +def sha256_of(path: pathlib.Path) -> str: + """Return the lower-case hex SHA-256 digest of ``path``.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify_sha256(path: pathlib.Path, expected: str) -> None: + """Fail unless ``path`` hashes to ``expected``. + + Parameters + ---------- + path + The downloaded archive. + expected + The pinned hex digest from the workflow. + + Raises + ------ + CiScriptError + If the digest differs; the message names both digests so a pin bump + and a tampered download are told apart in the log. + """ + actual = sha256_of(path) + if actual != expected.lower(): + message = ( + f"{path.name} does not match its pinned SHA-256: expected " + f"{expected}, got {actual}; the release archive changed or the " + f"pin in .github/workflows/ci.yml is stale" + ) + raise CiScriptError(message) + + +def extract_members( + archive: pathlib.Path, + destination: pathlib.Path, + members: tuple[str, ...], +) -> list[pathlib.Path]: + """Extract named regular-file members of a gzip tarball as executables. + + Parameters + ---------- + archive + A ``.tar.gz`` archive that has already been checksum-verified. + destination + The directory the members are written into, flat, under their own + base names. + members + The exact member names to extract; every one must be present and be + a regular file. Directories and links are refused so an archive can + never write outside ``destination``. + + Returns + ------- + list[pathlib.Path] + The extracted executables, in ``members`` order. + + Raises + ------ + CiScriptError + If a member is missing or is not a regular file. + """ + extracted = [] + with tarfile.open(archive, "r:gz") as tar: + available = {member.name: member for member in tar.getmembers()} + for name in members: + member = available.get(name) + if member is None: + message = f"{archive.name} has no member {name!r}" + raise CiScriptError(message) + if not member.isfile(): + message = f"{archive.name} member {name!r} is not a regular file" + raise CiScriptError(message) + source = tar.extractfile(member) + if source is None: # pragma: no cover - isfile() guarantees a stream + message = f"{archive.name} member {name!r} could not be read" + raise CiScriptError(message) + target = destination / pathlib.PurePosixPath(name).name + with source, target.open("wb") as handle: + shutil.copyfileobj(source, handle) + target.chmod(EXECUTABLE_MODE) + extracted.append(target) + return extracted diff --git a/scripts/ci/discard_instrumented_tree.py b/scripts/ci/discard_instrumented_tree.py new file mode 100755 index 000000000..ceca5e8a0 --- /dev/null +++ b/scripts/ci/discard_instrumented_tree.py @@ -0,0 +1,91 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.14" +# dependencies = ["cyclopts==4.25.2", "cuprum==0.1.0"] +# /// +"""Remove the instrumented coverage build trees before any cache save. + +Once the coverage report exists the ``llvm-cov`` trees have no consumer, and +they are the second build tree on a volume a sibling repository has exhausted +before. Disk usage is shown before and after so the log records what the +removal freed. A tree that does not exist is not an error. + +Parameters arrive from the environment: ``INPUT_TARGET_DIR`` overrides the +Cargo target directory and ``INPUT_SUBTREES`` the whitespace-separated names +removed beneath it. +""" + +import pathlib +import shutil +import sys + +from ci_support import CiScriptError, catalogue, describe_failure, run + +import cyclopts +from cyclopts import App + +DF = "df" +ALLOWED = catalogue(DF) +DEFAULT_SUBTREES = ("llvm-cov-target", "llvm-cov") + +app = App(config=cyclopts.config.Env("INPUT_", command=False)) + + +def show_disk_usage() -> None: + """Print ``df -h .`` so the log records the volume's state.""" + result = run(DF, "-h", ".", allowed=ALLOWED) + if not result.ok: + message = f"reading disk usage failed: {describe_failure(result)}" + raise CiScriptError(message) + + +def discard(target_dir: pathlib.Path, subtrees: tuple[str, ...]) -> list[pathlib.Path]: + """Remove each existing subtree under ``target_dir`` and return them.""" + removed = [] + for name in subtrees: + tree = target_dir / name + if tree.is_symlink() or tree.is_file(): + tree.unlink() + removed.append(tree) + elif tree.is_dir(): + shutil.rmtree(tree) + removed.append(tree) + return removed + + +@app.default +def main( + *, + target_dir: pathlib.Path = pathlib.Path("target"), + subtrees: list[str] | None = None, +) -> int: + """Discard the instrumented trees, reporting disk usage around the removal. + + Parameters + ---------- + target_dir + The Cargo target directory holding the instrumented trees. + subtrees + Names beneath ``target_dir`` to remove; the two ``llvm-cov`` trees by + default. + + Returns + ------- + int + ``0`` on success, ``1`` if disk usage or a removal fails. + """ + names = tuple(subtrees) if subtrees else DEFAULT_SUBTREES + try: + show_disk_usage() + removed = discard(target_dir, names) + show_disk_usage() + except (CiScriptError, OSError) as error: + print(f"discard_instrumented_tree: {error}", file=sys.stderr) + return 1 + for tree in removed: + print(f"removed {tree}") + return 0 + + +if __name__ == "__main__": + app() diff --git a/scripts/ci/install_actionlint.py b/scripts/ci/install_actionlint.py new file mode 100755 index 000000000..e68b1e50a --- /dev/null +++ b/scripts/ci/install_actionlint.py @@ -0,0 +1,151 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.14" +# dependencies = ["cyclopts==4.25.2", "cuprum==0.1.0"] +# /// +"""Install the pinned actionlint release, reusing a cached binary that matches. + +The Linux gate cache owns ``./actionlint``. A warm run finds it present and, +when it reports the pinned version, keeps it. Otherwise the pinned Linux amd64 +release archive is downloaded, its SHA-256 is verified against the pin, and +the single ``actionlint`` member is extracted as a regular executable. No +upstream installer script runs: the archive is the whole artefact, and the +checksum is what makes it trusted. + +Parameters arrive from the environment: ``INPUT_ACTIONLINT_VERSION`` and +``INPUT_ACTIONLINT_SHA256`` are the pins the workflow declares beside the +step; ``INPUT_INSTALL_DIR`` and ``INPUT_RELEASE_ROOT`` override the defaults. +""" + +import pathlib +import sys +import tempfile +import typing as typ + +from ci_support import ( + CiScriptError, + catalogue, + describe_failure, + download, + extract_members, + is_executable_file, + run, + verify_sha256, +) + +import cyclopts +from cyclopts import App, Parameter + +BINARY_NAME = "actionlint" +DEFAULT_RELEASE_ROOT = "https://github.com/rhysd/actionlint/releases/download" +DEFAULT_PLATFORM = "linux_amd64" + +app = App(config=cyclopts.config.Env("INPUT_", command=False)) + + +def reported_version(binary: pathlib.Path) -> str | None: + """Return the version ``binary --version`` reports, or ``None`` if it fails.""" + if not is_executable_file(binary): + return None + result = run(str(binary), "--version", allowed=catalogue(str(binary)), echo=False) + if not result.ok or not result.stdout: + return None + return result.stdout.splitlines()[0].strip() + + +def archive_url(release_root: str, version: str, platform: str) -> str: + """Return the release archive URL for ``version`` on ``platform``.""" + return f"{release_root}/v{version}/{BINARY_NAME}_{version}_{platform}.tar.gz" + + +def install( + binary: pathlib.Path, + url: str, + sha256: str, + scratch: pathlib.Path, +) -> None: + """Download, verify, and extract the release archive to ``binary``.""" + archive = scratch / pathlib.PurePosixPath(url).name + download(url, archive) + verify_sha256(archive, sha256) + extract_members(archive, binary.parent, (BINARY_NAME,)) + + +def ensure_installed_version(binary: pathlib.Path, expected: str) -> str: + """Return the version ``binary`` reports, failing unless it is ``expected``. + + Returns + ------- + str + The reported version, which equals ``expected``. + + Raises + ------ + CiScriptError + If the freshly extracted binary does not report the pinned version. + """ + installed = reported_version(binary) + if installed == expected: + return installed + result = run(str(binary), "--version", allowed=catalogue(str(binary)), echo=False) + message = ( + f"installed {BINARY_NAME} reports {installed!r}, expected " + f"{expected!r} ({describe_failure(result)})" + ) + raise CiScriptError(message) + + +# One keyword-only parameter per environment input is the Cyclopts contract. +@app.default +def main( # pylint: disable=too-many-arguments # one parameter per input + *, + actionlint_version: typ.Annotated[str, Parameter(required=True)], + actionlint_sha256: typ.Annotated[str, Parameter(required=True)], + install_dir: pathlib.Path = pathlib.Path(), + release_root: str = DEFAULT_RELEASE_ROOT, + platform: str = DEFAULT_PLATFORM, + runner_temp: typ.Annotated[ + pathlib.Path | None, Parameter(env_var="RUNNER_TEMP") + ] = None, +) -> int: + """Ensure ``install_dir/actionlint`` is the pinned release. + + Parameters + ---------- + actionlint_version + The release to install, for example ``1.7.12``. + actionlint_sha256 + The hex SHA-256 of the release archive for ``platform``. + install_dir + Where the binary lives; the gate cache restores and saves it here. + release_root + The releases URL prefix; tests point it at a ``file://`` fixture. + platform + The archive platform suffix. + runner_temp + Scratch space for the download; the system default when unset. + + Returns + ------- + int + ``0`` when the pinned binary is in place, ``1`` otherwise. + """ + # Resolved so the cached binary is executed by path, never looked up on PATH. + binary = (install_dir / BINARY_NAME).resolve() + try: + if reported_version(binary) == actionlint_version: + print(f"{BINARY_NAME} {actionlint_version} restored from the cache volume") + return 0 + url = archive_url(release_root, actionlint_version, platform) + with tempfile.TemporaryDirectory(dir=runner_temp) as scratch: + install(binary, url, actionlint_sha256, pathlib.Path(scratch)) + installed = ensure_installed_version(binary, actionlint_version) + print(f"installed {BINARY_NAME} {installed} from {url}") + except CiScriptError as error: + print(f"install_actionlint: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + app() diff --git a/scripts/ci/install_kani.py b/scripts/ci/install_kani.py new file mode 100755 index 000000000..ef98f0e8c --- /dev/null +++ b/scripts/ci/install_kani.py @@ -0,0 +1,197 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.14" +# dependencies = ["cyclopts==4.25.2", "cuprum==0.1.0"] +# /// +"""Install the pinned Kani front-end and verifier bundle from release archives. + +Kani is two payloads. The ``cargo-kani`` and ``kani`` front-end binaries come +from the cargo-quickinstall release archive; the verifier bundle comes from +the upstream Kani release and is installed with ``cargo kani setup +--use-local-bundle``. Both archives are pinned by SHA-256 and land under +version-qualified directories, so a version bump can never reuse a stale +payload, and each is downloaded only when its executables are absent, so a +warm cache restore skips the network entirely. + +Parameters arrive from the environment: ``INPUT_KANI_VERSION``, +``INPUT_FRONTEND_SHA256``, and ``INPUT_BUNDLE_SHA256`` are the pins the +workflow declares beside the step; ``CARGO_HOME``, ``KANI_HOME``, +``RUSTUP_HOME``, ``RUNNER_TEMP``, and ``GITHUB_PATH`` are the job's ambient +variables. The version must equal ``tools/kani/VERSION`` so the cache key, +the Makefile, and this installer cannot drift. +""" + +import pathlib +import sys +import typing as typ + +from ci_support import ( + CiScriptError, + append_github_path, + catalogue, + describe_failure, + download, + extract_members, + is_executable_file, + prepend_path, + run, + verify_sha256, +) + +import cyclopts +from cyclopts import App, Parameter + +CARGO = "cargo" +ALLOWED = catalogue(CARGO) +FRONTEND_MEMBERS = ("cargo-kani", "kani") +DRIVER = pathlib.PurePosixPath("bin/kani-driver") +DEFAULT_TARGET = "x86_64-unknown-linux-gnu" +DEFAULT_QUICKINSTALL_ROOT = "https://github.com/cargo-bins/cargo-quickinstall" +DEFAULT_UPSTREAM_ROOT = "https://github.com/model-checking/kani" + +app = App(config=cyclopts.config.Env("INPUT_", command=False)) + + +def check_pinned_version(version_file: pathlib.Path, kani_version: str) -> None: + """Fail unless ``version_file`` holds exactly ``kani_version``.""" + pinned = version_file.read_text(encoding="utf-8").strip() + if pinned != kani_version: + message = ( + f"{version_file} pins Kani {pinned!r} but the workflow passed " + f"{kani_version!r}; update both in the same commit" + ) + raise CiScriptError(message) + + +def frontend_url(root: str, version: str, target: str) -> str: + """Return the cargo-quickinstall front-end archive URL.""" + archive = f"kani-verifier-{version}-{target}.tar.gz" + return f"{root}/releases/download/kani-verifier-{version}/{archive}" + + +def bundle_url(root: str, version: str, target: str) -> str: + """Return the upstream verifier bundle URL.""" + return f"{root}/releases/download/kani-{version}/kani-{version}-{target}.tar.gz" + + +def fetch_verified(url: str, sha256: str, scratch: pathlib.Path) -> pathlib.Path: + """Download ``url`` into ``scratch`` and verify it against ``sha256``.""" + archive = scratch / pathlib.PurePosixPath(url).name + download(url, archive) + verify_sha256(archive, sha256) + return archive + + +def install_frontend(archive: pathlib.Path, frontend_bin: pathlib.Path) -> None: + """Extract the two front-end executables into ``frontend_bin``.""" + extract_members(archive, frontend_bin, FRONTEND_MEMBERS) + + +def install_bundle(archive: pathlib.Path) -> None: + """Install the verifier bundle through ``cargo kani setup``.""" + result = run( + CARGO, "kani", "setup", "--use-local-bundle", str(archive), allowed=ALLOWED + ) + if not result.ok: + message = f"installing the verifier bundle failed: {describe_failure(result)}" + raise CiScriptError(message) + + +class Homes(typ.NamedTuple): + """The version-qualified directories the two payloads occupy.""" + + frontend_bin: pathlib.Path + kani_dir: pathlib.Path + + def frontend_present(self) -> bool: + """Return whether both front-end executables are in place.""" + return all(is_executable_file(self.frontend_bin / m) for m in FRONTEND_MEMBERS) + + def bundle_present(self) -> bool: + """Return whether the verifier driver is in place.""" + return is_executable_file(self.kani_dir / DRIVER) + + +# One keyword-only parameter per environment input is the Cyclopts contract. +@app.default +def main( # pylint: disable=too-many-arguments # one parameter per input + *, + kani_version: typ.Annotated[str, Parameter(required=True)], + frontend_sha256: typ.Annotated[str, Parameter(required=True)], + bundle_sha256: typ.Annotated[str, Parameter(required=True)], + cargo_home: typ.Annotated[pathlib.Path, Parameter(env_var="CARGO_HOME")], + kani_home: typ.Annotated[pathlib.Path, Parameter(env_var="KANI_HOME")], + rustup_home: typ.Annotated[pathlib.Path, Parameter(env_var="RUSTUP_HOME")], + runner_temp: typ.Annotated[pathlib.Path, Parameter(env_var="RUNNER_TEMP")], + github_path: typ.Annotated[pathlib.Path, Parameter(env_var="GITHUB_PATH")], + version_file: pathlib.Path = pathlib.Path("tools/kani/VERSION"), + target: str = DEFAULT_TARGET, + quickinstall_root: str = DEFAULT_QUICKINSTALL_ROOT, + upstream_root: str = DEFAULT_UPSTREAM_ROOT, +) -> int: + """Install whichever Kani payloads the cache did not restore. + + Parameters + ---------- + kani_version + The pinned Kani release; must equal ``version_file``. + frontend_sha256 + Hex SHA-256 of the cargo-quickinstall front-end archive. + bundle_sha256 + Hex SHA-256 of the upstream verifier bundle. + cargo_home + The job's ``CARGO_HOME``; the front-end lands under ``frontend/``. + kani_home + The job's ``KANI_HOME``; the bundle is installed beneath it. + rustup_home + The job's ``RUSTUP_HOME``, created so the setup has somewhere to write. + runner_temp + Scratch space for the downloaded archives. + github_path + The ``GITHUB_PATH`` file that publishes the front-end directory. + version_file + The repository's pinned version file. + target + The Rust target triple the archives are built for. + quickinstall_root + The cargo-quickinstall repository URL; tests use a ``file://`` root. + upstream_root + The Kani repository URL; tests use a ``file://`` root. + + Returns + ------- + int + ``0`` when both payloads are in place, ``1`` otherwise. + """ + homes = Homes( + cargo_home / "frontend" / f"kani-{kani_version}", + kani_home / f"kani-{kani_version}", + ) + try: + check_pinned_version(version_file, kani_version) + for directory in (homes.frontend_bin, kani_home, rustup_home): + directory.mkdir(parents=True, exist_ok=True) + prepend_path(homes.frontend_bin) + append_github_path(homes.frontend_bin, github_path) + if homes.frontend_present(): + print(f"front-end {kani_version} present in {homes.frontend_bin}") + else: + url = frontend_url(quickinstall_root, kani_version, target) + install_frontend( + fetch_verified(url, frontend_sha256, runner_temp), homes.frontend_bin + ) + print(f"installed front-end {kani_version} from {url}") + if homes.bundle_present(): + print(f"verifier bundle {kani_version} present in {homes.kani_dir}") + else: + url = bundle_url(upstream_root, kani_version, target) + install_bundle(fetch_verified(url, bundle_sha256, runner_temp)) + print(f"installed verifier bundle {kani_version} from {url}") + except (CiScriptError, OSError) as error: + print(f"install_kani: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + app() diff --git a/scripts/ci/report_sccache_stats.py b/scripts/ci/report_sccache_stats.py new file mode 100755 index 000000000..82b100537 --- /dev/null +++ b/scripts/ci/report_sccache_stats.py @@ -0,0 +1,86 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.14" +# dependencies = ["cyclopts==4.25.2", "cuprum==0.1.0"] +# /// +"""Record sccache statistics as text, JSON, and a step-summary section. + +The compile-cache report runs after the last compile step, on success and on +failure alike. The human-readable statistics go to the log and to +``sccache-stats.txt``; the machine-readable form goes to ``sccache-stats.json``; +and the text is appended to the job summary under an ``sccache`` heading. + +Parameters arrive from the environment: ``GITHUB_STEP_SUMMARY`` is ambient in +GitHub Actions; ``INPUT_OUTPUT_DIR`` overrides where the two files land. +""" + +import pathlib +import sys +import typing as typ + +from ci_support import CiScriptError, catalogue, describe_failure, run + +import cyclopts +from cyclopts import App, Parameter + +SCCACHE = "sccache" +ALLOWED = catalogue(SCCACHE) +TEXT_NAME = "sccache-stats.txt" +JSON_NAME = "sccache-stats.json" +SUMMARY_HEADING = "### sccache" + +app = App(config=cyclopts.config.Env("INPUT_", command=False)) + + +def show_stats(*extra: str, echo: bool) -> str: + """Return ``sccache --show-stats`` output, failing on a non-zero exit.""" + result = run(SCCACHE, "--show-stats", *extra, allowed=ALLOWED, echo=echo) + if not result.ok: + message = f"reading statistics failed: {describe_failure(result)}" + raise CiScriptError(message) + return result.stdout or "" + + +def summary_section(text: str) -> str: + """Return the Markdown block appended to the job summary.""" + body = text if text.endswith("\n") else f"{text}\n" + return f"{SUMMARY_HEADING}\n\n```text\n{body}```\n" + + +@app.default +def main( + *, + github_step_summary: typ.Annotated[ + pathlib.Path, Parameter(env_var="GITHUB_STEP_SUMMARY") + ], + output_dir: pathlib.Path = pathlib.Path(), +) -> int: + """Write the statistics files and the summary section. + + Parameters + ---------- + github_step_summary + The ``GITHUB_STEP_SUMMARY`` file the section is appended to. + output_dir + Where ``sccache-stats.txt`` and ``sccache-stats.json`` are written. + + Returns + ------- + int + ``0`` when both reports were captured, ``1`` otherwise. + """ + try: + text = show_stats(echo=True) + machine = show_stats("--stats-format=json", echo=False) + except CiScriptError as error: + print(f"report_sccache_stats: {error}", file=sys.stderr) + return 1 + (output_dir / TEXT_NAME).write_text(text, encoding="utf-8") + (output_dir / JSON_NAME).write_text(machine, encoding="utf-8") + with github_step_summary.open("a", encoding="utf-8") as handle: + handle.write(summary_section(text)) + return 0 + + +if __name__ == "__main__": + app() diff --git a/scripts/ci/stage_test_shell.py b/scripts/ci/stage_test_shell.py new file mode 100755 index 000000000..5bf4c456d --- /dev/null +++ b/scripts/ci/stage_test_shell.py @@ -0,0 +1,141 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.14" +# dependencies = ["cyclopts==4.25.2", "cuprum==0.1.0"] +# /// +"""Stage GNU Awk as a regular ``awk`` executable for the test sandbox. + +The hermetic dev-fast sandbox probes ``awk`` through a capability-backed +directory handle, which deliberately cannot follow a symlink that escapes its +containing directory. Ubuntu exposes ``awk`` as exactly such an alternatives +symlink, so this script installs the ``gawk`` package, copies its binary into +``$RUNNER_TEMP/netsuke-test-bin/awk`` as a regular file, publishes that +directory through ``GITHUB_PATH``, and proves the staged copy runs. + +Parameters arrive from the environment: ``RUNNER_TEMP`` and ``GITHUB_PATH`` +are ambient in GitHub Actions; ``INPUT_PACKAGE`` and ``INPUT_STAGING_NAME`` +override the defaults. +""" + +import pathlib +import shutil +import sys +import typing as typ + +from ci_support import ( + CiScriptError, + append_github_path, + catalogue, + describe_failure, + run, +) + +import cyclopts +from cyclopts import App, Parameter + +SUDO = "sudo" +APT_GET = "apt-get" +PRIVILEGED = catalogue(SUDO) + +app = App(config=cyclopts.config.Env("INPUT_", command=False)) + + +def install_package(package: str) -> None: + """Install ``package`` through ``sudo apt-get`` without recommends.""" + for arguments in ( + ("update",), + ("install", "--yes", "--no-install-recommends", package), + ): + result = run(SUDO, APT_GET, *arguments, allowed=PRIVILEGED) + if not result.ok: + message = f"installing {package} failed: {describe_failure(result)}" + raise CiScriptError(message) + + +def locate_binary(package: str) -> pathlib.Path: + """Return the installed awk implementation ``package`` put on PATH. + + Returns + ------- + pathlib.Path + The executable ``shutil.which`` resolved for ``package``. + + Raises + ------ + CiScriptError + If no executable named ``package`` is on PATH after installation. + """ + source = shutil.which(package) + if source is None: + message = f"{package} is not on PATH after installation" + raise CiScriptError(message) + return pathlib.Path(source) + + +def stage_copy(source: pathlib.Path, staging_dir: pathlib.Path) -> pathlib.Path: + """Copy ``source`` into ``staging_dir`` as a regular executable ``awk``. + + ``shutil.copyfile`` follows symlinks and writes a new regular file, which + is the property the sandbox probe depends on. + + Returns + ------- + pathlib.Path + The staged ``awk`` executable. + """ + staging_dir.mkdir(parents=True, exist_ok=True) + staged = staging_dir / "awk" + shutil.copyfile(source, staged) + staged.chmod(0o755) + return staged + + +def verify_staged(staged: pathlib.Path) -> str: + """Run the staged ``awk --version`` and return its first output line.""" + result = run(str(staged), "--version", allowed=catalogue(str(staged))) + if not result.ok: + message = f"the staged awk does not run: {describe_failure(result)}" + raise CiScriptError(message) + return (result.stdout or "").strip().splitlines()[0] if result.stdout else "" + + +@app.default +def main( + *, + runner_temp: typ.Annotated[pathlib.Path, Parameter(env_var="RUNNER_TEMP")], + github_path: typ.Annotated[pathlib.Path, Parameter(env_var="GITHUB_PATH")], + package: str = "gawk", + staging_name: str = "netsuke-test-bin", +) -> int: + """Install the awk package and stage its binary for the sandbox. + + Parameters + ---------- + runner_temp + The runner's scratch directory; the staging directory lives under it. + github_path + The ``GITHUB_PATH`` file that publishes the staging directory. + package + The apt package providing the awk implementation. + staging_name + The staging directory's name under ``runner_temp``. + + Returns + ------- + int + ``0`` on success, ``1`` with a message on standard error otherwise. + """ + try: + install_package(package) + source = locate_binary(package) + staged = stage_copy(source, runner_temp / staging_name) + append_github_path(staged.parent, github_path) + print(f"staged {source} as {staged}: {verify_staged(staged)}") + except CiScriptError as error: + print(f"stage_test_shell: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + app() diff --git a/scripts/tests/ci_script_support.py b/scripts/tests/ci_script_support.py new file mode 100644 index 000000000..cf8157d1e --- /dev/null +++ b/scripts/tests/ci_script_support.py @@ -0,0 +1,84 @@ +"""Shared helpers for the ``scripts/ci`` test modules. + +The CI helper scripts are Cyclopts applications whose parameters come from +``INPUT_*`` and ambient GitHub Actions environment variables, and whose +external programs run through cuprum. These helpers load a script by name, +invoke its app so the entry point's return value comes back as an ``int`` +rather than a ``SystemExit``, and build the release-archive fixtures the +installers download over ``file://`` URLs. + +Run via ``make test-ci-scripts``. +""" + +import importlib +import io +import pathlib +import sys +import tarfile +import typing as typ + +if typ.TYPE_CHECKING: + import types + + from cyclopts import App + +CI_SCRIPT_DIRECTORY = pathlib.Path(__file__).resolve().parents[1] / "ci" +EXECUTABLE_MODE = 0o755 + + +def load_ci_script(name: str) -> types.ModuleType: + """Import ``scripts/ci/.py`` under its own module name.""" + directory = str(CI_SCRIPT_DIRECTORY) + if directory not in sys.path: + sys.path.insert(0, directory) + importlib.invalidate_caches() + return importlib.import_module(name) + + +def invoke(app: App) -> int: + """Run a script's app on the current environment and return its exit code.""" + result = app([], result_action="return_value") + assert isinstance(result, int), ( + f"the entry point must return an int, got {result!r}" + ) + return result + + +def write_executable(path: pathlib.Path, body: str) -> pathlib.Path: + """Write a POSIX shell executable at ``path`` and return it.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"#!/bin/sh\n{body}\n", encoding="utf-8") + path.chmod(EXECUTABLE_MODE) + return path + + +def script_reporting(version: str) -> bytes: + """Return the bytes of an executable that prints ``version``.""" + return f"#!/bin/sh\necho '{version}'\n".encode() + + +def write_tarball( + archive: pathlib.Path, + members: dict[str, bytes], + *, + symlinks: dict[str, str] | None = None, +) -> pathlib.Path: + """Write a gzip tarball holding regular ``members`` and optional symlinks.""" + archive.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(archive, "w:gz") as tar: + for name, content in members.items(): + info = tarfile.TarInfo(name) + info.size = len(content) + info.mode = EXECUTABLE_MODE + tar.addfile(info, io.BytesIO(content)) + for name, target in (symlinks or {}).items(): + info = tarfile.TarInfo(name) + info.type = tarfile.SYMTYPE + info.linkname = target + tar.addfile(info) + return archive + + +def file_url(path: pathlib.Path) -> str: + """Return the ``file://`` URL for ``path``.""" + return path.resolve().as_uri() diff --git a/scripts/tests/test_ci_discard_instrumented_tree.py b/scripts/tests/test_ci_discard_instrumented_tree.py new file mode 100644 index 000000000..dfb0eaf00 --- /dev/null +++ b/scripts/tests/test_ci_discard_instrumented_tree.py @@ -0,0 +1,116 @@ +"""Behavioural tests for ``scripts/ci/discard_instrumented_tree.py``.""" + +import typing as typ + +import pytest +from ci_script_support import invoke, load_ci_script + +if typ.TYPE_CHECKING: + import pathlib + + from cmd_mox import CmdMox + +script = load_ci_script("discard_instrumented_tree") + + +@pytest.fixture +def target(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> pathlib.Path: + """Provide a Cargo target directory with both instrumented trees.""" + target = tmp_path / "target" + (target / "llvm-cov-target" / "debug").mkdir(parents=True) + (target / "llvm-cov-target" / "debug" / "obj").write_bytes(b"x") + (target / "llvm-cov").mkdir() + (target / "debug").mkdir() + monkeypatch.setenv("INPUT_TARGET_DIR", str(target)) + return target + + +def _df(cmd_mox: CmdMox, *, exit_code: int = 0) -> None: + cmd_mox.mock("df").with_args("-h", ".").returns( + stdout="Filesystem 1%\n", exit_code=exit_code + ).times(2) + + +def test_removes_the_instrumented_trees_and_keeps_the_rest( + cmd_mox: CmdMox, target: pathlib.Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Both llvm-cov trees go; the ordinary debug tree stays; df runs twice.""" + _df(cmd_mox) + + exit_code = invoke(script.app) + + assert exit_code == 0, "expected: exit_code == 0" + assert not (target / "llvm-cov-target").exists(), ( + "expected: not (target / 'llvm-cov-target').exists()" + ) + assert not (target / "llvm-cov").exists(), ( + "expected: not (target / 'llvm-cov').exists()" + ) + assert (target / "debug").is_dir(), "expected: (target / 'debug').is_dir()" + out = capsys.readouterr().out + assert f"removed {target / 'llvm-cov-target'}" in out, ( + "expected: f'removed {target / 'llvm-cov-target'}' in out" + ) + + +def test_missing_trees_are_not_an_error( + cmd_mox: CmdMox, monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path +) -> None: + """A lane that never built coverage has nothing to discard and succeeds.""" + monkeypatch.setenv("INPUT_TARGET_DIR", str(tmp_path / "absent")) + _df(cmd_mox) + + assert invoke(script.app) == 0, "expected: invoke(script.app) == 0" + + +def test_symlinked_tree_is_unlinked_not_followed( + cmd_mox: CmdMox, target: pathlib.Path, tmp_path: pathlib.Path +) -> None: + """A link in place of a tree is removed as a link; its target survives.""" + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + (elsewhere / "keep").write_text("keep", encoding="utf-8") + link = target / "llvm-cov" + link.rmdir() + link.symlink_to(elsewhere) + _df(cmd_mox) + + assert invoke(script.app) == 0, "expected: invoke(script.app) == 0" + + assert not link.exists(), "expected: not link.exists()" + assert (elsewhere / "keep").exists(), "expected: (elsewhere / 'keep').exists()" + + +def test_subtrees_input_overrides_the_default_names( + cmd_mox: CmdMox, target: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """INPUT_SUBTREES selects which names are removed, whitespace separated.""" + monkeypatch.setenv("INPUT_SUBTREES", "debug llvm-cov") + _df(cmd_mox) + + assert invoke(script.app) == 0, "expected: invoke(script.app) == 0" + + assert not (target / "debug").exists(), "expected: not (target / 'debug').exists()" + assert not (target / "llvm-cov").exists(), ( + "expected: not (target / 'llvm-cov').exists()" + ) + assert (target / "llvm-cov-target").is_dir(), ( + "expected: (target / 'llvm-cov-target').is_dir()" + ) + + +def test_a_failing_df_fails_the_step_before_removing_anything( + cmd_mox: CmdMox, target: pathlib.Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Disk usage must be observable; if it is not, nothing is discarded.""" + cmd_mox.mock("df").with_args("-h", ".").returns(exit_code=1) + + exit_code = invoke(script.app) + + assert exit_code == 1, "expected: exit_code == 1" + assert "reading disk usage failed" in capsys.readouterr().err, ( + "expected: 'reading disk usage failed' in capsys.readouterr().err" + ) + assert (target / "llvm-cov-target").is_dir(), ( + "expected: (target / 'llvm-cov-target').is_dir()" + ) diff --git a/scripts/tests/test_ci_install_actionlint.py b/scripts/tests/test_ci_install_actionlint.py new file mode 100644 index 000000000..fc42b095b --- /dev/null +++ b/scripts/tests/test_ci_install_actionlint.py @@ -0,0 +1,208 @@ +"""Behavioural tests for ``scripts/ci/install_actionlint.py``.""" + +import hashlib +import os +import typing as typ + +import pytest +from ci_script_support import ( + file_url, + invoke, + load_ci_script, + script_reporting, + write_executable, + write_tarball, +) + +if typ.TYPE_CHECKING: + import pathlib + +script = load_ci_script("install_actionlint") + +VERSION = "1.7.12" + + +@pytest.fixture +def install_dir( + monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path +) -> pathlib.Path: + """Provide the install directory and the required INPUT_* pins.""" + install_dir = tmp_path / "workspace" + install_dir.mkdir() + monkeypatch.setenv("INPUT_INSTALL_DIR", str(install_dir)) + monkeypatch.setenv("INPUT_ACTIONLINT_VERSION", VERSION) + monkeypatch.setenv("RUNNER_TEMP", str(tmp_path)) + return install_dir + + +def _publish_release( + tmp_path: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, + members: dict[str, bytes], + *, + pin_matches: bool = True, +) -> pathlib.Path: + """Serve a release archive under the real URL layout and pin its digest.""" + root = tmp_path / "releases" + archive = write_tarball( + root / f"v{VERSION}" / f"actionlint_{VERSION}_linux_amd64.tar.gz", members + ) + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + monkeypatch.setenv("INPUT_RELEASE_ROOT", file_url(root)) + monkeypatch.setenv("INPUT_ACTIONLINT_SHA256", digest if pin_matches else "0" * 64) + return archive + + +def test_cold_cache_downloads_verifies_and_extracts_the_binary( + install_dir: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, + tmp_path: pathlib.Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """With no binary present, the pinned archive is fetched and installed.""" + _publish_release( + tmp_path, + monkeypatch, + {"actionlint": script_reporting(VERSION), "LICENSE": b"x"}, + ) + + exit_code = invoke(script.app) + + assert exit_code == 0, "expected: exit_code == 0" + binary = install_dir / "actionlint" + assert binary.is_file(), "expected: binary.is_file()" + assert os.access(binary, os.X_OK), "expected: os.access(binary, os.X_OK)" + assert not (install_dir / "LICENSE").exists(), ( + "expected: not (install_dir / 'LICENSE').exists()" + ) + assert f"installed actionlint {VERSION}" in capsys.readouterr().out, ( + "the install must be reported with its version" + ) + + +def test_warm_cache_reuses_a_binary_reporting_the_pinned_version( + install_dir: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, + tmp_path: pathlib.Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """A restored binary at the pin is kept and no download is attempted.""" + cached = write_executable(install_dir / "actionlint", f"echo '{VERSION}'") + before = cached.read_bytes() + monkeypatch.setenv("INPUT_RELEASE_ROOT", file_url(tmp_path / "absent")) + monkeypatch.setenv("INPUT_ACTIONLINT_SHA256", "0" * 64) + + exit_code = invoke(script.app) + + assert exit_code == 0, "expected: exit_code == 0" + assert cached.read_bytes() == before, "expected: cached.read_bytes() == before" + assert "restored from the cache volume" in capsys.readouterr().out, ( + "expected: 'restored from the cache volume' in capsys.readouterr().out" + ) + + +def test_stale_cached_binary_is_replaced_by_the_pinned_release( + install_dir: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, + tmp_path: pathlib.Path, +) -> None: + """A cached binary at another version is overwritten, not trusted.""" + write_executable(install_dir / "actionlint", "echo '1.6.0'") + _publish_release(tmp_path, monkeypatch, {"actionlint": script_reporting(VERSION)}) + + exit_code = invoke(script.app) + + assert exit_code == 0, "expected: exit_code == 0" + assert script.reported_version(install_dir / "actionlint") == VERSION, ( + "expected: script.reported_version(install_dir / 'actionlint') == VE..." + ) + + +def test_checksum_mismatch_installs_nothing( + install_dir: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, + tmp_path: pathlib.Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """An archive that does not match its pin never reaches the install dir.""" + _publish_release( + tmp_path, + monkeypatch, + {"actionlint": script_reporting(VERSION)}, + pin_matches=False, + ) + + exit_code = invoke(script.app) + + assert exit_code == 1, "expected: exit_code == 1" + assert not (install_dir / "actionlint").exists(), ( + "expected: not (install_dir / 'actionlint').exists()" + ) + assert "does not match its pinned SHA-256" in capsys.readouterr().err, ( + "expected: 'does not match its pinned SHA-256' in capsys.readouterr(..." + ) + + +def test_archive_without_the_binary_is_rejected( + install_dir: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, + tmp_path: pathlib.Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """A verified archive still has to contain the one member we extract.""" + _publish_release(tmp_path, monkeypatch, {"README": b"nothing here"}) + + exit_code = invoke(script.app) + + assert exit_code == 1, "expected: exit_code == 1" + assert "has no member 'actionlint'" in capsys.readouterr().err, ( + "expected: 'has no member 'actionlint'' in capsys.readouterr().err" + ) + + +def test_installed_binary_reporting_the_wrong_version_fails( + install_dir: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, + tmp_path: pathlib.Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """The post-install version probe guards against a mislabelled release.""" + _publish_release(tmp_path, monkeypatch, {"actionlint": script_reporting("9.9.9")}) + + exit_code = invoke(script.app) + + assert exit_code == 1, "expected: exit_code == 1" + err = capsys.readouterr().err + assert "reports '9.9.9'" in err, "expected: 'reports '9.9.9'' in err" + assert f"expected '{VERSION}'" in err, "the message must name the pinned version" + + +def test_missing_pins_are_reported_before_anything_runs( + monkeypatch: pytest.MonkeyPatch, + tmp_path: pathlib.Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Without the version pin the app fails at parameter parsing, exit 1.""" + monkeypatch.setenv("INPUT_INSTALL_DIR", str(tmp_path)) + monkeypatch.delenv("INPUT_ACTIONLINT_VERSION", raising=False) + monkeypatch.setenv("INPUT_ACTIONLINT_SHA256", "0" * 64) + + with pytest.raises(SystemExit) as caught: + script.app([]) + + assert caught.value.code == 1, "expected: caught.value.code == 1" + assert "actionlint-version" in capsys.readouterr().err, ( + "expected: 'actionlint-version' in capsys.readouterr().err" + ) + assert not (tmp_path / "actionlint").exists(), ( + "expected: not (tmp_path / 'actionlint').exists()" + ) + + +def test_archive_url_follows_the_release_layout() -> None: + """The URL is the versioned release path with the platform suffix.""" + url = script.archive_url("https://example.invalid/dl", "1.2.3", "linux_amd64") + + assert url == ( + "https://example.invalid/dl/v1.2.3/actionlint_1.2.3_linux_amd64.tar.gz" + ), url diff --git a/scripts/tests/test_ci_install_kani.py b/scripts/tests/test_ci_install_kani.py new file mode 100644 index 000000000..dc4d26d14 --- /dev/null +++ b/scripts/tests/test_ci_install_kani.py @@ -0,0 +1,256 @@ +"""Behavioural tests for ``scripts/ci/install_kani.py``.""" + +import dataclasses +import hashlib +import os +import typing as typ + +import pytest +from ci_script_support import ( + file_url, + invoke, + load_ci_script, + script_reporting, + write_executable, + write_tarball, +) + +if typ.TYPE_CHECKING: + import pathlib + + from cmd_mox import CmdMox + +script = load_ci_script("install_kani") + +VERSION = "0.67.0" +TARGET = "x86_64-unknown-linux-gnu" + + +class Homes(typ.NamedTuple): + """The job's home directories and the expected payload locations.""" + + cargo_home: pathlib.Path + kani_home: pathlib.Path + rustup_home: pathlib.Path + github_path: pathlib.Path + + @property + def frontend_bin(self) -> pathlib.Path: + """The version-qualified front-end directory.""" + return self.cargo_home / "frontend" / f"kani-{VERSION}" + + @property + def driver(self) -> pathlib.Path: + """The verifier driver the bundle install produces.""" + return self.kani_home / f"kani-{VERSION}" / "bin" / "kani-driver" + + +@dataclasses.dataclass(frozen=True, slots=True) +class Harness: + """The job's directories plus the means to publish release fixtures.""" + + homes: Homes + tmp_path: pathlib.Path + monkeypatch: pytest.MonkeyPatch + + def publish( + self, *, frontend_ok: bool = True, bundle_ok: bool = True + ) -> tuple[pathlib.Path, pathlib.Path]: + """Serve both archives under their real URL layouts and pin their digests.""" + quick = self.tmp_path / "quickinstall" + upstream = self.tmp_path / "upstream" + frontend = write_tarball( + quick + / "releases" + / "download" + / f"kani-verifier-{VERSION}" + / f"kani-verifier-{VERSION}-{TARGET}.tar.gz", + { + "cargo-kani": script_reporting("cargo-kani"), + "kani": script_reporting("kani"), + }, + ) + bundle = write_tarball( + upstream + / "releases" + / "download" + / f"kani-{VERSION}" + / f"kani-{VERSION}-{TARGET}.tar.gz", + {"bin/kani-driver": script_reporting("driver")}, + ) + self.monkeypatch.setenv("INPUT_QUICKINSTALL_ROOT", file_url(quick)) + self.monkeypatch.setenv("INPUT_UPSTREAM_ROOT", file_url(upstream)) + self.monkeypatch.setenv( + "INPUT_FRONTEND_SHA256", + hashlib.sha256(frontend.read_bytes()).hexdigest() + if frontend_ok + else "0" * 64, + ) + self.monkeypatch.setenv( + "INPUT_BUNDLE_SHA256", + hashlib.sha256(bundle.read_bytes()).hexdigest() if bundle_ok else "0" * 64, + ) + return frontend, bundle + + def restore_everything(self) -> None: + """Pretend the cache restored both payloads and point the roots nowhere.""" + for name in ("cargo-kani", "kani"): + write_executable(self.homes.frontend_bin / name, "true") + write_executable(self.homes.driver, "true") + absent = file_url(self.tmp_path / "absent") + self.monkeypatch.setenv("INPUT_QUICKINSTALL_ROOT", absent) + self.monkeypatch.setenv("INPUT_UPSTREAM_ROOT", absent) + self.monkeypatch.setenv("INPUT_FRONTEND_SHA256", "0" * 64) + self.monkeypatch.setenv("INPUT_BUNDLE_SHA256", "0" * 64) + + +@pytest.fixture +def kani(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> Harness: + """Provide the ambient job variables, a pinned version file, and fixtures.""" + homes = Homes( + tmp_path / "cargo", + tmp_path / "kani", + tmp_path / "rustup", + tmp_path / "github-path", + ) + homes.github_path.touch() + version_file = tmp_path / "VERSION" + version_file.write_text(f"{VERSION}\n", encoding="utf-8") + monkeypatch.setenv("CARGO_HOME", str(homes.cargo_home)) + monkeypatch.setenv("KANI_HOME", str(homes.kani_home)) + monkeypatch.setenv("RUSTUP_HOME", str(homes.rustup_home)) + monkeypatch.setenv("RUNNER_TEMP", str(tmp_path)) + monkeypatch.setenv("GITHUB_PATH", str(homes.github_path)) + monkeypatch.setenv("INPUT_KANI_VERSION", VERSION) + monkeypatch.setenv("INPUT_VERSION_FILE", str(version_file)) + monkeypatch.setenv("PATH", os.environ["PATH"]) + return Harness(homes, tmp_path, monkeypatch) + + +def _mock_setup(cmd_mox: CmdMox, bundle: pathlib.Path, *, exit_code: int = 0) -> None: + cmd_mox.mock("cargo").with_args( + "kani", "setup", "--use-local-bundle", str(bundle) + ).returns(exit_code=exit_code) + + +def test_cold_cache_installs_both_payloads(cmd_mox: CmdMox, kani: Harness) -> None: + """Front-end binaries are extracted and the bundle is handed to cargo kani.""" + _, bundle = kani.publish() + _mock_setup(cmd_mox, kani.tmp_path / bundle.name) + + exit_code = invoke(script.app) + + assert exit_code == 0, "expected: exit_code == 0" + for name in ("cargo-kani", "kani"): + assert os.access(kani.homes.frontend_bin / name, os.X_OK), ( + "expected: os.access(kani.homes.frontend_bin / name, os.X_OK)" + ) + assert kani.homes.rustup_home.is_dir(), "expected: kani.homes.rustup_home.is_dir()" + assert kani.homes.github_path.read_text(encoding="utf-8") == ( + f"{kani.homes.frontend_bin}\n" + ), "the front-end directory must be published through GITHUB_PATH exactly once" + assert os.environ["PATH"].split(os.pathsep)[0] == str(kani.homes.frontend_bin), ( + "expected: os.environ['PATH'].split(os.pathsep)[0] == str(kani.homes..." + ) + + +def test_warm_cache_downloads_nothing_and_skips_setup( + cmd_mox: CmdMox, kani: Harness, capsys: pytest.CaptureFixture[str] +) -> None: + """Restored payloads are detected by their executables; no network, no cargo.""" + kani.restore_everything() + cargo = cmd_mox.spy("cargo").returns(exit_code=0) + + exit_code = invoke(script.app) + + assert exit_code == 0, "expected: exit_code == 0" + cargo.assert_not_called() + out = capsys.readouterr().out + assert "front-end 0.67.0 present" in out, ( + "expected: 'front-end 0.67.0 present' in out" + ) + assert "verifier bundle 0.67.0 present" in out, ( + "expected: 'verifier bundle 0.67.0 present' in out" + ) + assert kani.homes.github_path.read_text(encoding="utf-8") == ( + f"{kani.homes.frontend_bin}\n" + ), "the front-end directory must be published through GITHUB_PATH exactly once" + + +def test_version_file_disagreeing_with_the_pin_stops_everything( + cmd_mox: CmdMox, kani: Harness, capsys: pytest.CaptureFixture[str] +) -> None: + """tools/kani/VERSION and the workflow pin must agree before any download.""" + kani.publish() + kani.monkeypatch.setenv("INPUT_KANI_VERSION", "0.66.0") + cargo = cmd_mox.spy("cargo").returns(exit_code=0) + + exit_code = invoke(script.app) + + assert exit_code == 1, "expected: exit_code == 1" + cargo.assert_not_called() + err = capsys.readouterr().err + assert "pins Kani '0.67.0' but the workflow passed '0.66.0'" in err, err + assert not kani.homes.cargo_home.exists(), ( + "expected: not kani.homes.cargo_home.exists()" + ) + + +def test_front_end_checksum_mismatch_extracts_nothing( + cmd_mox: CmdMox, kani: Harness, capsys: pytest.CaptureFixture[str] +) -> None: + """The front-end archive is verified before a single member is extracted.""" + kani.publish(frontend_ok=False) + cargo = cmd_mox.spy("cargo").returns(exit_code=0) + + exit_code = invoke(script.app) + + assert exit_code == 1, "expected: exit_code == 1" + cargo.assert_not_called() + assert not (kani.homes.frontend_bin / "cargo-kani").exists(), ( + "expected: not (kani.homes.frontend_bin / 'cargo-kani').exists()" + ) + assert "does not match its pinned SHA-256" in capsys.readouterr().err, ( + "expected: 'does not match its pinned SHA-256' in capsys.readouterr(..." + ) + + +def test_bundle_checksum_mismatch_never_reaches_cargo_kani_setup( + cmd_mox: CmdMox, kani: Harness +) -> None: + """The bundle is verified before it is handed to the installer.""" + kani.publish(bundle_ok=False) + cargo = cmd_mox.spy("cargo").returns(exit_code=0) + + exit_code = invoke(script.app) + + assert exit_code == 1, "expected: exit_code == 1" + cargo.assert_not_called() + assert os.access(kani.homes.frontend_bin / "kani", os.X_OK), ( + "the front-end still installs" + ) + + +def test_failed_bundle_setup_is_reported( + cmd_mox: CmdMox, kani: Harness, capsys: pytest.CaptureFixture[str] +) -> None: + """A non-zero cargo kani setup fails the step with its exit code.""" + _, bundle = kani.publish() + _mock_setup(cmd_mox, kani.tmp_path / bundle.name, exit_code=7) + + exit_code = invoke(script.app) + + assert exit_code == 1, "expected: exit_code == 1" + assert "installing the verifier bundle failed" in capsys.readouterr().err, ( + "expected: 'installing the verifier bundle failed' in capsys.readout..." + ) + + +def test_urls_follow_the_release_layouts() -> None: + """Both archive URLs are the versioned release paths for the target.""" + assert script.frontend_url("https://q.invalid", "1.0.0", "t") == ( + "https://q.invalid/releases/download/kani-verifier-1.0.0/kani-verifier-1.0.0-t.tar.gz" + ), "expected: script.frontend_url('https://q.invalid', '1.0.0', 't') ==..." + assert script.bundle_url("https://u.invalid", "1.0.0", "t") == ( + "https://u.invalid/releases/download/kani-1.0.0/kani-1.0.0-t.tar.gz" + ), "expected: script.bundle_url('https://u.invalid', '1.0.0', 't') == (..." diff --git a/scripts/tests/test_ci_report_sccache_stats.py b/scripts/tests/test_ci_report_sccache_stats.py new file mode 100644 index 000000000..88fc1abdf --- /dev/null +++ b/scripts/tests/test_ci_report_sccache_stats.py @@ -0,0 +1,100 @@ +"""Behavioural tests for ``scripts/ci/report_sccache_stats.py``.""" + +import typing as typ + +import pytest +from ci_script_support import invoke, load_ci_script + +if typ.TYPE_CHECKING: + import pathlib + + from cmd_mox import CmdMox + from cmd_mox.ipc import Invocation + +script = load_ci_script("report_sccache_stats") + +TEXT_STATS = "Compile requests 12\nCache hits 10\n" +JSON_STATS = '{"stats":{"compile_requests":12}}\n' + + +@pytest.fixture +def summary(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> pathlib.Path: + """Provide GITHUB_STEP_SUMMARY with prior content and the output directory.""" + summary = tmp_path / "summary.md" + summary.write_text("## Earlier\n", encoding="utf-8") + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + monkeypatch.setenv("INPUT_OUTPUT_DIR", str(tmp_path)) + return summary + + +def _sccache(cmd_mox: CmdMox, *, json_exit: int = 0) -> list[list[str]]: + """Mock ``sccache`` for the text and JSON reports, recording its calls.""" + calls: list[list[str]] = [] + + def handle(invocation: Invocation) -> tuple[str, str, int]: + calls.append(list(invocation.args)) + if "--stats-format=json" in invocation.args: + return (JSON_STATS, "", json_exit) + return (TEXT_STATS, "", 0) + + cmd_mox.mock("sccache").runs(handle).times(2) + return calls + + +def test_writes_text_json_and_the_summary_section( + cmd_mox: CmdMox, summary: pathlib.Path, tmp_path: pathlib.Path +) -> None: + """Both report files land in the output dir and the summary gains a block.""" + calls = _sccache(cmd_mox) + + exit_code = invoke(script.app) + + assert exit_code == 0, "expected: exit_code == 0" + assert calls == [["--show-stats"], ["--show-stats", "--stats-format=json"]], ( + "expected: calls == [['--show-stats'], ['--show-stats', '--stats-for..." + ) + assert (tmp_path / "sccache-stats.txt").read_text(encoding="utf-8") == TEXT_STATS, ( + "expected: (tmp_path / 'sccache-stats.txt').read_text(encoding='utf-..." + ) + assert (tmp_path / "sccache-stats.json").read_text( + encoding="utf-8" + ) == JSON_STATS, ( + "expected: (tmp_path / 'sccache-stats.json').read_text(encoding='utf..." + ) + assert summary.read_text(encoding="utf-8") == ( + "## Earlier\n### sccache\n\n```text\n" + TEXT_STATS + "```\n" + ), "expected: summary.read_text(encoding='utf-8') == ( '## Earlier\n###..." + + +def test_a_failing_report_leaves_no_files_and_fails_the_step( + cmd_mox: CmdMox, + summary: pathlib.Path, + tmp_path: pathlib.Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """A non-zero sccache exit is surfaced and nothing partial is written.""" + _sccache(cmd_mox, json_exit=2) + + exit_code = invoke(script.app) + + assert exit_code == 1, "expected: exit_code == 1" + assert "reading statistics failed" in capsys.readouterr().err, ( + "expected: 'reading statistics failed' in capsys.readouterr().err" + ) + assert not (tmp_path / "sccache-stats.txt").exists(), ( + "expected: not (tmp_path / 'sccache-stats.txt').exists()" + ) + assert not (tmp_path / "sccache-stats.json").exists(), ( + "expected: not (tmp_path / 'sccache-stats.json').exists()" + ) + assert summary.read_text(encoding="utf-8") == "## Earlier\n", ( + "expected: summary.read_text(encoding='utf-8') == '## Earlier\n'" + ) + + +def test_summary_section_terminates_unterminated_text() -> None: + """The fenced block always closes on its own line.""" + section = script.summary_section("hits 1") + + assert section.startswith("### sccache\n\n```text\n"), section + assert section.endswith("hits 1\n```\n"), "the fence must close on its own line" diff --git a/scripts/tests/test_ci_scripts_metadata.py b/scripts/tests/test_ci_scripts_metadata.py new file mode 100644 index 000000000..dbe7125a1 --- /dev/null +++ b/scripts/tests/test_ci_scripts_metadata.py @@ -0,0 +1,64 @@ +"""Hold every CI helper script to the shared dependency pins and entry shape.""" + +import re +import typing as typ + +import pytest +from ci_script_support import CI_SCRIPT_DIRECTORY + +if typ.TYPE_CHECKING: + import pathlib + +REPO_ROOT = CI_SCRIPT_DIRECTORY.parents[1] +MAKEFILE = REPO_ROOT / "Makefile" +SCRIPTS = sorted( + path for path in CI_SCRIPT_DIRECTORY.glob("*.py") if path.name != "ci_support.py" +) + + +def _makefile_variable(name: str) -> str: + text = MAKEFILE.read_text(encoding="utf-8") + matches = re.findall(rf"^{re.escape(name)} \?= (\S+)$", text, flags=re.MULTILINE) + assert len(matches) == 1, f"the Makefile must declare {name} exactly once" + return matches[0] + + +def test_there_are_scripts_to_hold() -> None: + """The glob is live; an empty directory would pass the other tests vacuously.""" + assert [path.name for path in SCRIPTS] == [ + "discard_instrumented_tree.py", + "install_actionlint.py", + "install_kani.py", + "report_sccache_stats.py", + "stage_test_shell.py", + ], "expected: [path.name for path in SCRIPTS] == [ 'discard_instrumente..." + + +@pytest.mark.parametrize("path", SCRIPTS, ids=[path.name for path in SCRIPTS]) +def test_script_pins_the_makefile_dependency_versions(path: pathlib.Path) -> None: + """Each PEP 723 block pins cyclopts and cuprum at the Makefile's versions.""" + text = path.read_text(encoding="utf-8") + cyclopts = _makefile_variable("CYCLOPTS_VERSION") + cuprum = _makefile_variable("CUPRUM_VERSION") + header, _, _ = text.partition('# ///\n"""') + assert header.startswith("#!/usr/bin/env -S uv run --script\n# /// script\n"), ( + path.name + ) + assert 'requires-python = ">=3.14"' in header, path.name + assert f'"cyclopts=={cyclopts}"' in header, ( + f"{path.name} must pin cyclopts=={cyclopts}, the Makefile CYCLOPTS_VERSION" + ) + assert f'"cuprum=={cuprum}"' in header, ( + f"{path.name} must pin cuprum=={cuprum}, the Makefile CUPRUM_VERSION" + ) + + +@pytest.mark.parametrize("path", SCRIPTS, ids=[path.name for path in SCRIPTS]) +def test_script_reads_inputs_from_the_environment_through_cyclopts( + path: pathlib.Path, +) -> None: + """Every script is an env-first Cyclopts app with a single default command.""" + text = path.read_text(encoding="utf-8") + assert 'App(config=cyclopts.config.Env("INPUT_", command=False))' in text, path.name + assert "@app.default" in text, path.name + assert text.rstrip().endswith('if __name__ == "__main__":\n app()'), path.name diff --git a/scripts/tests/test_ci_stage_test_shell.py b/scripts/tests/test_ci_stage_test_shell.py new file mode 100644 index 000000000..8d9a61356 --- /dev/null +++ b/scripts/tests/test_ci_stage_test_shell.py @@ -0,0 +1,166 @@ +"""Behavioural tests for ``scripts/ci/stage_test_shell.py``.""" + +import os +import pathlib +import sys +import typing as typ + +import pytest +from ci_script_support import invoke, load_ci_script, write_executable + +if typ.TYPE_CHECKING: + from cmd_mox import CmdMox + from cmd_mox.ipc import Invocation + +script = load_ci_script("stage_test_shell") + +GAWK_VERSION_LINE = "GNU Awk 5.3.1, API 4.0" + + +@pytest.fixture +def workflow_env( + monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path +) -> pathlib.Path: + """Provide the ambient GitHub Actions variables and an empty GITHUB_PATH.""" + runner_temp = tmp_path / "runner-temp" + runner_temp.mkdir() + github_path = tmp_path / "github-path" + github_path.touch() + monkeypatch.setenv("RUNNER_TEMP", str(runner_temp)) + monkeypatch.setenv("GITHUB_PATH", str(github_path)) + return tmp_path + + +@pytest.fixture +def system_gawk( + monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path +) -> pathlib.Path: + """Put a fake ``gawk`` on PATH behind an ``awk`` symlink, as Ubuntu does.""" + system_bin = tmp_path / "usr-bin" + gawk = write_executable(system_bin / "gawk", f"echo '{GAWK_VERSION_LINE}'") + (system_bin / "awk").symlink_to(gawk) + monkeypatch.setenv("PATH", f"{system_bin}{os.pathsep}{os.environ['PATH']}") + return gawk + + +def _apt_get(cmd_mox: CmdMox, *, install_exit: int = 0) -> list[list[str]]: + """Mock ``sudo apt-get`` twice, recording the argument lists it received.""" + calls: list[list[str]] = [] + + def handle(invocation: Invocation) -> tuple[str, str, int]: + calls.append(list(invocation.args)) + exit_code = install_exit if "install" in invocation.args else 0 + return ("", "E: install failed\n" if exit_code else "", exit_code) + + cmd_mox.mock("sudo").runs(handle).times(2) + return calls + + +def test_stages_gawk_as_a_regular_awk_and_publishes_the_directory( + cmd_mox: CmdMox, + workflow_env: pathlib.Path, + system_gawk: pathlib.Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Cold run: apt installs gawk, a regular copy is staged, verified, published.""" + calls = _apt_get(cmd_mox) + + exit_code = invoke(script.app) + + assert exit_code == 0, "expected: exit_code == 0" + assert calls == [ + ["apt-get", "update"], + ["apt-get", "install", "--yes", "--no-install-recommends", "gawk"], + ], "expected: calls == [ ['apt-get', 'update'], ['apt-get', 'install', ..." + staged = workflow_env / "runner-temp" / "netsuke-test-bin" / "awk" + assert staged.is_file(), "expected: staged.is_file()" + assert not staged.is_symlink(), "the sandbox probe cannot follow a symlink" + assert os.access(staged, os.X_OK), "expected: os.access(staged, os.X_OK)" + assert staged.read_bytes() == system_gawk.read_bytes(), ( + "expected: staged.read_bytes() == system_gawk.read_bytes()" + ) + github_path = (workflow_env / "github-path").read_text(encoding="utf-8") + assert github_path == f"{staged.parent}\n", ( + "expected: github_path == f'{staged.parent}\n'" + ) + assert GAWK_VERSION_LINE in capsys.readouterr().out, ( + "expected: GAWK_VERSION_LINE in capsys.readouterr().out" + ) + + +def test_reports_a_failed_package_install_and_stages_nothing( + cmd_mox: CmdMox, + workflow_env: pathlib.Path, + system_gawk: pathlib.Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """An apt failure stops the step with its reason; nothing is published.""" + _apt_get(cmd_mox, install_exit=100) + + exit_code = invoke(script.app) + + assert exit_code == 1, "expected: exit_code == 1" + captured = capsys.readouterr() + assert "installing gawk failed" in captured.err, ( + "expected: 'installing gawk failed' in captured.err" + ) + assert "exited 100" in captured.err, "expected: 'exited 100' in captured.err" + assert not (workflow_env / "runner-temp" / "netsuke-test-bin").exists(), ( + "expected: not (workflow_env / 'runner-temp' / 'netsuke-test-bin').e..." + ) + assert not (workflow_env / "github-path").read_text(encoding="utf-8"), ( + "expected: not (workflow_env / 'github-path').read_text(encoding='ut..." + ) + + +def test_reports_a_package_that_leaves_no_binary_on_path( + cmd_mox: CmdMox, + workflow_env: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """A successful install that provides no ``gawk`` is named, not staged.""" + shim_dir = cmd_mox.environment.shim_dir + assert shim_dir is not None, "cmd-mox must provide its shim directory" + # Keep the shims (and the interpreter they run under) reachable while + # hiding every real binary from the search. + interpreter_dir = pathlib.Path(sys.executable).parent + monkeypatch.setenv( + "PATH", + os.pathsep.join([ + str(workflow_env / "empty-bin"), + str(shim_dir), + str(interpreter_dir), + ]), + ) + _apt_get(cmd_mox) + + exit_code = invoke(script.app) + + assert exit_code == 1, "expected: exit_code == 1" + assert "gawk is not on PATH after installation" in capsys.readouterr().err, ( + "expected: 'gawk is not on PATH after installation' in capsys.readou..." + ) + + +def test_honours_package_and_staging_name_inputs( + cmd_mox: CmdMox, + workflow_env: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, + tmp_path: pathlib.Path, +) -> None: + """INPUT_PACKAGE and INPUT_STAGING_NAME select the package and directory.""" + system_bin = tmp_path / "usr-bin" + write_executable(system_bin / "mawk", "echo 'mawk 1.3'") + monkeypatch.setenv("PATH", f"{system_bin}{os.pathsep}{os.environ['PATH']}") + monkeypatch.setenv("INPUT_PACKAGE", "mawk") + monkeypatch.setenv("INPUT_STAGING_NAME", "custom-bin") + calls = _apt_get(cmd_mox) + + exit_code = invoke(script.app) + + assert exit_code == 0, "expected: exit_code == 0" + assert calls[1][-1] == "mawk", "expected: calls[1][-1] == 'mawk'" + assert (workflow_env / "runner-temp" / "custom-bin" / "awk").is_file(), ( + "expected: (workflow_env / 'runner-temp' / 'custom-bin' / 'awk').is_..." + ) diff --git a/scripts/tests/test_ci_support.py b/scripts/tests/test_ci_support.py new file mode 100644 index 000000000..2062e994b --- /dev/null +++ b/scripts/tests/test_ci_support.py @@ -0,0 +1,215 @@ +"""Exercise the helpers shared by the CI helper scripts.""" + +import hashlib +import os +import typing as typ + +import pytest +from ci_script_support import file_url, load_ci_script, write_executable, write_tarball + +if typ.TYPE_CHECKING: + import pathlib + + from cmd_mox import CmdMox + +support = load_ci_script("ci_support") + + +def test_run_returns_captured_output_for_an_allowlisted_program( + cmd_mox: CmdMox, +) -> None: + """A mocked program's stdout and exit code come back on the result.""" + cmd_mox.mock("frob").with_args("--flag", "value").returns(stdout="done\n") + + result = support.run( + "frob", "--flag", "value", allowed=support.catalogue("frob"), echo=False + ) + + assert result.ok, result + assert result.stdout == "done\n", "expected: result.stdout == 'done\n'" + + +def test_run_refuses_a_program_outside_the_catalogue() -> None: + """The catalogue is the allowlist: an unlisted program never spawns.""" + from cuprum import UnknownProgramError + + with pytest.raises(UnknownProgramError): + support.run("frob", allowed=support.catalogue("other"), echo=False) + + +def test_run_overlays_env_on_the_child(cmd_mox: CmdMox) -> None: + """A per-run env overlay reaches the child without touching os.environ.""" + seen: list[str | None] = [] + cmd_mox.mock("frob").runs( + lambda inv: (seen.append(inv.env.get("CI_PROBE")) or "", "", 0) + ) + + support.run( + "frob", allowed=support.catalogue("frob"), env={"CI_PROBE": "yes"}, echo=False + ) + + assert seen == ["yes"], "expected: seen == ['yes']" + assert "CI_PROBE" not in os.environ, "expected: 'CI_PROBE' not in os.environ" + + +def test_describe_failure_names_the_command_and_last_stderr_line( + cmd_mox: CmdMox, +) -> None: + """Failure text carries the argv, exit code, and the final stderr line.""" + cmd_mox.mock("frob").with_args("x").returns( + stderr="first\nlast line\n", exit_code=3 + ) + + result = support.run("frob", "x", allowed=support.catalogue("frob"), echo=False) + + assert support.describe_failure(result) == "`frob x` exited 3: last line", ( + "expected: support.describe_failure(result) == '`frob x` exited 3: l..." + ) + + +def test_append_github_path_appends_one_line(tmp_path: pathlib.Path) -> None: + """Each published directory is one line, appended after existing content.""" + github_path = tmp_path / "path" + github_path.write_text("/existing\n", encoding="utf-8") + + support.append_github_path(tmp_path / "bin", github_path) + + assert ( + github_path.read_text(encoding="utf-8") == f"/existing\n{tmp_path / 'bin'}\n" + ), "expected: github_path.read_text(encoding='utf-8') == f'/existing\n{..." + + +def test_prepend_path_puts_the_directory_first( + monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path +) -> None: + """The process PATH gains the directory ahead of everything else.""" + monkeypatch.setenv("PATH", "/usr/bin") + + support.prepend_path(tmp_path) + + assert os.environ["PATH"] == f"{tmp_path}{os.pathsep}/usr/bin", ( + "the directory must be first on PATH, ahead of the existing entries" + ) + + +def test_is_executable_file_rejects_directories_and_plain_files( + tmp_path: pathlib.Path, +) -> None: + """Only a regular file with the execute bit counts.""" + plain = tmp_path / "plain" + plain.write_text("x", encoding="utf-8") + executable = write_executable(tmp_path / "exe", "true") + + assert not support.is_executable_file(tmp_path), ( + "expected: not support.is_executable_file(tmp_path)" + ) + assert not support.is_executable_file(plain), ( + "expected: not support.is_executable_file(plain)" + ) + assert not support.is_executable_file(tmp_path / "missing"), ( + "expected: not support.is_executable_file(tmp_path / 'missing')" + ) + assert support.is_executable_file(executable), ( + "expected: support.is_executable_file(executable)" + ) + + +def test_download_copies_a_file_url(tmp_path: pathlib.Path) -> None: + """A ``file://`` fixture is fetched through the real download path.""" + source = tmp_path / "source.bin" + source.write_bytes(b"payload") + + support.download(file_url(source), tmp_path / "dest.bin") + + assert (tmp_path / "dest.bin").read_bytes() == b"payload", ( + "expected: (tmp_path / 'dest.bin').read_bytes() == b'payload'" + ) + + +@pytest.mark.parametrize("url", ["http://example.invalid/x", "ftp://example.invalid/x"]) +def test_download_refuses_insecure_schemes(url: str, tmp_path: pathlib.Path) -> None: + """Only https and file URLs are ever opened.""" + with pytest.raises(support.CiScriptError, match="URLs are permitted"): + support.download(url, tmp_path / "dest.bin") + assert not (tmp_path / "dest.bin").exists(), ( + "expected: not (tmp_path / 'dest.bin').exists()" + ) + + +def test_download_reports_a_missing_source(tmp_path: pathlib.Path) -> None: + """A failed fetch is an actionable error naming the URL.""" + url = file_url(tmp_path / "absent.bin") + with pytest.raises(support.CiScriptError, match=r"downloading .* failed"): + support.download(url, tmp_path / "dest.bin") + + +def test_verify_sha256_accepts_the_matching_digest(tmp_path: pathlib.Path) -> None: + """A matching pin, in either case, passes silently.""" + archive = tmp_path / "a.tar.gz" + archive.write_bytes(b"archive") + digest = hashlib.sha256(b"archive").hexdigest() + + support.verify_sha256(archive, digest) + support.verify_sha256(archive, digest.upper()) + + +def test_verify_sha256_names_both_digests_on_mismatch(tmp_path: pathlib.Path) -> None: + """The failure states the pinned and actual digests so drift is diagnosable.""" + archive = tmp_path / "a.tar.gz" + archive.write_bytes(b"archive") + actual = hashlib.sha256(b"archive").hexdigest() + + with pytest.raises(support.CiScriptError) as caught: + support.verify_sha256(archive, "0" * 64) + message = str(caught.value) + assert "0" * 64 in message, "expected: '0' * 64 in message" + assert actual in message, "expected: actual in message" + assert "ci.yml" in message, "expected: 'ci.yml' in message" + + +def test_extract_members_writes_named_files_as_executables( + tmp_path: pathlib.Path, +) -> None: + """Named members land flat under the destination with the execute bit set.""" + archive = write_tarball( + tmp_path / "a.tar.gz", + {"tool": b"#!/bin/sh\necho tool\n", "nested/other": b"other", "README": b"doc"}, + ) + destination = tmp_path / "bin" + destination.mkdir() + + extracted = support.extract_members(archive, destination, ("tool", "nested/other")) + + assert extracted == [destination / "tool", destination / "other"], ( + "expected: extracted == [destination / 'tool', destination / 'other']" + ) + assert (destination / "tool").read_bytes().startswith(b"#!/bin/sh"), ( + "expected: (destination / 'tool').read_bytes().startswith(b'#!/bin/sh')" + ) + assert os.access(destination / "tool", os.X_OK), ( + "expected: os.access(destination / 'tool', os.X_OK)" + ) + assert not (destination / "README").exists(), ( + "expected: not (destination / 'README').exists()" + ) + + +def test_extract_members_refuses_a_missing_member(tmp_path: pathlib.Path) -> None: + """An archive without the expected member is rejected by name.""" + archive = write_tarball(tmp_path / "a.tar.gz", {"README": b"doc"}) + + with pytest.raises(support.CiScriptError, match="has no member 'tool'"): + support.extract_members(archive, tmp_path, ("tool",)) + + +def test_extract_members_refuses_a_symlink_member(tmp_path: pathlib.Path) -> None: + """A link is never followed or written, so an archive cannot escape.""" + archive = write_tarball( + tmp_path / "a.tar.gz", {"README": b"doc"}, symlinks={"tool": "/etc/passwd"} + ) + + with pytest.raises(support.CiScriptError, match="is not a regular file"): + support.extract_members(archive, tmp_path, ("tool",)) + assert not (tmp_path / "tool").exists(), ( + "expected: not (tmp_path / 'tool').exists()" + ) diff --git a/tests/workflow_contracts/actionlint_installer_contract.py b/tests/workflow_contracts/actionlint_installer_contract.py index 3d37f9dd0..44057e79e 100644 --- a/tests/workflow_contracts/actionlint_installer_contract.py +++ b/tests/workflow_contracts/actionlint_installer_contract.py @@ -1,111 +1,32 @@ """Pinned installer contract for the actionlint workflow linter. -The Linux CI job downloads actionlint through a reviewed installer script at a -fixed revision, feeding it an archive whose SHA-256 the job verifies itself. -These constants spell out that script line by line so a silent change to the -version, endpoint, or checksum fails a test rather than shipping. -``github_actions_validation_test.py`` consumes them; this module holds no -tests of its own. +The Linux CI job installs actionlint through ``scripts/ci/install_actionlint.py``, +a Cyclopts helper that reuses the gate-cached binary when it reports the +pinned version and otherwise downloads the pinned Linux amd64 release archive, +verifies its SHA-256, and extracts the single ``actionlint`` member. The pins +live beside the step as ``INPUT_*`` environment variables so a silent change +to the version or checksum fails a test rather than shipping; the download, +verification, reuse, and failure behaviours are covered by +``scripts/tests/test_ci_install_actionlint.py``. ``github_actions_validation_test.py`` +consumes these constants; this module holds no tests of its own. Run via ``make test-workflow-contracts``. """ - -def shell_variable(name: str) -> str: - """Return a shell variable expansion for script contract expectations.""" - return f"${{{name}}}" - - ACTIONLINT_VERSION = "1.7.12" ACTIONLINT_SHA256 = "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8" -ACTIONLINT_INSTALLER_COMMIT = "914e7df21a07ef503a81201c76d2b11c789d3fca" -ACTIONLINT_ARCHIVE = ( - f"actionlint_{shell_variable('ACTIONLINT_VERSION')}_linux_amd64.tar.gz" -) -ACTIONLINT_RAW_BASE = "https://raw.githubusercontent.com/rhysd/actionlint" -ACTIONLINT_SCRIPT = "scripts/download-actionlint.bash" -ACTIONLINT_RELEASE_ROOT = "https://github.com/rhysd/actionlint/releases/download" - -ACTIONLINT_INSTALL_COMMAND = ( - f'bash "{shell_variable("ACTIONLINT_INSTALLER_PATH")}" ' - f'"{shell_variable("ACTIONLINT_VERSION")}"' -) -ACTIONLINT_CHECKSUM_COMMAND = ( - f"printf '%s %s\\n' \"{shell_variable('ACTIONLINT_SHA256')}\" " - f'"{shell_variable("ACTIONLINT_ARCHIVE_PATH")}" | sha256sum --check --' -) -ACTIONLINT_SCRIPT_CONTRACTS = ( - ( - f"readonly ACTIONLINT_VERSION='{ACTIONLINT_VERSION}'", - "the actionlint installer must pin the expected release version", - ), - ( - f"readonly ACTIONLINT_SHA256='{ACTIONLINT_SHA256}'", - "the actionlint installer must pin the expected release archive checksum", - ), - ( - f"readonly ACTIONLINT_INSTALLER_COMMIT='{ACTIONLINT_INSTALLER_COMMIT}'", - "the actionlint installer must pin its reviewed installer revision", - ), - ( - f'readonly ACTIONLINT_ARCHIVE="{ACTIONLINT_ARCHIVE}"', - "the actionlint installer must request the published Linux amd64 archive", - ), - ( - f"readonly ACTIONLINT_RAW_BASE='{ACTIONLINT_RAW_BASE}'", - "the actionlint installer must own its immutable raw-content endpoint", - ), - ( - f"readonly ACTIONLINT_SCRIPT='{ACTIONLINT_SCRIPT}'", - "the actionlint installer must pin its downloader script path", - ), - ( - ( - 'readonly ACTIONLINT_INSTALLER_URL="' - f"{shell_variable('ACTIONLINT_RAW_BASE')}/" - f"{shell_variable('ACTIONLINT_INSTALLER_COMMIT')}/" - f'{shell_variable("ACTIONLINT_SCRIPT")}"' - ), - "the actionlint installer URL must be constructed from its pinned inputs", - ), - ( - f"readonly ACTIONLINT_RELEASE_ROOT='{ACTIONLINT_RELEASE_ROOT}'", - "the actionlint installer must own its release endpoint", - ), - ( - ( - 'readonly ACTIONLINT_RELEASE_BASE="' - f"{shell_variable('ACTIONLINT_RELEASE_ROOT')}/" - f'v{shell_variable("ACTIONLINT_VERSION")}"' - ), - "the actionlint release base must select the pinned version", - ), - ( - ( - 'readonly ACTIONLINT_RELEASE_URL="' - f"{shell_variable('ACTIONLINT_RELEASE_BASE')}/" - f'{shell_variable("ACTIONLINT_ARCHIVE")}"' - ), - ("the actionlint release URL must be constructed from the pinned archive"), - ), - ( - ( - "command curl --fail --location --show-error --output " - f'"{shell_variable("ACTIONLINT_INSTALLER_PATH")}" \\\n' - f' "{shell_variable("ACTIONLINT_INSTALLER_URL")}"' - ), - "the actionlint installer download must use the installer endpoint", - ), +ACTIONLINT_INSTALL_SCRIPT = "scripts/ci/install_actionlint.py" +ACTIONLINT_INSTALL_COMMAND = f"uv run --script {ACTIONLINT_INSTALL_SCRIPT}" +#: The pins the step must pass to the installer, and the concern each protects. +ACTIONLINT_STEP_ENV_CONTRACTS = ( ( - ( - "command curl --fail --location --show-error --output " - f'"{shell_variable("ACTIONLINT_ARCHIVE_PATH")}" \\\n' - f' "{shell_variable("ACTIONLINT_RELEASE_URL")}"' - ), - "the actionlint archive download must use the release endpoint", + "INPUT_ACTIONLINT_VERSION", + ACTIONLINT_VERSION, + "the actionlint step must pin the expected release version", ), ( - ACTIONLINT_CHECKSUM_COMMAND, - "the actionlint archive checksum must verify the downloaded archive", + "INPUT_ACTIONLINT_SHA256", + ACTIONLINT_SHA256, + "the actionlint step must pin the expected release archive checksum", ), ) diff --git a/tests/workflow_contracts/ci_lint_test.py b/tests/workflow_contracts/ci_lint_test.py index 2803eab86..d91d7a598 100644 --- a/tests/workflow_contracts/ci_lint_test.py +++ b/tests/workflow_contracts/ci_lint_test.py @@ -32,6 +32,7 @@ CI_WORKFLOW_PATH, MAKEFILE_PATH, NEXTEST_JOBS, + REPO_ROOT, SETUP_RUST_JOBS, _WorkflowLoader, job_steps, @@ -50,14 +51,11 @@ #: silently drops `test_support` and the non-default targets from the gate. EXPECTED_CLIPPY_FLAGS = "--workspace --all-targets --all-features -- -D warnings" -#: The whole awk staging dance the sandbox probe depends on. Asserting the -#: fragments as one block reports every part a partial edit dropped, rather -#: than only the first. -AWK_STAGING_FRAGMENTS = ( - 'test_shell_bin="${RUNNER_TEMP}/netsuke-test-bin"', - 'install --mode=0755 "$(command -v gawk)"', - '"${test_shell_bin}/awk"', -) +#: The Cyclopts helper that installs gawk and stages a regular-file `awk` +#: the sandbox probe can follow. Its behaviour is covered by +#: `scripts/tests/test_ci_stage_test_shell.py`; the workflow contract is only +#: that the step runs it, through uv, after uv is set up. +TEST_SHELL_SCRIPT = "scripts/ci/stage_test_shell.py" def _tool_input(step: dict[str, object]) -> object: @@ -144,62 +142,18 @@ def test_load_workflow_rejects_invalid_document_roots( load_workflow(workflow_path) -@pytest.fixture -def test_shell_script() -> str: - """Return the run script of the test-shell dependency step.""" - step = named_step(job_steps(load_workflow(), "build-test"), TEST_SHELL_STEP) - match step.get("run"): - case str() as run: - return run - case _: - pytest.fail(f"{TEST_SHELL_STEP} must declare a run script") - - -def test_test_shell_step_installs_gawk(test_shell_script: str) -> None: - """The step installs gawk, the implementation awk is copied from.""" - assert re.search(r"apt-get install\b.*\bgawk\b", test_shell_script), ( - f"{TEST_SHELL_STEP} must apt-get install gawk, got:\n{test_shell_script}" - ) - - -def test_test_shell_step_copies_gawk_to_a_regular_awk_executable( - test_shell_script: str, -) -> None: - """Gawk is copied — not linked — to ${RUNNER_TEMP}/netsuke-test-bin/awk. - - The sandbox probe cannot follow a symlink out of its directory handle, so - the destination must be a regular executable file. - """ - missing = [ - fragment - for fragment in AWK_STAGING_FRAGMENTS - if fragment not in test_shell_script - ] - assert not missing, ( - f"{TEST_SHELL_STEP} must copy $(command -v gawk) into " - f"${{RUNNER_TEMP}}/netsuke-test-bin as a regular awk executable; " - f"missing {missing!r}, got:\n{test_shell_script}" - ) - - -def test_test_shell_step_exports_the_directory_to_github_path( - test_shell_script: str, -) -> None: - """Later steps see the staged awk because the directory joins PATH.""" - assert 'echo "${test_shell_bin}" >> "${GITHUB_PATH}"' in test_shell_script, ( - f"{TEST_SHELL_STEP} must append the staging directory to " - f"GITHUB_PATH, got:\n{test_shell_script}" - ) - +def test_test_shell_step_runs_the_staging_script_through_uv() -> None: + """The step delegates the awk staging to the checked-in, tested script.""" + steps = job_steps(load_workflow(), "build-test") + step = named_step(steps, TEST_SHELL_STEP) -def test_test_shell_step_verifies_the_staged_awk(test_shell_script: str) -> None: - """The step proves the staged awk resolves and runs before CI proceeds.""" - lines = [line.strip() for line in test_shell_script.splitlines()] - assert "command -v awk" in lines, ( - f"{TEST_SHELL_STEP} must run `command -v awk`, got:\n{test_shell_script}" + assert str(step.get("run", "")).strip() == f"uv run --script {TEST_SHELL_SCRIPT}", ( + f"{TEST_SHELL_STEP} must run {TEST_SHELL_SCRIPT} and nothing else, " + f"got {step.get('run')!r}" ) - assert "awk --version" in lines, ( - f"{TEST_SHELL_STEP} must run `awk --version`, got:\n{test_shell_script}" + assert (REPO_ROOT / TEST_SHELL_SCRIPT).is_file(), f"{TEST_SHELL_SCRIPT} must exist" + assert steps.index(named_step(steps, "Setup uv")) < steps.index(step), ( + f"Setup uv must precede {TEST_SHELL_STEP}, which runs through uv" ) diff --git a/tests/workflow_contracts/github_actions_validation_test.py b/tests/workflow_contracts/github_actions_validation_test.py index 49b440f14..1ae46f4f4 100644 --- a/tests/workflow_contracts/github_actions_validation_test.py +++ b/tests/workflow_contracts/github_actions_validation_test.py @@ -16,10 +16,9 @@ import yaml from actionlint_installer_contract import ( - ACTIONLINT_CHECKSUM_COMMAND, ACTIONLINT_INSTALL_COMMAND, - ACTIONLINT_SCRIPT_CONTRACTS, - shell_variable, + ACTIONLINT_INSTALL_SCRIPT, + ACTIONLINT_STEP_ENV_CONTRACTS, ) from cmd_mox import CmdMox from hypothesis import given, settings @@ -115,35 +114,28 @@ def _assert_yamllint_ci_contract(steps: list[dict[str, object]]) -> None: ), "the Linux CI job must install and expose the pinned yamllint binary" -ACTIONLINT_REUSE_GUARD = ( - "if [[ -x ./actionlint ]] \\\n" - ' && [[ "$(./actionlint --version | head --lines=1)" == ' - f'"{shell_variable("ACTIONLINT_VERSION")}" ]]; then' -) - - def _assert_actionlint_ci_contract(steps: list[dict[str, object]]) -> None: """Assert that Linux CI provisions actionlint and invokes trusted Make.""" download_actionlint = named_step(steps, "Download actionlint") lint = named_step(steps, "Lint") - download_script = download_actionlint.get("run") cached_paths = [line.strip() for line in _gate_cache_paths().splitlines()] assert "actionlint" in cached_paths, "the gate cache must own the actionlint binary" - assert isinstance(download_script, str), ( - "the actionlint cache-miss step must define its verified installer script" + assert ( + str(download_actionlint.get("run", "")).strip() == ACTIONLINT_INSTALL_COMMAND + ), "the actionlint step must run the checked-in installer script and nothing else" + assert (REPO_ROOT / ACTIONLINT_INSTALL_SCRIPT).is_file(), ( + f"{ACTIONLINT_INSTALL_SCRIPT} must exist; the workflow runs it" ) - assert ACTIONLINT_REUSE_GUARD in download_script, ( - "the cached actionlint must be reused only when it reports the pinned version" + env = require_mapping(download_actionlint.get("env"), "actionlint step env") + for key, expected, message in ACTIONLINT_STEP_ENV_CONTRACTS: + assert str(env.get(key)) == expected, f"{message}, got {env.get(key)!r}" + assert set(env) == {key for key, _, _ in ACTIONLINT_STEP_ENV_CONTRACTS}, ( + f"the actionlint step passes only its two pins, got {sorted(env)!r}" ) - assert download_script.index(ACTIONLINT_REUSE_GUARD) < download_script.index( - ACTIONLINT_INSTALL_COMMAND - ), "the cached-version guard must precede the installer invocation" - for expected, message in ACTIONLINT_SCRIPT_CONTRACTS: - assert expected in download_script, message - assert download_script.index(ACTIONLINT_CHECKSUM_COMMAND) < download_script.index( - ACTIONLINT_INSTALL_COMMAND - ), "the actionlint archive checksum must be verified before running the installer" + assert _step_position(steps, "Setup uv") < _step_position( + steps, "Download actionlint" + ), "uv must be set up before the installer script runs through it" assert lint.get("run") == ( '/usr/bin/make ACTIONLINT="$GITHUB_WORKSPACE/actionlint" lint' ), "the Linux CI job must use trusted `/usr/bin/make` with the cached actionlint" diff --git a/tests/workflow_contracts/kani_cache_test.py b/tests/workflow_contracts/kani_cache_test.py index 262afe064..1c2b2f1aa 100644 --- a/tests/workflow_contracts/kani_cache_test.py +++ b/tests/workflow_contracts/kani_cache_test.py @@ -9,8 +9,6 @@ Run via ``make test-workflow-contracts``. """ -import typing as typ - import yaml from workflow_loading import ( REPO_ROOT, @@ -106,63 +104,25 @@ def test_kani_payloads_share_one_versioned_cache_entry() -> None: assert len(paths) == 1, f"restore and save must claim the same paths: {paths!r}" -class _InstallFragment(typ.NamedTuple): - """A required substring of the install script, and the concern it protects.""" - - concern: str - text: str - +KANI_INSTALL_SCRIPT = "scripts/ci/install_kani.py" +KANI_VERSION_FILE = REPO_ROOT / "tools" / "kani" / "VERSION" -#: Required substrings of the install script, one row per concern so a -#: regression names the property it broke rather than a bare fragment. Each -#: concern groups the URLs, checksum prefixes, directory layout, or -#: executable probes that together protect one part of the binary-only, -#: cache-aware contract. -REQUIRED_INSTALL_FRAGMENTS = ( - _InstallFragment( - "quickinstall front-end download host", - "quickinstall='https://github.com/cargo-bins/cargo-quickinstall'", - ), - _InstallFragment( - "quickinstall front-end archive URL", - '"${quickinstall}/releases/download/kani-verifier-${kani_version}/', - ), - _InstallFragment( - "quickinstall front-end archive checksum prefix", +#: The pins the install step must pass to the script, one row per concern so +#: a regression names the property it broke. The checksum prefixes are the +#: same ones the earlier inline script carried; the script verifies each +#: archive before it is unpacked, which +#: `scripts/tests/test_ci_install_kani.py` proves for both payloads. +REQUIRED_INSTALL_PINS = ( + ( + "quickinstall front-end archive checksum", + "INPUT_FRONTEND_SHA256", "ed2bafc239b834e14c6b66fc4838e342", ), - _InstallFragment( - "upstream verifier download host", - "upstream='https://github.com/model-checking/kani'", - ), - _InstallFragment( - "upstream verifier bundle URL", - '"${upstream}/releases/download/kani-${kani_version}/', - ), - _InstallFragment( - "upstream verifier bundle checksum prefix", + ( + "upstream verifier bundle checksum", + "INPUT_BUNDLE_SHA256", "3b5f7afd3b51603ee720db7bc1bc4fe4", ), - _InstallFragment( - "front-end directory layout", - 'frontend_bin="${CARGO_HOME}/frontend/kani-${kani_version}"', - ), - _InstallFragment( - "verifier directory layout", - 'kani_dir="${KANI_HOME}/kani-${kani_version}"', - ), - _InstallFragment( - "front-end executable probe", - '[[ ! -x "${frontend_bin}/cargo-kani"', - ), - _InstallFragment( - "verifier executable probe", - '[[ ! -x "${kani_dir}/bin/kani-driver"', - ), - _InstallFragment( - "local bundle setup invocation", - 'cargo kani setup --use-local-bundle "${bundle}"', - ), ) @@ -183,44 +143,30 @@ def test_kani_uses_cached_prebuilt_frontend_and_release_bundle() -> None: assert steps.index(named_step(steps, "Restore Kani payloads")) < steps.index( install_step ), "the Kani cache must be restored before Kani is installed" + assert steps.index(named_step(steps, "Setup uv")) < steps.index(install_step), ( + "uv must be set up before the installer script runs through it" + ) + assert str(install_step.get("run", "")).strip() == ( + f"uv run --script {KANI_INSTALL_SCRIPT}" + ), "the Kani step must run the checked-in installer script and nothing else" + assert (REPO_ROOT / KANI_INSTALL_SCRIPT).is_file(), ( + f"{KANI_INSTALL_SCRIPT} must exist" + ) - install_command = str(install_step.get("run")) - missing_fragments = [ - fragment - for fragment in REQUIRED_INSTALL_FRAGMENTS - if fragment.text not in install_command + env = require_mapping(install_step.get("env"), "Kani install step env") + pinned = KANI_VERSION_FILE.read_text(encoding="utf-8").strip() + assert str(env.get("INPUT_KANI_VERSION")) == pinned, ( + f"the Kani step must pass tools/kani/VERSION ({pinned!r}) as " + f"INPUT_KANI_VERSION, got {env.get('INPUT_KANI_VERSION')!r}" + ) + missing = [ + f"{concern} ({key} starting {prefix!r})" + for concern, key, prefix in REQUIRED_INSTALL_PINS + if not str(env.get(key, "")).startswith(prefix) ] - assert not missing_fragments, ( - "Kani's binary-only cached installation is missing: " - + ", ".join( - f"{fragment.concern} ({fragment.text!r})" for fragment in missing_fragments - ) + assert not missing, ( + "Kani's binary-only cached installation is missing: " + ", ".join(missing) ) - _assert_kani_archives_are_verified_before_use(install_command) - - -def _assert_kani_archives_are_verified_before_use(install_command: str) -> None: - """Require each Kani archive's checksum to gate its own unpacking step. - - A bare `sha256sum --check` substring would also pass if the workflow - verified an unrelated file, so each assertion names the archive variable - and requires the verification to precede that archive's extraction. - """ - frontend_check = '"${frontend_archive}" | sha256sum --check --' - frontend_extract = 'tar --extract --gzip --file "${frontend_archive}"' - bundle_check = '"${bundle}" | sha256sum --check --' - bundle_use = 'cargo kani setup --use-local-bundle "${bundle}"' - for check, use, label in ( - (frontend_check, frontend_extract, "front-end archive"), - (bundle_check, bundle_use, "verifier bundle"), - ): - assert check in install_command, ( - f"the Kani {label} must be checksum-verified by name" - ) - assert use in install_command, f"the Kani {label} must be unpacked by name" - assert install_command.index(check) < install_command.index(use), ( - f"the Kani {label} must be verified before it is unpacked" - ) def test_kani_cache_action_requires_runner_image() -> None: diff --git a/tests/workflow_contracts/sccache_contract_test.py b/tests/workflow_contracts/sccache_contract_test.py index 458bf4fff..86a9860bc 100644 --- a/tests/workflow_contracts/sccache_contract_test.py +++ b/tests/workflow_contracts/sccache_contract_test.py @@ -31,6 +31,11 @@ workflow_job, ) +#: The Cyclopts helper the Linux gate reports through; it writes the JSON +#: export and the summary section, covered by +#: `scripts/tests/test_ci_report_sccache_stats.py`. +SCCACHE_REPORT_SCRIPT = "uv run --script scripts/ci/report_sccache_stats.py" + def _assert_sccache_contract(workflow_name: str, job_name: str) -> None: """Require one observable, binary-installed sccache owner for a job.""" @@ -64,9 +69,10 @@ def _assert_sccache_contract(workflow_name: str, job_name: str) -> None: f"{workflow_name} {job_name} must report sccache statistics on failure too" ) show_command = str(show.get("run")) - assert "sccache --show-stats --stats-format=json" in show_command, ( - f"{workflow_name} {job_name} must export machine-readable sccache statistics" - ) + assert ( + "sccache --show-stats --stats-format=json" in show_command + or SCCACHE_REPORT_SCRIPT in show_command + ), f"{workflow_name} {job_name} must export machine-readable sccache statistics" assert steps.index(reset) < steps.index(show), ( f"{workflow_name} {job_name} must reset its counters before reporting them" ) From f91dfe980ed628f15ac758e781fd9f7d52dc1a49 Mon Sep 17 00:00:00 2001 From: leynos Date: Mon, 14 Sep 2026 17:47:18 +0200 Subject: [PATCH 2/4] Pin cuprum to a leynos/cuprum commit and adopt RunOutputOptions cuprum 0.1.0 on PyPI predates the `RunOutputOptions` runtime the scripting standards describe; 0.2.0 is close but unreleased. The scripts and the Makefile's CI_SCRIPT_DEPS now pin `cuprum @ git+https://github.com/ leynos/cuprum@`, held equal by the metadata test, and the shared runner passes `output=RunOutputOptions(capture=True, echo=...)`. Bump CUPRUM_REF to the release pin once 0.2.0 ships. Co-Authored-By: Claude Fable 5.1 --- Makefile | 7 +++++-- scripts/ci/ci_support.py | 6 +++++- scripts/ci/discard_instrumented_tree.py | 2 +- scripts/ci/install_actionlint.py | 2 +- scripts/ci/install_kani.py | 2 +- scripts/ci/report_sccache_stats.py | 2 +- scripts/ci/stage_test_shell.py | 2 +- scripts/tests/test_ci_scripts_metadata.py | 9 +++++---- 8 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index b07ef7db6..9749b1d1b 100644 --- a/Makefile +++ b/Makefile @@ -89,9 +89,12 @@ TY_VERSION ?= 0.0.74 # The CI helper scripts under scripts/ci declare these in their PEP 723 # blocks; scripts/tests/test_ci_scripts_metadata.py holds the two in step, so # `uv run --script` in CI and the test and typecheck gates resolve one release. +# cuprum is pinned to a leynos/cuprum commit until 0.2.0 ships the +# `RunOutputOptions` runtime the scripts use. CYCLOPTS_VERSION ?= 4.25.2 -CUPRUM_VERSION ?= 0.1.0 -CI_SCRIPT_DEPS = --with cyclopts==$(CYCLOPTS_VERSION) --with cuprum==$(CUPRUM_VERSION) +CUPRUM_REF ?= a2134c7a3966b224eaed917efb94f8090ce5104a +CUPRUM_SOURCE = cuprum @ git+https://github.com/leynos/cuprum@$(CUPRUM_REF) +CI_SCRIPT_DEPS = --with cyclopts==$(CYCLOPTS_VERSION) --with '$(CUPRUM_SOURCE)' # Every Python source the repository owns. Ruff and Pylint resolve their own # configuration and exclusions from pyproject.toml, so these paths only bound # the walk. diff --git a/scripts/ci/ci_support.py b/scripts/ci/ci_support.py index 488af52d0..43a1d0945 100755 --- a/scripts/ci/ci_support.py +++ b/scripts/ci/ci_support.py @@ -25,6 +25,7 @@ Program, ProgramCatalogue, ProjectSettings, + RunOutputOptions, sh, ) @@ -97,7 +98,10 @@ def run( raised, so callers decide what it means. """ command = sh.make(Program(program), catalogue=allowed)(*args) - return command.run_sync(echo=echo, context=ExecutionContext(env=env)) + return command.run_sync( + output=RunOutputOptions(capture=True, echo=echo), + context=ExecutionContext(env=env), + ) def describe_failure(result: CommandResult) -> str: diff --git a/scripts/ci/discard_instrumented_tree.py b/scripts/ci/discard_instrumented_tree.py index ceca5e8a0..588d01853 100755 --- a/scripts/ci/discard_instrumented_tree.py +++ b/scripts/ci/discard_instrumented_tree.py @@ -1,7 +1,7 @@ #!/usr/bin/env -S uv run --script # /// script # requires-python = ">=3.14" -# dependencies = ["cyclopts==4.25.2", "cuprum==0.1.0"] +# dependencies = ["cyclopts==4.25.2", "cuprum @ git+https://github.com/leynos/cuprum@a2134c7a3966b224eaed917efb94f8090ce5104a"] # /// """Remove the instrumented coverage build trees before any cache save. diff --git a/scripts/ci/install_actionlint.py b/scripts/ci/install_actionlint.py index e68b1e50a..87bd026ac 100755 --- a/scripts/ci/install_actionlint.py +++ b/scripts/ci/install_actionlint.py @@ -1,7 +1,7 @@ #!/usr/bin/env -S uv run --script # /// script # requires-python = ">=3.14" -# dependencies = ["cyclopts==4.25.2", "cuprum==0.1.0"] +# dependencies = ["cyclopts==4.25.2", "cuprum @ git+https://github.com/leynos/cuprum@a2134c7a3966b224eaed917efb94f8090ce5104a"] # /// """Install the pinned actionlint release, reusing a cached binary that matches. diff --git a/scripts/ci/install_kani.py b/scripts/ci/install_kani.py index ef98f0e8c..d02d5001c 100755 --- a/scripts/ci/install_kani.py +++ b/scripts/ci/install_kani.py @@ -1,7 +1,7 @@ #!/usr/bin/env -S uv run --script # /// script # requires-python = ">=3.14" -# dependencies = ["cyclopts==4.25.2", "cuprum==0.1.0"] +# dependencies = ["cyclopts==4.25.2", "cuprum @ git+https://github.com/leynos/cuprum@a2134c7a3966b224eaed917efb94f8090ce5104a"] # /// """Install the pinned Kani front-end and verifier bundle from release archives. diff --git a/scripts/ci/report_sccache_stats.py b/scripts/ci/report_sccache_stats.py index 82b100537..3cdfc706a 100755 --- a/scripts/ci/report_sccache_stats.py +++ b/scripts/ci/report_sccache_stats.py @@ -1,7 +1,7 @@ #!/usr/bin/env -S uv run --script # /// script # requires-python = ">=3.14" -# dependencies = ["cyclopts==4.25.2", "cuprum==0.1.0"] +# dependencies = ["cyclopts==4.25.2", "cuprum @ git+https://github.com/leynos/cuprum@a2134c7a3966b224eaed917efb94f8090ce5104a"] # /// """Record sccache statistics as text, JSON, and a step-summary section. diff --git a/scripts/ci/stage_test_shell.py b/scripts/ci/stage_test_shell.py index 5bf4c456d..f64a93b98 100755 --- a/scripts/ci/stage_test_shell.py +++ b/scripts/ci/stage_test_shell.py @@ -1,7 +1,7 @@ #!/usr/bin/env -S uv run --script # /// script # requires-python = ">=3.14" -# dependencies = ["cyclopts==4.25.2", "cuprum==0.1.0"] +# dependencies = ["cyclopts==4.25.2", "cuprum @ git+https://github.com/leynos/cuprum@a2134c7a3966b224eaed917efb94f8090ce5104a"] # /// """Stage GNU Awk as a regular ``awk`` executable for the test sandbox. diff --git a/scripts/tests/test_ci_scripts_metadata.py b/scripts/tests/test_ci_scripts_metadata.py index dbe7125a1..ef8c1fd66 100644 --- a/scripts/tests/test_ci_scripts_metadata.py +++ b/scripts/tests/test_ci_scripts_metadata.py @@ -36,10 +36,11 @@ def test_there_are_scripts_to_hold() -> None: @pytest.mark.parametrize("path", SCRIPTS, ids=[path.name for path in SCRIPTS]) def test_script_pins_the_makefile_dependency_versions(path: pathlib.Path) -> None: - """Each PEP 723 block pins cyclopts and cuprum at the Makefile's versions.""" + """Each PEP 723 block pins cyclopts and cuprum at the Makefile's pins.""" text = path.read_text(encoding="utf-8") cyclopts = _makefile_variable("CYCLOPTS_VERSION") - cuprum = _makefile_variable("CUPRUM_VERSION") + cuprum_ref = _makefile_variable("CUPRUM_REF") + cuprum = f"cuprum @ git+https://github.com/leynos/cuprum@{cuprum_ref}" header, _, _ = text.partition('# ///\n"""') assert header.startswith("#!/usr/bin/env -S uv run --script\n# /// script\n"), ( path.name @@ -48,8 +49,8 @@ def test_script_pins_the_makefile_dependency_versions(path: pathlib.Path) -> Non assert f'"cyclopts=={cyclopts}"' in header, ( f"{path.name} must pin cyclopts=={cyclopts}, the Makefile CYCLOPTS_VERSION" ) - assert f'"cuprum=={cuprum}"' in header, ( - f"{path.name} must pin cuprum=={cuprum}, the Makefile CUPRUM_VERSION" + assert f'"{cuprum}"' in header, ( + f"{path.name} must pin {cuprum!r}, the Makefile CUPRUM_REF" ) From 05f74f4cef9fadc2ad82677ab47fa0b69d32ac71 Mon Sep 17 00:00:00 2001 From: leynos Date: Mon, 14 Sep 2026 17:48:24 +0200 Subject: [PATCH 3/4] Describe the cuprum commit pin in the developers guide Co-Authored-By: Claude Fable 5.1 --- docs/developers-guide.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 3b4eba3d3..caba197c1 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -1658,10 +1658,12 @@ pytest and [cmd-mox](https://github.com/leynos/cmd-mox), which shims the external programs (`sudo`, `sccache`, `df`, `cargo`) so each script's happy path and failure modes are exercised at the process boundary. CI runs the same target before `make check-fmt`. The scripts pin `cyclopts` and `cuprum` in -their PEP 723 blocks at the Makefile's `CYCLOPTS_VERSION` and `CUPRUM_VERSION`, +their PEP 723 blocks at the Makefile's `CYCLOPTS_VERSION` and `CUPRUM_REF`, held equal by `scripts/tests/test_ci_scripts_metadata.py`, and `make typecheck-python` checks them with ty alongside the rest of the Python -sources. +sources. `CUPRUM_REF` is a `leynos/cuprum` commit rather than a PyPI release +until 0.2.0 ships the `RunOutputOptions` runtime the scripts use; bump it to +the release pin then. ### GitHub Actions validation From 8fb93e920cd504bd0f215d87b329a0c9d1d6bc94 Mon Sep 17 00:00:00 2001 From: leynos Date: Mon, 14 Sep 2026 17:56:18 +0200 Subject: [PATCH 4/4] Report sccache statistics through the script on every lane ci-windows.yml, netsukefile-test.yml, and coverage-main.yml carried the same inline sccache report block as ci.yml; each now runs scripts/ci/report_sccache_stats.py, and coverage-main.yml runs scripts/ci/discard_instrumented_tree.py for its instrumented tree as well. The lanes that lacked uv gain a cache-less Setup uv step after their credential export. The sccache contract now requires the script on every compiling lane; the release smoke job, which only prints the JSON form and has no uv, is the one recorded exception. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci-windows.yml | 27 +++++++-------- .github/workflows/coverage-main.yml | 22 ++++++------- .github/workflows/netsukefile-test.yml | 17 +++++----- docs/developers-guide.md | 7 ++++ .../sccache_compile_step_data.py | 33 +++++++++++++++++++ .../sccache_contract_test.py | 13 ++------ 6 files changed, 72 insertions(+), 47 deletions(-) diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index 3fd867658..ed7e3eb01 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -193,14 +193,8 @@ jobs: } - name: Show sccache statistics if: always() - run: | - sccache --show-stats | tee sccache-stats.txt - sccache --show-stats --stats-format=json > sccache-stats.json - { - printf '### sccache\n\n```text\n' - cat sccache-stats.txt - printf '```\n' - } >> "$GITHUB_STEP_SUMMARY" + # Text and JSON files plus a step-summary section, from one script. + run: uv run --script scripts/ci/report_sccache_stats.py - name: Save gate caches # The action restricts each save to a push on `main` that missed the # key, so a pull request never publishes a competing generation. @@ -264,6 +258,13 @@ jobs: toolchain: ${{ env.NETSUKE_RUST_TOOLCHAIN }} mode: restore profile: gate + - name: Setup uv + # The sccache report is a scripts/ci helper run through + # `uv run --script`; uv's own cache is not part of this lane's keys. + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 + with: + python-version: '3.14' + enable-cache: 'false' - name: Install GNU Make shell: pwsh run: | @@ -342,14 +343,8 @@ jobs: } - name: Show sccache statistics if: always() - run: | - sccache --show-stats | tee sccache-stats.txt - sccache --show-stats --stats-format=json > sccache-stats.json - { - printf '### sccache\n\n```text\n' - cat sccache-stats.txt - printf '```\n' - } >> "$GITHUB_STEP_SUMMARY" + # Text and JSON files plus a step-summary section, from one script. + run: uv run --script scripts/ci/report_sccache_stats.py - name: Save gate caches # The action restricts each save to a push on `main` that missed the # key, so a pull request never publishes a competing generation. diff --git a/.github/workflows/coverage-main.yml b/.github/workflows/coverage-main.yml index c3610403f..7e83b4e30 100644 --- a/.github/workflows/coverage-main.yml +++ b/.github/workflows/coverage-main.yml @@ -83,6 +83,13 @@ jobs: # Before the installer, so before anything can start the server: a # server started without these stays on local disk all job. uses: ./.github/actions/sccache-gha-credentials + - name: Setup uv + # The sccache report is a scripts/ci helper run through + # `uv run --script`; uv's own cache is not part of this lane's keys. + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 + with: + python-version: '3.14' + enable-cache: 'false' - name: Start resource sampler uses: ./.github/actions/memory-sampler with: @@ -218,14 +225,8 @@ jobs: # reached the compiler cache, which is a failed integration rather # than a quiet no-op. if: always() - run: | - sccache --show-stats | tee sccache-stats.txt - sccache --show-stats --stats-format=json > sccache-stats.json - { - printf '### sccache\n\n```text\n' - cat sccache-stats.txt - printf '```\n' - } | tee -a "$GITHUB_STEP_SUMMARY" + # Text and JSON files plus a step-summary section, from one script. + run: uv run --script scripts/ci/report_sccache_stats.py - name: Discard the instrumented build tree # `llvm-cov`'s target tree has no consumer once the report is written, # and it is the second tree on a volume that a sibling repository @@ -233,10 +234,7 @@ jobs: # the archive never carries it and the sampler's least-free figure # reflects the job's real high-water mark. if: always() - run: | - df -h . - rm --recursive --force target/llvm-cov-target target/llvm-cov - df -h . + run: uv run --script scripts/ci/discard_instrumented_tree.py - name: Report peak resource use if: always() uses: ./.github/actions/memory-sampler diff --git a/.github/workflows/netsukefile-test.yml b/.github/workflows/netsukefile-test.yml index d48cf42b8..4c9c030a7 100644 --- a/.github/workflows/netsukefile-test.yml +++ b/.github/workflows/netsukefile-test.yml @@ -38,6 +38,13 @@ jobs: # Before the installer, so before anything can start the server: a # server started without these stays on local disk all job. uses: ./.github/actions/sccache-gha-credentials + - name: Setup uv + # The sccache report is a scripts/ci helper run through + # `uv run --script`; uv's own cache is not part of this lane's keys. + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 + with: + python-version: '3.14' + enable-cache: 'false' - name: Render cache keys # This lane owns its own key family: nothing else in the repository # runs on Ubuntu 22.04, so a 24.04 archive must never reach it. The @@ -187,14 +194,8 @@ jobs: run: scripts/assert-file-absent.sh unused.txt - name: Show sccache statistics if: always() - run: | - sccache --show-stats | tee sccache-stats.txt - sccache --show-stats --stats-format=json > sccache-stats.json - { - printf '### sccache\n\n```text\n' - cat sccache-stats.txt - printf '```\n' - } | tee -a "$GITHUB_STEP_SUMMARY" + # Text and JSON files plus a step-summary section, from one script. + run: uv run --script scripts/ci/report_sccache_stats.py # This job is the only writer of the Ubuntu 22.04 key family, and only # on the trunk, so a pull request never publishes a competing archive. - name: Save Cargo download store diff --git a/docs/developers-guide.md b/docs/developers-guide.md index caba197c1..7ddaf9d25 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -1647,6 +1647,13 @@ workflow contract tests can read it. | `discard_instrumented_tree.py` | Discard the instrumented build tree | | `install_kani.py` | Install prebuilt Kani (`kani-smoke`) | +`report_sccache_stats.py` also replaces the same report in `ci-windows.yml`, +`netsukefile-test.yml`, and `coverage-main.yml`, and `coverage-main.yml` runs +`discard_instrumented_tree.py` too; each of those jobs sets up uv for the +purpose. `tests/workflow_contracts/sccache_contract_test.py` requires the +script on every compiling lane except the release smoke job, which only prints +the JSON form. + `scripts/ci/ci_support.py` holds what they share: the catalogue, a synchronous runner, `GITHUB_PATH` publication, and checksum-verified download and extraction of release archives. Downloads accept `https://` and `file://` URLs diff --git a/tests/workflow_contracts/sccache_compile_step_data.py b/tests/workflow_contracts/sccache_compile_step_data.py index 8127cdfa5..2ec71335e 100644 --- a/tests/workflow_contracts/sccache_compile_step_data.py +++ b/tests/workflow_contracts/sccache_compile_step_data.py @@ -16,6 +16,39 @@ #: `cargo llvm-cov` build rather than a plain `run:` compiler invocation. GENERATE_COVERAGE_ACTION = "leynos/shared-actions/.github/actions/generate-coverage@" +#: The Cyclopts helper every compiling lane reports through; it writes the +#: text and JSON files and the summary section, covered by +#: `scripts/tests/test_ci_report_sccache_stats.py`. +SCCACHE_REPORT_SCRIPT = "uv run --script scripts/ci/report_sccache_stats.py" +#: The one lane that only prints the JSON form and keeps no files: the +#: release smoke job has no uv and nothing downstream reads its statistics. +JSON_ONLY_REPORTS = frozenset({("release.yml", "windows-native-recipe-smoke")}) + + +def assert_report_command(workflow_name: str, job_name: str, command: str) -> None: + """Assert a job's statistics step reports the way its lane is allowed to. + + Parameters + ---------- + workflow_name + Workflow file the job lives in. + job_name + The compiling job under test. + command + The ``run`` script of its ``Show sccache statistics`` step. + """ + command = command.strip() + if (workflow_name, job_name) in JSON_ONLY_REPORTS: + assert "sccache --show-stats --stats-format=json" in command, ( + f"{workflow_name} {job_name} must export machine-readable sccache " + "statistics" + ) + return + assert command == SCCACHE_REPORT_SCRIPT, ( + f"{workflow_name} {job_name} must report through the checked-in " + f"{SCCACHE_REPORT_SCRIPT!r} rather than inline shell, got {command!r}" + ) + def invokes_build_command(line: str) -> bool: """Return whether a shell line's first word is `make` or `cargo`. diff --git a/tests/workflow_contracts/sccache_contract_test.py b/tests/workflow_contracts/sccache_contract_test.py index 86a9860bc..28a37f0b2 100644 --- a/tests/workflow_contracts/sccache_contract_test.py +++ b/tests/workflow_contracts/sccache_contract_test.py @@ -22,7 +22,7 @@ declared_paths, lane_steps, ) -from sccache_compile_step_data import is_compile_step +from sccache_compile_step_data import assert_report_command, is_compile_step from workflow_loading import ( job_steps, load_workflow, @@ -31,11 +31,6 @@ workflow_job, ) -#: The Cyclopts helper the Linux gate reports through; it writes the JSON -#: export and the summary section, covered by -#: `scripts/tests/test_ci_report_sccache_stats.py`. -SCCACHE_REPORT_SCRIPT = "uv run --script scripts/ci/report_sccache_stats.py" - def _assert_sccache_contract(workflow_name: str, job_name: str) -> None: """Require one observable, binary-installed sccache owner for a job.""" @@ -68,11 +63,7 @@ def _assert_sccache_contract(workflow_name: str, job_name: str) -> None: assert show.get("if") == "always()", ( f"{workflow_name} {job_name} must report sccache statistics on failure too" ) - show_command = str(show.get("run")) - assert ( - "sccache --show-stats --stats-format=json" in show_command - or SCCACHE_REPORT_SCRIPT in show_command - ), f"{workflow_name} {job_name} must export machine-readable sccache statistics" + assert_report_command(workflow_name, job_name, str(show.get("run"))) assert steps.index(reset) < steps.index(show), ( f"{workflow_name} {job_name} must reset its counters before reporting them" )