feat: diff generated corpus output against an earlier ref

.config/nextest.toml

@@ -0,0 +1,12 @@
+# Many tests here generate a crate and compile it, so multi-minute runtimes are
+# expected rather than a symptom; raise the slow threshold so the ones worth
+# looking at stand out.
+[profile.default]
+slow-timeout = { period = "120s" }
+
+# CI reports every failure in one run: re-queuing a pull request to discover the
+# second broken test wastes more than finishing the suite does.
+[profile.ci]
+slow-timeout = { period = "120s" }
+fail-fast = false
+failure-output = "immediate-final"

.github/workflows/ci.yml

@@ -21,10 +21,14 @@ jobs:
           submodules: true
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
-      - run: cargo test --all-features
+      - uses: taiki-e/install-action@nextest
+      - run: cargo nextest run --profile ci --all-features
+      # nextest runs each test in its own process and cannot run doctests, so
+      # the crate's doctests still go through the built-in harness.
+      - run: cargo test --doc --all-features
       - name: Verify checked-in conformance reports
         run: |
-          CONFORMANCE_REPORT=1 cargo test --test conformance --test conformance_json_schema
+          CONFORMANCE_REPORT=1 cargo nextest run --test conformance --test conformance_json_schema
           git diff --exit-code -- tests/conformance/coverage-report.md tests/conformance/json-schema-2020-12-report.md
 
   openai-sdk-compat:
@@ -137,6 +141,63 @@ jobs:
   # Fast tier: every supported corpus document must generate, while the two
   # production-target specs must also compile from their exact dependency
   # fragments. The full compile tier runs separately on a schedule/manual run.
+  # Hash every file the generator emits for the pinned corpus and compare it
+  # with tests/corpus-manifest.txt. A change here means real-world output moved.
+  # Pull requests get this gate from corpus-diff instead, which generates the
+  # same corpus anyway and can also say what moved.
+  corpus-manifest:
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - run: scripts/corpus-manifest.sh --check
+
+  # Show the reviewer what a generator change does to real-world output, rather
+  # than only that something changed: regenerate the corpus at the base of the
+  # pull request and at its head, and diff the two. The manifest gate then runs
+  # off the head corpus this already produced.
+  corpus-diff:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # gen-diff.sh builds the base ref in a worktree, so its history has
+          # to be present.
+          fetch-depth: 0
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - name: Diff generated corpus against the pull request base
+        # `shell: bash` for pipefail: without it the pipe would report tee's
+        # status and hide a failed run.
+        shell: bash
+        run: |
+          scripts/gen-diff.sh "${{ github.event.pull_request.base.sha }}" \
+            | tee "$RUNNER_TEMP/gen-diff-summary.txt"
+      - name: Publish summary
+        if: always()
+        run: |
+          {
+            echo '## Generated corpus diff'
+            echo
+            echo '```'
+            cat "$RUNNER_TEMP/gen-diff-summary.txt" 2>/dev/null \
+              || echo '(gen-diff did not complete)'
+            echo '```'
+          } >>"$GITHUB_STEP_SUMMARY"
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: corpus-diff
+          path: tmp/gen-diff/report/
+          if-no-files-found: ignore
+      - name: Verify tests/corpus-manifest.txt matches the generated corpus
+        run: scripts/corpus-manifest.sh --check --from tmp/gen-diff/head
+
   spec-compile:
     if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'
     runs-on: ubuntu-latest

.gitignore

@@ -2,6 +2,8 @@
 **/target/
 # Build outputs from spec-compile.sh and ad-hoc generator runs.
 /tmp/spec-compile/
+/tmp/gen-diff/
+/tmp/corpus-manifest/
 /tmp/spec-compile-target/
 /tmp/gen-anthropic/
 /tmp/gen-openai/

CHANGELOG.md

@@ -6,6 +6,38 @@ when correcting output that was wrong or incomplete on the wire.
 
 ## [Unreleased]
 
+### Added
+
+- `scripts/gen-diff.sh <base-ref>` shows what a generator change does to the
+  code emitted for the pinned corpus. Generation is byte-deterministic and the
+  specs are pinned, so the earlier output is rebuilt on demand — the generator
+  at that ref is built in a throwaway worktree, both corpora are regenerated,
+  and the run reports per-spec churn plus any public item that appeared or
+  disappeared, with full diffs under `tmp/gen-diff/report/`. Nothing generated
+  is checked in; the base side is cached per commit.
+- `tests/corpus-manifest.txt` records one hashed line per generated file across
+  the corpus, so a change that moves real-world output cannot land unnoticed.
+  The `Generated by openapi-to-rust vX.Y.Z` stamp is normalized away, so a
+  version bump alone never touches the manifest.
+- `scripts/spec-compile.sh` and the corpus tooling now emit the generator config
+  from one shared definition, so the code the manifest hashes cannot drift from
+  the code the compile gate checks.
+- CI diffs the corpus on every pull request: the per-spec table lands in the job
+  summary, the full diffs upload as an artifact, and the manifest is verified
+  against the corpus that run already generated. Pushes to `main` verify the
+  manifest on its own. Both cover types and client output; server scaffolding
+  has no uniform corpus pass yet.
+
+### Changed
+
+- The test suite runs under [nextest](https://nexte.st): `cargo nextest run
+  --all-features`, which cut a warm local run from 11m40s to 7m25s by running
+  each test in its own process rather than one process per test binary.
+  Contributors need `cargo install cargo-nextest --locked`. Doctests keep going
+  through the built-in harness (`cargo test --doc --all-features`) because
+  nextest cannot run them — `cargo nextest run` alone would skip all seven
+  without saying so.
+
 ### Fixed
 
 - A struct whose schema states `additionalProperties: false` rejects undeclared

CONTRIBUTING.md

@@ -38,6 +38,8 @@ Reference the public GitHub issue in your pull request when one exists.
 - `tests/fixtures/` contains focused regression documents.
 - `tests/conformance/` contains the compatibility catalog and reports.
 - `specs/` contains the real-world corpus used by the compile gate.
+- `tests/corpus-manifest.txt` hashes the code that corpus generates, so an
+  unintended change to real-world output fails CI.
 
 ## Making a change
 
@@ -65,10 +67,17 @@ Run the standard gate before opening a pull request:
 ```bash
 cargo fmt --check
 cargo clippy --all-features -- -D warnings
-cargo test --all-features
+cargo nextest run --all-features
+cargo test --doc --all-features
 RUSTDOCFLAGS=-Dwarnings cargo doc --no-deps --all-features
 ```
 
+The suite runs under [nextest](https://nexte.st)
+(`cargo install cargo-nextest --locked`), which runs each test in its own
+process. It cannot run doctests, so those keep going through the built-in
+harness in the second command — running only `cargo nextest run` silently skips
+them.
+
 Also run the relevant distribution or corpus gate when touching these areas:
 
 ```bash
@@ -78,6 +87,40 @@ scripts/spec-compile.sh                  # broad generator/type changes
 scripts/untyped-census.sh                # anything that changes which fields get typed
 ```
 
+### Corpus output diffs
+
+Nothing generated is checked in, but generation is byte-deterministic and the
+specs under `specs/` are pinned, so any earlier revision's output can be
+rebuilt on demand:
+
+```bash
+scripts/gen-diff.sh                # vs the merge base with main
+scripts/gen-diff.sh v0.15.0        # vs a release
+GEN_DIFF_SPECS="anthropic openai" scripts/gen-diff.sh HEAD~1
+```
+
+It builds the generator at that ref in a throwaway worktree, regenerates the
+corpus on both sides, and prints per-spec churn plus any public item that
+appeared or disappeared. Full per-spec diffs land in `tmp/gen-diff/report/`.
+The base side is cached per commit, so repeat runs only pay for the working
+tree. A cold full-corpus run is roughly a minute and a half.
+
+`tests/corpus-manifest.txt` is the committed tripwire for the same thing: one
+hashed line per generated file. When a change legitimately moves output,
+inspect it with `gen-diff.sh`, then refresh the manifest with
+`scripts/corpus-manifest.sh` so the review shows which specs moved. The
+`Generated by openapi-to-rust vX.Y.Z` stamp is normalized away, so a version
+bump alone never touches the manifest.
+
+CI runs both. A pull request gets the `corpus-diff` job, which diffs against
+the base of the pull request, puts the per-spec table in the job summary,
+uploads the full diffs as an artifact, and then checks the manifest against the
+corpus it already generated (`--from tmp/gen-diff/head`). Pushes to `main` and
+scheduled runs get the manifest check on its own.
+
+Both cover the types and client output. Server scaffolding is generated from
+per-spec `[server].operations` selectors, so it has no uniform corpus pass yet.
+
 `scripts/untyped-census.sh` rewrites `tests/conformance/untyped-report.md`,
 which counts every generated field that carries `serde_json::Value` and says
 why. Regenerate it when a change types fields that used to be opaque (or stops

scripts/corpus-manifest.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+# Write or verify tests/corpus-manifest.txt: one hashed line per file generated
+# from every spec in specs/.
+#
+# The manifest is the tripwire, not the record. It is small enough to read in a
+# review and answers "did this generator change move real-world output, and for
+# which specs?" — it deliberately does not store the code. To see what actually
+# changed, run scripts/gen-diff.sh <base-ref>, which reconstructs the old output
+# from the pinned specs.
+#
+# Usage:
+#   scripts/corpus-manifest.sh            # regenerate the manifest in place
+#   scripts/corpus-manifest.sh --check    # fail if the manifest is out of date
+#   scripts/corpus-manifest.sh --check --from tmp/gen-diff/head
+#                                         # hash a corpus that is already on
+#                                         # disk instead of generating one
+#
+# Env:
+#   CORPUS_PROFILE=debug   build the generator without --release (slower to run)
+set -euo pipefail
+cd "$(dirname "$0")/.."
+source scripts/lib/corpus.sh
+
+MANIFEST="tests/corpus-manifest.txt"
+CHECK=0
+FROM=""
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --check) CHECK=1 ;;
+    --from) FROM="${2:?--from needs a directory}"; shift ;;
+    *) echo "usage: $0 [--check] [--from <corpus-dir>]" >&2; exit 2 ;;
+  esac
+  shift
+done
+
+WORK="tmp/corpus-manifest"
+rm -rf "$WORK"
+mkdir -p "$WORK"
+# Keep the scratch tree when something failed: corpus_generate leaves the
+# generator log for the spec that broke.
+cleanup() {
+  local code=$?
+  if [ "$code" -eq 0 ] || [ ! -d "$WORK/out" ]; then
+    rm -rf "$WORK"
+  else
+    echo "[corpus-manifest] work dir kept: $WORK" >&2
+  fi
+}
+trap cleanup EXIT
+
+if [ -n "$FROM" ]; then
+  # Reuse a corpus another script already generated (scripts/gen-diff.sh leaves
+  # one at tmp/gen-diff/head) rather than generating a third copy. It must be
+  # complete, or missing specs would read as deletions.
+  SRC="$FROM"
+  expected=$(corpus_specs | wc -l | tr -d ' ')
+  found=$(find "$SRC" -maxdepth 1 -mindepth 1 -type d | wc -l | tr -d ' ')
+  if [ "$expected" != "$found" ]; then
+    echo "[corpus-manifest] $SRC holds $found of $expected specs; refusing a partial manifest" >&2
+    exit 2
+  fi
+else
+  SRC="$WORK/out"
+  echo "[corpus-manifest] building generator..." >&2
+  GEN_BIN="$(corpus_build "$PWD" "$PWD/target" "${CORPUS_PROFILE:-release}")"
+
+  echo "[corpus-manifest] generating corpus..." >&2
+  corpus_generate "$GEN_BIN" "$SRC"
+fi
+
+OUT="$WORK/manifest.txt"
+{
+  echo "# openapi-to-rust corpus manifest"
+  echo "#"
+  echo "# SHA-256 (first 16 hex) of every file generated from the specs pinned in"
+  echo "# specs/. Regenerate with scripts/corpus-manifest.sh; CI verifies it with"
+  echo "# --check. A diff here means a generator change moved real-world output:"
+  echo "# confirm that was intended, then see exactly what moved with"
+  echo "#   scripts/gen-diff.sh <base-ref>"
+  echo "#"
+  echo "# The \"Generated by openapi-to-rust vX.Y.Z\" stamp is normalized away, so a"
+  echo "# version bump alone never touches this file."
+  echo "#"
+  echo "# columns: path  bytes  lines  sha256[0:16]"
+} >"$OUT"
+
+files=0
+bytes=0
+while IFS= read -r f; do
+  rel="${f#"$SRC/"}"
+  b=$(wc -c <"$f" | tr -d ' ')
+  l=$(wc -l <"$f" | tr -d ' ')
+  printf '%-52s %10s %8s %s\n' "$rel" "$b" "$l" "$(corpus_hash "$f")" >>"$OUT"
+  files=$((files + 1))
+  bytes=$((bytes + b))
+done < <(find "$SRC" -type f | sort)
+
+specs=$(find "$SRC" -maxdepth 1 -mindepth 1 -type d | wc -l | tr -d ' ')
+printf '#\n# specs: %s   files: %s   bytes: %s\n' "$specs" "$files" "$bytes" >>"$OUT"
+
+if [ "$CHECK" = "1" ]; then
+  if diff -u "$MANIFEST" "$OUT" >"$WORK/manifest.diff"; then
+    echo "[corpus-manifest] up to date ($specs specs, $files files)"
+    exit 0
+  fi
+  echo "[corpus-manifest] generated corpus output changed:" >&2
+  cat "$WORK/manifest.diff" >&2
+  cat >&2 <<'EOF'
+
+This generator change moves the code emitted for real-world specs. If that was
+intended, inspect the change with
+
+  scripts/gen-diff.sh <base-ref>
+
+then refresh the manifest with
+
+  scripts/corpus-manifest.sh
+EOF
+  exit 1
+fi
+
+mkdir -p "$(dirname "$MANIFEST")"
+cp "$OUT" "$MANIFEST"
+echo "[corpus-manifest] wrote $MANIFEST ($specs specs, $files files)"