diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 0000000..0789f06 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,12 @@ +# Many tests here generate a crate and compile it, so multi-minute runtimes are +# expected rather than a symptom; raise the slow threshold so the ones worth +# looking at stand out. +[profile.default] +slow-timeout = { period = "120s" } + +# CI reports every failure in one run: re-queuing a pull request to discover the +# second broken test wastes more than finishing the suite does. +[profile.ci] +slow-timeout = { period = "120s" } +fail-fast = false +failure-output = "immediate-final" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73c0f2a..81f9e08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,10 +21,14 @@ jobs: submodules: true - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - - run: cargo test --all-features + - uses: taiki-e/install-action@nextest + - run: cargo nextest run --profile ci --all-features + # nextest runs each test in its own process and cannot run doctests, so + # the crate's doctests still go through the built-in harness. + - run: cargo test --doc --all-features - name: Verify checked-in conformance reports run: | - CONFORMANCE_REPORT=1 cargo test --test conformance --test conformance_json_schema + CONFORMANCE_REPORT=1 cargo nextest run --test conformance --test conformance_json_schema git diff --exit-code -- tests/conformance/coverage-report.md tests/conformance/json-schema-2020-12-report.md openai-sdk-compat: @@ -137,6 +141,63 @@ jobs: # Fast tier: every supported corpus document must generate, while the two # production-target specs must also compile from their exact dependency # fragments. The full compile tier runs separately on a schedule/manual run. + # Hash every file the generator emits for the pinned corpus and compare it + # with tests/corpus-manifest.txt. A change here means real-world output moved. + # Pull requests get this gate from corpus-diff instead, which generates the + # same corpus anyway and can also say what moved. + corpus-manifest: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: scripts/corpus-manifest.sh --check + + # Show the reviewer what a generator change does to real-world output, rather + # than only that something changed: regenerate the corpus at the base of the + # pull request and at its head, and diff the two. The manifest gate then runs + # off the head corpus this already produced. + corpus-diff: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + # gen-diff.sh builds the base ref in a worktree, so its history has + # to be present. + fetch-depth: 0 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Diff generated corpus against the pull request base + # `shell: bash` for pipefail: without it the pipe would report tee's + # status and hide a failed run. + shell: bash + run: | + scripts/gen-diff.sh "${{ github.event.pull_request.base.sha }}" \ + | tee "$RUNNER_TEMP/gen-diff-summary.txt" + - name: Publish summary + if: always() + run: | + { + echo '## Generated corpus diff' + echo + echo '```' + cat "$RUNNER_TEMP/gen-diff-summary.txt" 2>/dev/null \ + || echo '(gen-diff did not complete)' + echo '```' + } >>"$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: corpus-diff + path: tmp/gen-diff/report/ + if-no-files-found: ignore + - name: Verify tests/corpus-manifest.txt matches the generated corpus + run: scripts/corpus-manifest.sh --check --from tmp/gen-diff/head + spec-compile: if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index c41cf2b..a0b4f76 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ **/target/ # Build outputs from spec-compile.sh and ad-hoc generator runs. /tmp/spec-compile/ +/tmp/gen-diff/ +/tmp/corpus-manifest/ /tmp/spec-compile-target/ /tmp/gen-anthropic/ /tmp/gen-openai/ diff --git a/CHANGELOG.md b/CHANGELOG.md index d0c963b..86e2d02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,38 @@ when correcting output that was wrong or incomplete on the wire. ## [Unreleased] +### Added + +- `scripts/gen-diff.sh ` shows what a generator change does to the + code emitted for the pinned corpus. Generation is byte-deterministic and the + specs are pinned, so the earlier output is rebuilt on demand — the generator + at that ref is built in a throwaway worktree, both corpora are regenerated, + and the run reports per-spec churn plus any public item that appeared or + disappeared, with full diffs under `tmp/gen-diff/report/`. Nothing generated + is checked in; the base side is cached per commit. +- `tests/corpus-manifest.txt` records one hashed line per generated file across + the corpus, so a change that moves real-world output cannot land unnoticed. + The `Generated by openapi-to-rust vX.Y.Z` stamp is normalized away, so a + version bump alone never touches the manifest. +- `scripts/spec-compile.sh` and the corpus tooling now emit the generator config + from one shared definition, so the code the manifest hashes cannot drift from + the code the compile gate checks. +- CI diffs the corpus on every pull request: the per-spec table lands in the job + summary, the full diffs upload as an artifact, and the manifest is verified + against the corpus that run already generated. Pushes to `main` verify the + manifest on its own. Both cover types and client output; server scaffolding + has no uniform corpus pass yet. + +### Changed + +- The test suite runs under [nextest](https://nexte.st): `cargo nextest run + --all-features`, which cut a warm local run from 11m40s to 7m25s by running + each test in its own process rather than one process per test binary. + Contributors need `cargo install cargo-nextest --locked`. Doctests keep going + through the built-in harness (`cargo test --doc --all-features`) because + nextest cannot run them — `cargo nextest run` alone would skip all seven + without saying so. + ### Fixed - A struct whose schema states `additionalProperties: false` rejects undeclared diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3b87951..d750abe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,6 +38,8 @@ Reference the public GitHub issue in your pull request when one exists. - `tests/fixtures/` contains focused regression documents. - `tests/conformance/` contains the compatibility catalog and reports. - `specs/` contains the real-world corpus used by the compile gate. +- `tests/corpus-manifest.txt` hashes the code that corpus generates, so an + unintended change to real-world output fails CI. ## Making a change @@ -65,10 +67,17 @@ Run the standard gate before opening a pull request: ```bash cargo fmt --check cargo clippy --all-features -- -D warnings -cargo test --all-features +cargo nextest run --all-features +cargo test --doc --all-features RUSTDOCFLAGS=-Dwarnings cargo doc --no-deps --all-features ``` +The suite runs under [nextest](https://nexte.st) +(`cargo install cargo-nextest --locked`), which runs each test in its own +process. It cannot run doctests, so those keep going through the built-in +harness in the second command — running only `cargo nextest run` silently skips +them. + Also run the relevant distribution or corpus gate when touching these areas: ```bash @@ -78,6 +87,40 @@ scripts/spec-compile.sh # broad generator/type changes scripts/untyped-census.sh # anything that changes which fields get typed ``` +### Corpus output diffs + +Nothing generated is checked in, but generation is byte-deterministic and the +specs under `specs/` are pinned, so any earlier revision's output can be +rebuilt on demand: + +```bash +scripts/gen-diff.sh # vs the merge base with main +scripts/gen-diff.sh v0.15.0 # vs a release +GEN_DIFF_SPECS="anthropic openai" scripts/gen-diff.sh HEAD~1 +``` + +It builds the generator at that ref in a throwaway worktree, regenerates the +corpus on both sides, and prints per-spec churn plus any public item that +appeared or disappeared. Full per-spec diffs land in `tmp/gen-diff/report/`. +The base side is cached per commit, so repeat runs only pay for the working +tree. A cold full-corpus run is roughly a minute and a half. + +`tests/corpus-manifest.txt` is the committed tripwire for the same thing: one +hashed line per generated file. When a change legitimately moves output, +inspect it with `gen-diff.sh`, then refresh the manifest with +`scripts/corpus-manifest.sh` so the review shows which specs moved. The +`Generated by openapi-to-rust vX.Y.Z` stamp is normalized away, so a version +bump alone never touches the manifest. + +CI runs both. A pull request gets the `corpus-diff` job, which diffs against +the base of the pull request, puts the per-spec table in the job summary, +uploads the full diffs as an artifact, and then checks the manifest against the +corpus it already generated (`--from tmp/gen-diff/head`). Pushes to `main` and +scheduled runs get the manifest check on its own. + +Both cover the types and client output. Server scaffolding is generated from +per-spec `[server].operations` selectors, so it has no uniform corpus pass yet. + `scripts/untyped-census.sh` rewrites `tests/conformance/untyped-report.md`, which counts every generated field that carries `serde_json::Value` and says why. Regenerate it when a change types fields that used to be opaque (or stops diff --git a/scripts/corpus-manifest.sh b/scripts/corpus-manifest.sh new file mode 100755 index 0000000..e6a13e9 --- /dev/null +++ b/scripts/corpus-manifest.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# Write or verify tests/corpus-manifest.txt: one hashed line per file generated +# from every spec in specs/. +# +# The manifest is the tripwire, not the record. It is small enough to read in a +# review and answers "did this generator change move real-world output, and for +# which specs?" — it deliberately does not store the code. To see what actually +# changed, run scripts/gen-diff.sh , which reconstructs the old output +# from the pinned specs. +# +# Usage: +# scripts/corpus-manifest.sh # regenerate the manifest in place +# scripts/corpus-manifest.sh --check # fail if the manifest is out of date +# scripts/corpus-manifest.sh --check --from tmp/gen-diff/head +# # hash a corpus that is already on +# # disk instead of generating one +# +# Env: +# CORPUS_PROFILE=debug build the generator without --release (slower to run) +set -euo pipefail +cd "$(dirname "$0")/.." +source scripts/lib/corpus.sh + +MANIFEST="tests/corpus-manifest.txt" +CHECK=0 +FROM="" +while [ $# -gt 0 ]; do + case "$1" in + --check) CHECK=1 ;; + --from) FROM="${2:?--from needs a directory}"; shift ;; + *) echo "usage: $0 [--check] [--from ]" >&2; exit 2 ;; + esac + shift +done + +WORK="tmp/corpus-manifest" +rm -rf "$WORK" +mkdir -p "$WORK" +# Keep the scratch tree when something failed: corpus_generate leaves the +# generator log for the spec that broke. +cleanup() { + local code=$? + if [ "$code" -eq 0 ] || [ ! -d "$WORK/out" ]; then + rm -rf "$WORK" + else + echo "[corpus-manifest] work dir kept: $WORK" >&2 + fi +} +trap cleanup EXIT + +if [ -n "$FROM" ]; then + # Reuse a corpus another script already generated (scripts/gen-diff.sh leaves + # one at tmp/gen-diff/head) rather than generating a third copy. It must be + # complete, or missing specs would read as deletions. + SRC="$FROM" + expected=$(corpus_specs | wc -l | tr -d ' ') + found=$(find "$SRC" -maxdepth 1 -mindepth 1 -type d | wc -l | tr -d ' ') + if [ "$expected" != "$found" ]; then + echo "[corpus-manifest] $SRC holds $found of $expected specs; refusing a partial manifest" >&2 + exit 2 + fi +else + SRC="$WORK/out" + echo "[corpus-manifest] building generator..." >&2 + GEN_BIN="$(corpus_build "$PWD" "$PWD/target" "${CORPUS_PROFILE:-release}")" + + echo "[corpus-manifest] generating corpus..." >&2 + corpus_generate "$GEN_BIN" "$SRC" +fi + +OUT="$WORK/manifest.txt" +{ + echo "# openapi-to-rust corpus manifest" + echo "#" + echo "# SHA-256 (first 16 hex) of every file generated from the specs pinned in" + echo "# specs/. Regenerate with scripts/corpus-manifest.sh; CI verifies it with" + echo "# --check. A diff here means a generator change moved real-world output:" + echo "# confirm that was intended, then see exactly what moved with" + echo "# scripts/gen-diff.sh " + echo "#" + echo "# The \"Generated by openapi-to-rust vX.Y.Z\" stamp is normalized away, so a" + echo "# version bump alone never touches this file." + echo "#" + echo "# columns: path bytes lines sha256[0:16]" +} >"$OUT" + +files=0 +bytes=0 +while IFS= read -r f; do + rel="${f#"$SRC/"}" + b=$(wc -c <"$f" | tr -d ' ') + l=$(wc -l <"$f" | tr -d ' ') + printf '%-52s %10s %8s %s\n' "$rel" "$b" "$l" "$(corpus_hash "$f")" >>"$OUT" + files=$((files + 1)) + bytes=$((bytes + b)) +done < <(find "$SRC" -type f | sort) + +specs=$(find "$SRC" -maxdepth 1 -mindepth 1 -type d | wc -l | tr -d ' ') +printf '#\n# specs: %s files: %s bytes: %s\n' "$specs" "$files" "$bytes" >>"$OUT" + +if [ "$CHECK" = "1" ]; then + if diff -u "$MANIFEST" "$OUT" >"$WORK/manifest.diff"; then + echo "[corpus-manifest] up to date ($specs specs, $files files)" + exit 0 + fi + echo "[corpus-manifest] generated corpus output changed:" >&2 + cat "$WORK/manifest.diff" >&2 + cat >&2 <<'EOF' + +This generator change moves the code emitted for real-world specs. If that was +intended, inspect the change with + + scripts/gen-diff.sh + +then refresh the manifest with + + scripts/corpus-manifest.sh +EOF + exit 1 +fi + +mkdir -p "$(dirname "$MANIFEST")" +cp "$OUT" "$MANIFEST" +echo "[corpus-manifest] wrote $MANIFEST ($specs specs, $files files)" diff --git a/scripts/gen-diff.sh b/scripts/gen-diff.sh new file mode 100755 index 0000000..51273e2 --- /dev/null +++ b/scripts/gen-diff.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# Diff the code this generator emits for every pinned spec against the code the +# generator at some earlier ref emitted for the same specs. +# +# Nothing generated is checked in. The specs in specs/ are pinned and generation +# is byte-deterministic, so the old output is reconstructed on demand: build the +# generator at in a throwaway worktree, regenerate the corpus, diff. +# The whole corpus regenerates in well under a minute, and the base side is +# cached per commit, so repeat runs against the same base only pay for HEAD. +# +# Usage: +# scripts/gen-diff.sh # vs merge-base with main +# scripts/gen-diff.sh v0.15.0 # vs a release tag +# scripts/gen-diff.sh HEAD~1 +# +# Env: +# GEN_DIFF_SPECS="anthropic openai" restrict to these specs +# GEN_DIFF_PROFILE=debug build without --release +# GEN_DIFF_MAX_DIFF_BYTES=0 do not truncate per-spec diffs (5MB cap) +# GEN_DIFF_REFRESH=1 ignore the cached base corpus +set -euo pipefail +cd "$(dirname "$0")/.." +source scripts/lib/corpus.sh + +ROOT="tmp/gen-diff" +PROFILE="${GEN_DIFF_PROFILE:-release}" +MAX_DIFF_BYTES="${GEN_DIFF_MAX_DIFF_BYTES:-5000000}" +read -r -a SPEC_FILTER <<<"${GEN_DIFF_SPECS:-}" + +BASE_REF="${1:-}" +if [ -z "$BASE_REF" ]; then + BASE_REF="$(git merge-base HEAD main 2>/dev/null || echo main)" +fi +BASE_SHA="$(git rev-parse --verify "${BASE_REF}^{commit}")" +echo "[gen-diff] base $BASE_REF ($(git log -1 --format='%h %s' "$BASE_SHA"))" + +BASE_OUT="$ROOT/base/$BASE_SHA" +HEAD_OUT="$ROOT/head" +REPORT="$ROOT/report" +rm -rf "$HEAD_OUT" "$REPORT" +mkdir -p "$HEAD_OUT" "$REPORT" + +# ---- base side (cached per commit) -------------------------------------- +if [ "${GEN_DIFF_REFRESH:-}" = "1" ]; then + rm -rf "$BASE_OUT" +fi +# What the run needs to compare. Taking the spec list from corpus_specs rather +# than from whatever directories exist keeps a cached wider base corpus from +# reading as deletions against a filtered head. +WANT_SPECS="$(corpus_specs "${SPEC_FILTER[@]}" | cut -d'|' -f1 | sort)" + +# The cache marker records which specs the cached corpus actually holds: keyed +# on the commit alone, a corpus left behind by an earlier GEN_DIFF_SPECS run +# would be reused for a wider run and every spec it lacks would look new. +CACHE_OK=0 +if [ -f "$BASE_OUT/.corpus-specs" ] \ + && [ -z "$(comm -23 <(echo "$WANT_SPECS") <(sort "$BASE_OUT/.corpus-specs"))" ]; then + CACHE_OK=1 +fi + +if [ "$CACHE_OK" = "1" ]; then + echo "[gen-diff] reusing cached base corpus ($BASE_OUT)" +else + rm -rf "$BASE_OUT" + mkdir -p "$BASE_OUT" + WT="$ROOT/worktrees/$BASE_SHA" + rm -rf "$WT" + git worktree add --detach --quiet "$WT" "$BASE_SHA" + # Remove the worktree even if the build or generation fails, or the next run + # trips over a stale registration. + trap 'git worktree remove --force "$WT" >/dev/null 2>&1 || true' EXIT + echo "[gen-diff] building generator at $BASE_SHA..." + # Both sides build into the workspace target dir: the dependency graph is + # identical, so reqwest and friends compile once instead of twice. Only the + # crate itself is rebuilt per side, and the base binary is stashed first + # because the head build overwrites it in place. + BASE_BIN="$(corpus_build "$PWD/$WT" "$PWD/target" "$PROFILE")" + cp "$BASE_BIN" "$ROOT/openapi-to-rust-$BASE_SHA" + BASE_BIN="$PWD/$ROOT/openapi-to-rust-$BASE_SHA" + echo "[gen-diff] generating base corpus..." + corpus_generate "$BASE_BIN" "$BASE_OUT" "${SPEC_FILTER[@]}" || true + echo "$WANT_SPECS" >"$BASE_OUT/.corpus-specs" + git worktree remove --force "$WT" >/dev/null 2>&1 || true + trap - EXIT +fi + +# ---- head side (always regenerated; the working tree may be dirty) ------- +echo "[gen-diff] building generator at working tree..." +HEAD_BIN="$(corpus_build "$PWD" "$PWD/target" "$PROFILE")" +echo "[gen-diff] generating working-tree corpus..." +corpus_generate "$HEAD_BIN" "$HEAD_OUT" "${SPEC_FILTER[@]}" || true + +# ---- compare ------------------------------------------------------------- +# Top-level items are what a consumer of the generated crate actually sees, so +# an added/removed name is worth far more attention than a churned line count. +items() { + local dir="$1" + [ -d "$dir" ] || return 0 + find "$dir" -name '*.rs' -exec cat {} + 2>/dev/null \ + | grep -oE '^pub (struct|enum|trait|type|fn|const) [A-Za-z0-9_]+' \ + | awk '{print $2, $3}' | sort -u +} + +rows="" +changed=0 +: >"$REPORT/items.txt" +for name in $WANT_SPECS; do + b="$BASE_OUT/$name" + h="$HEAD_OUT/$name" + mkdir -p "$b" "$h" + + # `git diff --no-index` between two directories lists every file, including + # identical ones (their paths differ, so it reports them as 0/0 renames); + # only rows with real churn count as a changed file. + numstat="$(git diff --no-index --numstat "$b" "$h" 2>/dev/null || true)" + added=$(echo "$numstat" | awk '{a += $1} END {print a + 0}') + removed=$(echo "$numstat" | awk '{r += $2} END {print r + 0}') + files=$(echo "$numstat" | awk '($1 + $2) > 0' | grep -c . || true) + + new_items="$(comm -13 <(items "$b") <(items "$h"))" + gone_items="$(comm -23 <(items "$b") <(items "$h"))" + n_new=$(echo "$new_items" | grep -c . || true) + n_gone=$(echo "$gone_items" | grep -c . || true) + + [ "$files" -eq 0 ] && continue + changed=$((changed + 1)) + git diff --no-index "$b" "$h" >"$REPORT/$name.diff" 2>/dev/null || true + # A sweeping change can diff hundreds of megabytes; keep a report a reviewer + # (and an artifact upload) can actually handle. + if [ "$MAX_DIFF_BYTES" -gt 0 ] \ + && [ "$(wc -c <"$REPORT/$name.diff")" -gt "$MAX_DIFF_BYTES" ]; then + head -c "$MAX_DIFF_BYTES" "$REPORT/$name.diff" >"$REPORT/$name.diff.cut" + mv "$REPORT/$name.diff.cut" "$REPORT/$name.diff" + echo "... [truncated at $MAX_DIFF_BYTES bytes; rerun with" \ + "GEN_DIFF_SPECS=$name GEN_DIFF_MAX_DIFF_BYTES=0]" >>"$REPORT/$name.diff" + fi + rows+="$((added + removed))|$name|$files|$added|$removed|$n_new|$n_gone"$'\n' + if [ -n "$new_items$gone_items" ]; then + { + echo "=== $name ===" + [ -n "$new_items" ] && echo "$new_items" | sed 's/^/ + /' + [ -n "$gone_items" ] && echo "$gone_items" | sed 's/^/ - /' + } >>"$REPORT/items.txt" + fi +done + +echo +if [ "$changed" -eq 0 ]; then + echo "[gen-diff] no change: every spec generates identical code at both refs." + exit 0 +fi + +printf '%-24s %6s %9s %9s %8s %8s\n' SPEC FILES +LINES -LINES +ITEMS -ITEMS +printf '%-24s %6s %9s %9s %8s %8s\n' ------------------------ ------ --------- --------- -------- -------- +echo "$rows" | grep . | sort -t'|' -k1,1nr | while IFS='|' read -r _ name files added removed n_new n_gone; do + printf '%-24s %6s %9s %9s %8s %8s\n' "$name" "$files" "$added" "$removed" "$n_new" "$n_gone" +done + +echo +echo "[gen-diff] $changed spec(s) changed." +echo " per-spec diffs: $REPORT/.diff" +if [ -s "$REPORT/items.txt" ]; then + echo " item add/remove: $REPORT/items.txt" + echo + head -40 "$REPORT/items.txt" +fi diff --git a/scripts/lib/corpus.sh b/scripts/lib/corpus.sh new file mode 100755 index 0000000..c5a67cf --- /dev/null +++ b/scripts/lib/corpus.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Shared corpus generation for scripts/corpus-manifest.sh and scripts/gen-diff.sh. +# +# Both consumers must produce byte-identical output for an unchanged generator, +# or the checked-in manifest and the on-demand diff disagree about what moved. +# Everything that decides those bytes lives here: +# +# * the per-spec TOML, which mirrors scripts/spec-compile.sh so the manifest +# describes the same generated code CI compiles; +# * the Swagger 2.0 skip, matching spec-compile.sh; +# * corpus_normalize, which erases the "Generated by openapi-to-rust vX.Y.Z. +# Source OpenAPI document: " stamp. Left alone, a version bump or a +# different checkout path would rewrite every hash and bury a real change +# in noise. +# +# Source this file; it defines functions only. + +# Every ordering these scripts produce — the manifest's line order, the spec +# lists `comm` compares — has to be identical on a contributor's machine and on +# a CI runner, and `sort` collates by locale. macOS sorts +# `anthropic/REQUIRED_DEPS.toml` after `anthropic/client.rs`; a C-locale runner +# sorts it before. Byte order everywhere, so the two agree. +export LC_ALL=C + +# Emit "name|relative-spec-path" per spec, sorted. Args, when given, whitelist +# spec names. +corpus_specs() { + local want=("$@") spec name keep w + while IFS= read -r spec; do + name="$(basename "$spec")" + name="${name%.*}" + if [ ${#want[@]} -gt 0 ]; then + keep=0 + for w in "${want[@]}"; do [ "$w" = "$name" ] && keep=1; done + [ $keep -eq 0 ] && continue + fi + # Swagger 2.0 is out of scope for this generator (same test as + # spec-compile.sh): a `swagger: 2.x` key with no `openapi:` key. + if grep -qE '("swagger"[[:space:]]*:|swagger[[:space:]]*:)[[:space:]]*"?2\.' "$spec" 2>/dev/null \ + && ! grep -qE '("openapi"[[:space:]]*:|openapi[[:space:]]*:)' "$spec" 2>/dev/null; then + continue + fi + printf '%s|%s\n' "$name" "$spec" + done < <(find specs -maxdepth 1 -type f \( -name '*.yaml' -o -name '*.json' \) | sort) +} + +# corpus_config — the generator config +# the corpus is built with. Both this file and scripts/spec-compile.sh emit it, +# so it lives in one place: if they drift, the manifest stops describing the +# code the compile gate actually checks. +corpus_config() { + cat < — strip the version/source stamp from every generated +# file in so hashes depend only on the generated code itself. +corpus_normalize() { + local dir="$1" + find "$dir" -type f \( -name '*.rs' -o -name '*.toml' \) -print0 \ + | xargs -0 perl -pi -e 's/(Generated by openapi-to-rust ).*/${1}/' +} + +# corpus_build [profile] — build the generator from +# the checkout at , into . Echoes the binary path. +# Default profile is release: debug generation of microsoft-graph alone takes +# ~56s against ~9s release, and the corpus is generated twice per diff. +corpus_build() { + local src="$1" target="$2" profile="${3:-release}" flag="" + [ "$profile" = "release" ] && flag="--release" + cargo build $flag --bin openapi-to-rust \ + --manifest-path "$src/Cargo.toml" --target-dir "$target" >&2 + printf '%s/%s/openapi-to-rust\n' "$target" "$profile" +} + +# corpus_generate [spec names...] — generate every spec +# into //, normalized. Returns non-zero if any spec fails; the +# failures are echoed to stderr as "GEN-FAIL ". +# +# Run from the repository root: spec paths are resolved against it, so both +# sides of a diff read the same pinned specs and the diff isolates the +# generator change. +corpus_generate() { + local gen_bin="$1" out_root="$2"; shift 2 + local name spec dir module rc=0 + # The generator resolves a relative output_dir against the config file's own + # directory, which would nest the output under a duplicated path. Anchor it. + mkdir -p "$out_root" + out_root="$(cd "$out_root" && pwd)" + while IFS='|' read -r name spec; do + [ -z "$name" ] && continue + dir="$out_root/$name" + mkdir -p "$dir" + module="$(echo "$name" | tr '-' '_')" + corpus_config "$PWD/$spec" "$dir" "$module" >"$dir/openapi-to-rust.toml" + if ! "$gen_bin" generate --config "$dir/openapi-to-rust.toml" --quiet \ + >"$dir/generate.log" 2>&1; then + echo "GEN-FAIL $name" >&2 + rc=1 + continue + fi + rm -f "$dir/openapi-to-rust.toml" "$dir/generate.log" + done < <(corpus_specs "$@") + corpus_normalize "$out_root" + return $rc +} + +# corpus_hash — first 16 hex chars of the file's SHA-256. +corpus_hash() { + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$1" | cut -c1-16 + else + sha256sum "$1" | cut -c1-16 + fi +} diff --git a/scripts/spec-compile.sh b/scripts/spec-compile.sh index 18b6b6f..bca5d33 100755 --- a/scripts/spec-compile.sh +++ b/scripts/spec-compile.sh @@ -33,6 +33,7 @@ # cold build. set -euo pipefail cd "$(dirname "$0")/.." +source scripts/lib/corpus.sh OFFLINE="" if [ "${SPEC_COMPILE_OFFLINE:-}" = "1" ]; then @@ -166,19 +167,10 @@ EOF # Sanitize module name (replace - with _). module_name="$(echo "$name" | tr '-' '_')" - cat >"$dir/openapi-to-rust.toml" <"$dir/openapi-to-rust.toml" # Generator step log="$dir/generate.log" diff --git a/tests/corpus-manifest.txt b/tests/corpus-manifest.txt new file mode 100644 index 0000000..6370860 --- /dev/null +++ b/tests/corpus-manifest.txt @@ -0,0 +1,238 @@ +# openapi-to-rust corpus manifest +# +# SHA-256 (first 16 hex) of every file generated from the specs pinned in +# specs/. Regenerate with scripts/corpus-manifest.sh; CI verifies it with +# --check. A diff here means a generator change moved real-world output: +# confirm that was intended, then see exactly what moved with +# scripts/gen-diff.sh +# +# The "Generated by openapi-to-rust vX.Y.Z" stamp is normalized away, so a +# version bump alone never touches this file. +# +# columns: path bytes lines sha256[0:16] +anthropic/REQUIRED_DEPS.toml 648 15 f82dfebf99e5b65c +anthropic/client.rs 230956 6016 b41d4815c287f252 +anthropic/mod.rs 444 17 5214bdbc37b918db +anthropic/types.rs 4151195 85587 fccedf56680b4aa4 +arcade/REQUIRED_DEPS.toml 507 12 755f2be80b920e95 +arcade/client.rs 213669 5708 7f57e226351b99af +arcade/mod.rs 438 17 b55cb3acf5fdbd02 +arcade/types.rs 105169 2609 7717f649b6d59880 +asana/REQUIRED_DEPS.toml 678 15 a15aa2931bb0b740 +asana/client.rs 1947690 49119 a5b6d639f5c9178b +asana/mod.rs 436 17 2390079a5e4e9f74 +asana/types.rs 612843 13322 8c9551b8dfd04f6e +box/REQUIRED_DEPS.toml 750 17 7638684f0c95ceda +box/client.rs 1634574 42616 c12317d2dea3581c +box/mod.rs 432 17 b40589a066c4b98b +box/types.rs 1166877 30496 6c34d244c87ec882 +browserbase/REQUIRED_DEPS.toml 678 15 a15aa2931bb0b740 +browserbase/client.rs 89147 2445 bde73f2cb70fe9cd +browserbase/mod.rs 448 17 7e721aa1bb802e26 +browserbase/types.rs 50190 1258 8a533e945c0a1773 +cal-com/REQUIRED_DEPS.toml 604 14 b19d0c69b16a5189 +cal-com/client.rs 1280907 33369 a52eacf23bf7c7b4 +cal-com/mod.rs 440 17 05fb649563c82805 +cal-com/types.rs 1395931 35605 63c9dea82070f9f4 +cartesia/REQUIRED_DEPS.toml 648 15 f82dfebf99e5b65c +cartesia/client.rs 250631 6881 410e1e08a70dc65b +cartesia/mod.rs 442 17 3c5001b7641597a2 +cartesia/types.rs 210349 5830 ecd4ce55e9bc8b1c +cerebras/REQUIRED_DEPS.toml 507 12 755f2be80b920e95 +cerebras/client.rs 41434 1109 81d193006d41b33f +cerebras/mod.rs 442 17 71f9d1b20905ad16 +cerebras/types.rs 225828 5803 740d8cc143b1af41 +circleci/REQUIRED_DEPS.toml 605 14 12f8c3bf73140bbb +circleci/client.rs 610871 15847 f36db0cb68de4927 +circleci/mod.rs 442 17 7a8b5691342a9d62 +circleci/types.rs 364570 9619 5dd0e36a1e3ca912 +cloudflare/REQUIRED_DEPS.toml 766 18 e1f1cd474716a2a2 +cloudflare/client.rs 13273871 347584 0a255ad312ac38e3 +cloudflare/mod.rs 446 17 ce0e733f646147da +cloudflare/types.rs 16273278 378321 231c0f88c435b00c +coda/REQUIRED_DEPS.toml 651 15 438a743a40eeab6d +coda/client.rs 847577 21833 a2d28305b44bc24a +coda/mod.rs 434 17 4e8b99caec52b623 +coda/types.rs 1990457 41947 788c94a3949d60b9 +coingecko/REQUIRED_DEPS.toml 558 13 91fa33352b5a78a2 +coingecko/client.rs 446879 12322 249bf7ea8de2ab6d +coingecko/mod.rs 444 17 0059623eabdef914 +coingecko/types.rs 273846 6498 1a5bef1d4bcea737 +datadog-v2/REQUIRED_DEPS.toml 679 15 4ea57404e188eeed +datadog-v2/client.rs 5195138 134568 afec246e2b500c5a +datadog-v2/mod.rs 446 17 56e45841c9753096 +datadog-v2/types.rs 3346743 85109 6233c943a806f68b +digitalocean/REQUIRED_DEPS.toml 715 17 9da205e740fd7edc +digitalocean/client.rs 3795427 97729 d361fff00f8197ad +digitalocean/mod.rs 450 17 3b99e8d4d3161f03 +digitalocean/types.rs 1492411 35105 8d22f01598c85c65 +discord/REQUIRED_DEPS.toml 678 15 a15aa2931bb0b740 +discord/client.rs 1051676 27491 fefa3ef8d4280fc6 +discord/mod.rs 440 17 1d9f7eee6c834b1c +discord/types.rs 1272025 32755 de44694314113c14 +gcore/REQUIRED_DEPS.toml 725 16 8b991cf496fc1fdf +gcore/client.rs 5111457 134937 79c203ec0a7e722f +gcore/mod.rs 436 17 d5c05d8821f23bcb +gcore/types.rs 5862313 136588 532f95a64ec49f61 +github/REQUIRED_DEPS.toml 604 14 b19d0c69b16a5189 +github/client.rs 5743782 145446 bcf2c833960a7f35 +github/mod.rs 438 17 ac4d40f51fb06aa3 +github/types.rs 7702391 207387 0019503f70f38bce +gitpod/REQUIRED_DEPS.toml 715 17 9da205e740fd7edc +gitpod/client.rs 1590920 42783 bf19ad8cd1aa3dc4 +gitpod/mod.rs 438 17 516880570ce5294f +gitpod/types.rs 849831 19869 c2ab58c21a1573e6 +google-calendar/REQUIRED_DEPS.toml 507 12 755f2be80b920e95 +google-calendar/client.rs 155887 4143 d81a31ab413e7693 +google-calendar/mod.rs 456 17 c31832b05087624c +google-calendar/types.rs 65472 1002 d25796f627e27a14 +google-drive/REQUIRED_DEPS.toml 507 12 755f2be80b920e95 +google-drive/client.rs 239596 6127 48ca0b9ffd7ec4e4 +google-drive/mod.rs 450 17 fdb9dcacd2e8d704 +google-drive/types.rs 128460 1946 75bee84db99d2863 +google-gmail/REQUIRED_DEPS.toml 507 12 755f2be80b920e95 +google-gmail/client.rs 295232 7578 72063b949b3bd3f5 +google-gmail/mod.rs 450 17 ce127037493417e3 +google-gmail/types.rs 65481 1177 aef0efe2b930b0dd +google-tasks/REQUIRED_DEPS.toml 507 12 755f2be80b920e95 +google-tasks/client.rs 61196 1627 0745ff11d3c4ca10 +google-tasks/mod.rs 450 17 d25368315b206134 +google-tasks/types.rs 10259 190 27998c94ecf7453b +google-youtube/REQUIRED_DEPS.toml 507 12 755f2be80b920e95 +google-youtube/client.rs 345735 9193 cf322900deede5f7 +google-youtube/mod.rs 454 17 dc6e7f7cef938c3e +google-youtube/types.rs 397395 9292 4b91fb45be6db00a +grafana/REQUIRED_DEPS.toml 604 14 b19d0c69b16a5189 +grafana/client.rs 1731914 44985 a507d06d534cfacd +grafana/mod.rs 440 17 7b59b00331edb605 +grafana/types.rs 391878 9248 e82c5cd4fb60daf2 +groq/REQUIRED_DEPS.toml 609 14 069dc9aaa57c8032 +groq/client.rs 92291 2480 3215c7bc387bf9b6 +groq/mod.rs 434 17 72fd6d54577d5597 +groq/types.rs 372951 9387 e3c08712828ff708 +imagekit/REQUIRED_DEPS.toml 652 15 305aa2fd565f9a65 +imagekit/client.rs 335746 8647 c0b0022abd1e7a39 +imagekit/mod.rs 442 17 40484a363f5391c3 +imagekit/types.rs 717309 15820 b158f06cd64572b6 +increase/REQUIRED_DEPS.toml 632 14 4543dfdd58abc04d +increase/client.rs 1284127 32560 9f5cb7fe0691a1a6 +increase/mod.rs 442 17 f6131a8548853a65 +increase/types.rs 2544062 56883 63883f4dff81d804 +knocklabs/REQUIRED_DEPS.toml 651 15 438a743a40eeab6d +knocklabs/client.rs 476400 12528 57939677a9b33387 +knocklabs/mod.rs 444 17 7822049ab73a5ed0 +knocklabs/types.rs 259668 6425 e15eeacb10637dc6 +langsmith/REQUIRED_DEPS.toml 725 16 8b991cf496fc1fdf +langsmith/client.rs 2201343 57228 5c2e8348ef0f4778 +langsmith/mod.rs 444 17 eb5007d340d34a8e +langsmith/types.rs 1161995 32610 fbf8b66781fcdab7 +launchdarkly/REQUIRED_DEPS.toml 679 15 4ea57404e188eeed +launchdarkly/client.rs 2523843 64222 cae8aa7cdceab366 +launchdarkly/mod.rs 450 17 d4ed5579304ccd39 +launchdarkly/types.rs 735300 18316 37aac538ad06edd7 +letta/REQUIRED_DEPS.toml 725 16 8b991cf496fc1fdf +letta/client.rs 1413641 38038 223005e4fc5da7ca +letta/mod.rs 436 17 5cc3226a7c317d2a +letta/types.rs 6858612 139380 79ac34e92dacd5fb +lithic/REQUIRED_DEPS.toml 667 16 e22e920265ec7964 +lithic/client.rs 1351761 35353 cc521651073da9b7 +lithic/mod.rs 438 17 eba70abf1f172ff1 +lithic/types.rs 1612473 39122 146638e5d90ddfa5 +luma/REQUIRED_DEPS.toml 651 15 438a743a40eeab6d +luma/client.rs 59403 1612 009906caac2964f9 +luma/mod.rs 434 17 d85018b8cdd8f625 +luma/types.rs 75620 2116 b98bae10c054ef3e +meta-llama/REQUIRED_DEPS.toml 609 14 069dc9aaa57c8032 +meta-llama/client.rs 45875 1230 69263f435a321bd5 +meta-llama/mod.rs 446 17 6e485725d643b7c2 +meta-llama/types.rs 97541 2408 2bd9da2ef77022e0 +microsoft-graph/REQUIRED_DEPS.toml 653 15 4a261fde9be44018 +microsoft-graph/client.rs 104447058 2552569 f6b639240330267a +microsoft-graph/mod.rs 456 17 b99d7781b4460b5d +microsoft-graph/types.rs 14189944 351429 700bb06df11c5302 +modern-treasury/REQUIRED_DEPS.toml 725 16 8b991cf496fc1fdf +modern-treasury/client.rs 955510 25670 71bef9ab6151f78d +modern-treasury/mod.rs 456 17 5f336a50419a709b +modern-treasury/types.rs 995584 26633 ce34002029faedaa +openai/REQUIRED_DEPS.toml 627 14 d48ef79f2f03af3a +openai/client.rs 1045741 28436 e11c4799e5ca95d2 +openai/mod.rs 438 17 59c0207149335235 +openai/types.rs 6310615 142383 a39f47ec0b9c4cd0 +opencode/REQUIRED_DEPS.toml 576 14 9228e95d38f5078e +opencode/client.rs 943247 25092 68cf5b4201ff50cf +opencode/mod.rs 442 17 98c109eb94b95dbf +opencode/types.rs 6695017 143213 ef443d0415e39591 +pagerduty/REQUIRED_DEPS.toml 651 15 438a743a40eeab6d +pagerduty/client.rs 2168383 58184 d2ee8b2e1fd33cbd +pagerduty/mod.rs 444 17 2edbc04dfe892a50 +pagerduty/types.rs 1696912 40730 4db4dd8ac98e4d1f +perplexity/REQUIRED_DEPS.toml 554 13 a6164504984d80b3 +perplexity/client.rs 52982 1401 b0a83d486acfd8bc +perplexity/mod.rs 446 17 1f9103fe36426530 +perplexity/types.rs 530449 12140 0456b118c51b5e00 +resend/REQUIRED_DEPS.toml 653 15 4a261fde9be44018 +resend/client.rs 269122 7292 001f79622d7afd71 +resend/mod.rs 438 17 f7a8a77315259e44 +resend/types.rs 133568 3604 b889f478d7735cfc +retell/REQUIRED_DEPS.toml 507 12 755f2be80b920e95 +retell/client.rs 484265 12604 c3b8c0fc3b474148 +retell/mod.rs 438 17 e5b430243537a3be +retell/types.rs 1225853 32915 436678419f67417f +runway/REQUIRED_DEPS.toml 651 15 438a743a40eeab6d +runway/client.rs 193884 5307 e81d3bb173f62912 +runway/mod.rs 438 17 67ec5d69926d7742 +runway/types.rs 963739 22398 587121e3f343ec8c +sentry/REQUIRED_DEPS.toml 678 15 a15aa2931bb0b740 +sentry/client.rs 967361 25033 146a1b22d0eee589 +sentry/mod.rs 438 17 2bdd2d5164ca444f +sentry/types.rs 2369613 66073 c9cbb5b2d1d6cbef +snyk/REQUIRED_DEPS.toml 667 16 e22e920265ec7964 +snyk/client.rs 1858782 48182 7326789a98545962 +snyk/mod.rs 434 17 eec62ea6f01f40f6 +snyk/types.rs 2717805 59426 5866664d242459a9 +spotify/REQUIRED_DEPS.toml 558 13 91fa33352b5a78a2 +spotify/client.rs 563386 14794 1e64690d1e855ae8 +spotify/mod.rs 440 17 954184fb7fd84586 +spotify/types.rs 261914 6374 ec3db89e1975c7b0 +storyden/REQUIRED_DEPS.toml 698 17 569479aa8197c912 +storyden/client.rs 967591 25555 3f5848b0ea037c1a +storyden/mod.rs 442 17 ff7a510276d50e59 +storyden/types.rs 884799 20120 38b50906e06a668d +stripe/REQUIRED_DEPS.toml 580 14 d04fcff74e41419c +stripe/client.rs 2892850 75103 49c4b8e2723364dd +stripe/mod.rs 438 17 cafa363545854cd3 +stripe/types.rs 10112467 248486 fb430bcdf91b98f7 +supabase/REQUIRED_DEPS.toml 676 16 f4c727cabf80fb53 +supabase/client.rs 602482 16285 b2936923215ac3ac +supabase/mod.rs 442 17 331ec75d430bc0a8 +supabase/types.rs 476436 13390 306a5b647643e04d +telnyx/REQUIRED_DEPS.toml 766 18 e1f1cd474716a2a2 +telnyx/client.rs 5179949 136395 757c1aec497e4d5e +telnyx/mod.rs 438 17 f3a44d93d532e621 +telnyx/types.rs 4496512 107989 7d19b35d8678c506 +terminal-shop/REQUIRED_DEPS.toml 508 13 95d5dce08c1f99a8 +terminal-shop/client.rs 208005 5558 daf63a66100730d7 +terminal-shop/mod.rs 452 17 7014abd4e66e4e4e +terminal-shop/types.rs 51936 1629 12f624a0585e437b +together/REQUIRED_DEPS.toml 741 17 ded5e2bd1cd63603 +together/client.rs 449000 11997 e2d4b2a703e5fc53 +together/mod.rs 442 17 69fe71a3ff6db06e +together/types.rs 565362 14607 c07c3f2fcae7972a +twilio/REQUIRED_DEPS.toml 629 15 7e3af3d5d13dfd94 +twilio/client.rs 843971 22153 fccd4e5e7fcf3dc3 +twilio/mod.rs 438 17 1e85ee5c08c46793 +twilio/types.rs 895681 20820 37d493aaebf60a09 +val-town/REQUIRED_DEPS.toml 699 16 a8b7f9dfad5ffd4c +val-town/client.rs 154262 4122 7fd7731be44ffd36 +val-town/mod.rs 442 17 8a07cd572c0c8638 +val-town/types.rs 68086 1978 78ac613c49efff35 +vercel/REQUIRED_DEPS.toml 652 15 305aa2fd565f9a65 +vercel/client.rs 1465481 38564 22213443ba598e84 +vercel/mod.rs 438 17 6e4cb11d3c843a23 +vercel/types.rs 12196079 315218 93755a3f454f3470 +writer/REQUIRED_DEPS.toml 699 16 a8b7f9dfad5ffd4c +writer/client.rs 141267 3797 75e8b378af67f185 +writer/mod.rs 438 17 0435bf1552e493d1 +writer/types.rs 182479 4509 36b85cce121b9439 +# +# specs: 56 files: 224 bytes: 313747115