morgaesis · Oct 20, 2026 · Oct 20, 2026 · Oct 20, 2026 · Oct 20, 2026 · Oct 20, 2026
diff --git a/.github/workflows/benchmark-calibration.yml b/.github/workflows/benchmark-calibration.yml
@@ -13,7 +13,7 @@ jobs:
     permissions:
       contents: write
     steps:
-      - name: Reserve the current main calibration source
+      - name: Verify the current main calibration source
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
@@ -31,10 +31,6 @@ jobs:
             echo "::error::Calibration source is not the current main commit."
             exit 1
           }
-          gh api --method POST "repos/${GITHUB_REPOSITORY}/git/refs" \
-            -f ref="refs/tags/postil-calibration-${GITHUB_SHA}" \
-            -f sha="${GITHUB_SHA}" \
-            --silent
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
         with:
@@ -45,10 +41,76 @@ jobs:
       - name: Verify embedded model admission
         working-directory: bench
         run: bun run verify-admission --allow-provisional
-      - name: Require an unpopulated Azure/US Luna baseline
-        run: >-
-          jq -e '.profiles["openai/gpt-5.6-luna"].populated == false'
-          bench/baseline-us.json >/dev/null
+      - name: Fingerprint the current evaluator contract
+        id: evaluator
+        working-directory: bench
+        run: |
+          bun -e 'import { evaluatorSourceSha256 } from "./src/live"; console.log("sha256=" + await evaluatorSourceSha256())' >> "$GITHUB_OUTPUT"
+      - name: Verify the calibration baseline input
+        env:
+          GH_TOKEN: ${{ github.token }}
+          CURRENT_EVALUATOR_SHA256: ${{ steps.evaluator.outputs.sha256 }}
+        run: |
+          set -euo pipefail
+          profile="openai/gpt-5.6-luna"
+          populated="$(jq -er --arg profile "$profile" '.profiles[$profile].populated | if type == "boolean" then tostring else error("Calibration populated state must be boolean") end' bench/baseline-us.json)"
+          case "$populated" in
+            false) exit 0 ;;
+            true) ;;
+            *) echo "::error::Calibration baseline must declare a boolean populated state."; exit 1 ;;
+          esac
+          test -s bench/baseline-us.attestation.json || {
+            echo "::error::Populated recalibration requires the retained baseline attestation."
+            exit 1
+          }
+          source_sha="$(jq -er --arg profile "$profile" '.profiles[$profile].calibration.sourceSha' bench/baseline-us.json)"
+          [[ "$source_sha" =~ ^[0-9a-f]{40,64}$ ]] || {
+            echo "::error::Prior calibration source digest is invalid."
+            exit 1
+          }
+          test "$source_sha" != "$GITHUB_SHA" || {
+            echo "::error::Recalibration requires a new main source commit."
+            exit 1
+          }
+          prior_evaluator="$(jq -er '.corpus.evaluatorSha256' bench/baseline-us.json)"
+          [[ "$prior_evaluator" =~ ^[0-9a-f]{64}$ && "$CURRENT_EVALUATOR_SHA256" =~ ^[0-9a-f]{64}$ ]] || {
+            echo "::error::Calibration evaluator digest is invalid."
+            exit 1
+          }
+          test "$prior_evaluator" != "$CURRENT_EVALUATOR_SHA256" || {
+            echo "::error::Populated recalibration requires a changed evaluator contract."
+            exit 1
+          }
+          registry_sha="$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/tags/postil-calibration-${source_sha}" --jq .object.sha)"
+          test "$registry_sha" = "$source_sha" || {
+            echo "::error::Prior calibration registry source mismatch."
+            exit 1
+          }
+          gh attestation verify bench/baseline-us.json \
+            --bundle bench/baseline-us.attestation.json \
+            --repo postil-dev/postil-cli \
+            --signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml \
+            --signer-digest "$source_sha" \
+            --source-digest "$source_sha" \
+            --source-ref refs/heads/main \
+            --cert-oidc-issuer https://token.actions.githubusercontent.com \
+            --predicate-type https://slsa.dev/provenance/v1 \
+            --deny-self-hosted-runners \
+            --format json >/dev/null
+      - name: Reserve the current main calibration source
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          main_sha="$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/heads/main" --jq .object.sha)"
+          test "${GITHUB_SHA}" = "$main_sha" || {
+            echo "::error::Calibration source is no longer the current main commit."
+            exit 1
+          }
+          gh api --method POST "repos/${GITHUB_REPOSITORY}/git/refs" \
+            -f ref="refs/tags/postil-calibration-${GITHUB_SHA}" \
+            -f sha="${GITHUB_SHA}" \
+            --silent
 
   prepare:
     needs: validate
@@ -149,6 +211,7 @@ jobs:
           name: benchmark-calibration-prepared-${{ github.run_id }}
           path: ${{ runner.temp }}/prepared
       - name: Verify prepared calibration provenance
+        id: verify-prepared
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
@@ -173,6 +236,7 @@ jobs:
       - name: Resolve canonical sample slot
         run: echo "CALIBRATION_SLOT=$(printf '%02d' '${{ matrix.sample }}')" >> "$GITHUB_ENV"
       - name: Reserve benchmark sample ${{ matrix.sample }}
+        id: reserve-sample
         working-directory: bench
         run: >-
           bun run bench:cohort-run --
@@ -193,6 +257,7 @@ jobs:
           cp "${RUNNER_TEMP}/prepared/slots/${CALIBRATION_SLOT}/receipt.json" \
             "${RUNNER_TEMP}/reservation.receipt.json"
       - name: Execute benchmark sample ${{ matrix.sample }}
+        id: execute-sample
         working-directory: bench
         env:
           OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
@@ -213,6 +278,7 @@ jobs:
             ${{ runner.temp }}/prepared/slots/${{ env.CALIBRATION_SLOT }}/report.json
             ${{ runner.temp }}/prepared/slots/${{ env.CALIBRATION_SLOT }}/receipt.json
       - name: Stage benchmark sample evidence
+        id: stage-sample
         if: always()
         run: |
           set -euo pipefail
@@ -228,13 +294,33 @@ jobs:
             cp "${{ steps.attest-result.outputs.bundle-path }}" "$evidence/result.attestation.json"
           fi
       - name: Upload benchmark sample evidence
+        id: upload-sample
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: benchmark-calibration-sample-${{ github.run_id }}-${{ matrix.sample }}
           path: ${{ runner.temp }}/benchmark-calibration-sample-${{ env.CALIBRATION_SLOT }}
           if-no-files-found: error
           retention-days: 90
+      - name: Summarize failed calibration sample
+        if: failure()
+        env:
+          SAMPLE: ${{ matrix.sample }}
+          STEP_RESULTS: ${{ toJSON(steps) }}
+          UPLOAD_OUTCOME: ${{ steps.upload-sample.outcome }}
+        run: |
+          {
+            echo "Calibration sample ${SAMPLE} failed; step outcomes:"
+            jq -r 'to_entries[] | "\(.key): \(.value.outcome)"' <<< "$STEP_RESULTS"
+            echo "Run logs (job ${GITHUB_JOB}, attempt ${GITHUB_RUN_ATTEMPT}): ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+            if [[ "$UPLOAD_OUTCOME" == "success" ]]; then
+              echo "Uploaded sample evidence: benchmark-calibration-sample-${GITHUB_RUN_ID}-${SAMPLE}."
+            else
+              echo "Sample artifact was not uploaded (upload outcome: ${UPLOAD_OUTCOME:-not run}); use the run/job logs."
+            fi
+            echo "Failed evidence is not a calibration."
+            echo "Fix the source on main and start a new first-attempt run. Do not reuse this reservation or rerun this cohort."
+          } >> "$GITHUB_STEP_SUMMARY"
 
   record:
     needs: [validate, prepare, sample]

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "postil-cli"
-version = "0.9.20"
+version = "0.9.21"
 edition = "2024"
 description = "Postil: a low-noise AI review gate. Silent on clean PRs, hard gate on real risk."
 license = "Apache-2.0"

diff --git a/bench/fixtures/cases.ts b/bench/fixtures/cases.ts
@@ -92,11 +92,11 @@ export function makeDiff(path: string, hunks: FixtureHunk[]): string {
   ].join("\n");
 }
 
-function makeDistantStreamingChurn(side: "prefix" | "suffix"): string {
+function makeDistantStreamingChurn(side: "prefix" | "suffix", additionalChangeLine?: number): string {
   return Array.from({ length: 3 }, (_, fileIndex) => {
     const lines = [` export function ordinary_${side}_${fileIndex}(actor: Actor) {`];
     for (let line = 2; line < 130; line += 1) {
-      if (line === 64) {
+      if (line === 64 || line === additionalChangeLine) {
         lines.push(
           `-  const ordinary_${side}_${fileIndex}_${line}=actor.id;`,
           `+  const ordinary_${side}_${fileIndex}_${line} = actor.id;`,
@@ -1109,7 +1109,7 @@ const fixtureSpecs: FixtureSpec[] = [
       "export async function bulkEdit(actor: Actor, changeSet: ChangeSet) { await applyBulkEdit(changeSet); }",
     policy: "Huge diffs still need authorization checks on privileged writes.",
     scoringLabels: ["huge-low-signal", "multi-hunk", "security", "authorization", "error"],
-    prefixDiff: makeDistantStreamingChurn("prefix"),
+    prefixDiff: makeDistantStreamingChurn("prefix", 70),
     suffixDiff: makeDistantStreamingChurn("suffix"),
     expectedCoverage: "bounded",
     finding: {

diff --git a/bench/src/compare-baseline.test.ts b/bench/src/compare-baseline.test.ts
@@ -1,4 +1,5 @@
 import { describe, expect, test } from "bun:test";
+import { execFileSync } from "node:child_process";
 import { createHash } from "node:crypto";
 import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
@@ -48,15 +49,23 @@ import {
   type ProviderContractEvidence,
 } from "./livemodels-score";
 
-test("EU baseline preserves its fixture identity but is stale for the calibration evaluator", async () => {
+const historicalImport = "7ae07f4c2c62256d837860deb76e498706e3ac5b";
+const historicalArtifact = (path: string): string => execFileSync(
+  "git", ["show", `${historicalImport}:${path}`],
+  { cwd: resolve(import.meta.dir, "../.."), encoding: "utf8" },
+);
+
+test("retained EU baseline preserves historical identity and rejects the current corpus", async () => {
   const baseline = parseBaselineFile(JSON.parse(
     await readFile(resolve(import.meta.dir, "..", "baseline.json"), "utf8"),
   ));
   const fixtureCorpusSha256 = createHash("sha256")
     .update(JSON.stringify(cases.map((input) => benchmarkCase.parse(input))))
     .digest("hex");
 
-  expect(baseline.corpus.fixtureCorpusSha256).toBe(fixtureCorpusSha256);
+  const historical = parseBaselineFile(JSON.parse(historicalArtifact("bench/baseline.json")));
+  expect(baseline.corpus).toEqual(historical.corpus);
+  expect(baseline.corpus.fixtureCorpusSha256).not.toBe(fixtureCorpusSha256);
   expect(baseline.corpus.evaluatorSha256).not.toBe(await evaluatorSourceSha256());
 });
 
@@ -115,12 +124,12 @@ test("release prerequisite rejects an unpopulated US baseline before attestation
   }
 });
 
-test("attested US calibration binds its source and route while the embedded provider stays on Azure/EU", async () => {
+test("retained attested US calibration binds its source and route while the embedded provider stays on Azure/EU", async () => {
   const root = resolve(import.meta.dir, "..", "..");
   const [usBytes, euBytes, attestationBytes, us, eu] = await Promise.all([
-    readFile(resolve(root, "bench/baseline-us.json"), "utf8"),
-    readFile(resolve(root, "bench/baseline.json"), "utf8"),
-    readFile(resolve(root, "bench/baseline-us.attestation.json"), "utf8"),
+    historicalArtifact("bench/baseline-us.json"),
+    historicalArtifact("bench/baseline.json"),
+    historicalArtifact("bench/baseline-us.attestation.json"),
     screeningProfileMetadata(resolve(root, "provisional-models-us.json")),
     screeningProfileMetadata(resolve(root, "provisional-models-eu.json")),
   ]);
@@ -163,7 +172,7 @@ test("attested US calibration binds its source and route while the embedded prov
   expect(await readFile(resolve(root, "provisional-models.json"), "utf8")).toBe(await readFile(resolve(root, "provisional-models-eu.json"), "utf8"));
   expect(usBaseline.corpus.fixtureCorpusSha256).toBe(euBaseline.corpus.fixtureCorpusSha256);
   expect(usBaseline.corpus.evaluatorSha256).not.toBe(euBaseline.corpus.evaluatorSha256);
-  expect(usBaseline.corpus.evaluatorSha256).toBe(await evaluatorSourceSha256());
+  expect(usBaseline.corpus.evaluatorSha256).not.toBe(await evaluatorSourceSha256());
   expect(eu.upstreamProviderRoute).toBe("azure/eu");
   expect(us.upstreamProviderRoute).toBe("azure/us");
   for (const profile of [us, eu]) {
@@ -322,7 +331,7 @@ function cloneReport(report: LiveReportForComparison): LiveReportForComparison {
   return structuredClone(report);
 }
 
-async function inputBoundReport(): Promise<LiveReportForComparison> {
+async function inputBoundReport(screeningProfilePath = resolve(import.meta.dir, "..", "..", "provisional-models.json")): Promise<LiveReportForComparison> {
   const report = fakeReport();
   const parsedCases = cases.map((input) => benchmarkCase.parse(input));
   let detected = 0;
@@ -346,7 +355,6 @@ async function inputBoundReport(): Promise<LiveReportForComparison> {
       exitCode: caseDetected && finding?.severity === "error" ? 1 : 0,
     };
   });
-  const screeningProfilePath = resolve(import.meta.dir, "..", "..", "provisional-models.json");
   const [binary, evaluatorSha256, profile] = await Promise.all([
     readFile(process.execPath),
     evaluatorSourceSha256(),
@@ -637,6 +645,77 @@ function fakeReleaseCohort(reports: readonly LiveReportForComparison[]): {
   };
 }
 
+test("recording replaces a populated historical baseline only after complete current-input cohort verification", async () => {
+  const directory = await mkdtemp(resolve(tmpdir(), "postil-populated-record-"));
+  try {
+    const profilePath = resolve(import.meta.dir, "../..", "provisional-models-us.json");
+    const original = await inputBoundReport(profilePath);
+    const reports = Array.from({ length: 10 }, (_, index) => {
+      const report = cloneReport(original);
+      report.summary.runId = `recalibration-${index + 1}`;
+      report.summary.ranAt = `2026-08-25T00:10:0${index}.000Z`;
+      report.summary.providerGenerationIds = [`gen-recalibration-${index + 1}`];
+      return report;
+    });
+    const cohort = fakeReleaseCohort(reports);
+    for (const field of ["binarySha256", "evaluatorSha256", "fixtureCorpusSha256", "screeningProfileSha256", "providerContractSha256"] as const) {
+      cohort.manifest[field] = original.summary[field];
+    }
+    const manifestBytes = JSON.stringify(cohort.manifest);
+    const manifestPath = resolve(directory, "cohort.json");
+    await writeFile(manifestPath, manifestBytes);
+    const baselinePath = resolve(directory, "baseline.json");
+    const historical = historicalArtifact("bench/baseline-us.json");
+    await writeFile(baselinePath, historical);
+    const arguments_: string[] = [];
+    for (const [index, report] of reports.entries()) {
+      const reportPath = resolve(directory, `report-${index}.json`);
+      const receiptPath = resolve(directory, `receipt-${index}.json`);
+      cohort.receipts[index]!.receipt.manifestSha256 = sha256(manifestBytes);
+      await writeFile(reportPath, JSON.stringify(report));
+      await writeFile(receiptPath, JSON.stringify(cohort.receipts[index]!.receipt));
+      arguments_.push("--expected-run-id", report.summary.runId, "--result", reportPath, "--receipt", receiptPath);
+    }
+    const run = async () => {
+      const execution = cohort.manifest.execution;
+      const child = Bun.spawn([
+        process.execPath, "--no-env-file", resolve(import.meta.dir, "compare-baseline.ts"),
+        "--binary", process.execPath, "--baseline", baselinePath, "--screen-profile", profilePath,
+        "--cohort-manifest", manifestPath, ...arguments_, "--record",
+      ], { stdout: "pipe", stderr: "pipe", timeout: 10_000, env: {
+        GITHUB_REPOSITORY: execution.repository, GITHUB_SHA: execution.sourceSha,
+        GITHUB_REF: execution.sourceRef, GITHUB_RUN_ID: execution.runId, GITHUB_RUN_ATTEMPT: execution.runAttempt,
+      } });
+      const [code, stdout, stderr] = await Promise.all([
+        child.exited, new Response(child.stdout).text(), new Response(child.stderr).text(),
+      ]);
+      return { code, output: stdout + stderr };
+    };
+    const receiptPath = resolve(directory, "receipt-9.json");
+    const receiptBytes = await readFile(receiptPath);
+    await writeFile(receiptPath, JSON.stringify({ ...cohort.receipts[9]!.receipt, state: "failed", exitCode: 1 }));
+    expect((await run()).code).not.toBe(0);
+    expect(await readFile(baselinePath, "utf8")).toBe(historical);
+    await writeFile(receiptPath, receiptBytes);
+    const recorded = await run();
+    expect(recorded.code).toBe(0);
+    expect(recorded.output).toContain("Recorded baseline");
+    const baseline = parseBaselineFile(JSON.parse(await readFile(baselinePath, "utf8")));
+    expect(baseline.corpus.evaluatorSha256).toBe(original.summary.evaluatorSha256);
+    expect(baseline.corpus.fixtureCorpusSha256).toBe(original.summary.fixtureCorpusSha256);
+    const profile = baseline.profiles[original.summary.model]!;
+    expect(profile.populated).toBe(true);
+    if (!isCalibratedBaselineProfile(profile)) throw new Error("recorded calibration is missing");
+    expect(profile.calibration.reportCount).toBe(10);
+    expect(profile.calibration.sourceSha).toBe(cohort.manifest.execution.sourceSha);
+    for (const [name, prior] of Object.entries(JSON.parse(historical).profiles)) {
+      if (name !== original.summary.model) expect(baseline.profiles[name]).toEqual(prior);
+    }
+  } finally {
+    await rm(directory, { recursive: true, force: true });
+  }
+});
+
 describe("predeclared cohort evidence", () => {
   test("accepts every completed immutable slot", () => {
     const reports = Array.from({ length: 5 }, (_, index) => fakeReport({