Record review timing and correction outcomes in benchmark results

bench/src/compare-baseline.test.ts

@@ -7,6 +7,8 @@ import { resolve } from "node:path";
 import { cases } from "../fixtures/cases";
 import {
   aggregateObservedMetrics,
+  assertAdjudicationCapture,
+  formatAdjudicationCapture,
   assertCompleteCohortEvidence,
   assertDistinctRunIdentities,
   assertDistinctRawReportDigests,
@@ -225,6 +227,15 @@ interface FakeReportOptions {
   runId?: string;
 }
 
+function capturedAdjudicationAttempt() {
+  return { outerAttempt: 1, attemptResult: "scored" as const, phaseStart: "unknown" as const,
+    captureStatus: "valid" as const, authenticity: "perAttemptNonceMatched" as const,
+    summary: { version: 1 as const, phase: "adjudication" as const, outcome: "validated" as const,
+      callsStarted: 1 as const, correctionKind: null, phaseElapsedMs: 503, initialBudgetMs: 60000,
+      firstCallElapsedMs: 500, firstCallRemainingMs: 59500, promptTokens: 10, completionTokens: 2,
+      costProviderDecimal: "0.000003", usageAccountingComplete: true, incidents: [] } };
+}
+
 let fakeRunSequence = 0;
 
 function fakeReport(options: FakeReportOptions = {}): LiveReportForComparison {
@@ -248,6 +259,8 @@ function fakeReport(options: FakeReportOptions = {}): LiveReportForComparison {
       observedProviderCostUsdDecimal: costPerCase,
       costAccountingComplete: true,
       attemptCount: 1,
+      adjudicationEvidenceVersion: 1,
+      adjudicationAttempts: [capturedAdjudicationAttempt()],
       recoveredErrors: [],
       exitCode: truthSeverity === "error" && caseDetected === true ? 1 : 0,
     };
@@ -351,6 +364,8 @@ async function inputBoundReport(screeningProfilePath = resolve(import.meta.dir,
       observedProviderCostUsdDecimal: "0.001",
       costAccountingComplete: true,
       attemptCount: 1,
+      adjudicationEvidenceVersion: 1,
+      adjudicationAttempts: [capturedAdjudicationAttempt()],
       recoveredErrors: [],
       exitCode: caseDetected && finding?.severity === "error" ? 1 : 0,
     };
@@ -697,6 +712,17 @@ test("recording replaces a populated historical baseline only after complete cur
     expect((await run()).code).not.toBe(0);
     expect(await readFile(baselinePath, "utf8")).toBe(historical);
     await writeFile(receiptPath, receiptBytes);
+    const reportPath = resolve(directory, "report-0.json");
+    const reportBytes = await readFile(reportPath);
+    const malformedCapture = JSON.parse(reportBytes.toString());
+    malformedCapture.results[0].adjudicationAttempts[0].summary.phaseElapsedMs = -1;
+    await writeFile(reportPath, JSON.stringify(malformedCapture));
+    const invalidCapture = await run();
+    expect(invalidCapture.code).not.toBe(0);
+    expect(invalidCapture.output).toContain("original attempt 1: invalid capture field summary.phaseElapsedMs");
+    expect(invalidCapture.output).toContain("No replacement call is permitted");
+    expect(await readFile(baselinePath, "utf8")).toBe(historical);
+    await writeFile(reportPath, reportBytes);
     const recorded = await run();
     expect(recorded.code).toBe(0);
     expect(recorded.output).toContain("Recorded baseline");
@@ -1427,3 +1453,61 @@ describe("CLI report selection", () => {
     })).toContain("predeclared ten-report calibration cohort");
   });
 });
+
+
+describe("formal adjudication capture acceptance", () => {
+  test("retains failed unknowns and requires complete final capture without changing verdict", () => {
+    const report = fakeReport();
+    const result = report.results[0]!;
+    const final = capturedAdjudicationAttempt();
+    final.outerAttempt = 2;
+    result.attemptCount = 2;
+    result.recoveredErrors = ["operational envelope: review/invalidOutput"];
+    result.adjudicationAttempts = [{ outerAttempt: 1, attemptResult: "failed", failureClass: "processTimeout",
+      phaseStart: "unknown", captureStatus: "notObserved", producerFrameObserved: true,
+      captureDetail: "unterminatedAuthenticatedFrame" }, final];
+    expect(() => assertAdjudicationCapture(report, 1)).not.toThrow();
+    expect(formatAdjudicationCapture([report])).toContain("1 attempts with unknown measurements");
+    expect(formatAdjudicationCapture([report])).toContain("Correction recovery (captured subset): unmeasured");
+    const preserved = structuredClone(result);
+    for (const status of ["notObserved", "invalid", "duplicate"] as const) {
+      result.adjudicationAttempts[1] = { outerAttempt: 2, attemptResult: "scored", phaseStart: "unknown", captureStatus: status };
+      expect(() => assertAdjudicationCapture(report, 1)).toThrow(/case defect-01, original attempt 2/);
+      expect(result.scored).toBe(true);
+    }
+    Object.assign(result, preserved);
+    for (const status of ["invalid", "duplicate"] as const) {
+      result.adjudicationAttempts![0] = { outerAttempt: 1, attemptResult: "failed", failureClass: "operational", phaseStart: "unknown", captureStatus: status };
+      expect(() => assertAdjudicationCapture(report, 1)).toThrow("authenticated capture");
+    }
+  });
+
+  test("calibration and release share the capture refusal before recording or publication", () => {
+    for (const count of [5, 10]) {
+      const reports = Array.from({ length: count }, () => fakeReport());
+      const check = () => { const cohort = fakeReleaseCohort(reports); return assertCompleteCohortEvidence({
+        ...cohort, manifestSha256: "9".repeat(64), reports, record: count === 10,
+      }); };
+      expect(check).not.toThrow();
+      delete reports[0]!.results[0]!.adjudicationAttempts;
+      expect(check).toThrow("No replacement call is permitted");
+      expect(check).toThrow("Preserve original samples and any spent release tag");
+    }
+  });
+
+  test("zero candidates and correction populations retain explicit denominators", () => {
+    const report = fakeReport();
+    const first = report.results[0]!.adjudicationAttempts![0]!;
+    first.summary = { version: 1, phase: "adjudication", outcome: "notRequired", callsStarted: 0,
+      correctionKind: null, phaseElapsedMs: null, initialBudgetMs: null, firstCallElapsedMs: null,
+      firstCallRemainingMs: null, promptTokens: null, completionTokens: null, costProviderDecimal: null,
+      usageAccountingComplete: null, incidents: [] };
+    expect(() => assertAdjudicationCapture(report, 1)).not.toThrow();
+    const correction = report.results[1]!.adjudicationAttempts![0]!.summary!;
+    correction.callsStarted = 2; correction.correctionKind = "semanticRetry";
+    expect(formatAdjudicationCapture([report])).toContain("1 zero-candidate runs");
+    expect(formatAdjudicationCapture([report])).toContain("Correction recovery (captured subset): 1/1");
+    first.summary.outcome = "failed";
+    expect(() => assertAdjudicationCapture(report, 1)).toThrow();
+  });
+});

bench/src/compare-baseline.ts

@@ -43,6 +43,7 @@ import {
 } from "./cohort";
 import {
   ADMISSION_API_BASE,
+  adjudicationAttemptSchema,
   evaluatorSourceSha256,
   screeningProfileMetadata,
 } from "./live";
@@ -140,6 +141,8 @@ const providerContractSchema = z.object({
 });
 
 const liveCaseResultSchema = z.object({
+  adjudicationEvidenceVersion: z.literal(1).optional(),
+  adjudicationAttempts: z.array(adjudicationAttemptSchema).optional(),
   id: nonemptyStringSchema,
   type: z.enum(["defect", "clean"]),
   scored: z.boolean(),
@@ -1287,6 +1290,9 @@ async function loadReports(paths: readonly string[]): Promise<LoadedReports> {
     }
     const report = liveReportSchema.safeParse(parsed);
     if (!report.success) {
+      if (report.error.issues.some((issue) => issue.path[0] === "results" && String(issue.path[2]).startsWith("adjudication"))) {
+        assertAdjudicationCapture(parsed as LiveReportForComparison, 1);
+      }
       throw new Error(`invalid release benchmark report at ${path}: ${report.error.message}`);
     }
     try {
@@ -1304,6 +1310,49 @@ async function loadReports(paths: readonly string[]): Promise<LoadedReports> {
   };
 }
 
+export function assertAdjudicationCapture(report: LiveReportForComparison, retries: number): void {
+  for (const result of report.results) {
+    const refuse = (attempt: number, defect: string): never => {
+      throw new Error(`adjudication evidence qualification failed: case ${result.id}, original attempt ${attempt}: ${defect}; case verdict remains ${result.scored ? "scored" : "failed"}. No replacement call is permitted. Preserve original samples and any spent release tag; return the failure for a separately authorized future release decision.`);
+    };
+    const attempts = result.adjudicationAttempts;
+    if (result.adjudicationEvidenceVersion !== 1 || !Array.isArray(attempts) || attempts.length !== result.attemptCount || result.attemptCount > retries + 1) {
+      refuse(result.attemptCount, "missing capture capability or original attempt lineage");
+    }
+    for (const [index, attempt] of attempts!.entries()) {
+      const parsed = adjudicationAttemptSchema.safeParse(attempt);
+      if (!parsed.success) refuse(index + 1, `invalid capture field ${parsed.error.issues[0]?.path.join(".") || "provenance"}`);
+      if (attempt.outerAttempt !== index + 1 ||
+          attempt.attemptResult !== (index === attempts!.length - 1 && result.scored ? "scored" : "failed")) {
+        refuse(index + 1, "invalid capture provenance or attempt order");
+      }
+      if (attempt.captureStatus === "invalid" || attempt.captureStatus === "duplicate") refuse(index + 1, `${attempt.captureStatus} authenticated capture`);
+      if (index === attempts!.length - 1 && result.scored && (attempt.captureStatus !== "valid" || attempt.summary?.outcome === "failed")) refuse(index + 1, "missing complete successful final capture");
+    }
+  }
+}
+
+export function formatAdjudicationCapture(reports: readonly LiveReportForComparison[]): string {
+  const attempts = reports.flatMap((report) => report.results.flatMap((result) => result.adjudicationAttempts ?? []));
+  const invoked = attempts.filter((a) => a.captureStatus === "valid" && a.summary?.outcome !== "notRequired");
+  const zeroCandidates = attempts.filter((a) => a.captureStatus === "valid" && a.summary?.outcome === "notRequired").length;
+  const notStarted = attempts.filter((a) => a.phaseStart === "notStarted").length;
+  const unknown = attempts.filter((a) => a.captureStatus === "notObserved" && a.phaseStart !== "notStarted").length;
+  const invalid = attempts.filter((a) => a.captureStatus === "invalid" || a.captureStatus === "duplicate").length;
+  const corrections = invoked.filter((a) => a.summary?.callsStarted === 2);
+  const recovered = corrections.filter((a) => a.summary?.outcome === "validated").length;
+  const finalCaptured = attempts.filter((a) => a.attemptResult === "scored" && a.captureStatus === "valid").length;
+  const firstValid = invoked.filter((a) => a.summary?.callsStarted === 1 && a.summary.outcome === "validated").length;
+  const selected = invoked.filter((a) => a.summary?.callsStarted === 1 && a.summary.correctionKind !== null).length;
+  const firstFailed = invoked.length - corrections.length - firstValid - selected;
+  return `Adjudication evidence: ${attempts.length} original attempts; ${finalCaptured} final summaries captured; ` +
+    `${invoked.length} invoked terminal summaries; ${zeroCandidates} zero-candidate runs; ${unknown} attempts with unknown measurements; ` +
+    `${notStarted} proven spawn failures; ${invalid} integrity defects.\n` +
+    `Captured logical calls: ${firstValid} first-call successes; ${selected} selected corrections not entered; ${firstFailed} first-call failures without correction.\n` +
+    `Correction recovery (captured subset): ${corrections.length === 0 ? "unmeasured" : `${recovered}/${corrections.length}`}; ` +
+    `${unknown} unknown attempts retain unknown correction incidence and recovery.`;
+}
+
 export interface VerifiedCohortEvidence {
   manifest: CohortManifest;
   manifestSha256: string;
@@ -1371,6 +1420,7 @@ export function assertCompleteCohortEvidence(options: {
         `benchmark report slot ${slot.slot} exceeds its manifest outer retry allowance`,
       );
     }
+    assertAdjudicationCapture(report, manifest.caseRetries);
     if (receipt.reportRawSha256 !== options.rawReportSha256[index]) {
       throw new Error(`benchmark report slot ${slot.slot} raw digest does not match its receipt`);
     }
@@ -1542,6 +1592,7 @@ async function main() {
       record,
     });
   }
+  if (cohortEvidence !== undefined) console.log(formatAdjudicationCapture(reports));
   const observed = aggregateObservedMetrics(reports);
   const rebaselineGuidance = formatRebaselineGuidance({
     binaryPath,