From 583cc32441816fdd311d8342d85a7fe460a9d8cf Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Tue, 20 Oct 2026 14:40:08 +0000 Subject: [PATCH] Capture adjudication evidence for benchmark qualification Retain nonce-bound terminal measurements for each original benchmark attempt, including explicit zero-candidate results and unknown failed-attempt capture. Validate final successful capture before calibration recording or release comparison, with diagnostic transport isolated from existing retry decisions. --- bench/src/compare-baseline.test.ts | 84 +++++ bench/src/compare-baseline.ts | 51 +++ bench/src/live.test.ts | 208 +++++++++++ bench/src/live.ts | 144 +++++++- bench/src/scorer-eval.test.ts | 137 +++++++ bench/src/verify-calibration.test.ts | 46 ++- src/llm.rs | 531 +++++++++++++++++++-------- src/review.rs | 2 + 8 files changed, 1050 insertions(+), 153 deletions(-) diff --git a/bench/src/compare-baseline.test.ts b/bench/src/compare-baseline.test.ts index c768422..39d0786 100644 --- a/bench/src/compare-baseline.test.ts +++ b/bench/src/compare-baseline.test.ts @@ -7,6 +7,8 @@ import { resolve } from "node:path"; import { cases } from "../fixtures/cases"; import { aggregateObservedMetrics, + assertAdjudicationCapture, + formatAdjudicationCapture, assertCompleteCohortEvidence, assertDistinctRunIdentities, assertDistinctRawReportDigests, @@ -225,6 +227,15 @@ interface FakeReportOptions { runId?: string; } +function capturedAdjudicationAttempt() { + return { outerAttempt: 1, attemptResult: "scored" as const, phaseStart: "unknown" as const, + captureStatus: "valid" as const, authenticity: "perAttemptNonceMatched" as const, + summary: { version: 1 as const, phase: "adjudication" as const, outcome: "validated" as const, + callsStarted: 1 as const, correctionKind: null, phaseElapsedMs: 503, initialBudgetMs: 60000, + firstCallElapsedMs: 500, firstCallRemainingMs: 59500, promptTokens: 10, completionTokens: 2, + costProviderDecimal: "0.000003", usageAccountingComplete: true, incidents: [] } }; +} + let fakeRunSequence = 0; function fakeReport(options: FakeReportOptions = {}): LiveReportForComparison { @@ -248,6 +259,8 @@ function fakeReport(options: FakeReportOptions = {}): LiveReportForComparison { observedProviderCostUsdDecimal: costPerCase, costAccountingComplete: true, attemptCount: 1, + adjudicationEvidenceVersion: 1, + adjudicationAttempts: [capturedAdjudicationAttempt()], recoveredErrors: [], exitCode: truthSeverity === "error" && caseDetected === true ? 1 : 0, }; @@ -351,6 +364,8 @@ async function inputBoundReport(screeningProfilePath = resolve(import.meta.dir, observedProviderCostUsdDecimal: "0.001", costAccountingComplete: true, attemptCount: 1, + adjudicationEvidenceVersion: 1, + adjudicationAttempts: [capturedAdjudicationAttempt()], recoveredErrors: [], exitCode: caseDetected && finding?.severity === "error" ? 1 : 0, }; @@ -697,6 +712,17 @@ test("recording replaces a populated historical baseline only after complete cur expect((await run()).code).not.toBe(0); expect(await readFile(baselinePath, "utf8")).toBe(historical); await writeFile(receiptPath, receiptBytes); + const reportPath = resolve(directory, "report-0.json"); + const reportBytes = await readFile(reportPath); + const malformedCapture = JSON.parse(reportBytes.toString()); + malformedCapture.results[0].adjudicationAttempts[0].summary.phaseElapsedMs = -1; + await writeFile(reportPath, JSON.stringify(malformedCapture)); + const invalidCapture = await run(); + expect(invalidCapture.code).not.toBe(0); + expect(invalidCapture.output).toContain("original attempt 1: invalid capture field summary.phaseElapsedMs"); + expect(invalidCapture.output).toContain("No replacement call is permitted"); + expect(await readFile(baselinePath, "utf8")).toBe(historical); + await writeFile(reportPath, reportBytes); const recorded = await run(); expect(recorded.code).toBe(0); expect(recorded.output).toContain("Recorded baseline"); @@ -1427,3 +1453,61 @@ describe("CLI report selection", () => { })).toContain("predeclared ten-report calibration cohort"); }); }); + + +describe("formal adjudication capture acceptance", () => { + test("retains failed unknowns and requires complete final capture without changing verdict", () => { + const report = fakeReport(); + const result = report.results[0]!; + const final = capturedAdjudicationAttempt(); + final.outerAttempt = 2; + result.attemptCount = 2; + result.recoveredErrors = ["operational envelope: review/invalidOutput"]; + result.adjudicationAttempts = [{ outerAttempt: 1, attemptResult: "failed", failureClass: "processTimeout", + phaseStart: "unknown", captureStatus: "notObserved", producerFrameObserved: true, + captureDetail: "unterminatedAuthenticatedFrame" }, final]; + expect(() => assertAdjudicationCapture(report, 1)).not.toThrow(); + expect(formatAdjudicationCapture([report])).toContain("1 attempts with unknown measurements"); + expect(formatAdjudicationCapture([report])).toContain("Correction recovery (captured subset): unmeasured"); + const preserved = structuredClone(result); + for (const status of ["notObserved", "invalid", "duplicate"] as const) { + result.adjudicationAttempts[1] = { outerAttempt: 2, attemptResult: "scored", phaseStart: "unknown", captureStatus: status }; + expect(() => assertAdjudicationCapture(report, 1)).toThrow(/case defect-01, original attempt 2/); + expect(result.scored).toBe(true); + } + Object.assign(result, preserved); + for (const status of ["invalid", "duplicate"] as const) { + result.adjudicationAttempts![0] = { outerAttempt: 1, attemptResult: "failed", failureClass: "operational", phaseStart: "unknown", captureStatus: status }; + expect(() => assertAdjudicationCapture(report, 1)).toThrow("authenticated capture"); + } + }); + + test("calibration and release share the capture refusal before recording or publication", () => { + for (const count of [5, 10]) { + const reports = Array.from({ length: count }, () => fakeReport()); + const check = () => { const cohort = fakeReleaseCohort(reports); return assertCompleteCohortEvidence({ + ...cohort, manifestSha256: "9".repeat(64), reports, record: count === 10, + }); }; + expect(check).not.toThrow(); + delete reports[0]!.results[0]!.adjudicationAttempts; + expect(check).toThrow("No replacement call is permitted"); + expect(check).toThrow("Preserve original samples and any spent release tag"); + } + }); + + test("zero candidates and correction populations retain explicit denominators", () => { + const report = fakeReport(); + const first = report.results[0]!.adjudicationAttempts![0]!; + first.summary = { version: 1, phase: "adjudication", outcome: "notRequired", callsStarted: 0, + correctionKind: null, phaseElapsedMs: null, initialBudgetMs: null, firstCallElapsedMs: null, + firstCallRemainingMs: null, promptTokens: null, completionTokens: null, costProviderDecimal: null, + usageAccountingComplete: null, incidents: [] }; + expect(() => assertAdjudicationCapture(report, 1)).not.toThrow(); + const correction = report.results[1]!.adjudicationAttempts![0]!.summary!; + correction.callsStarted = 2; correction.correctionKind = "semanticRetry"; + expect(formatAdjudicationCapture([report])).toContain("1 zero-candidate runs"); + expect(formatAdjudicationCapture([report])).toContain("Correction recovery (captured subset): 1/1"); + first.summary.outcome = "failed"; + expect(() => assertAdjudicationCapture(report, 1)).toThrow(); + }); +}); diff --git a/bench/src/compare-baseline.ts b/bench/src/compare-baseline.ts index bdef18a..98b818b 100644 --- a/bench/src/compare-baseline.ts +++ b/bench/src/compare-baseline.ts @@ -43,6 +43,7 @@ import { } from "./cohort"; import { ADMISSION_API_BASE, + adjudicationAttemptSchema, evaluatorSourceSha256, screeningProfileMetadata, } from "./live"; @@ -140,6 +141,8 @@ const providerContractSchema = z.object({ }); const liveCaseResultSchema = z.object({ + adjudicationEvidenceVersion: z.literal(1).optional(), + adjudicationAttempts: z.array(adjudicationAttemptSchema).optional(), id: nonemptyStringSchema, type: z.enum(["defect", "clean"]), scored: z.boolean(), @@ -1287,6 +1290,9 @@ async function loadReports(paths: readonly string[]): Promise { } const report = liveReportSchema.safeParse(parsed); if (!report.success) { + if (report.error.issues.some((issue) => issue.path[0] === "results" && String(issue.path[2]).startsWith("adjudication"))) { + assertAdjudicationCapture(parsed as LiveReportForComparison, 1); + } throw new Error(`invalid release benchmark report at ${path}: ${report.error.message}`); } try { @@ -1304,6 +1310,49 @@ async function loadReports(paths: readonly string[]): Promise { }; } +export function assertAdjudicationCapture(report: LiveReportForComparison, retries: number): void { + for (const result of report.results) { + const refuse = (attempt: number, defect: string): never => { + throw new Error(`adjudication evidence qualification failed: case ${result.id}, original attempt ${attempt}: ${defect}; case verdict remains ${result.scored ? "scored" : "failed"}. No replacement call is permitted. Preserve original samples and any spent release tag; return the failure for a separately authorized future release decision.`); + }; + const attempts = result.adjudicationAttempts; + if (result.adjudicationEvidenceVersion !== 1 || !Array.isArray(attempts) || attempts.length !== result.attemptCount || result.attemptCount > retries + 1) { + refuse(result.attemptCount, "missing capture capability or original attempt lineage"); + } + for (const [index, attempt] of attempts!.entries()) { + const parsed = adjudicationAttemptSchema.safeParse(attempt); + if (!parsed.success) refuse(index + 1, `invalid capture field ${parsed.error.issues[0]?.path.join(".") || "provenance"}`); + if (attempt.outerAttempt !== index + 1 || + attempt.attemptResult !== (index === attempts!.length - 1 && result.scored ? "scored" : "failed")) { + refuse(index + 1, "invalid capture provenance or attempt order"); + } + if (attempt.captureStatus === "invalid" || attempt.captureStatus === "duplicate") refuse(index + 1, `${attempt.captureStatus} authenticated capture`); + if (index === attempts!.length - 1 && result.scored && (attempt.captureStatus !== "valid" || attempt.summary?.outcome === "failed")) refuse(index + 1, "missing complete successful final capture"); + } + } +} + +export function formatAdjudicationCapture(reports: readonly LiveReportForComparison[]): string { + const attempts = reports.flatMap((report) => report.results.flatMap((result) => result.adjudicationAttempts ?? [])); + const invoked = attempts.filter((a) => a.captureStatus === "valid" && a.summary?.outcome !== "notRequired"); + const zeroCandidates = attempts.filter((a) => a.captureStatus === "valid" && a.summary?.outcome === "notRequired").length; + const notStarted = attempts.filter((a) => a.phaseStart === "notStarted").length; + const unknown = attempts.filter((a) => a.captureStatus === "notObserved" && a.phaseStart !== "notStarted").length; + const invalid = attempts.filter((a) => a.captureStatus === "invalid" || a.captureStatus === "duplicate").length; + const corrections = invoked.filter((a) => a.summary?.callsStarted === 2); + const recovered = corrections.filter((a) => a.summary?.outcome === "validated").length; + const finalCaptured = attempts.filter((a) => a.attemptResult === "scored" && a.captureStatus === "valid").length; + const firstValid = invoked.filter((a) => a.summary?.callsStarted === 1 && a.summary.outcome === "validated").length; + const selected = invoked.filter((a) => a.summary?.callsStarted === 1 && a.summary.correctionKind !== null).length; + const firstFailed = invoked.length - corrections.length - firstValid - selected; + return `Adjudication evidence: ${attempts.length} original attempts; ${finalCaptured} final summaries captured; ` + + `${invoked.length} invoked terminal summaries; ${zeroCandidates} zero-candidate runs; ${unknown} attempts with unknown measurements; ` + + `${notStarted} proven spawn failures; ${invalid} integrity defects.\n` + + `Captured logical calls: ${firstValid} first-call successes; ${selected} selected corrections not entered; ${firstFailed} first-call failures without correction.\n` + + `Correction recovery (captured subset): ${corrections.length === 0 ? "unmeasured" : `${recovered}/${corrections.length}`}; ` + + `${unknown} unknown attempts retain unknown correction incidence and recovery.`; +} + export interface VerifiedCohortEvidence { manifest: CohortManifest; manifestSha256: string; @@ -1371,6 +1420,7 @@ export function assertCompleteCohortEvidence(options: { `benchmark report slot ${slot.slot} exceeds its manifest outer retry allowance`, ); } + assertAdjudicationCapture(report, manifest.caseRetries); if (receipt.reportRawSha256 !== options.rawReportSha256[index]) { throw new Error(`benchmark report slot ${slot.slot} raw digest does not match its receipt`); } @@ -1542,6 +1592,7 @@ async function main() { record, }); } + if (cohortEvidence !== undefined) console.log(formatAdjudicationCapture(reports)); const observed = aggregateObservedMetrics(reports); const rebaselineGuidance = formatRebaselineGuidance({ binaryPath, diff --git a/bench/src/live.test.ts b/bench/src/live.test.ts index b3a3703..4346a48 100644 --- a/bench/src/live.test.ts +++ b/bench/src/live.test.ts @@ -7,6 +7,9 @@ import { cases } from "../fixtures/cases"; import type { Envelope } from "./harness"; import { aggregateAttemptAccounting, + adjudicationSummarySchema, + decodeAdjudicationCapture, + isTransientFailure, boundedCoverageFailure, envelopeOperationalFailure, exactProviderCost, @@ -480,12 +483,20 @@ describe("live benchmark operational envelopes", () => { }; const second = envelope(20, 3, "0.00018", 90); const binary = join(root, "fake-postil"); + const skipped = { version: 1, phase: "adjudication", outcome: "notRequired", callsStarted: 0, + correctionKind: null, phaseElapsedMs: null, initialBudgetMs: null, firstCallElapsedMs: null, + firstCallRemainingMs: null, promptTokens: null, completionTokens: null, costProviderDecimal: null, + usageAccountingComplete: null, incidents: [] }; await writeFile(binary, `#!/bin/sh +nonce=$(printf '%s' "$POSTIL_BENCH_ADJUDICATION_NONCE" | tr '0123456789abcdef' '!#$%&()*+,\\-./:;<') if [ -f '${marker}' ]; then printf 2 > '${marker}' + printf '\\036%s%s %s %s %s\\n' '${capturePayload("postil: adjudication-metrics ")}' "$nonce" '${capturePayload(JSON.stringify(skipped))}' "$nonce" '${capturePayload("end")}' >&2 printf '%s\\n' '${JSON.stringify(second)}' else printf 1 > '${marker}' + printf '\\036%s%s %s' '${capturePayload("postil: adjudication-metrics ")}' "$nonce" '${capturePayload('{"phaseElapsedMs":503')}' >&2 + printf 'postil: finding adjudication unavailable; preserving all generated findings\\n' >&2 printf '%s\\n' '${JSON.stringify(first)}' fi `, { mode: 0o700 }); @@ -511,6 +522,10 @@ fi attemptCount: 2, recoveredErrors: ["operational envelope: review/invalidOutput"], }); + expect(report.results[0]!.adjudicationAttempts).toMatchObject([ + { outerAttempt: 1, attemptResult: "failed", captureStatus: "notObserved", phaseStart: "unknown", producerFrameObserved: true }, + { outerAttempt: 2, attemptResult: "scored", captureStatus: "valid", summary: { outcome: "notRequired" } }, + ]); expect(report.summary.totalTokens).toEqual({ prompt: 30, completion: 5, total: 35 }); expect(report.summary.observedProviderCostUsdDecimal).toBe("0.0003"); expect(report.summary.costAccountingComplete).toBe(true); @@ -519,6 +534,25 @@ fi retriedCases: 1, recoveredErrors: [{ error: "operational envelope: review/invalidOutput", count: 1 }], }); + + for (const [index, diagnostic, retries] of [ + [0, "time\x1e*!out", 0], [1, "5\x1e*!03", 0], + [2, "time\x1e*!out\n503", 1], [3, "5\x1e*!03\ntimeout", 1], + ] as const) { + const failed = { ...first, modelIncidents: [{ phase: "review", category: "providerError", recovered: false }] }; + await writeFile(binary, `#!/bin/sh +printf '%s' '${diagnostic}' >&2 +printf '%s\\n' '${JSON.stringify(failed)}' +`, { mode: 0o700 }); + const observed = await runLive([cases[0]!], { binary, model: "test/model", rootDir: root, + runId: `diagnostic-boundary-${index}`, concurrency: 1, retries: 1 }); + expect(observed.results[0]).toMatchObject({ scored: false, attemptCount: 1 + retries, + durationMs: 110 * (1 + retries), promptTokens: 10 * (1 + retries), completionTokens: 2 * (1 + retries), + observedProviderCostUsdDecimal: retries ? "0.00024" : "0.00012", costAccountingComplete: true }); + expect(observed.results[0]!.adjudicationAttempts).toHaveLength(1 + retries); + expect(observed.results[0]!.adjudicationAttempts!.every((attempt) => + attempt.phaseStart === "unknown" && attempt.captureStatus === "notObserved")).toBe(true); + } } finally { if (previousKey === undefined) delete process.env.MODEL_API_KEY; else process.env.MODEL_API_KEY = previousKey; @@ -614,3 +648,177 @@ fi }, "qualified/scorer")).toContain("identity missing"); }); }); + + +const captureAlphabet = "!#$%&()*+,-./:;<"; +const captureChallenge = (nonce: string) => [...nonce].map((nibble) => captureAlphabet[parseInt(nibble, 16)]).join(""); +const capturePayload = (text: string) => [...Buffer.from(text)].map((byte) => captureAlphabet[byte >> 4]! + captureAlphabet[byte & 15]!).join(""); +const captureFrame = (record: unknown, nonce: string) => + `\x1e${capturePayload("postil: adjudication-metrics ")}${captureChallenge(nonce)} ${capturePayload(JSON.stringify(record))} ${captureChallenge(nonce)} ${capturePayload("end")}\n`; + +describe("adjudication terminal capture", () => { + test("only a positively refused child spawn establishes notStarted", async () => { + const root = await mkdtemp(join(tmpdir(), "postil-capture-no-start-")); + const previous = process.env.MODEL_API_KEY; + process.env.MODEL_API_KEY = crypto.randomUUID(); + try { + const binary = join(root, "nonexecutable-child"); + await writeFile(binary, "child cannot execute\n", { mode: 0o600 }); + const report = await runLive([cases[0]!], { binary, model: "test/model", rootDir: root, + runId: "spawn-refused", concurrency: 1, retries: 0 }); + expect(report.results[0]!.adjudicationAttempts).toEqual([{ + outerAttempt: 1, attemptResult: "failed", failureClass: "spawnFailure", + phaseStart: "notStarted", captureStatus: "notObserved", + }]); + expect(report.results[0]!.attemptCount).toBe(1); + expect(report.results[0]!.costAccountingComplete).toBe(false); + } finally { + if (previous === undefined) delete process.env.MODEL_API_KEY; else process.env.MODEL_API_KEY = previous; + await rm(root, { recursive: true }); + } + }); + + const nonce = crypto.randomUUID().replaceAll("-", ""); + const summary = (overrides = {}) => ({ version: 1, phase: "adjudication", outcome: "validated", + callsStarted: 1, correctionKind: null, phaseElapsedMs: 503, initialBudgetMs: 60000, + firstCallElapsedMs: 500, firstCallRemainingMs: 59500, promptTokens: 10, completionTokens: 2, + costProviderDecimal: "0.000003", usageAccountingComplete: true, incidents: [], ...overrides }); + const frame = (record: unknown, challenge = nonce) => captureFrame(record, challenge); + + test("accepts exactly reachable outcome/call/correction tuples", () => { + for (const outcome of ["validated", "failed", "notRequired"]) { + for (const callsStarted of [0, 1, 2]) for (const correctionKind of [null, "schemaRepair", "semanticRetry"]) { + const skipped = outcome === "notRequired"; + const record = summary({ outcome, callsStarted, correctionKind, ...(skipped ? { + phaseElapsedMs: null, initialBudgetMs: null, firstCallElapsedMs: null, firstCallRemainingMs: null, + promptTokens: null, completionTokens: null, costProviderDecimal: null, usageAccountingComplete: null, + } : {}) }); + const reachable = skipped ? callsStarted === 0 && correctionKind === null + : callsStarted === 1 ? outcome === "failed" || correctionKind === null + : callsStarted === 2 && correctionKind !== null; + expect(adjudicationSummarySchema.safeParse(record).success).toBe(reachable); + } + } + for (const invalid of [summary({ phaseElapsedMs: -1 }), summary({ promptTokens: 2 ** 53 }), + summary({ firstCallRemainingMs: 60001 }), summary({ extra: 1 }), summary({ costProviderDecimal: "0.00" })]) { + expect(decodeAdjudicationCapture(frame(invalid), nonce).evidence.captureStatus).toBe("invalid"); + } + }); + + test("isolates matching transport from retry decisions and preserves caller diagnostics", async () => { + for (let elapsed = 0; elapsed <= 1200; elapsed++) { + for (const ordinary of ["", "postil: failure\n", "postil: provider returned 503\n", "timeout\n"]) { + const baseline = { scored: false, error: "other", stderr: ordinary } as Parameters[0]; + const captured = decodeAdjudicationCapture(ordinary + frame(summary({ phaseElapsedMs: elapsed, firstCallElapsedMs: 0, firstCallRemainingMs: 60000 })), nonce); + expect(captured.evidence.captureStatus).toBe("valid"); + expect(isTransientFailure({ ...baseline, stderr: captured.classifierStderr })).toBe(isTransientFailure(baseline)); + } + } + const full = frame(summary({ outcome: "failed" })); + const headerLength = `\x1e${capturePayload("postil: adjudication-metrics ")}${captureChallenge(nonce)} `.length; + const closingLength = full.trimEnd().length; + const payloadEnd = full.indexOf(" ", headerLength); + const callers = ["", "503", "429", "timeout", "econnreset", "network", "ordinary suffix", "Error: time\x1eout", + "postil: finding adjudication unavailable; preserving all generated findings\n", + "postil: finding adjudication unavailable; preserving 1 finding (HTTP 503)\n", + "postil: failed to build adjudication prompt; preserving all generated findings\n", + "Error: provider returned 503\n", "postil: provider returned 503\n"]; + const callerSource = await readFile(join(import.meta.dir, "../../src/review.rs"), "utf8"); + const adjudicationCaller = callerSource.slice(callerSource.indexOf("let adjudicated = match adjudication_user"), callerSource.indexOf("staged_adjudication = Some")); + const actualDiagnostics = [...adjudicationCaller.matchAll(/eprintln!\(\s*"((?:[^"\\]|\\.)*)"/gu)] + .map((match) => match[1]!.replace(/\{[^}]*\}/gu, "503") + "\n"); + expect(actualDiagnostics).toHaveLength(3); + for (let cut = 0; cut <= full.length; cut++) for (const caller of [...callers, ...actualDiagnostics]) { + const decoded = decodeAdjudicationCapture(full.slice(0, cut) + caller, nonce); + const baseline = { scored: false, error: "other", stderr: caller } as Parameters[0]; + expect(isTransientFailure({ ...baseline, stderr: decoded.classifierStderr })).toBe(isTransientFailure(baseline)); + const retained = cut < headerLength ? full.slice(0, cut) : cut === full.length ? "\n\n" + : "\n" + (cut < closingLength && cut > payloadEnd ? full.slice(payloadEnd, cut) : ""); + expect(decoded.classifierStderr).toBe(retained + caller); + expect(decoded.evidence.captureStatus).toBe(cut >= closingLength ? "valid" : "notObserved"); + if (cut >= headerLength && cut < closingLength) expect(decoded.evidence).toMatchObject({ producerFrameObserved: true }); + if (cut < closingLength) expect(decoded.evidence.summary).toBeUndefined(); + } + // A raw status appended exactly after the encoded partial elapsed field + // remains the caller's signature, never part of the uncompleted record. + const elapsedPrefix = `\x1e${capturePayload("postil: adjudication-metrics ")}${captureChallenge(nonce)} ${capturePayload('{"phaseElapsedMs":')}`; + expect(decodeAdjudicationCapture(elapsedPrefix + "503", nonce).classifierStderr).toBe("\n503"); + }); + + test("preserves unrecognized bytes and prevents retry signatures across removed transport", () => { + const retry = (stderr: string) => isTransientFailure({ scored: false, error: "other", stderr } as Parameters[0]); + for (const input of ["time\x1e*!out", "5\x1e*!03"]) { + const decoded = decodeAdjudicationCapture(input, nonce); + expect(retry(input)).toBe(false); + expect(decoded.classifierStderr).toBe(input); + expect(retry(decoded.classifierStderr)).toBe(false); + expect(decoded.evidence).toEqual({ phaseStart: "unknown", captureStatus: "notObserved" }); + } + const full = frame(summary()); + const header = `\x1e${capturePayload("postil: adjudication-metrics ")}${captureChallenge(nonce)} `; + const trailer = ` ${captureChallenge(nonce)} ${capturePayload("end")}`; + const payloadEnd = full.indexOf(" ", header.length); + const closingLength = full.trimEnd().length; + const wrong = frame(summary(), crypto.randomUUID().replaceAll("-", "")); + const signatures = ["503", "429", "rate limit", "timeout", "time out", "network", "econnreset", + "too many requests", "temporarily unavailable", "service unavailable", "bad gateway", "gateway time-out", + "overloaded", "connection reset", "econnrefused", "etimedout", "socket hang up"]; + for (const signature of signatures) { + for (let split = 1; split < signature.length; split++) { + const left = signature.slice(0, split), right = signature.slice(split); + for (const fragment of ["\x1e", "\x1e*!", "\x1e?", header.slice(0, -8), wrong]) { + const input = left + fragment + right; + const decoded = decodeAdjudicationCapture(input, nonce); + expect(decoded.classifierStderr).toBe(input); + expect(retry(decoded.classifierStderr)).toBe(retry(input)); + expect(decoded.evidence.captureStatus).toBe("notObserved"); + } + } + // Every producer cut is surrounded by ordinary bytes. Genuine complete + // signatures on either side survive, while split keywords stay split. + const split = Math.floor(signature.length / 2); + for (let cut = 0; cut <= full.length; cut++) { + const fragment = full.slice(0, cut); + for (const [left, right] of [[signature.slice(0, split), signature.slice(split)], + [`${signature}\n`, "ordinary"], ["ordinary", `\n${signature}`]]) { + const input = left + fragment + right; + const decoded = decodeAdjudicationCapture(input, nonce); + expect(retry(decoded.classifierStderr)).toBe(retry(input)); + if (cut === header.length - 1 && right.startsWith(" ")) { + // The suffix supplies the final separator of a complete matching + // header; the remaining ordinary suffix still survives. + expect(decoded.classifierStderr).toBe(left + "\n" + right.slice(1)); + } else if (cut < header.length) expect(decoded.classifierStderr).toBe(input); + else { + const retained = cut === full.length ? "\n\n" : "\n" + + (cut < closingLength && cut > payloadEnd ? full.slice(payloadEnd, cut) : ""); + expect(decoded.classifierStderr).toBe(left + retained + right); + } + } + } + const malformed = header + signature + trailer; + expect(decodeAdjudicationCapture(malformed, nonce)).toEqual({ + evidence: { phaseStart: "unknown", captureStatus: "invalid" }, classifierStderr: malformed, + }); + expect(retry(decodeAdjudicationCapture(malformed, nonce).classifierStderr)).toBe(true); + } + for (const malformed of [header + "!" + trailer, header + capturePayload("not json") + trailer]) { + const decoded = decodeAdjudicationCapture("time" + malformed + "out", nonce); + expect(decoded.evidence.captureStatus).toBe("invalid"); + expect(decoded.classifierStderr).toBe("time\nout"); + expect(retry(decoded.classifierStderr)).toBe(false); + } + }); + + test("ignores stale nonces and rejects authenticated malformed and duplicate frames", () => { + const wrong = crypto.randomUUID().replaceAll("-", ""); + expect(decodeAdjudicationCapture(frame(summary(), wrong), nonce).evidence.captureStatus).toBe("notObserved"); + const result = decodeAdjudicationCapture(frame(summary(), wrong) + frame(summary()), nonce); + expect(result.evidence.captureStatus).toBe("valid"); + expect(JSON.stringify(result.evidence)).not.toContain(nonce); + expect(decodeAdjudicationCapture(frame(summary()) + frame(summary()), nonce).evidence.captureStatus).toBe("duplicate"); + expect(decodeAdjudicationCapture(`\x1e${capturePayload("postil: adjudication-metrics ")}${captureChallenge(nonce)} {?} ${captureChallenge(nonce)} ${capturePayload("end")}\n`, nonce).evidence.captureStatus).toBe("invalid"); + expect(decodeAdjudicationCapture(frame(summary()).trimEnd(), nonce).evidence.captureStatus).toBe("valid"); + expect(decodeAdjudicationCapture(frame({ extra: "x".repeat(2048) }), nonce).evidence.captureStatus).toBe("invalid"); + }); +}); diff --git a/bench/src/live.ts b/bench/src/live.ts index 7fa6d21..812d9c8 100644 --- a/bench/src/live.ts +++ b/bench/src/live.ts @@ -25,8 +25,9 @@ // tier (info<->warn and warn<->error treated as a match). See the severity-tier // helpers below for the rationale. +import { z } from "zod"; import { execFile as execFileCb } from "node:child_process"; -import { createHash } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join, relative, resolve, sep } from "node:path"; import { promisify } from "node:util"; @@ -129,7 +130,121 @@ interface GroundTruth { severity: string | null; } +const CAPTURE_ALPHABET = "!#$%&()*+,-./:;<"; +const encodeCaptureText = (text: string) => [...Buffer.from(text)] + .map((byte) => CAPTURE_ALPHABET[byte >> 4]! + CAPTURE_ALPHABET[byte & 15]!).join(""); +const ADJUDICATION_PREFIX = "\x1e" + encodeCaptureText("postil: adjudication-metrics "); +const CAPTURE_MAX_BYTES = 2048; +const safeCounter = z.number().int().safe().nonnegative(); +export const adjudicationSummarySchema = z.object({ + version: z.literal(1), phase: z.literal("adjudication"), + outcome: z.enum(["validated", "failed", "notRequired"]), + callsStarted: z.union([z.literal(0), z.literal(1), z.literal(2)]), + correctionKind: z.enum(["schemaRepair", "semanticRetry"]).nullable(), + phaseElapsedMs: safeCounter.nullable(), initialBudgetMs: safeCounter.nullable(), + firstCallElapsedMs: safeCounter.nullable(), firstCallRemainingMs: safeCounter.nullable(), + promptTokens: safeCounter.nullable(), completionTokens: safeCounter.nullable(), + costProviderDecimal: z.string().max(128).refine((value) => { + try { return formatCanonicalDecimal(parseCanonicalDecimal(value)) === value; } catch { return false; } + }).nullable(), + usageAccountingComplete: z.boolean().nullable(), + incidents: z.array(z.object({ + phase: z.literal("scorer"), category: z.enum(["providerError", "invalidOutput", "timeout", "deadline"]), + recovered: z.boolean(), recovery: z.enum(["repair", "fallback"]).optional(), + }).strict()).max(2), +}).strict().superRefine((value, ctx) => { + const times = [value.phaseElapsedMs, value.initialBudgetMs, value.firstCallElapsedMs, value.firstCallRemainingMs]; + const counters = [value.promptTokens, value.completionTokens]; + const skipped = value.outcome === "notRequired"; + const shape = skipped + ? value.callsStarted === 0 && value.correctionKind === null && [...times, ...counters].every((v) => v === null) && + value.costProviderDecimal === null && value.usageAccountingComplete === null && value.incidents.length === 0 + : value.callsStarted > 0 && [...times, ...counters].every((v) => v !== null) && value.usageAccountingComplete !== null && + (value.callsStarted !== 2 || value.correctionKind !== null) && + !(value.outcome === "validated" && value.callsStarted === 1 && value.correctionKind !== null) && + value.phaseElapsedMs! >= value.firstCallElapsedMs! && value.firstCallRemainingMs! <= value.initialBudgetMs! && + Math.abs(value.firstCallRemainingMs! - Math.max(0, value.initialBudgetMs! - value.firstCallElapsedMs!)) <= 1; + if (!shape) ctx.addIssue({ code: "custom", message: "unreachable adjudication measurement combination" }); +}); + +export const adjudicationAttemptSchema = z.object({ + outerAttempt: safeCounter.positive(), attemptResult: z.enum(["scored", "failed"]), + failureClass: z.enum(["envelopeInvalid", "operational", "processTimeout", "spawnFailure", "other"]).optional(), + phaseStart: z.enum(["unknown", "notStarted"]), + captureStatus: z.enum(["valid", "notObserved", "invalid", "duplicate"]), + captureDetail: z.literal("unterminatedAuthenticatedFrame").optional(), + producerFrameObserved: z.literal(true).optional(), + authenticity: z.literal("perAttemptNonceMatched").optional(), + summary: adjudicationSummarySchema.optional(), +}).strict().superRefine((value, ctx) => { + if ((value.captureStatus === "valid") !== (value.summary !== undefined && value.authenticity !== undefined) || + (value.captureStatus !== "valid" && (value.summary !== undefined || value.authenticity !== undefined)) || + (value.phaseStart === "notStarted" && (value.failureClass !== "spawnFailure" || value.captureStatus !== "notObserved" || value.producerFrameObserved)) || + (value.attemptResult === "scored" ? value.failureClass !== undefined : value.failureClass === undefined)) { + ctx.addIssue({ code: "custom", message: "inconsistent adjudication capture provenance" }); + } +}); +export type AdjudicationAttempt = z.infer; + +export function decodeAdjudicationCapture(stderr: string, nonce: string): { + evidence: Omit; + classifierStderr: string; +} { + const challenge = [...nonce].map((nibble) => CAPTURE_ALPHABET[parseInt(nibble, 16)]).join(""); + const header = `${ADJUDICATION_PREFIX}${challenge} `; + const trailer = ` ${challenge} ${encodeCaptureText("end")}`; + let classifierStderr = "", cursor = 0, count = 0; + let evidence: ReturnType["evidence"] = { phaseStart: "unknown", captureStatus: "notObserved" }; + for (;;) { + const start = stderr.indexOf("\x1e", cursor); + if (start < 0) { classifierStderr += stderr.slice(cursor); break; } + classifierStderr += stderr.slice(cursor, start); + // Public, partial and foreign prefixes are ordinary diagnostic bytes. + if (!stderr.startsWith(header, start)) { + classifierStderr += stderr[start]; + cursor = start + 1; + continue; + } + count++; + const payloadStart = start + header.length; + const lineEnd = stderr.indexOf("\n", payloadStart); + const end = lineEnd < 0 ? stderr.length : lineEnd; + const close = stderr.indexOf(trailer, payloadStart); + if (close < 0 || close > end) { + cursor = payloadStart; + while (cursor < end && CAPTURE_ALPHABET.includes(stderr[cursor]!)) cursor++; + classifierStderr += "\n"; + evidence = count > 1 ? { phaseStart: "unknown", captureStatus: "duplicate" } + : { phaseStart: "unknown", captureStatus: "notObserved", producerFrameObserved: true, captureDetail: "unterminatedAuthenticatedFrame" }; + continue; + } + cursor = close + trailer.length; + const payload = stderr.slice(payloadStart, close); + const encoded = [...payload].every((char) => CAPTURE_ALPHABET.includes(char)); + // Newlines keep neighboring diagnostics from forming a retry signature. + // Unencoded payload bytes can be ordinary errors and remain unchanged. + classifierStderr += encoded ? "\n" : stderr.slice(start, cursor); + if (count > 1) { evidence = { phaseStart: "unknown", captureStatus: "duplicate" }; continue; } + let parsed: ReturnType | undefined; + if (Buffer.byteLength(stderr.slice(start, cursor)) + 1 <= CAPTURE_MAX_BYTES) { + if (encoded && payload.length % 2 === 0) { + const bytes = Buffer.alloc(payload.length / 2); + for (let i = 0; i < bytes.length; i++) { + bytes[i] = CAPTURE_ALPHABET.indexOf(payload[i * 2]!) * 16 + CAPTURE_ALPHABET.indexOf(payload[i * 2 + 1]!); + } + try { parsed = adjudicationSummarySchema.safeParse(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes))); } catch { /* Complete malformed frame. */ } + } + } + evidence = parsed?.success + ? { phaseStart: "unknown", captureStatus: "valid", authenticity: "perAttemptNonceMatched", summary: parsed.data } + : { phaseStart: "unknown", captureStatus: "invalid" }; + } + return { evidence, classifierStderr }; +} + export interface LiveCaseResult { + adjudicationEvidenceVersion?: 1; + adjudicationAttempts?: AdjudicationAttempt[]; id: string; name: string; type: "defect" | "clean"; @@ -550,6 +665,7 @@ export function aggregateAttemptAccounting( const durationComplete = attempts.every((attempt) => attempt.durationMs !== null); return { ...finalAttempt, + adjudicationAttempts: attempts.flatMap((attempt) => attempt.adjudicationAttempts ?? []), durationMs: durationComplete ? attempts.reduce((sum, attempt) => sum + attempt.durationMs!, 0) : null, @@ -596,7 +712,7 @@ const TRANSIENT_STDERR = new RegExp( /** True when an unscored case is worth one outer retry: the binary emitted a * transient signature, produced no valid envelope, or explicitly reported an * unrecovered review/invalidOutput incident. */ -function isTransientFailure(result: LiveCaseResult): boolean { +export function isTransientFailure(result: LiveCaseResult): boolean { if (result.scored) return false; if (result.stderr && TRANSIENT_STDERR.test(result.stderr)) return true; if (result.error === "operational envelope: review/invalidOutput") return true; @@ -697,6 +813,8 @@ async function runLiveCase( const diffPath = join(runDir, "pull.diff"); await writeFile(diffPath, c.diff, { mode: 0o600 }); + const nonce = randomBytes(16).toString("hex"); + let processFailure: "processTimeout" | "spawnFailure" | undefined; let stdout = ""; let stderr = ""; let exitCode: number | undefined; @@ -706,7 +824,7 @@ async function runLiveCase( liveReviewArguments(diffPath, options.bounded), { cwd: runDir, - env: liveEnv( + env: { ...liveEnv( options.model, options.scorerModel, options.screenProfilePath, @@ -715,7 +833,7 @@ async function runLiveCase( provider, timeoutOverrides, captureApiBase, - ), + ), POSTIL_BENCH_ADJUDICATION_NONCE: nonce }, timeout: timeoutOverrides.caseProcessMilliseconds, maxBuffer: 8 * 1024 * 1024, }, @@ -727,18 +845,28 @@ async function runLiveCase( // Exit 1 with a valid envelope is the gate failing on an error-severity // finding, not a transport failure. Keep the stdout and score it. The // stderr is captured so a genuine transport failure can be retried. - const e = err as { code?: unknown; stdout?: string; stderr?: string; message?: string }; + const e = err as { code?: unknown; stdout?: string; stderr?: string; message?: string; killed?: boolean }; + if (e.killed) processFailure = "processTimeout"; + else if (e.code === "ENOENT" || e.code === "EACCES" || e.code === "ENOEXEC") processFailure = "spawnFailure"; exitCode = typeof e.code === "number" ? e.code : undefined; stdout = e.stdout ?? ""; stderr = e.stderr ?? e.message ?? ""; } base.exitCode = exitCode; - base.stderr = stderr; + const capture = decodeAdjudicationCapture(stderr, nonce); + base.stderr = capture.classifierStderr; + base.adjudicationEvidenceVersion = 1; + const evidence: AdjudicationAttempt = { + ...capture.evidence, outerAttempt: attempt, attemptResult: "failed", failureClass: processFailure ?? "other", + ...(processFailure === "spawnFailure" ? { phaseStart: "notStarted" as const } : {}), + }; + base.adjudicationAttempts = [evidence]; await writeFile(join(runDir, "stdout.json"), stdout, { mode: 0o600 }); await writeFile(join(runDir, "stderr.log"), stderr, { mode: 0o600 }); const parsed = envelopeV1.safeParse(safeJson(stdout)); if (!parsed.success) { + evidence.failureClass = processFailure ?? "envelopeInvalid"; base.error = `no valid v1 envelope (exit ${exitCode ?? "unknown"})`; return base; } @@ -750,11 +878,13 @@ async function runLiveCase( base.costAccountingComplete = observedCost.complete; const operationalFailure = envelopeOperationalFailure(parsed.data, options.model); if (operationalFailure) { + evidence.failureClass = "operational"; base.error = operationalFailure; return base; } const scorerFailure = scorerOperationalFailure(parsed.data, options.scorerModel); if (scorerFailure) { + evidence.failureClass = "operational"; base.error = scorerFailure; return base; } @@ -766,6 +896,8 @@ async function runLiveCase( return base; } } + evidence.attemptResult = "scored"; + delete evidence.failureClass; return scoreCase(base, truth, parsed.data); } diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index 7be962f..a26bb53 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -1,4 +1,8 @@ import { describe, expect, test } from "bun:test"; +import { runLive, decodeAdjudicationCapture, type LiveReport } from "./live"; +import { cohortSlotPaths, createCohortManifest, readCohortReceipt, reportSemanticSha256, sha256 } from "./cohort"; +import { reserveCohortSlot, executeReservedCohortSlot } from "./cohort-run"; +import { assertAdjudicationCapture, type LiveReportForComparison } from "./compare-baseline"; import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; import { execFile as execFileCallback } from "node:child_process"; import { createHash } from "node:crypto"; @@ -2687,3 +2691,136 @@ test("diagnostic custody rejects dirty, stale and mismatched inputs before gener await rm(dirty, { recursive: true, force: true }); } }, 120_000); + + +test("real diff-file child retains authentic adjudication measurements through live reporting", async () => { + const root = await mkdtemp(join(tmpdir(), "postil-capture-child-")); + const c = fixture("clean-comment-only"); + let mode = "firstValid", calls = 0; + const requests: string[] = []; + const server = createServer(async (req, res) => { + const body = await requestBody(req); + requests.push(body); + const adjudication = adjudicationResponse(body); + const finding = falseFinding(c); + const findings = mode === "zeroCandidates" ? [] : mode === "promptFailure" + ? Array.from({ length: 20 }, (_, index) => ({ ...finding, title: `${finding.title} ${String.fromCharCode(97 + index)}`, + body: `${finding.body} ${String.fromCharCode(97 + index)} ${"界".repeat(1000)}.` })) : [finding]; + let content = JSON.stringify({ summary: mode === "zeroCandidates" ? "" : "Local capture fixture", findings }); + if (adjudication !== null) { + calls++; + content = adjudication; + if ((mode === "schemaRepair" && calls === 1) || mode === "terminalFailure") content = "not an array"; + if (mode === "semanticRetry" && calls === 1) content = JSON.stringify([{ ...JSON.parse(adjudication)[0], candidateId: "unknown-candidate" }]); + } + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ choices: [{ finish_reason: "stop", message: { content } }], + usage: { prompt_tokens: 30, completion_tokens: 10, cost: 0.000045 } })); + }); + const apiBase = await listen(server); + const previousKey = process.env.MODEL_API_KEY; + const previousBase = process.env.POSTIL_API_BASE; + const previousNonce = process.env.POSTIL_BENCH_ADJUDICATION_NONCE; + process.env.MODEL_API_KEY = crypto.randomUUID(); + process.env.POSTIL_API_BASE = apiBase; + const inheritedNonce = crypto.randomUUID().replaceAll("-", ""); + process.env.POSTIL_BENCH_ADJUDICATION_NONCE = inheritedNonce; + const binary = join(root, "local-child"); + await writeFile(binary, `#!/bin/sh\nexport POSTIL_ALLOW_PRIVATE_API_BASE=1\nexec '${postilBinaryPath()}' "$@"\n`, { mode: 0o700 }); + try { + const execution = { kind: "github-sigstore-v1", repository: "postil-dev/postil-cli", + signerWorkflow: ".github/workflows/release.yml", sourceSha: "b".repeat(40), + sourceRef: "refs/tags/v1.0.0", runId: "123", runAttempt: "1" } as const; + const screeningProfilePath = resolve(import.meta.dir, "../../provisional-models.json"); + const manifest = await createCohortManifest({ purpose: "release", binaryPath: binary, screeningProfilePath, + runPrefix: "local-capture", execution }); + const manifestPath = join(root, "manifest.json"); + await writeFile(manifestPath, JSON.stringify(manifest)); + const slotOptions = { manifestPath, slot: 1, binaryPath: binary, screeningProfilePath, + environment: { GITHUB_REPOSITORY: execution.repository, GITHUB_SHA: execution.sourceSha, + GITHUB_REF: execution.sourceRef, GITHUB_RUN_ID: execution.runId, GITHUB_RUN_ATTEMPT: execution.runAttempt } }; + for (mode of ["firstValid", "schemaRepair", "semanticRetry", "terminalFailure", "zeroCandidates", "promptFailure"]) { + calls = 0; requests.length = 0; + const runId = mode === "firstValid" ? manifest.slots[0]!.runId : mode; + let report: LiveReport; + if (mode === "firstValid") { + await reserveCohortSlot(slotOptions); + expect(await executeReservedCohortSlot({ ...slotOptions, executeBenchmark: async ({ reportPath, runId }) => { + report = await runLive([c], { binary, model: "test/model", rootDir: root, runId, concurrency: 1, retries: 0 }); + await writeFile(reportPath, JSON.stringify(report)); + return report.results[0]!.scored ? 0 : 1; + } })).toBe(0); + const paths = cohortSlotPaths(manifestPath, 1); + const { receipt } = await readCohortReceipt(paths.receiptPath); + const raw = await readFile(paths.reportPath); + expect(receipt.state).toBe("completed"); + expect(receipt.state === "completed" && receipt.reportRawSha256).toBe(sha256(raw)); + report = JSON.parse(raw.toString()); + const mutated = structuredClone(report); + delete mutated.results[0]!.adjudicationAttempts; + expect(() => assertAdjudicationCapture(mutated as unknown as LiveReportForComparison, 1)).toThrow("missing capture capability"); + expect(reportSemanticSha256(mutated)).not.toBe(reportSemanticSha256(report)); + expect(sha256(JSON.stringify(mutated))).not.toBe(receipt.state === "completed" && receipt.reportRawSha256); + const originalCalls = requests.length; + await expect(executeReservedCohortSlot(slotOptions)).rejects.toThrow("already completed"); + expect(requests).toHaveLength(originalCalls); + } else { + report = await runLive([c], { binary, model: "test/model", rootDir: root, + runId, concurrency: 1, retries: 0 }); + } + const result = report.results[0]!; + const attempt = result.adjudicationAttempts![0]!; + const caseDirectory = (await readdir(join(root, "live", runId), { withFileTypes: true })).find((entry) => entry.isDirectory())!.name; + const artifacts = join(root, "live", runId, caseDirectory, "attempt-1"); + const stderr = await readFile(join(artifacts, "stderr.log"), "utf8"); + if (mode === "promptFailure") { + expect(calls).toBe(0); + expect(result.scored).toBe(false); + expect(attempt).toMatchObject({ captureStatus: "notObserved", phaseStart: "unknown", attemptResult: "failed" }); + expect(attempt.summary).toBeUndefined(); + expect(stderr).toContain("finding adjudication input exceeded its admitted bound"); + continue; + } + const alphabet = "!#$%&()*+,-./:;<"; + const prefix = "\x1e" + [...Buffer.from("postil: adjudication-metrics ")].map((byte) => alphabet[byte >> 4]! + alphabet[byte & 15]!).join(""); + const encodedNonce = stderr.split(prefix)[1]?.slice(0, 32); + const nonce = encodedNonce && [...encodedNonce].map((char) => alphabet.indexOf(char).toString(16)).join(""); + expect(nonce).toBeDefined(); + expect(nonce).not.toBe(inheritedNonce); + expect(stderr.split(prefix)).toHaveLength(2); + expect(attempt.captureStatus).toBe("valid"); + const decoded = decodeAdjudicationCapture(stderr, nonce!); + expect(decoded.evidence.summary).toEqual(attempt.summary); + expect(decoded.classifierStderr).not.toContain(encodedNonce!); + expect(stderr).not.toContain(nonce!); + expect(JSON.stringify(report)).not.toContain(nonce!); + expect(requests.every((body) => !body.includes(nonce!) && !body.includes(inheritedNonce))).toBe(true); + expect(attempt.summary?.outcome).toBe(mode === "zeroCandidates" ? "notRequired" : mode === "terminalFailure" ? "failed" : "validated"); + expect(calls).toBe(mode === "zeroCandidates" ? 0 : mode === "firstValid" ? 1 : 2); + expect(attempt.summary?.callsStarted).toBe(calls); + expect(attempt.summary?.correctionKind).toBe(mode === "semanticRetry" ? "semanticRetry" : + mode === "schemaRepair" || mode === "terminalFailure" ? "schemaRepair" : null); + expect(result.scored, `${mode}: ${result.error ?? "no operational failure"}`).toBe(mode !== "terminalFailure"); + if (mode !== "terminalFailure") expect(() => assertAdjudicationCapture(report as unknown as LiveReportForComparison, 1)).not.toThrow(); + if (mode === "schemaRepair" || mode === "semanticRetry") expect(attempt.summary?.incidents).toEqual([ + { phase: "scorer", category: "invalidOutput", recovered: true, recovery: "repair" }, + ]); + if (mode !== "zeroCandidates") expect(attempt.summary?.promptTokens).toBe(30 * calls); + } + // The same binary's ordinary invocation carries no added frame. + const noCapture = join(root, "ordinary-child"); + await writeFile(noCapture, `#!/bin/sh\nunset POSTIL_BENCH_ADJUDICATION_NONCE\nexport POSTIL_ALLOW_PRIVATE_API_BASE=1\nexec '${postilBinaryPath()}' "$@"\n`, { mode: 0o700 }); + mode = "zeroCandidates"; + const report = await runLive([c], { binary: noCapture, model: "test/model", rootDir: root, + runId: "capture-disabled", concurrency: 1, retries: 0 }); + expect(report.results[0]!.scored).toBe(true); + expect(report.results[0]!.adjudicationAttempts![0]!.captureStatus).toBe("notObserved"); + expect(() => assertAdjudicationCapture(report as unknown as LiveReportForComparison, 1)).toThrow("missing complete successful final capture"); + } finally { + if (previousKey === undefined) delete process.env.MODEL_API_KEY; else process.env.MODEL_API_KEY = previousKey; + if (previousBase === undefined) delete process.env.POSTIL_API_BASE; else process.env.POSTIL_API_BASE = previousBase; + if (previousNonce === undefined) delete process.env.POSTIL_BENCH_ADJUDICATION_NONCE; else process.env.POSTIL_BENCH_ADJUDICATION_NONCE = previousNonce; + await close(server); + await rm(root, { recursive: true, force: true }); + } +}, 45_000); diff --git a/bench/src/verify-calibration.test.ts b/bench/src/verify-calibration.test.ts index c021f93..caaf7d6 100644 --- a/bench/src/verify-calibration.test.ts +++ b/bench/src/verify-calibration.test.ts @@ -215,16 +215,60 @@ fi expect(refused.calls.some(args => args.includes("POST"))).toBe(false); } await writeFile(baselinePath, baselineBytes); - await writeFile(resolve(directory, "bench/baseline-us.attestation.json"), ""); + const retainedBundlePath = resolve(directory, "bench/baseline-us.attestation.json"); + const retainedBundleBytes = await readFile(retainedBundlePath); + await writeFile(retainedBundlePath, ""); const missingBundle = await run(); expect(missingBundle.output).toContain("requires the retained baseline attestation"); expect(missingBundle.calls.some(args => args.includes("POST"))).toBe(false); + await writeFile(baselinePath, baselineBytes); + await writeFile(retainedBundlePath, retainedBundleBytes); + const restored = await run(); + expect(restored.code).toBe(0); + expect(restored.calls.find(args => args[0] === "attestation")).toEqual(verification); + expect(await readFile(baselinePath)).toEqual(baselineBytes); + expect(await readFile(retainedBundlePath)).toEqual(retainedBundleBytes); + // The exact retained current pair also supports a later changed evaluator. + const currentPair = await readFile(resolve(root, "bench/baseline-us.json")); + const currentBundle = await readFile(resolve(root, "bench/baseline-us.attestation.json")); + const currentSource = JSON.parse(currentPair.toString()).profiles["openai/gpt-5.6-luna"].calibration.sourceSha; + await writeFile(baselinePath, currentPair); + await writeFile(retainedBundlePath, currentBundle); + const restoredCurrent = await run({ REGISTRY_SHA: currentSource }); + expect(restoredCurrent.code).toBe(0); + const currentVerification = restoredCurrent.calls.find(args => args[0] === "attestation")!; + expect(currentVerification[currentVerification.indexOf("--source-digest") + 1]).toBe(currentSource); + expect(await readFile(baselinePath)).toEqual(currentPair); + expect(await readFile(retainedBundlePath)).toEqual(currentBundle); await writeFile(baselinePath, JSON.stringify({ profiles: { "openai/gpt-5.6-luna": { populated: false } } })); const bootstrap = await run(); expect(bootstrap.code).toBe(0); expect(bootstrap.calls.some(args => args[0] === "attestation")).toBe(false); }); +test("source rollback restores the exact historical pair and refuses partial or cross-paired restoration", async () => { + const context = await transport(); + const sourcePath = resolve(context.root, "bench/src/live.ts"); + const baselinePath = resolve(context.root, "bench/baseline-us.json"); + const bundlePath = resolve(context.root, "bench/baseline-us.attestation.json"); + const [source, baseline, bundle] = await Promise.all([readFile(sourcePath), readFile(baselinePath), readFile(bundlePath)]); + const original = await verifyLegacyCalibration(context); + await cp(resolve(root, "bench/src/live.ts"), sourcePath); + await expect(verifyLegacyCalibration(context)).rejects.toThrow("evaluator is stale"); + // An incomplete source/pair rollback and a substituted pair remain refused. + await writeFile(baselinePath, JSON.stringify({ ...JSON.parse(baseline.toString()), schemaVersion: 3 })); + await writeFile(sourcePath, source); + await expect(verifyLegacyCalibration(context)).rejects.toThrow("Unsupported legacy calibration artifact pair"); + await writeFile(baselinePath, baseline); + await writeFile(bundlePath, JSON.stringify({ dsseEnvelope: { payload: "cross-paired" } })); + await expect(verifyLegacyCalibration(context)).rejects.toThrow("Unsupported legacy calibration artifact pair"); + await writeFile(bundlePath, bundle); + expect(await verifyLegacyCalibration(context)).toEqual(original); + expect(await readFile(baselinePath)).toEqual(baseline); + expect(await readFile(bundlePath)).toEqual(bundle); + expect(context.calls.at(-1)).toContain("--source-digest"); +}); + test("actual evaluator, effective-launch and workflow mutations invalidate legacy freshness", async () => { const directory = await fixture(); const context = await transport(); diff --git a/src/llm.rs b/src/llm.rs index 90ed530..810302d 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -67,6 +67,93 @@ pub struct ScorerReview { pub usage_accounting_complete: bool, } +const ADJUDICATION_METRICS_PREFIX: &str = "postil: adjudication-metrics "; +const ADJUDICATION_METRICS_MAX_BYTES: usize = 2048; +const ADJUDICATION_METRICS_ALPHABET: &[u8; 16] = b"!#$%&()*+,-./:;<"; + +fn encode_adjudication_metrics(bytes: &[u8]) -> String { + bytes + .iter() + .flat_map(|byte| [byte >> 4, byte & 15]) + .map(|nibble| char::from(ADJUDICATION_METRICS_ALPHABET[usize::from(nibble)])) + .collect() +} + +fn encode_adjudication_nonce(nonce: &str) -> String { + nonce + .bytes() + .map(|byte| { + let nibble = match byte { + b'0'..=b'9' => byte - b'0', + b'a'..=b'f' => byte - b'a' + 10, + _ => return '?', + }; + char::from(ADJUDICATION_METRICS_ALPHABET[usize::from(nibble)]) + }) + .collect() +} + +fn adjudication_timing( + start: Instant, + deadline: Instant, + first: Instant, + end: Instant, +) -> [u128; 4] { + [ + end.saturating_duration_since(start).as_millis(), + deadline.saturating_duration_since(start).as_millis(), + first.saturating_duration_since(start).as_millis(), + deadline.saturating_duration_since(first).as_millis(), + ] +} + +fn adjudication_metrics_nonce(value: Option) -> Option { + value.filter(|nonce| { + nonce.len() == 32 + && nonce + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }) +} + +fn write_adjudication_metrics( + writer: &mut impl std::io::Write, + nonce: &str, + record: &serde_json::Value, +) { + let Ok(payload) = serde_json::to_string(record) else { + return; + }; + // Punctuation encoding keeps even short-written telemetry out of retry + // signatures. The closing challenge authenticates complete records. + let nonce = encode_adjudication_nonce(nonce); + let payload = encode_adjudication_metrics(payload.as_bytes()); + let header = encode_adjudication_metrics(ADJUDICATION_METRICS_PREFIX.as_bytes()); + let end = encode_adjudication_metrics(b"end"); + let frame = format!("\u{1e}{header}{nonce} {payload} {nonce} {end}\n"); + if frame.len() <= ADJUDICATION_METRICS_MAX_BYTES { + let _ = writer.write(frame.as_bytes()); + } +} + +fn emit_adjudication_metrics(record: serde_json::Value) { + if let Some(nonce) = + adjudication_metrics_nonce(std::env::var("POSTIL_BENCH_ADJUDICATION_NONCE").ok()) + { + write_adjudication_metrics(&mut std::io::stderr().lock(), &nonce, &record); + } +} + +pub(crate) fn adjudication_not_required() { + emit_adjudication_metrics(json!({ + "version": 1, "phase": "adjudication", "outcome": "notRequired", + "callsStarted": 0, "correctionKind": null, "phaseElapsedMs": null, + "initialBudgetMs": null, "firstCallElapsedMs": null, "firstCallRemainingMs": null, + "promptTokens": null, "completionTokens": null, "costProviderDecimal": null, + "usageAccountingComplete": null, "incidents": [], + })); +} + #[derive(Debug, Clone)] pub(crate) struct AdjudicationReview { pub(crate) application: crate::adjudication::AdjudicationApplication, @@ -3300,60 +3387,71 @@ impl LlmClient { F: Fn(Vec) -> Result, { let mut client = self.clone(); - let deadline = Instant::now() + timeout; + let phase_started = Instant::now(); + let deadline = phase_started + timeout; client.scorer_deadline = Some( self.total_deadline .map_or(deadline, |total| deadline.min(total)), ); - let mut usage = Usage::default(); - let mut model_usage = Vec::new(); - let mut model_incidents: Vec = Vec::new(); - let mut usage_accounting_complete = true; - let mut call_phase = LlmCallPhase::Initial; - let mut request_user = user.to_owned(); - let mut last_validation_error = None; - let failure = |cause: anyhow::Error, - usage, - events: Vec, - incidents, - complete, - validation_error: Option<&anyhow::Error>| { - let cause = match validation_error { - Some(validation) => { - match validation.downcast_ref::() { - Some(violation) => cause.context(*violation), - None => cause.context(validation.to_string()), + let mut first_settled = None; + let mut calls_started = 0; + let mut correction_kind = None; + let result = async { + let mut usage = Usage::default(); + let mut model_usage = Vec::new(); + let mut model_incidents: Vec = Vec::new(); + let mut usage_accounting_complete = true; + let mut call_phase = LlmCallPhase::Initial; + let mut request_user = user.to_owned(); + let mut last_validation_error = None; + let failure = |cause: anyhow::Error, + usage, + events: Vec, + incidents, + complete, + validation_error: Option<&anyhow::Error>| { + let cause = match validation_error { + Some(validation) => { + match validation + .downcast_ref::() + { + Some(violation) => cause.context(*violation), + None => cause.context(validation.to_string()), + } } - } - None => cause, + None => cause, + }; + let mut error = ModelError::new(cause, usage, complete); + error.model_usage = events; + error.model_incidents = incidents; + error }; - let mut error = ModelError::new(cause, usage, complete); - error.model_usage = events; - error.model_incidents = incidents; - error - }; - // Syntax and semantic correction share this one operation's budget. - for call in 0..2 { - Self::adjudication_trace_for_call( - adjudication_record("callStarted", "chatTerminal", "started"), - call_phase, - ); - let previous_events = model_usage.len(); - let content = client - .chat_with_temperature( - model, - system, - &request_user, - &mut usage, - &mut model_usage, - &mut usage_accounting_complete, - MAX_ADJUDICATION_OUTPUT_TOKENS, - 0.0, - LlmPhase::Adjudication, + // Syntax and semantic correction share this one operation's budget. + for call in 0..2 { + Self::adjudication_trace_for_call( + adjudication_record("callStarted", "chatTerminal", "started"), call_phase, - ) - .await - .map_err(|cause| { + ); + let previous_events = model_usage.len(); + calls_started += 1; + let content = client + .chat_with_temperature( + model, + system, + &request_user, + &mut usage, + &mut model_usage, + &mut usage_accounting_complete, + MAX_ADJUDICATION_OUTPUT_TOKENS, + 0.0, + LlmPhase::Adjudication, + call_phase, + ) + .await; + if call == 0 { + first_settled = Some(Instant::now()); + } + let content = content.map_err(|cause| { Self::adjudication_trace_for_call( adjudication_chat_record(&cause, "callFailed", "chatTerminal", None), call_phase, @@ -3376,109 +3474,147 @@ impl LlmClient { } error })?; - let (application, correction_phase) = - match parse_adjudication_results(&content, expected_len, Some(call_phase)) { - Ok(results) => { - let application = validate(results); - let recoverable = application.as_ref().err().is_some_and(|error| { - error - .downcast_ref::() - .is_some() - }); - ( + let (application, correction_phase) = + match parse_adjudication_results(&content, expected_len, Some(call_phase)) { + Ok(results) => { + let application = validate(results); + let recoverable = application.as_ref().err().is_some_and(|error| { + error + .downcast_ref::() + .is_some() + }); + ( + application, + recoverable.then_some(LlmCallPhase::SemanticRetry), + ) + } + Err(_) => ( + Err(anyhow!("finding adjudication syntax is invalid")), + Some(LlmCallPhase::SchemaRepair), + ), + }; + match application { + Ok(application) => { + client + .remaining_budget(LlmPhase::Adjudication) + .map_err(|cause| { + let mut error = failure( + cause, + usage, + model_usage.clone(), + model_incidents.clone(), + usage_accounting_complete, + last_validation_error.as_ref(), + ); + error + .model_incidents + .push(error.incident(ModelIncidentPhase::Scorer)); + error + })?; + for incident in &mut model_incidents { + incident.recovered = true; + incident.recovery = Some(ModelIncidentRecovery::Repair); + } + return Ok(AdjudicationReview { application, - recoverable.then_some(LlmCallPhase::SemanticRetry), - ) + usage, + model_usage, + model_incidents, + usage_accounting_complete, + }); } - Err(_) => ( - Err(anyhow!("finding adjudication syntax is invalid")), - Some(LlmCallPhase::SchemaRepair), - ), + Err(cause) => { + eprintln!("postil: finding adjudication validation failed: {cause:#}"); + last_validation_error = Some(cause); + model_incidents.push(ModelIncident { + phase: ModelIncidentPhase::Scorer, + category: ModelIncidentCategory::InvalidOutput, + recovered: false, + recovery: None, + }); + } + } + let Some(next_phase) = correction_phase.filter(|_| call == 0) else { + break; }; - match application { - Ok(application) => { - client - .remaining_budget(LlmPhase::Adjudication) - .map_err(|cause| { - let mut error = failure( - cause, - usage, - model_usage.clone(), - model_incidents.clone(), - usage_accounting_complete, - last_validation_error.as_ref(), - ); + correction_kind = Some(match next_phase { + LlmCallPhase::SchemaRepair => "schemaRepair", + LlmCallPhase::SemanticRetry => "semanticRetry", + LlmCallPhase::Initial => "initial", + }); + let corrected = client + .remaining_budget(LlmPhase::Adjudication) + .and_then(|remaining| { + ensure!( + remaining.is_none_or(|remaining| remaining >= RETRY_ATTEMPT_RESERVE), + "finding adjudication has no viable correction slot before its deadline" + ); + adjudication_correction_user(user) + }) + .map_err(|cause| { + let mut error = failure( + cause, + usage, + model_usage.clone(), + model_incidents.clone(), + usage_accounting_complete, + last_validation_error.as_ref(), + ); + if error.is_deadline_exceeded() { error .model_incidents .push(error.incident(ModelIncidentPhase::Scorer)); - error - })?; - for incident in &mut model_incidents { - incident.recovered = true; - incident.recovery = Some(ModelIncidentRecovery::Repair); - } - return Ok(AdjudicationReview { - application, - usage, - model_usage, - model_incidents, - usage_accounting_complete, - }); - } - Err(cause) => { - eprintln!("postil: finding adjudication validation failed: {cause:#}"); - last_validation_error = Some(cause); - model_incidents.push(ModelIncident { - phase: ModelIncidentPhase::Scorer, - category: ModelIncidentCategory::InvalidOutput, - recovered: false, - recovery: None, - }); - } - } - let Some(next_phase) = correction_phase.filter(|_| call == 0) else { - break; - }; - let corrected = client - .remaining_budget(LlmPhase::Adjudication) - .and_then(|remaining| { - ensure!( - remaining.is_none_or(|remaining| remaining >= RETRY_ATTEMPT_RESERVE), - "finding adjudication has no viable correction slot before its deadline" - ); - adjudication_correction_user(user) - }) - .map_err(|cause| { - let mut error = failure( - cause, - usage, - model_usage.clone(), - model_incidents.clone(), - usage_accounting_complete, - last_validation_error.as_ref(), - ); - if error.is_deadline_exceeded() { + } error - .model_incidents - .push(error.incident(ModelIncidentPhase::Scorer)); - } - error - })?; - request_user = corrected; - call_phase = next_phase; - // Actual residual exposure is reserved atomically by the same - // provider path as the initial call, immediately before contact. - } - Err(failure( - last_validation_error.unwrap_or_else(|| { - anyhow!("finding adjudication output did not satisfy its admitted contract") - }), - usage, - model_usage, - model_incidents, - usage_accounting_complete, - None, - )) + })?; + request_user = corrected; + call_phase = next_phase; + // Actual residual exposure is reserved atomically by the same + // provider path as the initial call, immediately before contact. + } + Err(failure( + last_validation_error.unwrap_or_else(|| { + anyhow!("finding adjudication output did not satisfy its admitted contract") + }), + usage, + model_usage, + model_incidents, + usage_accounting_complete, + None, + )) + } + .await; + let settled = Instant::now(); + let (usage, incidents, complete) = match &result { + Ok(review) => ( + review.usage, + &review.model_incidents, + review.usage_accounting_complete, + ), + Err(error) => ( + error.usage(), + &error.model_incidents, + error.usage_accounting_complete, + ), + }; + if let Some(first) = first_settled { + let effective_deadline = client.scorer_deadline.unwrap_or(deadline); + let [elapsed, budget, first_elapsed, remaining] = + adjudication_timing(phase_started, effective_deadline, first, settled); + emit_adjudication_metrics(json!({ + "version": 1, "phase": "adjudication", + "outcome": if result.is_ok() { "validated" } else { "failed" }, + "callsStarted": calls_started, "correctionKind": correction_kind, + "phaseElapsedMs": elapsed, + "initialBudgetMs": budget, + "firstCallElapsedMs": first_elapsed, + "firstCallRemainingMs": remaining, + "promptTokens": usage.prompt_tokens, "completionTokens": usage.completion_tokens, + "costProviderDecimal": usage.provider_cost.map(|cost| cost.to_string()), + "usageAccountingComplete": complete, "incidents": incidents, + })); + } + result } pub async fn resolve_uncertainty( @@ -10097,6 +10233,102 @@ mod tests { client } + #[test] + fn adjudication_metrics_timing_preserves_effective_deadline_and_rounding() { + let start = Instant::now(); + for deadline_ns in [0, 1, 999_999, 1_000_000, 1_000_001, 59_999_999_999] { + for first_ns in [0, 1, 999_999, 1_000_000, 1_999_999, 60_000_000_000] { + let first = start + Duration::from_nanos(first_ns); + let deadline = start + Duration::from_nanos(deadline_ns); + let [elapsed, budget, first_elapsed, remaining] = adjudication_timing( + start, + deadline, + first, + first + Duration::from_nanos(999_999), + ); + assert!(elapsed >= first_elapsed); + assert!(remaining <= budget); + assert!(budget.saturating_sub(first_elapsed).abs_diff(remaining) <= 1); + assert_eq!(budget, u128::from(deadline_ns / 1_000_000)); + } + } + let [_, budget, _, remaining] = + adjudication_timing(start, start - Duration::from_nanos(1), start, start); + assert_eq!((budget, remaining), (0, 0)); + } + + #[test] + fn adjudication_metrics_writer_is_bounded_and_nonpanicking() { + struct ShortWriter { + limit: usize, + bytes: Vec, + broken: bool, + } + impl std::io::Write for ShortWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if self.broken { + return Err(std::io::ErrorKind::BrokenPipe.into()); + } + let count = self.limit.min(bytes.len()); + self.bytes.extend_from_slice(&bytes[..count]); + Ok(count) + } + fn flush(&mut self) -> std::io::Result<()> { + panic!("capture must not flush") + } + } + let nonce = format!("{:032x}", 17); + assert!(adjudication_metrics_nonce(None).is_none()); + for invalid in [ + String::new(), + "x".repeat(32), + "A".repeat(32), + "0".repeat(31), + "0".repeat(33), + ] { + assert!(adjudication_metrics_nonce(Some(invalid)).is_none()); + } + assert_eq!( + adjudication_metrics_nonce(Some(nonce.clone())), + Some(nonce.clone()) + ); + let record = json!({"version": 1, "phase": "adjudication", "outcome": "failed", + "callsStarted": 2, "correctionKind": "semanticRetry", "phaseElapsedMs": 503, + "initialBudgetMs": 60000, "firstCallElapsedMs": 500, "firstCallRemainingMs": 59500, + "promptTokens": 9007199254740991u64, "completionTokens": 9007199254740991u64, + "costProviderDecimal": "340282366920938463463.374607431768211455", + "usageAccountingComplete": false, "incidents": [ + {"phase": "scorer", "category": "invalidOutput", "recovered": false}, + {"phase": "scorer", "category": "deadline", "recovered": false}]}); + let mut full = Vec::new(); + write_adjudication_metrics(&mut full, &nonce, &record); + assert!(full.len() < ADJUDICATION_METRICS_MAX_BYTES); + assert!(String::from_utf8_lossy(&full).ends_with(&format!( + " {} {}\n", + encode_adjudication_nonce(&nonce), + encode_adjudication_metrics(b"end") + ))); + for limit in 0..=full.len() { + let mut writer = ShortWriter { + limit, + bytes: Vec::new(), + broken: false, + }; + write_adjudication_metrics(&mut writer, &nonce, &record); + assert_eq!(writer.bytes, full[..limit]); + } + let mut broken = ShortWriter { + limit: usize::MAX, + bytes: Vec::new(), + broken: true, + }; + write_adjudication_metrics(&mut broken, &nonce, &record); + assert!(broken.bytes.is_empty()); + let mut oversized = Vec::new(); + write_adjudication_metrics(&mut oversized, &nonce, &json!({"extra": "x".repeat(2048)})); + assert!(oversized.is_empty()); + } + #[tokio::test] async fn adjudication_recovery_generated_outcomes_share_one_correction() { // Exhaustive generated outcome pairs exercise the real parser and the @@ -10357,11 +10589,14 @@ mod tests { #[tokio::test] async fn adjudication_recovery_preparation_refusals_retain_both_causes() { - for prompt_bound in [false, true] { + for (prompt_bound, outcome) in [false, true] + .into_iter() + .flat_map(|bound| [(bound, 1), (bound, 2)]) + { let mut fixture = AdjudicationFixture::new(false, 1); // Reach the prompt bound within an already supported context window. fixture.cfg.model = "openai/gpt-5-mini".into(); - let server = adjudication_server(vec![fixture.output(2)]).await; + let server = adjudication_server(vec![fixture.output(outcome)]).await; let client = adjudication_client(&fixture, &server); let user = if prompt_bound { "x".repeat(crate::adjudication::MAX_ADJUDICATION_PROMPT_BYTES) @@ -10388,7 +10623,11 @@ mod tests { error .error .downcast_ref::(), - Some(&crate::adjudication::AdjudicationViolation::ContradictoryRefutation) + if outcome == 1 { + None + } else { + Some(&crate::adjudication::AdjudicationViolation::ContradictoryRefutation) + } ); assert!(format!("{error:#}").contains(if prompt_bound { "finding adjudication correction exceeds its admitted input bound" diff --git a/src/review.rs b/src/review.rs index febf2db..bef2416 100644 --- a/src/review.rs +++ b/src/review.rs @@ -2888,6 +2888,8 @@ async fn review_diff_at( baseline_candidate_indices, fallback_application, )); + } else { + crate::llm::adjudication_not_required(); } if !cfg.scorer_enabled() && !adjudication_incomplete { // Local scoring is opt-in. An adjudication-only exclusion