From 494a2af1455383d8938f314b11de4b18b92a6d69 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Sun, 18 Oct 2026 18:23:03 +0000 Subject: [PATCH 1/4] Retain finite adjudication failure provenance --- bench/src/scorer-eval.test.ts | 283 +++++++++++++++++++++++- bench/src/scorer-eval.ts | 181 +++++++++++++++- src/llm.rs | 393 ++++++++++++++++++++++++++++++++-- tests/e2e.rs | 11 + 4 files changed, 844 insertions(+), 24 deletions(-) diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index 7b59dbf..5f5dc56 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -1,5 +1,8 @@ import { describe, expect, test } from "bun:test"; -import { mkdtemp, readFile, readdir, rm } from "node:fs/promises"; +import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { execFile as execFileCallback } from "node:child_process"; +import { createHash } from "node:crypto"; +import { promisify } from "node:util"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import type { AddressInfo } from "node:net"; import { tmpdir } from "node:os"; @@ -12,6 +15,10 @@ import { parseUnifiedDiffFiles, } from "./harness"; import { + ADJUDICATION_OUTPUT_PREFIX, + adjudicationOutput, + assertScorerDiagnosticCustody, + runScorerDiagnostic, canonicalProviderCost, FALSE_FINDING_CASES, GENERATOR_MODEL, @@ -2324,3 +2331,277 @@ describe("formatReport", () => { )).toBe("0.000000123"); }); }); + + +function provenanceRecord(overrides: Record = {}) { + return { + attempt: null, call: 1, code: "noArray", event: "parserRejected", expected: 1, + field: null, finish: null, observed: null, phase: "adjudication", stage: "arrayExtraction", version: 1, + ...overrides, + }; +} +const provenanceLine = (value: unknown) => `${ADJUDICATION_OUTPUT_PREFIX}${JSON.stringify(value)}`; + +describe("finite adjudication provenance", () => { + test("accepts only canonical closed bounded records and retains ordered duplicates", () => { + const line = provenanceLine(provenanceRecord()); + expect(adjudicationOutput("")).toBeUndefined(); + expect(adjudicationOutput(`${line}\n${line}`)).toMatchObject({ + records: [provenanceRecord(), provenanceRecord()], malformed: 0, duplicates: 1, omitted: 0, + }); + const hostile = String.fromCharCode(115, 107, 45) + crypto.randomUUID(); + const mutations = [ + { version: 2 }, { phase: hostile }, { event: hostile }, { stage: hostile }, { code: hostile }, + { call: 0 }, { call: 65 }, { attempt: 1 }, { expected: 21 }, { expected: -1 }, + { field: hostile }, { finish: hostile }, { observed: 1 }, { [hostile]: true }, + { code: { secret: hostile } }, { expected: [] }, + ]; + for (const mutation of mutations) { + const parsed = adjudicationOutput(provenanceLine(provenanceRecord(mutation))); + expect(parsed?.records.length).toBe(0); + expect(parsed?.malformed).toBe(1); + expect(JSON.stringify(parsed).includes(hostile)).toBe(false); + } + for (const bad of [ + line.replace('"version":1', '"version":2,"version":1'), + line + " ", + ADJUDICATION_OUTPUT_PREFIX + "x".repeat(513), + ADJUDICATION_OUTPUT_PREFIX + "é".repeat(256), + ]) expect(adjudicationOutput(bad)?.malformed).toBe(1); + const bounded = adjudicationOutput(Array.from({ length: 40 }, () => line).join("\n")); + expect(bounded).toMatchObject({ malformed: 0, duplicates: 31, omitted: 8 }); + expect(bounded?.records.length).toBe(32); + expect(adjudicationOutput(`${line}\n`.repeat(66_000))?.omitted).toBe(65535); + const invalidEnvelope = scorerCaseDiagnostics({ + child: { exitCode: 1, stderr: line, timedOut: false }, attempts: [], passed: false, + }); + expect(invalidEnvelope.failureSignals).toContain("invalidEnvelope"); + expect(invalidEnvelope.adjudicationOutput?.records).toEqual([provenanceRecord()]); + const events = [ + provenanceRecord({ code: "started", event: "callStarted", stage: "chatTerminal", expected: null }), + provenanceRecord({ code: "emptyContent", event: "attemptRejected", stage: "endpointContent", attempt: 1, expected: null }), + provenanceRecord({ code: "accepted", event: "parsed", stage: "resultCount", observed: 1 }), + ]; + expect(adjudicationOutput(events.map(provenanceLine).join("\n"))?.records).toEqual(events); + }); + + test("real child preserves baseline behavior with enabled counted scorer and mock publication", async () => { + const baseBinary = process.env.POSTIL_PROVENANCE_BASE_BIN; + const binaries = baseBinary ? [baseBinary, postilBinaryPath()] : [postilBinaryPath()]; + const hostile = String.fromCharCode(115, 107, 45) + crypto.randomUUID(); + const modes = ["accepted", "fenced", "noArray", "unclosedArray", "jsonSyntax", "resultCountMismatch", + "missingKnownField", "unknownField", "invalidEnum", "endpointMalformed", "emptyRecovered", + "missingChoicesRecovered", "emptyTerminal", "length", "tool_calls", "absentFinish"]; + const keyName = "POSTIL_PROVENANCE_LOCAL_KEY"; + const previous = process.env[keyName]; + process.env[keyName] = crypto.randomUUID(); + const pricing = { promptUsdPerToken: 0.000001, completionUsdPerToken: 0.000002, + inputMicrosPerMillionTokens: 1_000_000, outputMicrosPerMillionTokens: 2_000_000 }; + const root = await mkdtemp(join(tmpdir(), "postil-provenance-e2e-")); + let mode = "accepted"; + let adjudicationCalls = 0; + let scorerCalls = 0; + let requests: string[] = []; + const upstream = createServer(async (req, res) => { + const body = await requestBody(req); + requests.push(body); + const adjudication = adjudicationResponse(body); + let content = adjudication ?? JSON.stringify({ scores: [{ confidence: 0.2, kind: "uncertainty", + reason: "The claimed runtime break is unsupported by the change." }] }); + let finish: string | undefined = "stop"; + if (adjudication !== null) { + adjudicationCalls++; + if (mode === "endpointMalformed") { res.writeHead(200); res.end("{?"); return; } + if (mode === "missingChoicesRecovered" && adjudicationCalls === 1) { + res.writeHead(200); res.end(JSON.stringify({ usage: { prompt_tokens: 30, completion_tokens: 10, cost: 0.000045 } })); return; + } + if (mode === "noArray") content = `refusal ${hostile} https://example.invalid/${hostile}`; + if (mode === "unclosedArray") content = "["; + if (mode === "jsonSyntax") content = "[?]"; + if (mode === "resultCountMismatch") content = "[]"; + if (mode === "missingKnownField") content = "[{}]"; + if (mode === "unknownField") content = JSON.stringify([{ ...JSON.parse(adjudication)[0], [hostile]: hostile }]); + if (mode === "invalidEnum") content = JSON.stringify([{ ...JSON.parse(adjudication)[0], status: hostile }]); + if (mode === "emptyTerminal" || (mode === "emptyRecovered" && adjudicationCalls === 1)) content = " "; + if (mode === "length" || mode === "tool_calls") finish = mode; + if (mode === "absentFinish") finish = undefined; + if (mode === "fenced") content = `prose \`\`\`json\n${adjudication}\n\`\`\``; + } else scorerCalls++; + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ model: TEST_SCORER_MODEL, provider: "Azure", + choices: [{ finish_reason: finish, message: { content } }], + usage: { prompt_tokens: 30, completion_tokens: 10, cost: 0.000045 } })); + }); + const apiBase = await listen(upstream); + try { + for (mode of modes) { + const observations = []; + for (const [index, binary] of binaries.entries()) { + adjudicationCalls = 0; scorerCalls = 0; requests = []; + const runRoot = join(root, mode, String(index)); + const c = fixture("clean-comment-only"); + const evaluation = await runScorerEvalCase(c, "falseFinding", TEST_SCORER_MODEL, 1, + binary, runRoot, apiBase, keyName, pricing); + const artifacts = join(runRoot, safeSegment(TEST_SCORER_MODEL), "repeat-1", c.id, "artifacts"); + const envelope = envelopeV1.parse(JSON.parse(await readFile(join(artifacts, "stdout.json"), "utf8"))); + const success = ["accepted", "fenced", "emptyRecovered", "missingChoicesRecovered"].includes(mode); + expect(evaluation.envelopeProduced).toBe(true); + expect(envelope.findings.some((finding) => finding.path === ".postil/model-output")).toBe(!success); + const publication = JSON.parse(await readFile(join(artifacts, "publication-telemetry.json"), "utf8")); + expect(publication.checkRunCreates).toBe(2); + expect(publication.checkRunCompletions).toBe(2); + expect(publication.finalFindings).toBe(success ? 0 : 2); + expect(scorerCalls).toBe(success ? 1 : 0); + expect(evaluation.gateFailing).toBe(!success); + expect(evaluation.findingPublished).toBe(!success); + expect(evaluation.finalConfidence).toBe(success ? 0.2 : 0.95); + expect(evaluation.diagnostics?.scorer.attempts).toBe(success ? 1 : 0); + const projection = evaluation.diagnostics?.adjudicationOutput; + if (baseBinary && index === 0) expect(projection).toBeUndefined(); + else { + expect(projection?.malformed).toBe(0); + expect(projection?.records[0]?.event).toBe("callStarted"); + expect(projection?.records.every((record) => record.call === 1)).toBe(true); + const expectedCode = success ? "accepted" : + mode === "emptyTerminal" ? "emptyContent" : + ["length", "tool_calls", "absentFinish"].includes(mode) ? "nonterminalStop" : mode; + expect(projection?.records.at(-1)?.code).toBe(expectedCode); + if (["length", "tool_calls", "absentFinish"].includes(mode)) { + expect(projection?.records.at(-1)?.finish).toBe(mode === "absentFinish" ? "absent" : mode); + expect(adjudicationCalls).toBe(1); + } + expect(projection?.records.at(-1)?.event).toBe(success ? "parsed" : + ["endpointMalformed", "emptyTerminal", "length", "tool_calls", "absentFinish"].includes(mode) ? "callFailed" : "parserRejected"); + expect(JSON.stringify(projection).includes(hostile)).toBe(false); + if (mode === "emptyRecovered" || mode === "missingChoicesRecovered") { + expect(projection?.records.map((record) => record.event)).toEqual(["callStarted", "attemptRejected", "parsed"]); + expect(projection?.records[1]?.attempt).toBe(1); + } + const jsonOut = join(root, "finite.json"); + await writeScorerEvalCheckpoint(jsonOut, [TEST_SCORER_MODEL], 1, 36, [evaluation]); + const partial = JSON.parse(await readFile(scorerCheckpointPath(jsonOut), "utf8")); + expect(partial.cases[0].diagnostics.adjudicationOutput).toEqual(projection); + await finalizeScorerEvalReport(jsonOut, JSON.stringify({ diagnostics: evaluation.diagnostics })); + expect((await readFile(jsonOut, "utf8")).includes(hostile)).toBe(false); + const clean = { ...evaluation, diagnostics: undefined }; + const replaced = { ...evaluation, diagnostics: { ...evaluation.diagnostics!, adjudicationOutput: undefined } }; + expect(scorerCasePasses(clean)).toBe(scorerCasePasses(evaluation)); + expect(isAdmissionFatalStructuralResult(replaced, TEST_SCORER_MODEL)).toBe(isAdmissionFatalStructuralResult(evaluation, TEST_SCORER_MODEL)); + expect(aggregate(TEST_SCORER_MODEL, [clean], 3)).toEqual(aggregate(TEST_SCORER_MODEL, [evaluation], 3)); + const decision = (entry: ScorerEvalCase) => qualificationExitCode({ + ...scorerReportContract(), generatedAt: "", qualificationSourceSha: "a".repeat(40), + cliBinarySha256: "b".repeat(64), apiBase, upstreamProvider: "Azure", upstreamProviderRoute: "azure/us", + repeats: 3, completedCases: 1, totalCases: 36, matrixComplete: false, passed: false, + models: [aggregate(TEST_SCORER_MODEL, [entry], 3)], cases: [entry], + }); + expect(decision(clean)).toBe(decision(evaluation)); + expect(decision(replaced)).toBe(1); + } + observations.push({ requests: requests.map((body) => createHash("sha256").update(body).digest("hex")), adjudicationCalls, scorerCalls, + semantic: { findings: envelope.findings, gate: envelope.gate, counts: envelope.counts, + modelUsage: envelope.modelUsage, modelIncidents: envelope.modelIncidents, + usage: envelope.usage, publication, publicationValid: evaluation.publicationValid } }); + } + if (baseBinary) expect(observations[1]).toEqual(observations[0]); + } + } finally { + if (previous === undefined) delete process.env[keyName]; else process.env[keyName] = previous; + await close(upstream); + await rm(root, { recursive: true, force: true }); + } + }, 240_000); +}); + + +test("diagnostic custody rejects dirty, stale and mismatched inputs before generation and uses the strict route locally", async () => { + const exec = promisify(execFileCallback); + const repositoryRoot = resolve(import.meta.dir, "..", ".."); + const sourceSha = (await exec("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot })).stdout.trim(); + const paths = (await exec("git", ["ls-files", "-z", "--", "bench/src", "bench/fixtures", + "bench/evaluator-contract-sources.json", "bench/review-contract-sources.json", "bench/package.json", + "bench/bun.lock", "config.toml", "provisional-models.json"], { cwd: repositoryRoot })).stdout.split("\0").filter(Boolean); + const evaluatorSha256 = scorerEvaluatorDigest(await Promise.all(paths.map(async (path) => ({ path, + contents: await readFile(join(repositoryRoot, path)) })))); + const binary = postilBinaryPath(); + const cliBinarySha256 = createHash("sha256").update(await readFile(binary)).digest("hex"); + const input = { binary, sourceSha, evaluatorSha256, cliBinarySha256, keyName: "POSTIL_API_KEY" }; + const keyPrevious = process.env.POSTIL_API_KEY; + process.env.POSTIL_API_KEY = crypto.randomUUID(); + const dirty = await mkdtemp(join(tmpdir(), "postil-diagnostic-dirty-")); + await exec("git", ["init", dirty]); + await writeFile(join(dirty, "source.rs"), "dirty source\n"); + const originalFetch = globalThis.fetch; + let calls = 0; + let pricingCalls = 0; + let adjudicationCalls = 0; + let scorerCalls = 0; + let identityFailures = 0; + let identityMode = "valid"; + const requests: string[] = []; + const upstream = createServer(async (req, res) => { + const body = await requestBody(req); + requests.push(body); + const adjudication = adjudicationResponse(body); + if (adjudication !== null) adjudicationCalls++; else scorerCalls++; + const omitIdentity = adjudication !== null && (identityMode === "missingTerminal" || + identityMode === "missingRecovered" && adjudicationCalls === 1); + if (omitIdentity) identityFailures++; + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ ...(omitIdentity ? {} : { model: GENERATOR_MODEL, provider: "Azure" }), + choices: [{ finish_reason: "stop", message: { content: adjudication ?? JSON.stringify({ + scores: [{ confidence: 0.2, kind: "uncertainty", reason: "The claimed runtime break is unsupported by the change." }], + }) } }], usage: { prompt_tokens: 30, completion_tokens: 10, cost: 0.000045 } })); + }); + const localBase = await listen(upstream); + globalThis.fetch = (async (url: Parameters[0], init?: RequestInit) => { + const target = String(url); + if (new URL(target).origin === "https://openrouter.ai" && new URL(target).pathname === "/api/v1/endpoints/zdr") { + calls++; pricingCalls++; + return Response.json({ data: [{ model_id: GENERATOR_MODEL, provider_name: "Azure", tag: "azure/us", status: 0, + supported_parameters: ["max_tokens", "max_completion_tokens", "reasoning", "reasoning_effort", "temperature", "response_format", "structured_outputs"], + pricing: { prompt: "0.000001", completion: "0.000002" } }] }); + } + if (new URL(target).origin === "https://openrouter.ai" && new URL(target).pathname === "/api/v1/chat/completions") { + calls++; + return originalFetch(`${localBase}/chat/completions`, init); + } + // Mock forge/capture traffic is loopback; no other destination is allowed. + if (new URL(target).hostname !== "127.0.0.1") throw new Error("unexpected diagnostic destination"); + return originalFetch(url, init); + }) as typeof fetch; + try { + await expect(assertScorerDiagnosticCustody({ ...input, repositoryRoot: dirty })).rejects.toThrow(); + await expect(runScorerDiagnostic({ ...input, sourceSha: "0".repeat(40) })).rejects.toThrow(); + await expect(runScorerDiagnostic({ ...input, evaluatorSha256: "0".repeat(64) })).rejects.toThrow(); + await expect(runScorerDiagnostic({ ...input, cliBinarySha256: "0".repeat(64) })).rejects.toThrow(); + expect(calls).toBe(0); + for (identityMode of ["valid", "missingRecovered", "missingTerminal"]) { + adjudicationCalls = 0; scorerCalls = 0; identityFailures = 0; + const diagnostic = await runScorerDiagnostic(input); + expect(diagnostic).toMatchObject({ diagnosticOnly: true, matrixComplete: false, + fixture: "clean-comment-only", scenario: "falseFinding", repeat: 1, + upstreamTimeoutMs: 30_000, childTimeoutMs: 65_000, + custody: { qualificationSourceSha: sourceSha, cliBinarySha256, evaluatorSha256 }, + providerContract: { upstreamProviderIdentity: "Azure", upstreamProviderRoute: "azure/us", + dataCollection: "deny", zeroDataRetention: true, allowFallbacks: false }, + }); + expect(diagnostic.diagnostics?.adjudicationOutput?.malformed).toBe(0); + const records = diagnostic.diagnostics?.adjudicationOutput?.records ?? []; + expect(records.at(-1)?.event).toBe(identityMode === "missingTerminal" ? "callFailed" : "parsed"); + expect(scorerCalls).toBe(identityMode === "missingTerminal" ? 0 : 1); + expect(identityFailures).toBe(identityMode === "valid" ? 0 : identityMode === "missingRecovered" ? 1 : 3); + expect(diagnostic.findingPublished).toBe(identityMode === "missingTerminal"); + expect(diagnostic.gateFailing).toBe(identityMode === "missingTerminal"); + if (identityMode === "missingRecovered") expect(records[1]).toMatchObject({ + event: "attemptRejected", code: "identityMissing", attempt: 1, call: 1, + }); + } + expect(pricingCalls).toBe(3); + expect(requests.length).toBeGreaterThan(0); + } finally { + globalThis.fetch = originalFetch; + if (keyPrevious === undefined) delete process.env.POSTIL_API_KEY; else process.env.POSTIL_API_KEY = keyPrevious; + await close(upstream); + await rm(dirty, { recursive: true, force: true }); + } +}, 120_000); diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index ce36857..568bfc7 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -8,9 +8,10 @@ import { execFile as execFileCb } from "node:child_process"; import { createHash } from "node:crypto"; -import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { promisify } from "node:util"; import { cases as fixtureInputs } from "../fixtures/cases"; @@ -42,6 +43,9 @@ import { type QualificationPair, } from "./livemodels-score"; import { + assertEvaluatorRuntime, + resolveBinaryQualificationMetadata, + resolveQualificationSourceAuthority, fetchPricing as fetchQualificationPricing, normalizeApiBase, qualificationCandidateDocument, @@ -194,6 +198,7 @@ interface ScorerPhaseDiagnostics { } export interface ScorerCaseDiagnostics { + adjudicationOutput?: AdjudicationOutput; childExitCode: number | null; requiredUpstreamAttempts: 2; adjudication: ScorerPhaseDiagnostics; @@ -209,6 +214,101 @@ export interface ScorerCaseDiagnostics { export const SCORER_DIAGNOSTIC_RESPONSE_LIMIT = 32; const SCORER_DIAGNOSTIC_ERROR_CODE_LIMIT = 8; +// Canonical compact JSON is part of this internal wire contract. Re-encoding the +// validated primitives also rejects duplicate keys, alternate escapes and members. +export const ADJUDICATION_OUTPUT_PREFIX = "postil: adjudication-output "; +const OUTPUT_FIELDS = ["candidateId", "status", "revisedTitle", "revisedBody", "evidence", + "duplicateOf", "scope", "disposition", "cause", "reason", "path", "side", "line", + "byteOffset", "diffLine", "corpusSha256", "other"] as const; +const OUTPUT_CODES = { + callStarted: { chatTerminal: ["started"] }, + attemptRejected: { + endpointErrorContract: ["endpointMalformed", "nonterminalStop"], + endpointContent: ["endpointMalformed", "missingChoices", "emptyContent", "nonterminalStop", "identityMissing", "identityMismatch"], + }, + callFailed: { chatTerminal: ["endpointMalformed", "missingChoices", "emptyContent", "nonterminalStop", "identityMissing", "identityMismatch", "contextExceeded", "providerFailure", "timeout", "deadline", "otherChatFailure"] }, + parserRejected: { + arrayExtraction: ["noArray", "unclosedArray"], + resultDeserialization: ["jsonSyntax", "jsonEof", "unexpectedIo", "missingKnownField", "unknownField", "duplicateKnownField", "invalidEnum", "wrongType", "otherData"], + resultCount: ["resultCountMismatch"], + }, + parsed: { resultCount: ["accepted"] }, +} as const; +type OutputCount = number | "overflow" | null; +export interface AdjudicationOutputRecord { + attempt: number | null; // Local to this adjudication call, never a proxy ordinal. + call: 1; // One adjudication call per review child; not a proxy ordinal. + code: string; + event: keyof typeof OUTPUT_CODES; + expected: OutputCount; + field: typeof OUTPUT_FIELDS[number] | null; + finish: "stop" | "length" | "tool_calls" | "content_filter" | "error" | "reported" | "absent" | "end_turn" | "stop_sequence" | null; + observed: OutputCount; + phase: "adjudication"; + stage: string; + version: 1; +} +interface AdjudicationOutput { + records: AdjudicationOutputRecord[]; + malformed: number; + duplicates: number; + omitted: number; +} + +export function adjudicationOutput(stderr: string): AdjudicationOutput | undefined { + const records: AdjudicationOutputRecord[] = []; + let malformed = 0; + let duplicates = 0; + const accepted = new Set(); + let omitted = 0; + let seen = 0; + const boundedIncrement = (value: number) => Math.min(65535, value + 1); + const ordinal = (value: unknown, max: number) => value === null || + (typeof value === "number" && Number.isInteger(value) && value >= 1 && value <= max); + const count = (value: unknown): value is OutputCount => value === null || value === "overflow" || + (typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 20); + // Ordered records intentionally bypass the legacy Set used for coarse signals. + for (const line of stderr.split(/\r?\n/u)) { + if (!line.startsWith(ADJUDICATION_OUTPUT_PREFIX)) continue; + if (++seen > SCORER_DIAGNOSTIC_RESPONSE_LIMIT) { omitted = boundedIncrement(omitted); continue; } + const raw = line.slice(ADJUDICATION_OUTPUT_PREFIX.length); + if (Buffer.byteLength(line, "utf8") > 512) { malformed = boundedIncrement(malformed); continue; } + try { + const value = JSON.parse(raw); + if (!value || typeof value !== "object" || Array.isArray(value)) throw 0; + const { attempt, call, code, event, expected, field, finish, observed, phase, stage, version } = value; + if (version !== 1 || phase !== "adjudication" || !Object.hasOwn(OUTPUT_CODES, event) || + call !== 1 || !ordinal(attempt, 13) || !count(expected) || !count(observed) || + !(field === null || OUTPUT_FIELDS.includes(field)) || + !(finish === null || ["stop", "length", "tool_calls", "content_filter", "error", "reported", "absent", "end_turn", "stop_sequence"].includes(finish))) throw 0; + const stages = OUTPUT_CODES[event as keyof typeof OUTPUT_CODES] as Record; + if (!Object.hasOwn(stages, stage) || !stages[stage]!.includes(code)) throw 0; + if ((event === "attemptRejected") !== (attempt !== null)) throw 0; + if (event !== "parserRejected" && event !== "parsed" && (expected !== null || observed !== null)) throw 0; + if (stage !== "resultDeserialization" && field !== null) throw 0; + if (stage === "resultDeserialization") { + if (field === null) throw 0; + if (["missingKnownField", "duplicateKnownField"].includes(code)) { + if (field === "other") throw 0; + } else if (code === "invalidEnum" ? field !== "status" : field !== "other") throw 0; + } + if (event === "parserRejected" || event === "parsed") { + if (expected === null || (stage === "resultCount" ? observed === null : observed !== null)) throw 0; + } + if (code !== "nonterminalStop" && finish !== null) throw 0; + if (code === "nonterminalStop" && finish === null) throw 0; + const record: AdjudicationOutputRecord = { + attempt, call, code, event, expected, field, finish, observed, phase, stage, version, + }; + if (JSON.stringify(record) !== raw) throw 0; + if (accepted.has(raw)) duplicates = boundedIncrement(duplicates); + accepted.add(raw); + records.push(record); + } catch { malformed = boundedIncrement(malformed); } + } + return seen === 0 ? undefined : { records, malformed, duplicates, omitted }; +} + interface ScorerResponseMetadata { jsonShape: "object" | "other" | "invalid" | "unobserved"; errorLocations: Array<"topLevel" | "choice">; @@ -359,6 +459,7 @@ export function scorerCaseDiagnostics(input: { }; }; return { + adjudicationOutput: adjudicationOutput(input.child.stderr), childExitCode: input.child.exitCode ?? null, requiredUpstreamAttempts: 2, adjudication: phaseCounts("adjudication"), @@ -644,11 +745,10 @@ export function scorerEvaluatorDigest( return hash.digest("hex"); } -async function scorerEvalSourceAuthority(): Promise<{ +async function scorerEvalSourceAuthority(repositoryRoot = resolve(import.meta.dir, "..", "..")): Promise<{ qualificationSourceSha: string; evaluatorSha256: string; }> { - const repositoryRoot = resolve(import.meta.dir, "..", ".."); const gitOptions = { cwd: repositoryRoot, timeout: 15_000 }; const [{ stdout: sourceOutput }, { stdout: trackedOutput }] = await Promise.all([ execFile("git", ["rev-parse", "--verify", "HEAD^{commit}"], gitOptions), @@ -973,6 +1073,81 @@ function evalCase( return { case: c, scenario }; } +// A single diagnostic is deliberately separate from formal matrix admission. +// The caller supplies immutable expected identities; no raw child artifact is +// returned. This function does not select a different model, route or timeout. +export async function assertScorerDiagnosticCustody(input: { + repositoryRoot: string; binary: string; sourceSha: string; + evaluatorSha256: string; cliBinarySha256: string; +}) { + const gitOptions = { cwd: input.repositoryRoot, timeout: 15_000 }; + const { stdout: status } = await execFile("git", ["status", "--porcelain=v1", "--untracked-files=all"], gitOptions); + assertCleanScorerEvaluatorStatus(status); + const authority = await scorerEvalSourceAuthority(input.repositoryRoot); + const source = await resolveQualificationSourceAuthority(input.repositoryRoot); + const runtime = await assertEvaluatorRuntime(input.repositoryRoot); + const binaryHash = await sha256File(input.binary); + if (source.sourceSha !== input.sourceSha || authority.qualificationSourceSha !== input.sourceSha || + authority.evaluatorSha256 !== input.evaluatorSha256 || binaryHash !== input.cliBinarySha256) { + throw new Error("diagnostic custody identity mismatch"); + } + const metadata = await resolveBinaryQualificationMetadata(input.binary); + for (const [actual, expected] of [ + [metadata.modelDefaultsSha256, source.configHash], + [metadata.reviewContractSha256, source.reviewContractHash], + [metadata.fixtureSetSha256, source.fixtureHash], + [metadata.evaluatorContractSha256, source.evaluatorContractHash], + [metadata.evaluatorRuntimeIdentity, runtime], + [metadata.defaultApiBase, normalizeApiBase(DEFAULT_API_BASE)], + [metadata.defaultApiFormat, "openai-compatible"], + ]) { + if (actual !== expected) throw new Error("diagnostic binary contract mismatch"); + } + return { ...authority, ...source, cliBinarySha256: binaryHash, evaluatorRuntimeIdentity: runtime }; +} + +export async function runScorerDiagnostic(input: { + binary: string; sourceSha: string; evaluatorSha256: string; + cliBinarySha256: string; keyName: string; +}) { + const custodyInput = { ...input, repositoryRoot: resolve(import.meta.dir, "..", "..") }; + const custody = await assertScorerDiagnosticCustody(custodyInput); + const embedded = await loadEmbeddedScorerDefaults(); + if (!embedded.enabled || embedded.qualification_candidates.length !== 1) { + throw new Error("diagnostic requires the unchanged single embedded scorer candidate"); + } + const model = embedded.qualification_candidates[0]!; + const c = selectEvalCases(fixtureInputs.map((value) => benchmarkCase.parse(value))) + .find((value) => value.case.id === "clean-comment-only" && value.scenario === "falseFinding")?.case; + if (!c || !FALSE_FINDING_CASES.includes(c.id)) throw new Error("diagnostic public fixture is unavailable"); + if (!process.env[input.keyName]) throw new Error("diagnostic credential is unavailable"); + const provider = "Azure"; + const route = "azure/us"; + const pricing = await fetchQualificationPricing(DEFAULT_API_BASE, "openai-compatible", + scorerQualificationModels([model]), provider, scorerQualificationRequiredParameters([model], provider), route); + assertQualificationPreflight([model], 1, pricing); + const providerContract = providerContractEvidence(provider, route, pricing, [GENERATOR_MODEL], [model]); + // mkdtemp owns this root. runScorerEvalCase may remove only its derived child. + const root = await mkdtemp(join(tmpdir(), "postil-scorer-diagnostic-")); + try { + const result = await runScorerEvalCase(c, "falseFinding", model, 1, input.binary, root, + DEFAULT_API_BASE, input.keyName, pricing.get(model) ?? null, SCORER_CASE_EXEC_TIMEOUT_MS, + provider, pricing.get(GENERATOR_MODEL) ?? null, route); + await assertScorerDiagnosticCustody(custodyInput); + return { + version: 1 as const, diagnosticOnly: true as const, matrixComplete: false as const, + custody, providerContract, providerContractSha256: providerContractSha256(providerContract), + fixture: c.id, scenario: "falseFinding" as const, repeat: 1 as const, + upstreamTimeoutMs: SCORER_PROXY_UPSTREAM_TIMEOUT_MS, childTimeoutMs: SCORER_CASE_EXEC_TIMEOUT_MS, + envelopeProduced: result.envelopeProduced, findingPublished: result.findingPublished, + gateFailing: result.gateFailing, scorerConfidence: result.scorerConfidence, + publicationValid: result.publicationValid, diagnostics: result.diagnostics, + }; + } finally { + await rm(root, { recursive: true, force: true }); + } +} + export async function runScorerEvalCase( c: BenchmarkCase, scenario: Scenario, diff --git a/src/llm.rs b/src/llm.rs index 680b58e..a88f510 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -3260,6 +3260,10 @@ impl LlmClient { })) } + fn adjudication_trace(record: serde_json::Value) { + eprintln!("postil: adjudication-output {}", record); + } + pub(crate) async fn adjudicate_findings( &self, model: &str, @@ -3269,6 +3273,11 @@ impl LlmClient { timeout: Duration, ) -> std::result::Result { let mut client = self.clone(); + Self::adjudication_trace(adjudication_record( + "callStarted", + "chatTerminal", + "started", + )); let deadline = Instant::now() + timeout; client.scorer_deadline = Some( self.total_deadline @@ -3292,6 +3301,12 @@ impl LlmClient { ) .await .map_err(|error| { + Self::adjudication_trace(adjudication_chat_record( + &error, + "callFailed", + "chatTerminal", + None, + )); let mut model_error = ModelError::new(error, usage, usage_accounting_complete); model_error.model_usage = model_usage.clone(); model_error @@ -3299,21 +3314,22 @@ impl LlmClient { .push(model_error.incident(ModelIncidentPhase::Scorer)); model_error })?; - let results = parse_adjudication_results(&content, expected_len).map_err(|detail| { - let mut error = ModelError::new( - anyhow!("finding adjudication output invalid: {detail}"), - usage, - usage_accounting_complete, - ); - error.model_usage = model_usage.clone(); - error.model_incidents.push(ModelIncident { - phase: ModelIncidentPhase::Scorer, - category: ModelIncidentCategory::InvalidOutput, - recovered: false, - recovery: None, - }); - error - })?; + let results = + parse_adjudication_results(&content, expected_len, true).map_err(|detail| { + let mut error = ModelError::new( + anyhow!("finding adjudication output invalid: {detail}"), + usage, + usage_accounting_complete, + ); + error.model_usage = model_usage.clone(); + error.model_incidents.push(ModelIncident { + phase: ModelIncidentPhase::Scorer, + category: ModelIncidentCategory::InvalidOutput, + recovered: false, + recovery: None, + }); + error + })?; Ok(AdjudicationReview { results, usage, @@ -4514,6 +4530,14 @@ impl LlmClient { match openai_response_error_status(&response.text) { Ok(status) => status.unwrap_or(response.status), Err(error) => { + if matches!(phase, LlmPhase::Adjudication) { + Self::adjudication_trace(adjudication_chat_record( + &error, + "attemptRejected", + "endpointErrorContract", + Some(attempt), + )); + } if let Some(response_usage) = summary.usage { add_usage(usage, response_usage); } else { @@ -4568,6 +4592,14 @@ impl LlmClient { expected_provider, ) { + if matches!(phase, LlmPhase::Adjudication) { + Self::adjudication_trace(adjudication_chat_record( + &error, + "attemptRejected", + "endpointContent", + Some(attempt), + )); + } if let Some(response_usage) = summary.usage { add_usage(usage, response_usage); } else { @@ -4625,6 +4657,14 @@ impl LlmClient { }); } Err(error) => { + if matches!(phase, LlmPhase::Adjudication) { + Self::adjudication_trace(adjudication_chat_record( + &error, + "attemptRejected", + "endpointContent", + Some(attempt), + )); + } let parse_added_usage = usage.prompt_tokens != usage_before_parse.prompt_tokens || usage.completion_tokens @@ -6295,13 +6335,202 @@ fn parse_scores(content: &str, expected_len: usize) -> Result, .collect::, String>>() } +// This wire object has fixed keys and finite values only. Null means unavailable; +// counts saturate independently of the authoritative parser and retries. The +// private review entry point calls adjudicate_findings once per child process; +// call=1 is scoped to that child, not a shared or proxy request counter. +fn adjudication_record( + event: &'static str, + stage: &'static str, + code: &'static str, +) -> serde_json::Value { + json!({"version":1,"phase":"adjudication","event":event,"stage":stage,"code":code, + "call":1,"attempt":null,"expected":null,"observed":null,"field":null,"finish":null}) +} + +fn adjudication_chat_record( + error: &anyhow::Error, + event: &'static str, + stage: &'static str, + attempt: Option, +) -> serde_json::Value { + let code = if error.downcast_ref::().is_some() { + "deadline" + } else if error.downcast_ref::().is_some() { + "contextExceeded" + } else if let Some(identity) = error.downcast_ref::() { + match identity { + AtomicAttributionIdentityFailure::Missing => "identityMissing", + AtomicAttributionIdentityFailure::Mismatch => "identityMismatch", + } + } else if let Some(content) = error.downcast_ref::() { + match content { + ModelContentFailure::Empty => "emptyContent", + ModelContentFailure::Malformed => "endpointMalformed", + ModelContentFailure::MissingChoices => "missingChoices", + ModelContentFailure::NonTerminal { .. } => "nonterminalStop", + } + } else if error.chain().any(|cause| { + cause.downcast_ref::().is_some() + || cause + .downcast_ref::() + .is_some_and(reqwest::Error::is_timeout) + || cause + .downcast_ref::() + .is_some_and(|failure| timeout_status(failure.0.as_u16())) + }) { + "timeout" + } else if error.downcast_ref::().is_some() { + "providerFailure" + } else { + "otherChatFailure" + }; + let mut record = adjudication_record(event, stage, code); + record["attempt"] = json!(attempt.filter(|value| (1..=13).contains(value))); + if let Some(ModelContentFailure::NonTerminal { reason }) = + error.downcast_ref::() + { + record["finish"] = json!(match reason.as_str() { + "missing finish_reason" => "absent", + "end_turn" => "end_turn", + "stop_sequence" => "stop_sequence", + value => safe_finish_reason(value), + }); + } + record +} + +const ADJUDICATION_FIELDS: &[&str] = &[ + "candidateId", + "status", + "revisedTitle", + "revisedBody", + "evidence", + "duplicateOf", + "scope", + "disposition", + "cause", + "reason", + "path", + "side", + "line", + "byteOffset", + "diffLine", + "corpusSha256", +]; + +fn adjudication_json_class(error: &serde_json::Error) -> (&'static str, &'static str) { + use serde_json::error::Category; + match error.classify() { + Category::Syntax => return ("jsonSyntax", "other"), + Category::Eof => return ("jsonEof", "other"), + Category::Io => return ("unexpectedIo", "other"), + Category::Data => {} + } + // Only serde's Data errors enter template refinement. Match the complete + // pinned template, including its generated location suffix; never scan an + // arbitrary provider string or expose the message/key/enum value. + let detail = error.to_string(); + let suffix = format!(" at line {} column {}", error.line(), error.column()); + let Some(message) = detail.strip_suffix(&suffix) else { + return ("otherData", "other"); + }; + for &field in ADJUDICATION_FIELDS { + if message == format!("missing field `{field}`") { + return ("missingKnownField", field); + } + if message == format!("duplicate field `{field}`") { + return ("duplicateKnownField", field); + } + } + let quoted_atom = |prefix: &str, suffix: &str| { + message + .strip_prefix(prefix) + .and_then(|text| text.strip_suffix(suffix)) + .is_some_and(|atom| { + !atom.is_empty() + && atom.len() <= 128 + && !atom.chars().any(|c| c == '`' || c.is_control()) + }) + }; + if quoted_atom( + "unknown field `", + "`, expected one of `candidateId`, `status`, `revisedTitle`, `revisedBody`, `evidence`, `duplicateOf`, `scope`", + ) { + return ("unknownField", "other"); + } + if quoted_atom( + "unknown variant `", + "`, expected one of `confirmed`, `refuted`, `unresolved`", + ) { + return ("invalidEnum", "status"); + } + if [ + "invalid type: null, expected a string", + "invalid type: null, expected a sequence", + "invalid type: boolean `true`, expected a string", + "invalid type: boolean `false`, expected a string", + ] + .contains(&message) + { + return ("wrongType", "other"); + } + ("otherData", "other") +} + +fn adjudication_count(value: usize) -> serde_json::Value { + if value <= 20 { + json!(value) + } else { + json!("overflow") + } +} + fn parse_adjudication_results( content: &str, expected_len: usize, + trace: bool, ) -> Result, String> { - let json = extract_json_array(content).ok_or("no JSON array found")?; - let results = - serde_json::from_str::>(json).map_err(|error| error.to_string())?; + let emit = |mut record: serde_json::Value| { + record["expected"] = adjudication_count(expected_len); + if trace { + LlmClient::adjudication_trace(record); + } + }; + let json = extract_json_array(content).ok_or_else(|| { + emit(adjudication_record( + "parserRejected", + "arrayExtraction", + if content.contains('[') { + "unclosedArray" + } else { + "noArray" + }, + )); + "no JSON array found".to_owned() + })?; + let results = serde_json::from_str::>(json).map_err(|error| { + let (code, field) = adjudication_json_class(&error); + let mut record = adjudication_record("parserRejected", "resultDeserialization", code); + record["field"] = json!(field); + emit(record); + error.to_string() + })?; + let mut record = adjudication_record( + if results.len() == expected_len { + "parsed" + } else { + "parserRejected" + }, + "resultCount", + if results.len() == expected_len { + "accepted" + } else { + "resultCountMismatch" + }, + ); + record["observed"] = adjudication_count(results.len()); + emit(record); if results.len() != expected_len { return Err(format!( "expected {expected_len} adjudication result(s), got {}", @@ -9310,15 +9539,139 @@ mod tests { #[test] fn extracts_adjudication_array_from_fenced_output() { let text = "Here you go:\n```json\n[{\"candidateId\":\"candidate\",\"status\":\"unresolved\"}]\n```"; - let results = parse_adjudication_results(text, 1).unwrap(); + let results = parse_adjudication_results(text, 1, false).unwrap(); assert_eq!(results.len(), 1); assert_eq!(results[0].candidate_id, "candidate"); } + #[test] + fn adjudication_provenance_classifies_only_generated_serde_errors() { + let hostile = format!( + "{}{}", + String::from_utf8(vec![115, 107, 45]).unwrap(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + let cases = [ + (json!([{"status":"confirmed"}]).to_string(), ("missingKnownField", "candidateId")), + (r#"[{"candidateId":"x","candidateId":"y","status":"confirmed"}]"#.to_owned(), ("duplicateKnownField", "candidateId")), + (json!([{"candidateId":null,"status":"confirmed"}]).to_string(), ("wrongType", "other")), + (json!([{"candidateId":"x","status":&hostile}]).to_string(), ("invalidEnum", "status")), + (json!([{"candidateId":"x","status":"confirmed",&hostile:true}]).to_string(), ("unknownField", "other")), + (json!([{"candidateId":"x","status":"confirmed","scope":{"disposition":"introducedOrWorsened","reason":"x","cause":{"unexpected":&hostile}}}]).to_string(), ("otherData", "other")), + ]; + for (input, expected) in cases { + let error = serde_json::from_str::>(&input).unwrap_err(); + let actual = adjudication_json_class(&error); + assert_eq!(actual, expected); + assert!( + !json!({"code":actual.0,"field":actual.1}) + .to_string() + .contains(&hostile) + ); + } + for (input, expected) in [("[", "jsonEof"), ("[?]", "jsonSyntax")] { + let error = serde_json::from_str::>(input).unwrap_err(); + assert_eq!(adjudication_json_class(&error).0, expected); + } + assert_eq!(adjudication_count(20), json!(20)); + assert_eq!(adjudication_count(21), json!("overflow")); + assert_eq!(adjudication_count(usize::MAX), json!("overflow")); + } + + #[tokio::test] + async fn adjudication_provenance_pre_request_failures_have_no_attempt_usage() { + for (oversized, code) in [(true, "contextExceeded"), (false, "deadline")] { + let server = MockServer::start().await; + let config = Config { + api_base: server.uri(), + model: "example/unknown-context".into(), + ..Config::default() + }; + let mut client = LlmClient::build( + &config, + std::process::id().to_string(), + Duration::from_secs(1), + None, + None, + ) + .unwrap(); + if !oversized { + client.total_deadline = Some(Instant::now()); + } + let user = if oversized { + "x".repeat(40_000) + } else { + "[]".to_owned() + }; + let error = match client + .adjudicate_findings(&config.model, "system", &user, 1, Duration::from_secs(1)) + .await + { + Ok(_) => panic!("pre-request rejection unexpectedly succeeded"), + Err(error) => error, + }; + assert!(error.model_usage.is_empty()); + assert!(server.received_requests().await.unwrap().is_empty()); + let record = adjudication_chat_record(&error.error, "callFailed", "chatTerminal", None); + assert_eq!(record["code"], code); + assert_eq!(record["attempt"], serde_json::Value::Null); + } + } + + #[test] + fn adjudication_provenance_terminal_types_do_not_infer_success() { + let cases: Vec<(anyhow::Error, &str)> = vec![ + (DeadlineExceeded(LlmPhase::Adjudication).into(), "deadline"), + ( + ReviewContextExceeded { + required: 2, + limit: 1, + } + .into(), + "contextExceeded", + ), + (RequestTimedOut.into(), "timeout"), + ( + AtomicAttributionIdentityFailure::Missing.into(), + "identityMissing", + ), + ( + AtomicAttributionIdentityFailure::Mismatch.into(), + "identityMismatch", + ), + (anyhow!("unclassified local failure"), "otherChatFailure"), + ]; + for (error, expected) in cases { + let record = adjudication_chat_record(&error, "callFailed", "chatTerminal", None); + assert_eq!(record["code"], expected); + assert_eq!(record["attempt"], serde_json::Value::Null); + assert!(record.to_string().len() + "postil: adjudication-output ".len() <= 512); + } + for (reason, expected) in [ + ("length", "length"), + ("missing finish_reason", "absent"), + ("end_turn", "end_turn"), + ("stop_sequence", "stop_sequence"), + ("unrecognized", "reported"), + ] { + let error = ModelContentFailure::NonTerminal { + reason: reason.to_owned(), + } + .into(); + let record = + adjudication_chat_record(&error, "attemptRejected", "endpointContent", Some(1)); + assert_eq!(record["finish"], expected); + } + } + #[test] fn rejects_adjudication_output_without_a_json_array() { assert_eq!( - parse_adjudication_results("I could not adjudicate this finding.", 1).unwrap_err(), + parse_adjudication_results("I could not adjudicate this finding.", 1, false) + .unwrap_err(), "no JSON array found" ); } diff --git a/tests/e2e.rs b/tests/e2e.rs index 94d62bb..9b0a25b 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -4221,6 +4221,17 @@ async fn malformed_adjudication_output_blocks_under_advisory_provider_policy() { .args(["--output", "json"]) .assert() .code(1); + let records: Vec = String::from_utf8_lossy(&out.get_output().stderr) + .lines() + .filter_map(|line| line.strip_prefix("postil: adjudication-output ")) + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!(records.len(), 2); + assert_eq!(records[0]["event"], "callStarted"); + assert_eq!(records[1]["event"], "parserRejected"); + assert_eq!(records[1]["code"], "noArray"); + assert_eq!(records[1]["call"], records[0]["call"]); + assert_eq!(records[1]["attempt"], Value::Null); let envelope: Value = serde_json::from_slice(&out.get_output().stdout).unwrap(); assert!( From 92222fd4ce052f0186fe03a321ad06a2b7126501 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Sun, 18 Oct 2026 18:24:25 +0000 Subject: [PATCH 2/4] Normalize the diagnostic endpoint before provider preflight --- bench/src/scorer-eval.test.ts | 1 + bench/src/scorer-eval.ts | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index 5f5dc56..59bd107 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -2456,6 +2456,7 @@ describe("finite adjudication provenance", () => { expect(evaluation.findingPublished).toBe(!success); expect(evaluation.finalConfidence).toBe(success ? 0.2 : 0.95); expect(evaluation.diagnostics?.scorer.attempts).toBe(success ? 1 : 0); + expect(evaluation.diagnostics?.childExitCode).toBe(success ? 0 : 1); const projection = evaluation.diagnostics?.adjudicationOutput; if (baseBinary && index === 0) expect(projection).toBeUndefined(); else { diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index 568bfc7..6aa04b9 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -1123,7 +1123,8 @@ export async function runScorerDiagnostic(input: { if (!process.env[input.keyName]) throw new Error("diagnostic credential is unavailable"); const provider = "Azure"; const route = "azure/us"; - const pricing = await fetchQualificationPricing(DEFAULT_API_BASE, "openai-compatible", + const apiBase = normalizeApiBase(DEFAULT_API_BASE); + const pricing = await fetchQualificationPricing(apiBase, "openai-compatible", scorerQualificationModels([model]), provider, scorerQualificationRequiredParameters([model], provider), route); assertQualificationPreflight([model], 1, pricing); const providerContract = providerContractEvidence(provider, route, pricing, [GENERATOR_MODEL], [model]); @@ -1131,7 +1132,7 @@ export async function runScorerDiagnostic(input: { const root = await mkdtemp(join(tmpdir(), "postil-scorer-diagnostic-")); try { const result = await runScorerEvalCase(c, "falseFinding", model, 1, input.binary, root, - DEFAULT_API_BASE, input.keyName, pricing.get(model) ?? null, SCORER_CASE_EXEC_TIMEOUT_MS, + apiBase, input.keyName, pricing.get(model) ?? null, SCORER_CASE_EXEC_TIMEOUT_MS, provider, pricing.get(GENERATOR_MODEL) ?? null, route); await assertScorerDiagnosticCustody(custodyInput); return { From 05b51b1d5f53045796a706cfc6f8f30221f8b7a4 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Sun, 18 Oct 2026 18:35:44 +0000 Subject: [PATCH 3/4] Reject coerced diagnostic event and stage values --- bench/src/scorer-eval.test.ts | 2 ++ bench/src/scorer-eval.ts | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index 59bd107..28acfe3 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -2352,6 +2352,8 @@ describe("finite adjudication provenance", () => { const hostile = String.fromCharCode(115, 107, 45) + crypto.randomUUID(); const mutations = [ { version: 2 }, { phase: hostile }, { event: hostile }, { stage: hostile }, { code: hostile }, + { event: ["callStarted"], stage: "chatTerminal", code: "started", expected: null }, + { event: "callStarted", stage: ["chatTerminal"], code: "started", expected: null }, { call: 0 }, { call: 65 }, { attempt: 1 }, { expected: 21 }, { expected: -1 }, { field: hostile }, { finish: hostile }, { observed: 1 }, { [hostile]: true }, { code: { secret: hostile } }, { expected: [] }, diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index 6aa04b9..1676609 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -277,7 +277,8 @@ export function adjudicationOutput(stderr: string): AdjudicationOutput | undefin const value = JSON.parse(raw); if (!value || typeof value !== "object" || Array.isArray(value)) throw 0; const { attempt, call, code, event, expected, field, finish, observed, phase, stage, version } = value; - if (version !== 1 || phase !== "adjudication" || !Object.hasOwn(OUTPUT_CODES, event) || + if (version !== 1 || phase !== "adjudication" || typeof event !== "string" || + typeof stage !== "string" || typeof code !== "string" || !Object.hasOwn(OUTPUT_CODES, event) || call !== 1 || !ordinal(attempt, 13) || !count(expected) || !count(observed) || !(field === null || OUTPUT_FIELDS.includes(field)) || !(finish === null || ["stop", "length", "tool_calls", "content_filter", "error", "reported", "absent", "end_turn", "stop_sequence"].includes(finish))) throw 0; From a3d7105f473ef8457bf731eca35175f75f6f324c Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Sun, 18 Oct 2026 19:41:09 +0000 Subject: [PATCH 4/4] Pin benchmark CI to Bun 1.3.14 --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76881d9..c832bc6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,6 +56,8 @@ jobs: - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 - run: cargo build --quiet --release - run: bun install --frozen-lockfile working-directory: bench