From 7a26317a13dc8d743886cbd366a952bd3fc62d15 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 26 Oct 2026 16:15:35 -0700 Subject: [PATCH 1/2] fix(drift): probe K9 exhaustion on gpt-5.4-mini after o4-mini retirement OpenAI shut down o4-mini on 2026-10-23 (its /v1/models shutdown_date); the K9 HTTP and WS probes have answered 404 model_not_found since. Move the probe model into one OPENAI_K9_PROBE_MODEL constant, grade the retained o4-mini replays against OPENAI_K9_REPLAY_MODEL, and accept output_tokens equal to reasoning_tokens instead of exactly 0. --- CHANGELOG.md | 1 + src/__tests__/drift/helpers.ts | 28 +++++- src/__tests__/drift/openai-responses.drift.ts | 92 +++++++++++++------ src/__tests__/drift/ws-responses.drift.ts | 66 +++++++++---- 4 files changed, 135 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc926330..d1f670b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Client disconnects stop delivery (no more chunks or waits) on OpenAI chat, Anthropic, Responses, Gemini/Vertex, Gemini Interactions, Cohere, Bedrock, Ollama, transcription streams and the Responses, Realtime and Gemini Live WebSockets - Client disconnects on those surfaces are journaled as `interrupted` with `interruptReason: "client aborted"` (HTTP `status: 0` when nothing was sent); aimock-caused teardowns record `"disconnect"`. Journal snapshots of hung-up requests now carry these fields - WebSocket: a client TCP FIN without a close frame now closes the connection at once (1006) and ends the turn +- Daily drift: the OpenAI Responses K9 reasoning-exhaustion probes (HTTP and WebSocket) request `gpt-5.4-mini`. OpenAI retired `o4-mini` on 2026-10-23 (its `/v1/models` `shutdown_date`), and it has answered `404 model_not_found` since. The probe model is one constant, `OPENAI_K9_PROBE_MODEL`. The usage check accepts `output_tokens` equal to `output_tokens_details.reasoning_tokens` (gpt-5.4-mini spends the 32-token cap on reasoning; o4-mini reported 0). The event-order check is unchanged, and the retained o4-mini replay fixtures are still graded against o4-mini. Drift harness only; aimock behavior is unchanged. - Docker image: the base image is pinned to `node:22.23.4-alpine` by digest. Node 22.23.4 fixes CVE-2026-111497 (Brotli decompression memory bug); the 1.46.0 image shipped Node 22.23.3. ## [1.46.0] - 2026-10-20 diff --git a/src/__tests__/drift/helpers.ts b/src/__tests__/drift/helpers.ts index e9bbdd94..82b72fb6 100644 --- a/src/__tests__/drift/helpers.ts +++ b/src/__tests__/drift/helpers.ts @@ -318,16 +318,34 @@ export async function collectMockWSMessages( // OpenAI Responses K9 (reasoning-only max_output_tokens exhaustion) // --------------------------------------------------------------------------- +/** + * The OpenAI model the live K9 exhaustion probes (HTTP and WS) request. Change + * only this line when OpenAI retires it: `o4-mini` hit its `/v1/models` + * `shutdown_date` of 2026-10-23 and has answered `404 model_not_found` since. + */ +export const OPENAI_K9_PROBE_MODEL = "gpt-5.4-mini"; + +/** + * The model the retained K9 replay fixtures were captured from (2026-10-10). + * The fixtures are frozen evidence, so they stay graded against this model. + */ +export const OPENAI_K9_REPLAY_MODEL = "o4-mini"; + /** * The reasoning-summary event runs that live OpenAI streams for the K9 - * exhaustion request (o4-mini, effort high, summary auto, max_output_tokens 32) - * between `response.output_item.added` and `response.output_item.done`. + * exhaustion request (OPENAI_K9_PROBE_MODEL, effort high, summary auto, + * max_output_tokens 32) between `response.output_item.added` and + * `response.output_item.done`. * - * The same request picks one at random. Measured 2026-10-10, every run ended - * `response.incomplete`/`max_output_tokens` with `output_tokens: 0` on - * o4-mini-2025-04-16: + * The same request picks one at random. Measured 2026-10-10 on + * o4-mini-2025-04-16 (OPENAI_K9_REPLAY_MODEL), every run ended + * `response.incomplete`/`max_output_tokens`: * - HTTP, 22 runs: full-summary 16, no-summary 4, unclosed-summary 2. * - WS, 10 runs: full-summary 5, no-summary 3, unclosed-summary 2. + * o4-mini reported `output_tokens: 0`; gpt-5.4-mini-2026-03-17 reports + * `output_tokens: 32`, all of them `reasoning_tokens`. Either way the cap is + * spent on reasoning only, so grading requires + * `output_tokens === output_tokens_details.reasoning_tokens`. * * Live/native grading accepts exactly these three. aimock's localhost stream * must still emit `full-summary`. diff --git a/src/__tests__/drift/openai-responses.drift.ts b/src/__tests__/drift/openai-responses.drift.ts index b61a001a..dd0cd4ff 100644 --- a/src/__tests__/drift/openai-responses.drift.ts +++ b/src/__tests__/drift/openai-responses.drift.ts @@ -48,6 +48,8 @@ import { startDriftServer, stopDriftServer, OPENAI_K9_SUMMARY_SHAPES, + OPENAI_K9_PROBE_MODEL, + OPENAI_K9_REPLAY_MODEL, matchOpenAIK9SummaryShape, isOpenAIModelSnapshot, } from "./helpers.js"; @@ -87,18 +89,22 @@ const OPENAI_MODELS_URL = "https://api.openai.com/v1/models"; // Credential presence enables two bounded paid exhaustion probes in daily // drift. An explicit capture directory instead replays the hash-pinned real -// transcripts, with their exact requests/status metadata checked below. +// transcripts, with their exact requests/status metadata checked below. Each +// capture was taken against `capturedModel`, which can trail the live probe +// model after a retirement. const EXHAUSTION_CAPTURE_DIR = process.env.OPENAI_RESPONSES_EXHAUSTION_CAPTURE_DIR; const exhaustionCases: { kind: "K5" | "K9"; prefix: string; sha256: string; + capturedModel: string; request: ResponseCreateParamsStreaming; }[] = [ { kind: "K5", prefix: "k5-20261008T170311Z", sha256: "8095336aeb8908dd4647bbecd49a5a06959f1e5dac06b7a9a17416a30c3e7152", + capturedModel: "gpt-4.1-mini", request: { model: "gpt-4.1-mini", input: @@ -126,8 +132,9 @@ const exhaustionCases: { kind: "K9", prefix: "k9-20261008T170312Z", sha256: "f96bf0db1dae1cab1956af15d589d26cace087a5893604bd6c727b6347c0d0ae", + capturedModel: OPENAI_K9_REPLAY_MODEL, request: { - model: "o4-mini", + model: OPENAI_K9_PROBE_MODEL, input: "Find all positive integers n below 100000 for which n squared plus n plus 41 is prime. Explain a rigorous strategy before computing.", reasoning: { effort: "high", summary: "auto" }, @@ -138,7 +145,12 @@ const exhaustionCases: { }, ]; -function inspectExhaustion(raw: string, kind: "K5" | "K9", side: "provider" | "localhost") { +function inspectExhaustion( + raw: string, + kind: "K5" | "K9", + side: "provider" | "localhost", + requested: string, +) { const blocks = raw.replace(/\r\n/g, "\n").split("\n\n"); expect(blocks.pop(), "Responses terminal SSE event must be terminated").toBe(""); const events: ResponseStreamEvent[] = blocks.filter(Boolean).map((block) => { @@ -257,10 +269,14 @@ function inspectExhaustion(raw: string, kind: "K5" | "K9", side: "provider" | "l } expect(done[0].item.summary).toEqual(item.summary); if (side === "provider") { - // Every accepted live shape spent no output tokens on the requested model. - expect(terminal.response.usage?.output_tokens).toBe(0); - const requested = exhaustionCases.find((scenario) => scenario.kind === kind)!.request.model; - expect(isOpenAIModelSnapshot(terminal.response.model, String(requested))).toBe(true); + // The cap went to reasoning alone: no visible output item, and every + // output token is a reasoning token (0 on o4-mini, the full cap on + // gpt-5.4-mini). + expect(terminal.response.output.every((output) => output.type === "reasoning")).toBe(true); + const usage = terminal.response.usage; + if (!usage) throw new Error("Missing terminal usage"); + expect(usage.output_tokens).toBe(usage.output_tokens_details.reasoning_tokens); + expect(isOpenAIModelSnapshot(terminal.response.model, requested)).toBe(true); for (const event of events) { if (event.type === "response.created" || event.type === "response.in_progress") { expect(event.response.model).toBe(terminal.response.model); @@ -279,9 +295,14 @@ function inspectExhaustion(raw: string, kind: "K5" | "K9", side: "provider" | "l }; } -function gradeExhaustion(raw: string, kind: "K5" | "K9", side: "provider" | "localhost") { +function gradeExhaustion( + raw: string, + kind: "K5" | "K9", + side: "provider" | "localhost", + requested: string, +) { try { - return inspectExhaustion(raw, kind, side); + return inspectExhaustion(raw, kind, side, requested); } catch (error) { throw new Error( formatDriftReport( @@ -302,7 +323,10 @@ function gradeExhaustion(raw: string, kind: "K5" | "K9", side: "provider" | "loc } } -async function exhaustionProviderWire(scenario: (typeof exhaustionCases)[number]) { +/** The provider stream plus the model it was requested from. */ +async function exhaustionProviderWire( + scenario: (typeof exhaustionCases)[number], +): Promise<{ raw: string; requested: string }> { if (EXHAUSTION_CAPTURE_DIR) { const raw = await readFile( join(EXHAUSTION_CAPTURE_DIR, `${scenario.prefix}-response.sse`), @@ -315,7 +339,7 @@ async function exhaustionProviderWire(scenario: (typeof exhaustionCases)[number] const status: { status: string; content_type: string } = JSON.parse( await readFile(join(EXHAUSTION_CAPTURE_DIR, `${scenario.prefix}-status.json`), "utf8"), ); - expect(request.body).toEqual(scenario.request); + expect(request.body).toEqual({ ...scenario.request, model: scenario.capturedModel }); expect(status.status).toBe("200"); expect(status.content_type).toContain("text/event-stream"); console.log( @@ -326,7 +350,7 @@ async function exhaustionProviderWire(scenario: (typeof exhaustionCases)[number] request: request.body, }), ); - return raw; + return { raw, requested: scenario.capturedModel }; } const controller = new AbortController(); const deadline = setTimeout(() => controller.abort(), 45_000); @@ -354,7 +378,7 @@ async function exhaustionProviderWire(scenario: (typeof exhaustionCases)[number] } raw += decoder.decode(); completed = true; - return raw; + return { raw, requested: String(scenario.request.model) }; } finally { clearTimeout(deadline); controller.abort(); @@ -376,14 +400,15 @@ describe.skipIf(!OPENAI_API_KEY && !EXHAUSTION_CAPTURE_DIR)( it.each(exhaustionCases)( "$kind provider terminal agrees with actual localhost", async (scenario) => { - await compareExhaustion(scenario, await exhaustionProviderWire(scenario)); + const wire = await exhaustionProviderWire(scenario); + await compareExhaustion(scenario, wire.raw, wire.requested); }, ); }, ); -// Retained live o4-mini K9 provider streams captured 2026-10-10 from the exact -// K9 request above, one per accepted provider summary shape. They replay +// Retained live o4-mini K9 provider streams captured 2026-10-10 from the K9 +// request above (then on OPENAI_K9_REPLAY_MODEL), one per accepted provider summary shape. They replay // without a credential so the variant grading and its rejections stay checked. const K9_PROVIDER_REPLAYS = ["full-summary", "no-summary", "unclosed-summary"] as const; const k9Case = exhaustionCases.find((scenario) => scenario.kind === "K9")!; @@ -414,12 +439,12 @@ describe("Responses K9 provider variant replay", () => { it.each(K9_PROVIDER_REPLAYS)( "retained %s provider stream agrees with actual localhost", async (shape) => { - await compareExhaustion(k9Case, await readK9ProviderReplay(shape)); + await compareExhaustion(k9Case, await readK9ProviderReplay(shape), OPENAI_K9_REPLAY_MODEL); }, ); it.each(K9_PROVIDER_REPLAYS)( - "retained %s provider stream rejects a wrong terminal status, nonzero output tokens or another model", + "retained %s provider stream rejects a wrong terminal status, non-reasoning output tokens or another model", async (shape) => { const raw = await readK9ProviderReplay(shape); expect(raw).toContain('"status":"incomplete"'); @@ -428,6 +453,7 @@ describe("Responses K9 provider variant replay", () => { raw.replaceAll('"status":"incomplete"', '"status":"completed"'), "K9", "provider", + OPENAI_K9_REPLAY_MODEL, ), ).toThrow(); // Only the terminal usage changes; created/in_progress keep their @@ -439,13 +465,16 @@ describe("Responses K9 provider variant replay", () => { expect(spentTokens.slice(0, spentTokens.indexOf("event: response.incomplete"))).toBe( raw.slice(0, raw.indexOf("event: response.incomplete")), ); - expect(() => inspectExhaustion(spentTokens, "K9", "provider")).toThrow(); + expect(() => + inspectExhaustion(spentTokens, "K9", "provider", OPENAI_K9_REPLAY_MODEL), + ).toThrow(); expect(raw).toContain('"model":"o4-mini-2025-04-16"'); expect(() => inspectExhaustion( raw.replaceAll('"model":"o4-mini-2025-04-16"', '"model":"gpt-4.1-mini-2025-04-14"'), "K9", "provider", + OPENAI_K9_REPLAY_MODEL, ), ).toThrow(); // A valid-looking snapshot on the terminal alone must still disagree @@ -453,7 +482,9 @@ describe("Responses K9 provider variant replay", () => { const terminalModel = withTerminalEvent(raw, (terminal) => { terminal.response.model = "o4-mini-2099-01-01"; }); - expect(() => inspectExhaustion(terminalModel, "K9", "provider")).toThrow(); + expect(() => + inspectExhaustion(terminalModel, "K9", "provider", OPENAI_K9_REPLAY_MODEL), + ).toThrow(); }, ); @@ -472,22 +503,28 @@ describe("Responses K9 provider variant replay", () => { ); expect(textDone).toHaveLength(1); expect(blocks.length - reordered.length).toBe(1); - expect(() => inspectExhaustion(reordered.join("\n\n"), "K9", "provider")).toThrow( - /matches none of the accepted summary shapes/, - ); + expect(() => + inspectExhaustion(reordered.join("\n\n"), "K9", "provider", OPENAI_K9_REPLAY_MODEL), + ).toThrow(/matches none of the accepted summary shapes/); }); it.each(["no-summary", "unclosed-summary"] as const)( "localhost grading still rejects the %s provider shape", async (shape) => { const raw = await readK9ProviderReplay(shape); - expect(() => inspectExhaustion(raw, "K9", "localhost")).toThrow(/deeply equal/); + expect(() => inspectExhaustion(raw, "K9", "localhost", OPENAI_K9_REPLAY_MODEL)).toThrow( + /deeply equal/, + ); }, ); }); -async function compareExhaustion(scenario: (typeof exhaustionCases)[number], provider: string) { - const expected = gradeExhaustion(provider, scenario.kind, "provider"); +async function compareExhaustion( + scenario: (typeof exhaustionCases)[number], + provider: string, + requested: string, +) { + const expected = gradeExhaustion(provider, scenario.kind, "provider", requested); // Deliberate mutation of authentic provider bytes tests this reader's // sensitivity, not a new product regression or a fabricated SDK response. expect(() => @@ -495,6 +532,7 @@ async function compareExhaustion(scenario: (typeof exhaustionCases)[number], pro provider.replaceAll("response.incomplete", "response.completed"), scenario.kind, "provider", + requested, ), ).toThrow(); console.log( @@ -543,7 +581,7 @@ async function compareExhaustion(scenario: (typeof exhaustionCases)[number], pro raw: local.body, }), ); - const actual = gradeExhaustion(local.body, scenario.kind, "localhost"); + const actual = gradeExhaustion(local.body, scenario.kind, "localhost", requested); expect(actual.frame).toEqual(expected.frame); expect(actual.status).toBe(expected.status); expect(actual.reason).toBe(expected.reason); diff --git a/src/__tests__/drift/ws-responses.drift.ts b/src/__tests__/drift/ws-responses.drift.ts index 2b08449b..04e79d3b 100644 --- a/src/__tests__/drift/ws-responses.drift.ts +++ b/src/__tests__/drift/ws-responses.drift.ts @@ -30,6 +30,8 @@ import { stopDriftServer, collectMockWSMessages, OPENAI_K9_SUMMARY_SHAPES, + OPENAI_K9_PROBE_MODEL, + OPENAI_K9_REPLAY_MODEL, matchOpenAIK9SummaryShape, isOpenAIModelSnapshot, } from "./helpers.js"; @@ -277,7 +279,7 @@ const exhaustionCases: { }, { kind: "K9", - model: "o4-mini", + model: OPENAI_K9_PROBE_MODEL, input: [ { role: "user", @@ -350,6 +352,7 @@ function inspectWSExhaustion( events: ResponseStreamEvent[], kind: "K5" | "K9", side: "native" | "localhost", + requested: string, ) { const order = events .map((event) => event.type) @@ -467,9 +470,13 @@ function inspectWSExhaustion( } expect(done[0].item.summary).toEqual(item.summary); if (side === "native") { - // Every accepted live shape spent no output tokens on the requested model. - expect(terminal.response.usage?.output_tokens).toBe(0); - const requested = exhaustionCases.find((scenario) => scenario.kind === kind)!.model; + // The cap went to reasoning alone: no visible output item, and every + // output token is a reasoning token (0 on o4-mini, the full cap on + // gpt-5.4-mini). + expect(terminal.response.output.every((output) => output.type === "reasoning")).toBe(true); + const usage = terminal.response.usage; + if (!usage) throw new Error("Missing terminal usage"); + expect(usage.output_tokens).toBe(usage.output_tokens_details.reasoning_tokens); expect(isOpenAIModelSnapshot(terminal.response.model, requested)).toBe(true); for (const event of events) { if (event.type === "response.created" || event.type === "response.in_progress") { @@ -491,9 +498,10 @@ function gradeWSExhaustion( events: ResponseStreamEvent[], kind: "K5" | "K9", side: "native" | "localhost", + requested: string, ) { try { - return inspectWSExhaustion(events, kind, side); + return inspectWSExhaustion(events, kind, side, requested); } catch (error) { throw new Error( formatDriftReport( @@ -520,15 +528,19 @@ describe("Responses WS exhaustion local contract", () => { it(`WS ${scenario.kind} actual localhost preserves incomplete contract`, async () => { const events = await localWSExhaustion(scenario); console.log(JSON.stringify({ kind: scenario.kind, mode: "actual localhost WS", events })); - gradeWSExhaustion(events, scenario.kind, "localhost"); + gradeWSExhaustion(events, scenario.kind, "localhost", scenario.model); }); } }); -async function compareWSExhaustion(scenario: ExhaustionCase, nativeEvents: ResponseStreamEvent[]) { - const native = gradeWSExhaustion(nativeEvents, scenario.kind, "native"); +async function compareWSExhaustion( + scenario: ExhaustionCase, + nativeEvents: ResponseStreamEvent[], + requested: string, +) { + const native = gradeWSExhaustion(nativeEvents, scenario.kind, "native", requested); const localEvents = await localWSExhaustion(scenario); - expect(gradeWSExhaustion(localEvents, scenario.kind, "localhost")).toEqual(native); + expect(gradeWSExhaustion(localEvents, scenario.kind, "localhost", requested)).toEqual(native); const sdk = [ ...openaiResponsesTextEventShapes().filter( (event) => event.type === "response.created" || event.type === "response.in_progress", @@ -551,8 +563,8 @@ async function compareWSExhaustion(scenario: ExhaustionCase, nativeEvents: Respo ).toEqual([]); } -// Retained live o4-mini K9 WS native events captured 2026-10-10 from the exact -// K9 request above, one per accepted native summary shape. They replay without +// Retained live o4-mini K9 WS native events captured 2026-10-10 from the K9 +// request above (then on OPENAI_K9_REPLAY_MODEL), one per accepted native summary shape. They replay without // a credential so the variant grading and its rejections stay checked. const K9_NATIVE_REPLAYS = ["full-summary", "no-summary", "unclosed-summary"] as const; const k9Case = exhaustionCases.find((scenario) => scenario.kind === "K9")!; @@ -569,12 +581,12 @@ describe("Responses WS K9 native variant replay", () => { it.each(K9_NATIVE_REPLAYS)( "retained %s native events agree with actual localhost", async (shape) => { - await compareWSExhaustion(k9Case, await readK9NativeReplay(shape)); + await compareWSExhaustion(k9Case, await readK9NativeReplay(shape), OPENAI_K9_REPLAY_MODEL); }, ); it.each(K9_NATIVE_REPLAYS)( - "retained %s native events reject a wrong terminal status, nonzero output tokens or another model", + "retained %s native events reject a wrong terminal status, non-reasoning output tokens or another model", async (shape) => { const events = await readK9NativeReplay(shape); const terminal = events.at(-1); @@ -585,13 +597,17 @@ describe("Responses WS K9 native variant replay", () => { const statusTerminal = wrongStatus.at(-1); if (statusTerminal?.type !== "response.incomplete") throw new Error("Missing terminal"); statusTerminal.response.status = "completed"; - expect(() => inspectWSExhaustion(wrongStatus, "K9", "native")).toThrow(); + expect(() => + inspectWSExhaustion(wrongStatus, "K9", "native", OPENAI_K9_REPLAY_MODEL), + ).toThrow(); const spentTokens = structuredClone(events); const tokenTerminal = spentTokens.at(-1); if (tokenTerminal?.type !== "response.incomplete" || !tokenTerminal.response.usage) throw new Error("Missing terminal usage"); tokenTerminal.response.usage.output_tokens = 7; - expect(() => inspectWSExhaustion(spentTokens, "K9", "native")).toThrow(); + expect(() => + inspectWSExhaustion(spentTokens, "K9", "native", OPENAI_K9_REPLAY_MODEL), + ).toThrow(); expect(terminal.response.model).toBe("o4-mini-2025-04-16"); const otherModel = structuredClone(events); for (const event of otherModel) { @@ -602,14 +618,18 @@ describe("Responses WS K9 native variant replay", () => { ) event.response.model = "gpt-4.1-mini-2025-04-14"; } - expect(() => inspectWSExhaustion(otherModel, "K9", "native")).toThrow(); + expect(() => + inspectWSExhaustion(otherModel, "K9", "native", OPENAI_K9_REPLAY_MODEL), + ).toThrow(); // A valid-looking snapshot on the terminal alone must still disagree // with created/in_progress. const terminalModel = structuredClone(events); const modelTerminal = terminalModel.at(-1); if (modelTerminal?.type !== "response.incomplete") throw new Error("Missing terminal"); modelTerminal.response.model = "o4-mini-2099-01-01"; - expect(() => inspectWSExhaustion(terminalModel, "K9", "native")).toThrow(); + expect(() => + inspectWSExhaustion(terminalModel, "K9", "native", OPENAI_K9_REPLAY_MODEL), + ).toThrow(); }, ); @@ -629,7 +649,7 @@ describe("Responses WS K9 native variant replay", () => { ); expect(textDone).toHaveLength(1); expect(events.length - reordered.length).toBe(1); - expect(() => inspectWSExhaustion(reordered, "K9", "native")).toThrow( + expect(() => inspectWSExhaustion(reordered, "K9", "native", OPENAI_K9_REPLAY_MODEL)).toThrow( /matches none of the accepted summary shapes/, ); }); @@ -638,7 +658,9 @@ describe("Responses WS K9 native variant replay", () => { "localhost grading still rejects the %s native shape", async (shape) => { const events = await readK9NativeReplay(shape); - expect(() => inspectWSExhaustion(events, "K9", "localhost")).toThrow(/deeply equal/); + expect(() => inspectWSExhaustion(events, "K9", "localhost", OPENAI_K9_REPLAY_MODEL)).toThrow( + /deeply equal/, + ); }, ); }); @@ -677,7 +699,11 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses WS exhaustion live drift", () ); // The SDK union supplies event discriminators; semantic validation below // still grades the untrusted native response, including error terminals. - await compareWSExhaustion(scenario, result.rawMessages as ResponseStreamEvent[]); + await compareWSExhaustion( + scenario, + result.rawMessages as ResponseStreamEvent[], + scenario.model, + ); } catch (error) { exhaustionStopped = true; if (error instanceof WSHandshakeError && isInfraSkip(error.status)) { From 9e564c0ad2f4d700b67f74165bbdc9eccc139169 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 26 Oct 2026 16:16:01 -0700 Subject: [PATCH 2/2] docs(changelog): link the K9 probe-model fix to #584 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1f670b3..ca062632 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ - Client disconnects stop delivery (no more chunks or waits) on OpenAI chat, Anthropic, Responses, Gemini/Vertex, Gemini Interactions, Cohere, Bedrock, Ollama, transcription streams and the Responses, Realtime and Gemini Live WebSockets - Client disconnects on those surfaces are journaled as `interrupted` with `interruptReason: "client aborted"` (HTTP `status: 0` when nothing was sent); aimock-caused teardowns record `"disconnect"`. Journal snapshots of hung-up requests now carry these fields - WebSocket: a client TCP FIN without a close frame now closes the connection at once (1006) and ends the turn -- Daily drift: the OpenAI Responses K9 reasoning-exhaustion probes (HTTP and WebSocket) request `gpt-5.4-mini`. OpenAI retired `o4-mini` on 2026-10-23 (its `/v1/models` `shutdown_date`), and it has answered `404 model_not_found` since. The probe model is one constant, `OPENAI_K9_PROBE_MODEL`. The usage check accepts `output_tokens` equal to `output_tokens_details.reasoning_tokens` (gpt-5.4-mini spends the 32-token cap on reasoning; o4-mini reported 0). The event-order check is unchanged, and the retained o4-mini replay fixtures are still graded against o4-mini. Drift harness only; aimock behavior is unchanged. +- Daily drift: the OpenAI Responses K9 reasoning-exhaustion probes (HTTP and WebSocket) request `gpt-5.4-mini`. OpenAI retired `o4-mini` on 2026-10-23 (its `/v1/models` `shutdown_date`), and it has answered `404 model_not_found` since. The probe model is one constant, `OPENAI_K9_PROBE_MODEL`. The usage check accepts `output_tokens` equal to `output_tokens_details.reasoning_tokens` (gpt-5.4-mini spends the 32-token cap on reasoning; o4-mini reported 0). The event-order check is unchanged, and the retained o4-mini replay fixtures are still graded against o4-mini. Drift harness only; aimock behavior is unchanged (#584). - Docker image: the base image is pinned to `node:22.23.4-alpine` by digest. Node 22.23.4 fixes CVE-2026-111497 (Brotli decompression memory bug); the 1.46.0 image shipped Node 22.23.3. ## [1.46.0] - 2026-10-20