From c4bd1babcdd301879eec702c8bf81c206a44bb03 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sat, 24 Oct 2026 11:32:47 -0700 Subject: [PATCH 1/7] feat(types): add RawResponse and RawWireId, export WireId (#549) --- src/helpers.ts | 39 +++++++++++++++++++++++++++++++++++++++ src/index.ts | 3 +++ src/types.ts | 45 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+) diff --git a/src/helpers.ts b/src/helpers.ts index 796a4ac2..768dee86 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -39,6 +39,7 @@ import type { TranscriptionResponse, VideoResponse, RawJSONResponse, + RawResponse, SSEChunk, ToolCall, FixtureToolCall, @@ -2002,6 +2003,44 @@ export function isJSONResponse(r: FixtureResponse): r is RawJSONResponse { return "json" in r && (r as RawJSONResponse).json !== undefined; } +/** + * Every key some other response guard reads. A response carrying any of them + * is that type (with a stray `raw` key the loader has always ignored), never + * a raw response — so adding raw support changes nothing for such fixtures. + */ +const NON_RAW_RESPONSE_MARKERS = [ + "content", + "toolCalls", + "blocks", + "responsesBlocks", + "customToolCalls", + "error", + "embedding", + "image", + "images", + "audio", + "transcription", + "video", + "json", + "live", +] as const; + +/** + * @internal Whether `r` is a raw passthrough response: `raw` is a non-null, + * non-array object and no other response type's marker key is present. The + * shape of `raw` itself is checked by the loader and again at serve time. + * Exported for in-repo use only; not part of the public API. + * + * A fixture added in code can hold a response that is not an object (null, a + * string). `in` throws on such a value, so it is simply not raw. + */ +export function isRawResponse(r: FixtureResponse): r is RawResponse { + if (typeof r !== "object" || r === null || !("raw" in r)) return false; + const raw: unknown = r.raw; + if (raw === null || typeof raw !== "object" || Array.isArray(raw)) return false; + return !NON_RAW_RESPONSE_MARKERS.some((marker) => marker in r); +} + export function extractOverrides( response: TextResponse | ToolCallResponse | ContentWithToolCallsResponse, ): ResponseOverrides { diff --git a/src/index.ts b/src/index.ts index 91f65e31..a22298f8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -547,6 +547,9 @@ export type { TranscriptionResponse, VideoResponse, RawJSONResponse, + RawResponse, + RawWireId, + WireId, ResponseOverrides, FixtureCitation, FixtureUrlCitation, diff --git a/src/types.ts b/src/types.ts index db585486..24864103 100644 --- a/src/types.ts +++ b/src/types.ts @@ -742,6 +742,49 @@ export interface RawJSONResponse extends ResponseOverrides { billableUnits?: number; } +/** + * Pass-through LLM response. aimock sends `events` (streaming) or `json` + * (non-streaming) as written, adding only the named wire's framing. No + * overrides, reasoning gating, provider shaping or drift checks apply. + */ +export interface RawResponse { + /** + * The passthrough body for one wire: `events` for a streaming request, + * `json` for a non-streaming one, plus optional `status`, `headers` and + * (`openai-chat` only) `done`. The router offers the fixture only to the + * handler for `wire`. Checked at load (`validateFixtures`) and again when + * served; a fixture that fails when served gets a 500 with the code + * `aimock_raw_fixture_invalid`. + */ + raw: { + /** + * The wire these events belong to. The router skips the fixture on every + * other wire; a factory that returns it for another wire gets a 500. + */ + wire: RawWireId; + /** Streamed body: one entry per frame, each framed per `wire`. */ + events?: Record[]; + /** Non-streaming body, sent as `JSON.stringify(json)`. */ + json?: unknown; + /** HTTP status. Default 200. */ + status?: number; + /** + * Extra response headers. Without a Content-Type here, a stream gets the + * wire's own type and a non-streaming reply gets `application/json`. + */ + headers?: Record; + /** `openai-chat` only: send the terminal `data: [DONE]`. Default true. */ + done?: boolean; + }; +} + +/** + * The wires a {@link RawResponse} can target: every HTTP LLM wire. Ollama has + * two: `ollama` is `/api/chat` and `ollama-generate` is `/api/generate`, whose + * frames have a different shape. + */ +export type RawWireId = Exclude | "ollama-generate"; + export type FixtureResponse = | TextResponse | ToolCallResponse @@ -753,6 +796,7 @@ export type FixtureResponse = | TranscriptionResponse | VideoResponse | RawJSONResponse + | RawResponse | LiveFixtureResponse; // GA Realtime session types @@ -1149,6 +1193,7 @@ export type FixtureFileResponse = | TranscriptionResponse | VideoResponse | RawJSONResponse + | RawResponse | LiveFixtureResponse; export interface FixtureFile { From 2693bd573584ed3a87d437a156c96beef2fc98e2 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sat, 24 Oct 2026 11:32:47 -0700 Subject: [PATCH 2/7] feat(router): scope raw fixtures to their wire and skip non-object responses (#549) --- src/__tests__/raw-response-guard.test.ts | 82 ++++++++ src/__tests__/raw-router-wire-filter.test.ts | 194 ++++++++++++++++++ .../router-invalid-response-skip.test.ts | 68 ++++++ src/router.ts | 44 ++++ 4 files changed, 388 insertions(+) create mode 100644 src/__tests__/raw-response-guard.test.ts create mode 100644 src/__tests__/raw-router-wire-filter.test.ts create mode 100644 src/__tests__/router-invalid-response-skip.test.ts diff --git a/src/__tests__/raw-response-guard.test.ts b/src/__tests__/raw-response-guard.test.ts new file mode 100644 index 00000000..59e2b3f8 --- /dev/null +++ b/src/__tests__/raw-response-guard.test.ts @@ -0,0 +1,82 @@ +import { afterEach, describe, expect, expectTypeOf, it } from "vitest"; +import { isRawResponse } from "../helpers.js"; +import { LLMock } from "../llmock.js"; +import type { FixtureResponse } from "../types.js"; + +// The internal raw-response guard and fixture kind (#549 T2). + +/** Built from JSON so deliberately odd shapes need no cast. */ +function response(json: string): FixtureResponse { + return JSON.parse(json); +} + +const raw = '"raw":{"wire":"openai-chat","json":{}}'; + +describe("isRawResponse", () => { + it.each([ + ["a raw object", `{${raw}}`, true], + ["a raw object with events only", '{"raw":{"wire":"anthropic","events":[]}}', true], + ["an empty raw object (shape checked later)", '{"raw":{}}', true], + ["raw: null", '{"raw":null}', false], + ["raw: array", '{"raw":[]}', false], + ["raw: string", '{"raw":"x"}', false], + ["no raw key", '{"other":1}', false], + ])("%s -> %s", (_label, json, expected) => { + expect(isRawResponse(response(json))).toBe(expected); + }); + + it.each([ + "content", + "toolCalls", + "blocks", + "responsesBlocks", + "customToolCalls", + "error", + "embedding", + "image", + "images", + "audio", + "transcription", + "video", + "json", + "live", + ])("is false when the existing marker %s is present", (marker) => { + expect(isRawResponse(response(`{${raw},"${marker}":null}`))).toBe(false); + }); +}); + +describe("raw fixture kind", () => { + let mock: LLMock | undefined; + afterEach(async () => { + await mock?.stop(); + mock = undefined; + }); + + it('/__aimock/fixtures?include=fixtures reports responseKind "raw"', async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.addFixture({ + match: { userMessage: "x" }, + response: { raw: { wire: "openai-chat", json: { ok: true } } }, + }); + await mock.start(); + const res = await fetch(`${mock.url}/__aimock/fixtures?include=fixtures`); + const body = (await res.json()) as { fixtures: { responseKind: string }[] }; + expect(body.fixtures.map((f) => f.responseKind)).toEqual(["raw"]); + }); +}); + +describe("public surface", () => { + it("exports the RawResponse, RawWireId and WireId types", () => { + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf< + Exclude | "ollama-generate" + >(); + expectTypeOf().toEqualTypeOf< + import("../types.js").RawResponse + >(); + }); + + it("does not export isRawResponse", async () => { + expect("isRawResponse" in (await import("../index.js"))).toBe(false); + }); +}); diff --git a/src/__tests__/raw-router-wire-filter.test.ts b/src/__tests__/raw-router-wire-filter.test.ts new file mode 100644 index 00000000..02904a70 --- /dev/null +++ b/src/__tests__/raw-router-wire-filter.test.ts @@ -0,0 +1,194 @@ +import { describe, expect, it } from "vitest"; +import { LLMock } from "../llmock.js"; +import { matchFixture, matchFixtureDiagnostic } from "../router.js"; +import type { ChatCompletionRequest, Fixture, FixtureResponse } from "../types.js"; + +// The router's raw wire filter (#549 T3): a raw fixture is offered only to the +// handler whose MatchOptions.wire equals raw.wire. + +function req(text: string): ChatCompletionRequest { + return { model: "m", messages: [{ role: "user", content: text }], _endpointType: "chat" }; +} + +const rawFixture: Fixture = { + match: { userMessage: "q" }, + response: { raw: { wire: "openai-responses", events: [{ type: "x" }] } }, +}; +const typedFallback: Fixture = { match: { userMessage: "q" }, response: { content: "typed" } }; + +describe("raw wire filter", () => { + it("offers a raw fixture to the matching wire", () => { + const result = matchFixtureDiagnostic( + [rawFixture, typedFallback], + req("q"), + undefined, + undefined, + { + wire: "openai-responses", + }, + ); + expect(result.fixture).toBe(rawFixture); + }); + + it("skips a raw fixture for another wire, so the typed fallback wins", () => { + const result = matchFixtureDiagnostic( + [rawFixture, typedFallback], + req("q"), + undefined, + undefined, + { + wire: "openai-chat", + }, + ); + expect(result.fixture).toBe(typedFallback); + }); + + it("a raw fixture skipped for its wire is a plain miss, not a sequence/turn near-miss", () => { + const result = matchFixtureDiagnostic([rawFixture], req("q"), undefined, undefined, { + wire: "openai-chat", + }); + expect(result.fixture).toBeNull(); + expect(result.skippedBySequenceOrTurn).toBe(0); + expect(result.matchedBy).toBeUndefined(); + expect(result.turnIndexRelaxed).toBeUndefined(); + }); + + it("skips a raw fixture when no wire is given", () => { + expect(matchFixture([rawFixture, typedFallback], req("q"))).toBe(typedFallback); + expect(matchFixture([rawFixture], req("q"))).toBeNull(); + }); + + it("never filters a typed fixture with a stray raw key", () => { + const stray: Fixture = { + match: { userMessage: "q" }, + response: JSON.parse( + '{"content":"hi","raw":{"wire":"openai-responses","events":[]}}', + ) as FixtureResponse, + }; + for (const wire of [undefined, "openai-chat", "anthropic"] as const) { + const result = matchFixtureDiagnostic([stray], req("q"), undefined, undefined, { wire }); + expect(result.fixture).toBe(stray); + } + }); + + it("never filters a factory", () => { + const factory: Fixture = { + match: { userMessage: "q" }, + response: () => ({ raw: { wire: "openai-responses", json: {} } }), + }; + for (const wire of [undefined, "openai-chat", "openai-responses"] as const) { + expect( + matchFixtureDiagnostic([factory], req("q"), undefined, undefined, { wire }).fixture, + ).toBe(factory); + } + }); + + it("returns identical results for non-raw fixtures with and without a wire", () => { + const fixtures: Fixture[] = [ + { match: { userMessage: "a" }, response: { content: "A" } }, + { match: { userMessage: "b" }, response: { toolCalls: [{ name: "t", arguments: "{}" }] } }, + { match: { userMessage: "c", sequenceIndex: 1 }, response: { content: "C" } }, + ]; + for (const text of ["a", "b", "c", "none"]) { + const without = matchFixtureDiagnostic(fixtures, req(text), new Map()); + const withWire = matchFixtureDiagnostic(fixtures, req(text), new Map(), undefined, { + wire: "anthropic", + }); + expect(withWire).toEqual(without); + } + }); +}); + +describe("raw wire mismatch diagnostic", () => { + function sink() { + const messages: string[] = []; + return { messages, logger: { warn: (...args: unknown[]) => messages.push(args.join(" ")) } }; + } + + it("warns with both wires when a raw fixture matched everything but its wire", () => { + const { messages, logger } = sink(); + const result = matchFixtureDiagnostic([rawFixture], req("q"), undefined, undefined, { + wire: "openai-chat", + logger, + }); + expect(result.fixture).toBeNull(); + expect(result.skippedBySequenceOrTurn).toBe(0); + expect(messages).toHaveLength(1); + expect(messages[0]).toContain('raw.wire="openai-responses"'); + expect(messages[0]).toContain('request wire "openai-chat"'); + }); + + it("names a missing raw.wire", () => { + const { messages, logger } = sink(); + const noWire: Fixture = { + match: { userMessage: "q" }, + response: JSON.parse('{"raw":{"events":[]}}') as FixtureResponse, + }; + const result = matchFixtureDiagnostic([noWire], req("q"), undefined, undefined, { + wire: "anthropic", + logger, + }); + expect(result.fixture).toBeNull(); + expect(messages).toHaveLength(1); + expect(messages[0]).toContain("raw.wire is missing"); + expect(messages[0]).toContain('request wire "anthropic"'); + }); + + it("stays silent when another fixture matched", () => { + const { messages, logger } = sink(); + const result = matchFixtureDiagnostic( + [rawFixture, typedFallback], + req("q"), + undefined, + undefined, + { wire: "openai-chat", logger }, + ); + expect(result.fixture).toBe(typedFallback); + expect(messages).toHaveLength(0); + }); + + it("stays silent when the raw fixture's content does not match either", () => { + const { messages, logger } = sink(); + const result = matchFixtureDiagnostic([rawFixture], req("other"), undefined, undefined, { + wire: "openai-chat", + logger, + }); + expect(result.fixture).toBeNull(); + expect(messages).toHaveLength(0); + }); + + it("leaves the HTTP 404 status and body unchanged, and logs the warning", async () => { + const warnings: string[] = []; + const original = console.warn; + console.warn = (...args: unknown[]) => { + warnings.push(args.join(" ")); + }; + const mock = new LLMock({ port: 0, logLevel: "warn" }); + try { + mock.addFixture(rawFixture); + await mock.start(); + const res = await fetch(`${mock.url}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-4.1", messages: [{ role: "user", content: "q" }] }), + }); + expect(res.status).toBe(404); + const body: unknown = await res.json(); + expect(body).toEqual({ + error: { + message: "No fixture matched", + type: "invalid_request_error", + code: "no_fixture_match", + param: null, + }, + }); + } finally { + console.warn = original; + await mock.stop(); + } + const hits = warnings.filter((w) => w.includes("raw.wire")); + expect(hits).toHaveLength(1); + expect(hits[0]).toContain('raw.wire="openai-responses"'); + expect(hits[0]).toContain('request wire "openai-chat"'); + }); +}); diff --git a/src/__tests__/router-invalid-response-skip.test.ts b/src/__tests__/router-invalid-response-skip.test.ts new file mode 100644 index 00000000..d93e5ecd --- /dev/null +++ b/src/__tests__/router-invalid-response-skip.test.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { LLMock } from "../llmock.js"; +import { matchFixture } from "../router.js"; +import type { ChatCompletionRequest, Fixture } from "../types.js"; + +// A fixture added in code can carry a response that is not an object (null, a +// string). Fixture-file loading validates it, but addFixture() does not. When +// such a fixture does not match the request, the router must skip it and serve +// the next fixture, as it always has. Evaluating `"key" in response` on it +// throws a TypeError and fails every request that reaches it. + +function req(text: string): ChatCompletionRequest { + return { model: "m", messages: [{ role: "user", content: text }], _endpointType: "chat" }; +} + +/** A fixture whose response is not an object, built from JSON (no casts on the value). */ +function invalidFixture(responseJson: string): Fixture { + return JSON.parse(`{"match":{"userMessage":"other"},"response":${responseJson}}`) as Fixture; +} + +const valid: Fixture = { match: { userMessage: "q" }, response: { content: "typed" } }; + +describe("router skips a non-matching fixture whose response is not an object", () => { + for (const json of ["null", '"a string"', "42"]) { + it(`response ${json}`, () => { + expect(matchFixture([invalidFixture(json), valid], req("q"))).toBe(valid); + }); + } +}); + +describe("LLMock over HTTP serves the next fixture after a null-response fixture", () => { + let mock: LLMock | undefined; + afterEach(async () => { + await mock?.stop(); + mock = undefined; + }); + + const wires = [ + { + path: "/v1/chat/completions", + body: { model: "gpt-4.1", messages: [{ role: "user", content: "q" }] }, + }, + { + path: "/v1/messages", + body: { + model: "claude-sonnet-4-20250514", + max_tokens: 64, + messages: [{ role: "user", content: "q" }], + }, + }, + ] as const; + + for (const wire of wires) { + it(wire.path, async () => { + mock = new LLMock({ port: 0, logLevel: "silent" }); + mock.addFixture(invalidFixture("null")); + mock.addFixture(valid); + await mock.start(); + const res = await fetch(`${mock.url}${wire.path}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(wire.body), + }); + expect(res.status).toBe(200); + expect(await res.text()).toContain("typed"); + }); + } +}); diff --git a/src/router.ts b/src/router.ts index 292e7e0f..ded14738 100644 --- a/src/router.ts +++ b/src/router.ts @@ -8,6 +8,8 @@ import type { CustomToolDefinition, Fixture, ToolDefinition, + RawWireId, + WireId, } from "./types.js"; import { describeMatch, @@ -17,6 +19,7 @@ import { isTranscriptionResponse, isVideoResponse, isJSONResponse, + isRawResponse, isErrorResponse, isExtendedResponsesToolsList, markExtendedResponsesToolsRequest, @@ -239,6 +242,12 @@ export interface MatchOptions { * own level gate keeps a passing programmatic run (silent default) quiet. */ logger?: { warn(...args: unknown[]): void }; + /** + * @internal The wire of the calling LLM handler. Set by handlers, never + * read from the request body. A raw fixture (see `isRawResponse`) matches + * only when its `raw.wire` equals this; with no wire, raw never matches. + */ + wire?: WireId | RawWireId; } /** @@ -345,6 +354,9 @@ export function matchFixtureDiagnostic( // Every fixture whose content / shape predicates (and sequenceIndex gate) // pass. turnIndex is applied afterwards as a non-fatal disambiguator. const contentMatches: Fixture[] = []; + // Raw fixtures whose content matched but whose `raw.wire` is not this + // request's wire, kept only for the miss warning below. + const rawWireMisses: { fixture: Fixture; wire: unknown }[] = []; for (const fixture of fixtures) { const { match } = fixture; @@ -382,6 +394,17 @@ export function matchFixtureDiagnostic( } else if (liveResponse) { continue; } + // A raw fixture is wire-specific: never offer it to another wire's handler. + // The wire comes from the calling handler via MatchOptions, never from the + // request body: body fields are journaled, and a `requestTransform` copy + // may drop them. No wire given (direct matchFixture callers, non-LLM + // handlers) means raw never matches. Factories are checked at serve time. + // The skip is deferred until every content predicate has run, so a fixture + // that failed ONLY on its wire can be named in a miss warning below. + const rawWire = + typeof response !== "function" && isRawResponse(response) + ? { wire: response.raw.wire } + : undefined; if (match.endpoint !== undefined) { if (match.endpoint !== reqEndpoint) continue; // A declared endpoint used to END the shape discussion — the router @@ -631,6 +654,13 @@ export function matchFixtureDiagnostic( if (currentTurnHasToolResult(effective.messages) !== match.hasToolResult) continue; } + // A raw fixture for another wire is skipped here, after its content + // matched and before the sequence/turn gates (it is not a count skip). + if (rawWire !== undefined && rawWire.wire !== options?.wire) { + rawWireMisses.push({ fixture, wire: rawWire.wire }); + continue; + } + // At this point every SHAPE / CONTENT predicate above has passed, so this // fixture is a genuine CONTENT match for the request. The sequenceIndex and // turnIndex constraints below are POSITION state, not request shape. @@ -674,6 +704,20 @@ export function matchFixtureDiagnostic( } if (contentMatches.length === 0) { + // Near-miss: a raw fixture matched this request in every way except its + // `raw.wire`. Name both wires so a wrong or missing wire is not a silent 404. + if (options?.logger) { + for (const miss of rawWireMisses) { + const fixtureWire = + miss.wire === undefined ? "raw.wire is missing" : `raw.wire=${JSON.stringify(miss.wire)}`; + const requestWire = + options.wire === undefined ? "request wire is not set" : `request wire "${options.wire}"`; + options.logger.warn( + `raw fixture not served: ${describeMatch(miss.fixture.match, fixtures.indexOf(miss.fixture))} ` + + `matched the request except its wire (${fixtureWire}, ${requestWire})`, + ); + } + } return { fixture: null, skippedBySequenceOrTurn }; } From e10d7cf445ee2c1260a755ec99f5dfa9db7d3ca8 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sat, 24 Oct 2026 11:32:47 -0700 Subject: [PATCH 3/7] feat(loader): validate raw fixtures with a shared raw checker (#549) --- .../custom-tool-call-validation.test.ts | 2 +- .../fixture-validation-parity.test.ts | 138 +++++ src/__tests__/raw-fixture-validation.test.ts | 526 ++++++++++++++++++ src/fixture-loader.ts | 63 ++- src/raw-validation.ts | 267 +++++++++ 5 files changed, 993 insertions(+), 3 deletions(-) create mode 100644 src/__tests__/raw-fixture-validation.test.ts create mode 100644 src/raw-validation.ts diff --git a/src/__tests__/custom-tool-call-validation.test.ts b/src/__tests__/custom-tool-call-validation.test.ts index 0d6e9907..6a7a1605 100644 --- a/src/__tests__/custom-tool-call-validation.test.ts +++ b/src/__tests__/custom-tool-call-validation.test.ts @@ -585,7 +585,7 @@ describe("default responsesTools: validating doors treat the new keys as 1.44.0 findings: [ [ "error", - "response is not a recognized type (must have content, toolCalls, error, embedding, image, audio, transcription, video, json, or live)", + "response is not a recognized type (must have content, toolCalls, error, embedding, image, audio, transcription, video, json, live, or raw)", ], ], }, diff --git a/src/__tests__/fixture-validation-parity.test.ts b/src/__tests__/fixture-validation-parity.test.ts index 11442e3f..51611f47 100644 --- a/src/__tests__/fixture-validation-parity.test.ts +++ b/src/__tests__/fixture-validation-parity.test.ts @@ -14,6 +14,10 @@ * validator gives for a representative set of usage shapes, and that every * usage the recorder sanitizer writes passes that validator. */ +import { execFile } from "node:child_process"; +import { existsSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { entryToFixture, @@ -675,3 +679,137 @@ describe("recorder sanitizer output always passes the validator", () => { expect(sanitizeRecordedUsage(null)).toBeUndefined(); }); }); + +/** + * #549 — raw passthrough fixtures get the same findings on every validating + * entry path: the CLI's `--validate-on-load` file load, `addFixturesFromJSON` + * and `POST /__aimock/fixtures`. `on()` is deliberately not in the triple: it + * has never validated and must not start throwing (raw-no-regression.test.ts). + */ +describe("raw fixtures: validation parity across entry paths", () => { + const CLI_PATH = resolve(__dirname, "../../dist/cli.js"); + const SOURCES = ["../fixture-loader.ts", "../raw-validation.ts", "../helpers.ts"].map((s) => + resolve(__dirname, s), + ); + + /** The built CLI is a different program when older than the sources it bundles. */ + function assertBuiltCliIsCurrent(): void { + const builtAt = statSync(CLI_PATH).mtimeMs; + for (const source of SOURCES) { + if (builtAt < statSync(source).mtimeMs) { + throw new Error(`dist/cli.js is older than ${source}: run \`pnpm build\` first`); + } + } + } + + const VALID: [string, Record][] = [ + ["openai-chat json", { wire: "openai-chat", json: { ok: true } }], + [ + "anthropic events + headers", + { wire: "anthropic", events: [{ type: "message_stop" }], headers: { "request-id": "r1" } }, + ], + [ + "bedrock-converse events + status", + { wire: "bedrock-converse", events: [{ messageStop: {} }], json: {}, status: 429 }, + ], + ]; + const INVALID: [string, Record, string][] = [ + [ + "forbidden content-length header", + { wire: "openai-chat", json: {}, headers: { "content-length": "5" } }, + 'raw.headers["content-length"] is not allowed: aimock frames the body and sets this header itself', + ], + [ + "status 99", + { wire: "openai-chat", json: {}, status: 99 }, + "raw.status must be an integer from 200 to 599; got 99", + ], + [ + "done on cohere", + { wire: "cohere", events: [{ type: "message-end" }], done: true }, + "raw.done applies only to openai-chat (the only wire with a [DONE] terminal), not cohere", + ], + ]; + + function entries(raw: Record): FixtureFileEntry[] { + return JSON.parse(JSON.stringify([{ match: { userMessage: "parity" }, response: { raw } }])); + } + + /** Error messages from the CLI's --validate-on-load (exit 1 + `Fixture N: msg` lines). */ + async function cliErrors(raw: Record): Promise { + assertBuiltCliIsCurrent(); + const dir = mkdtempSync(join(tmpdir(), "aimock-raw-parity-")); + try { + const file = join(dir, "fixtures.json"); + writeFileSync(file, JSON.stringify({ fixtures: entries(raw) })); + const { stderr, code } = await new Promise<{ stderr: string; code: number | null }>( + (done) => { + // A valid file starts a server: stop it once it is listening. + const cp = execFile("node", [CLI_PATH, "-f", file, "-p", "0", "--validate-on-load"]); + let err = ""; + let out = ""; + cp.stderr?.on("data", (d) => (err += d)); + cp.stdout?.on("data", (d) => { + out += d; + if (/listening on/i.test(out)) cp.kill("SIGTERM"); + }); + cp.on("close", (code) => done({ stderr: err, code })); + }, + ); + const errors = [...stderr.matchAll(/Fixture 0: (.+)/g)].map((m) => m[1]); + expect(code !== 1 || /Validation failed/.test(stderr)).toBe(true); + return code === 1 ? errors : []; + } finally { + rmSync(dir, { recursive: true, force: true }); + } + } + + function addFromJSONErrors(raw: Record): string[] { + try { + new LLMock({ logLevel: "silent" }).addFixturesFromJSON(entries(raw)); + return []; + } catch (error) { + const text = error instanceof Error ? error.message : String(error); + const details: { message: string }[] = JSON.parse( + text.replace(/^Fixture validation failed: /, ""), + ); + return details.map((d) => d.message); + } + } + + async function controlApiErrors(raw: Record): Promise { + const mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + await mock.start(); + try { + const res = await fetch(`${mock.url}/__aimock/fixtures`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ fixtures: entries(raw) }), + }); + if (res.status === 200) return []; + expect(res.status).toBe(400); + const body = (await res.json()) as { details: { message: string }[] }; + return body.details.map((d) => d.message); + } finally { + await mock.stop(); + } + } + + it.skipIf(!existsSync(CLI_PATH)).each(VALID)( + "%s: accepted on all three paths", + async (_label, raw) => { + expect(await cliErrors(raw)).toEqual([]); + expect(addFromJSONErrors(raw)).toEqual([]); + expect(await controlApiErrors(raw)).toEqual([]); + }, + ); + + it.skipIf(!existsSync(CLI_PATH)).each(INVALID)( + "%s: the same error on all three paths", + async (_label, raw, message) => { + expect(await cliErrors(raw)).toEqual([message]); + expect(addFromJSONErrors(raw)).toEqual([message]); + expect(await controlApiErrors(raw)).toEqual([message]); + }, + ); +}); diff --git a/src/__tests__/raw-fixture-validation.test.ts b/src/__tests__/raw-fixture-validation.test.ts new file mode 100644 index 00000000..817309a7 --- /dev/null +++ b/src/__tests__/raw-fixture-validation.test.ts @@ -0,0 +1,526 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { + enableHeldFixtureMisbehavior, + entryToFixture, + validateFixtures, +} from "../fixture-loader.js"; +import { LLMock } from "../llmock.js"; +import { checkRawResponse } from "../raw-validation.js"; +import type { FixtureFileEntry, RawResponse, ResponseFactory } from "../types.js"; + +// Load-time checks for raw passthrough fixtures (#549 T4). + +const WIRES = + "openai-responses, openai-chat, anthropic, gemini, gemini-interactions, cohere, ollama, ollama-generate, bedrock-invoke, bedrock-converse"; + +/** Loader findings for one file entry given as JSON (deliberately invalid shapes need no cast). */ +function load(json: string) { + const entry: FixtureFileEntry = JSON.parse(json); + return validateFixtures([entryToFixture(entry)]).map((r) => [r.severity, r.message]); +} + +function check(json: string) { + return checkRawResponse(JSON.parse(json)).map((f) => [f.severity, f.message]); +} + +describe("checkRawResponse", () => { + it.each([ + ['{"wire":"openai-chat","json":{}}'], + ['{"wire":"openai-chat","events":[{"choices":[]}],"done":false}'], + ['{"wire":"openai-responses","events":[{"type":"response.created"}]}'], + ['{"wire":"bedrock-converse","events":[{"messageStart":{"role":"assistant"}}]}'], + ['{"wire":"ollama","events":[{"done":true}],"json":{},"status":429}'], + ['{"wire":"ollama-generate","json":{"response":"x","done":true}}'], + [ + '{"wire":"gemini","json":{},"headers":{"x-request-id":"r1","Content-Type":"application/json"}}', + ], + ])("accepts %s", (raw) => { + expect(check(raw)).toEqual([]); + }); + + it.each([ + ['"x"', "raw must be an object"], + ["[]", "raw must be an object"], + [ + '{"wire":"openai-realtime","json":{}}', + `raw.wire must be one of ${WIRES}; got "openai-realtime"`, + ], + ['{"json":{}}', `raw.wire must be one of ${WIRES}; got undefined`], + ['{"wire":"openai-chat"}', 'raw needs "events" (streaming), "json" (non-streaming), or both'], + ['{"wire":"openai-chat","events":{}}', "raw.events must be an array"], + ['{"wire":"openai-chat","events":[1]}', "raw.events[0] must be a JSON object"], + ['{"wire":"gemini","events":[{},[]]}', "raw.events[1] must be a JSON object"], + [ + '{"wire":"openai-chat","json":{},"status":99}', + "raw.status must be an integer from 200 to 599; got 99", + ], + [ + '{"wire":"openai-chat","json":{},"status":600}', + "raw.status must be an integer from 200 to 599; got 600", + ], + [ + '{"wire":"openai-chat","json":{},"status":200.5}', + "raw.status must be an integer from 200 to 599; got 200.5", + ], + [ + '{"wire":"openai-chat","json":{},"status":"200"}', + 'raw.status must be an integer from 200 to 599; got "200"', + ], + [ + '{"wire":"openai-chat","json":{},"headers":[]}', + "raw.headers must be an object of string values", + ], + ['{"wire":"openai-chat","json":{},"headers":{"x-a":1}}', 'raw.headers["x-a"] must be a string'], + [ + '{"wire":"openai-chat","json":{},"headers":{"bad name":"v"}}', + 'raw.headers["bad name"] is not a valid header name', + ], + [ + '{"wire":"openai-chat","json":{},"headers":{"x-a":"line\\nbreak"}}', + 'raw.headers["x-a"] has an invalid header value', + ], + ...["content-length", "Transfer-Encoding", "CONNECTION", "content-encoding"].map((name) => [ + `{"wire":"openai-chat","json":{},"headers":{"${name}":"5"}}`, + `raw.headers["${name}"] is not allowed: aimock frames the body and sets this header itself`, + ]), + [ + '{"wire":"cohere","json":{},"done":true}', + "raw.done applies only to openai-chat (the only wire with a [DONE] terminal), not cohere", + ], + [ + '{"wire":"openai-responses","events":[{"delta":"x"}]}', + 'raw.events[0] needs a string "type" on openai-responses', + ], + [ + '{"wire":"anthropic","events":[{"type":1}]}', + 'raw.events[0] needs a string "type" on anthropic', + ], + [ + '{"wire":"cohere","events":[{"type":"message-start"},{}]}', + 'raw.events[1] needs a string "type" on cohere', + ], + [ + '{"wire":"bedrock-converse","events":[{"a":{},"b":{}}]}', + "raw.events[0] must have exactly one top-level key (the event type) on bedrock-converse", + ], + [ + '{"wire":"bedrock-converse","events":[{}]}', + "raw.events[0] must have exactly one top-level key (the event type) on bedrock-converse", + ], + [ + '{"wire":"bedrock-converse","events":[{"messageStop":"end_turn"}]}', + "raw.events[0].messageStop must be a JSON object (the frame payload) on bedrock-converse", + ], + ])("rejects %s", (raw, message) => { + expect(check(raw)).toEqual([["error", message]]); + }); + + it("warns on empty events", () => { + expect(check('{"wire":"openai-chat","events":[]}')).toEqual([ + ["warning", "raw.events is empty; a streaming request gets no frames"], + ]); + }); + + it("does not need a type on data-only wires", () => { + expect(check('{"wire":"openai-chat","events":[{}]}')).toEqual([]); + expect(check('{"wire":"gemini-interactions","events":[{"event_type":"x"}]}')).toEqual([]); + }); +}); + +describe("validateFixtures with raw fixtures", () => { + it("accepts a valid raw fixture with no findings", () => { + expect( + load('{"match":{"userMessage":"x"},"response":{"raw":{"wire":"openai-chat","json":{}}}}'), + ).toEqual([]); + }); + + it("reports the checker's findings for an invalid raw fixture", () => { + expect( + load( + '{"match":{"userMessage":"x"},"response":{"raw":{"wire":"openai-chat","json":{},"headers":{"content-length":"5"}}}}', + ), + ).toEqual([ + [ + "error", + 'raw.headers["content-length"] is not allowed: aimock frames the body and sets this header itself', + ], + ]); + }); + + it.each([ + ['"chunkSize":3', "chunkSize is ignored for raw fixtures: each event is one frame"], + ['"openRouterProcessing":true', "openRouterProcessing is ignored for raw fixtures"], + ])("warns when a raw fixture sets %s", (field, message) => { + expect( + load( + `{"match":{"userMessage":"x"},"response":{"raw":{"wire":"openai-chat","json":{}}},${field}}`, + ), + ).toEqual([["warning", message]]); + }); + + it("names raw in the unrecognized-type message", () => { + expect(load('{"match":{"userMessage":"x"},"response":{"nothing":1}}')).toEqual([ + [ + "error", + "response is not a recognized type (must have content, toolCalls, error, embedding, image, audio, transcription, video, json, live, or raw)", + ], + ]); + }); + + it("gives a typed fixture with a stray raw key no new finding", () => { + expect( + load( + '{"match":{"userMessage":"x"},"response":{"content":"hi","raw":{"wire":"nope","headers":{"content-length":"1"}}},"chunkSize":3}', + ), + ).toEqual([]); + }); +}); + +// ─── Round-1 validation gaps: each checked at load and at serve time ──────── + +const RAW_INVALID = "aimock_raw_fixture_invalid"; + +let servers: LLMock[] = []; + +afterEach(async () => { + await Promise.all(servers.map((s) => s.stop())); + servers = []; +}); + +async function server(misbehavior = false): Promise { + const s = new LLMock({ + port: 0, + host: "127.0.0.1", + logLevel: "silent", + ...(misbehavior ? { enableMisbehavior: true } : {}), + }); + servers.push(s); + return s; +} + +const CHAT_PATH = "/v1/chat/completions"; +const MESSAGES_PATH = "/v1/messages"; + +function chatBody(text: string, stream: boolean) { + return { model: "m", stream, messages: [{ role: "user", content: text }] }; +} + +function messagesBody(text: string) { + return { model: "m", max_tokens: 10, stream: true, messages: [{ role: "user", content: text }] }; +} + +async function post(url: string, path: string, body: unknown) { + const res = await fetch(url + path, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + return { status: res.status, text: await res.text() }; +} + +/** Load one file entry (JSON) and return its findings. */ +function loadEntry(json: string) { + return validateFixtures([entryToFixture(JSON.parse(json))]).map((r) => [r.severity, r.message]); +} + +describe("raw.done must be a boolean (round-1 #1)", () => { + const raw = '{"wire":"openai-chat","events":[{"x":1}],"done":"false"}'; + const message = 'raw.done must be a boolean; got "false"'; + + it("is an error from the checker and the loader", () => { + expect(check(raw)).toEqual([["error", message]]); + expect(loadEntry(`{"match":{"userMessage":"x"},"response":{"raw":${raw}}}`)).toEqual([ + ["error", message], + ]); + }); + + it("answers the coded 500 at serve time, with no [DONE]", async () => { + const s = await server(); + s.on({ userMessage: "x" }, JSON.parse(`{"raw":${raw}}`)); + await s.start(); + const got = await post(s.url, CHAT_PATH, chatBody("x", true)); + expect(got.status).toBe(500); + expect(got.text).toContain(RAW_INVALID); + expect(got.text).toContain(message.replace(/"/g, '\\"')); + expect(got.text).not.toContain("[DONE]"); + }); +}); + +describe("an event type with a line break (round-1 #2)", () => { + const raw = '{"wire":"anthropic","events":[{"type":"ping\\ndata: {\\"injected\\":true}"}]}'; + const message = "raw.events[0].type must not contain a line break (CR or LF) on anthropic"; + + it.each([ + ["LF", '"a\\nb"'], + ["CR", '"a\\rb"'], + ])("is an error from the checker and the loader (%s)", (_, type) => { + const r = `{"wire":"anthropic","events":[{"type":${type}}]}`; + expect(check(r)).toEqual([["error", message]]); + expect(loadEntry(`{"match":{"userMessage":"x"},"response":{"raw":${r}}}`)).toEqual([ + ["error", message], + ]); + }); + + it("answers the coded 500 at serve time and writes no injected frame", async () => { + const s = await server(); + s.on({ userMessage: "x" }, JSON.parse(`{"raw":${raw}}`)); + await s.start(); + const got = await post(s.url, MESSAGES_PATH, messagesBody("x")); + expect(got.status).toBe(500); + expect(got.text).toContain(RAW_INVALID); + expect(got.text).not.toContain("event: ping"); + }); +}); + +describe("status 204 or 304 with a body (round-1 #3)", () => { + it.each([204, 304])("%i is an error from the checker and the loader", (status) => { + const raw = `{"wire":"openai-chat","json":{"x":1},"status":${status}}`; + const message = `raw.status ${status} cannot carry a body, so "events" and "json" would be dropped; use a status that allows a body`; + expect(check(raw)).toEqual([["error", message]]); + expect(loadEntry(`{"match":{"userMessage":"x"},"response":{"raw":${raw}}}`)).toEqual([ + ["error", message], + ]); + }); + + it("answers the coded 500 at serve time instead of an empty 204", async () => { + const s = await server(); + s.on( + { userMessage: "x" }, + JSON.parse('{"raw":{"wire":"openai-chat","json":{"x":1},"status":204}}'), + ); + await s.start(); + const got = await post(s.url, CHAT_PATH, chatBody("x", false)); + expect(got.status).toBe(500); + expect(got.text).toContain(RAW_INVALID); + }); +}); + +describe("an unknown key inside raw (round-1 #4)", () => { + const raw = '{"wire":"openai-chat","evnets":[{"x":1}],"json":{"x":1}}'; + const message = + 'raw["evnets"] is ignored: not a raw field (wire, events, json, status, headers, done)'; + + it("is a warning from the checker and the loader", () => { + expect(check(raw)).toEqual([["warning", message]]); + expect(loadEntry(`{"match":{"userMessage":"x"},"response":{"raw":${raw}}}`)).toEqual([ + ["warning", message], + ]); + }); + + it("still serves at serve time (a warning, not an error)", async () => { + const s = await server(); + s.on({ userMessage: "x" }, JSON.parse(`{"raw":${raw}}`)); + await s.start(); + const got = await post(s.url, CHAT_PATH, chatBody("x", false)); + expect(got.status).toBe(200); + expect(JSON.parse(got.text)).toEqual({ x: 1 }); + }); +}); + +describe("raw combined with another response-type key (round-1 #5)", () => { + it.each([ + ["responsesBlocks", '[{"type":"text","text":"t"}]'], + ["customToolCalls", '[{"name":"t","input":"i"}]'], + ])("{raw, %s} is a load error", (key, value) => { + expect( + loadEntry( + `{"match":{"userMessage":"x"},"response":{"raw":{"wire":"openai-chat","json":{}},"${key}":${value}}}`, + ), + ).toEqual([ + ["error", `raw cannot be combined with "${key}": a raw fixture has no other response type`], + ]); + }); + + it("answers the coded 500 at serve time for a fixture that skipped load validation", async () => { + const s = await server(); + s.addFixture( + entryToFixture( + JSON.parse( + '{"match":{"userMessage":"x"},"response":{"raw":{"wire":"openai-chat","json":{"x":1}},"customToolCalls":[{"name":"t","input":"i"}]}}', + ), + ), + ); + await s.start(); + const got = await post(s.url, CHAT_PATH, chatBody("x", false)); + expect(got.status).toBe(500); + expect(got.text).toContain(RAW_INVALID); + expect(got.text).toContain("raw cannot be combined with"); + }); +}); + +describe("a raw fixture with its own misbehavior (round-1 #6)", () => { + const message = "misbehavior does not apply to raw fixtures"; + const entry = + '{"match":{"userMessage":"x"},"response":{"raw":{"wire":"openai-chat","events":[{"x":1}]}},"misbehavior":"empty-response"}'; + + it("is a load error from validateFixtures", () => { + expect(loadEntry(entry)).toEqual([["error", message]]); + }); + + it("is a load error naming raw fixtures when misbehavior is enabled", () => { + const fixture = entryToFixture(JSON.parse(entry)); + expect(() => enableHeldFixtureMisbehavior(fixture)).toThrow(message); + }); + + it("answers the coded 500, not a 501, for a factory raw fixture with misbehavior", async () => { + const s = await server(true); + const response: ResponseFactory = () => ({ + raw: { wire: "openai-chat", events: [{ x: 1 }] }, + }); + s.on({ userMessage: "x" }, response, { misbehavior: "empty-response" }); + await s.start(); + const got = await post(s.url, CHAT_PATH, chatBody("x", true)); + expect(got.status).toBe(500); + expect(got.text).toContain(RAW_INVALID); + expect(got.text).toContain(message); + }); + + it("answers the coded 500 for a static raw fixture with misbehavior", async () => { + const s = await server(); + s.on( + { userMessage: "x" }, + { raw: { wire: "openai-chat", events: [{ x: 1 }] } }, + { misbehavior: "empty-response" }, + ); + await s.start(); + const got = await post(s.url, CHAT_PATH, chatBody("x", true)); + expect(got.status).toBe(500); + expect(got.text).toContain(RAW_INVALID); + expect(got.text).toContain(message); + }); +}); + +// ─── Consumer-probe gaps: events must be plain JSON the encoders can frame ── + +/** Findings for a code-built raw fixture (shapes JSON cannot express). */ +function loadCode(raw: RawResponse["raw"]) { + return validateFixtures([{ match: { userMessage: "x" }, response: { raw } }]).map((r) => [ + r.severity, + r.message, + ]); +} + +/** A code-built raw fixture served on `path`: status and body text. */ +async function serveCode(raw: RawResponse["raw"], path: string, body: unknown) { + const s = await server(); + s.addFixture({ match: { userMessage: "x" }, response: { raw } }); + await s.start(); + return post(s.url, path, body); +} + +describe("sparse or undefined events (probe #7)", () => { + // eslint-disable-next-line no-sparse-arrays + const sparse = [{ x: 1 }, , { x: 2 }] as unknown as Record[]; + const withUndefined = [{ x: 1 }, undefined, { x: 2 }] as unknown as Record[]; + + it.each([ + ["a hole", sparse], + ["an undefined entry", withUndefined], + ])("%s is an error from the checker and the loader", (_, events) => { + const raw: RawResponse["raw"] = { wire: "openai-chat", events }; + const expected = [["error", "raw.events[1] must be a JSON object"]]; + expect(checkRawResponse(raw).map((f) => [f.severity, f.message])).toEqual(expected); + expect(loadCode(raw)).toEqual(expected); + }); + + it("a nested hole is an error", () => { + // eslint-disable-next-line no-sparse-arrays + const raw: RawResponse["raw"] = { wire: "openai-chat", events: [{ a: [1, , 3] }] }; + expect(loadCode(raw)).toEqual([ + ["error", 'raw.events[0]["a"][1] must not be a hole in a sparse array'], + ]); + }); + + it("answers the coded 500 at serve time, with no data: undefined frame", async () => { + const got = await serveCode( + { wire: "openai-chat", events: sparse }, + CHAT_PATH, + chatBody("x", true), + ); + expect(got.status).toBe(500); + expect(got.text).toContain(RAW_INVALID); + expect(got.text).not.toContain("data: undefined"); + }); +}); + +describe("non-plain event values (probe #8)", () => { + it.each([ + [ + "a Date event", + { wire: "ollama", events: [new Date(0) as unknown as Record] }, + "raw.events[0] must be a JSON object", + ], + [ + "a Map event", + { wire: "ollama", events: [new Map() as unknown as Record] }, + "raw.events[0] must be a JSON object", + ], + [ + "a nested Date", + { wire: "ollama", events: [{ at: new Date(0) }] }, + 'raw.events[0]["at"] must be a plain JSON object', + ], + [ + "a nested function", + { wire: "ollama", events: [{ f: () => 1 }] }, + 'raw.events[0]["f"] must be a JSON value; got function', + ], + [ + "a NaN", + { wire: "ollama", events: [{ n: Number.NaN }] }, + 'raw.events[0]["n"] must be a finite number', + ], + ] satisfies [string, RawResponse["raw"], string][])( + "%s is an error from the checker and the loader", + (_, raw, message) => { + expect(checkRawResponse(raw).map((f) => [f.severity, f.message])).toEqual([ + ["error", message], + ]); + expect(loadCode(raw)).toEqual([["error", message]]); + }, + ); + + it("a cyclic event is an error, not a crash", () => { + const event: Record = { a: 1 }; + event.self = event; + expect(loadCode({ wire: "ollama", events: [event] })).toEqual([ + ["error", 'raw.events[0]["self"] must not contain itself'], + ]); + }); + + it("answers the coded 500 at serve time instead of a bare-string Date frame", async () => { + const got = await serveCode( + { wire: "ollama", events: [new Date(0) as unknown as Record] }, + "/api/chat", + { model: "llama3.1", messages: [{ role: "user", content: "x" }] }, + ); + expect(got.status).toBe(500); + expect(got.text).toContain(RAW_INVALID); + expect(got.text).not.toContain("1970-01-01"); + }); +}); + +describe("a bedrock-converse event type over the event-stream header limit (probe #9)", () => { + const long = "k".repeat(65_536); + const message = + "raw.events[0] has an event type key over 65535 bytes, the event-stream header limit, on bedrock-converse"; + + it("is an error from the checker and the loader; 65535 bytes is accepted", () => { + const raw: RawResponse["raw"] = { wire: "bedrock-converse", events: [{ [long]: {} }] }; + expect(checkRawResponse(raw).map((f) => [f.severity, f.message])).toEqual([["error", message]]); + expect(loadCode(raw)).toEqual([["error", message]]); + expect( + checkRawResponse({ wire: "bedrock-converse", events: [{ [long.slice(1)]: {} }] }), + ).toEqual([]); + }); + + it("answers the coded 500 at serve time instead of a socket hang-up", async () => { + const got = await serveCode( + { wire: "bedrock-converse", events: [{ [long]: {} }] }, + "/model/anthropic.claude-3-haiku-20240307-v1:0/converse-stream", + { messages: [{ role: "user", content: [{ text: "x" }] }] }, + ); + expect(got.status).toBe(500); + expect(got.text).toContain(RAW_INVALID); + }); +}); diff --git a/src/fixture-loader.ts b/src/fixture-loader.ts index b9707db6..840fa787 100644 --- a/src/fixture-loader.ts +++ b/src/fixture-loader.ts @@ -10,6 +10,7 @@ import type { FixtureFileResponse, FixtureMatch, FixtureResponse, + RawResponse, ResponseOverrides, } from "./types.js"; import { @@ -24,11 +25,13 @@ import { isTranscriptionResponse, isVideoResponse, isJSONResponse, + isRawResponse, fixtureListTarget, fixtureCitationsProblem, fixtureFilesProblem, resolveServedBlockOutcome, } from "./helpers.js"; +import { checkRawResponse, type RawCheckContext } from "./raw-validation.js"; import type { Logger } from "./logger.js"; import { CHAOS_FIELDS, CHAOS_FIELD_NAMES, parseChaosField } from "./chaos.js"; import { Message, build, fixed, jsonText, msg, quote, within } from "./message-text.js"; @@ -1203,16 +1206,41 @@ export function validateFixtures( !isTranscriptionResponse(response) && !isVideoResponse(response) && !isJSONResponse(response) && - !isLiveResponse(response) + !isLiveResponse(response) && + !isRawResponse(response) ) { results.push({ severity: "error", fixtureIndex: i, message: - "response is not a recognized type (must have content, toolCalls, error, embedding, image, audio, transcription, video, json, or live)", + "response is not a recognized type (must have content, toolCalls, error, embedding, image, audio, transcription, video, json, live, or raw)", }); } + // Raw passthrough checks. isRawResponse is false whenever another type's + // marker is present, so a typed fixture with a stray `raw` key gets no + // finding here, exactly as before raw fixtures existed. + if (isRawResponse(response)) { + const context = rawFixtureContext(original, response); + for (const finding of checkRawResponse(response.raw, context)) { + results.push({ severity: finding.severity, fixtureIndex: i, message: finding.message }); + } + if (f.chunkSize !== undefined) { + results.push({ + severity: "warning", + fixtureIndex: i, + message: "chunkSize is ignored for raw fixtures: each event is one frame", + }); + } + if (f.openRouterProcessing !== undefined) { + results.push({ + severity: "warning", + fixtureIndex: i, + message: "openRouterProcessing is ignored for raw fixtures", + }); + } + } + // When a non-empty ordered `blocks` array is present, the builders stream // `blocks` and IGNORE the legacy `content` mirror (see validateBlocks's // divergence note + isCombinedFixtureResponse's BLOCKS-ONLY clause). @@ -2259,6 +2287,15 @@ const misbehaviorEnabledFixtures = new WeakSet(); export function enableHeldFixtureMisbehavior(fixture: Fixture): void { const held = heldFixtureMisbehavior.get(fixture); if (held === undefined) return; + // A raw fixture's own misbehavior is a fixture error, named as such + // (checkRawResponse reports the same at load and at serve time). + if (typeof fixture.response !== "function" && isRawResponse(fixture.response)) { + throw new MisbehaviorConfigError({ + rule: "misbehavior/not-applicable", + file: held.file, + detail: `${held.path}: misbehavior does not apply to raw fixtures`, + }); + } const parsed = parseMisbehavior(held.value, held.path); if (parsed.ok) fixture.misbehavior = parsed.config; const issue = parsed.ok ? validateFixtureMisbehavior(fixture, held.path) : parsed.issue; @@ -2269,6 +2306,28 @@ export function enableHeldFixtureMisbehavior(fixture: Fixture): void { misbehaviorEnabledFixtures.add(fixture); } +/** + * @internal The facts about a raw fixture that {@link checkRawResponse} checks + * beyond `raw` itself, for `served`, the response as this server reads it. + * Load validation and serve time both use it, so they agree: + * + * - `otherResponseKeys`: top-level keys of the fixture's response as written + * that `served` lacks. Only `customToolCalls` / `responsesBlocks` are ever + * dropped (on a legacy server), and a fixture that writes either next to + * `raw` is not a raw fixture to the router. + * - `fixtureMisbehavior`: the fixture sets its own `misbehavior`, recognized + * or still held. + */ +export function rawFixtureContext(fixture: Fixture, served: RawResponse): RawCheckContext { + const source = responsesToolsViewSource(fixture); + const written = fixture.response; + return { + otherResponseKeys: + typeof written === "function" ? [] : Object.keys(written).filter((key) => !(key in served)), + fixtureMisbehavior: fixture.misbehavior !== undefined || heldFixtureMisbehavior.has(source), + }; +} + /** * @internal Mark a fixture whose `misbehavior` a server with misbehavior * enabled has recognized, so `validateFixtures` checks it. diff --git a/src/raw-validation.ts b/src/raw-validation.ts new file mode 100644 index 00000000..059a9ae1 --- /dev/null +++ b/src/raw-validation.ts @@ -0,0 +1,267 @@ +/** + * @internal Shape checks for raw passthrough responses (#549). One pure + * checker serves load-time validation (`validateFixtures`) and the serve-time + * check, which runs on every raw fixture served (including factories and + * `on()` fixtures, which skip load validation). Not exported from the package + * entry point. + */ +import { validateHeaderName, validateHeaderValue } from "node:http"; +import type { RawResponse, RawWireId } from "./types.js"; + +export interface RawFinding { + severity: "error" | "warning"; + message: string; +} + +/** Every {@link RawWireId}; the `satisfies` + exhaustiveness check keep it in sync. */ +export const RAW_WIRE_IDS = [ + "openai-responses", + "openai-chat", + "anthropic", + "gemini", + "gemini-interactions", + "cohere", + "ollama", + "ollama-generate", + "bedrock-invoke", + "bedrock-converse", +] as const satisfies readonly RawWireId[]; + +// Compile-time: fails if RawWireId gains a member RAW_WIRE_IDS lacks. +type MissingRawWire = Exclude; +const rawWireIdsComplete: MissingRawWire extends never ? true : never = true; +void rawWireIdsComplete; + +export function isRawWireId(value: unknown): value is RawWireId { + return typeof value === "string" && (RAW_WIRE_IDS as readonly string[]).includes(value); +} + +/** Headers that break or misdescribe a body aimock frames itself. */ +const FORBIDDEN_HEADERS = new Set([ + "content-length", + "transfer-encoding", + "connection", + "content-encoding", +]); + +/** Wires whose framing takes the SSE event name from the event's string `type`. */ +const TYPE_NAMED_WIRES: ReadonlySet = new Set([ + "openai-responses", + "anthropic", + "cohere", +]); + +/** The fields of `raw`; any other key is reported as ignored. */ +const RAW_FIELDS: readonly string[] = [ + "wire", + "events", + "json", + "status", + "headers", + "done", +] satisfies (keyof RawResponse["raw"])[]; + +/** Statuses whose replies carry no body: Node drops anything written for them. */ +const BODYLESS_STATUSES: ReadonlySet = new Set([204, 304]); + +/** + * The longest AWS event-stream header value, in bytes. The encoder writes the + * length as a uint16 (aws-event-stream.ts), so it cannot frame a longer one. + */ +const EVENT_STREAM_HEADER_VALUE_MAX_BYTES = 0xffff; + +/** A plain object, as JSON.parse makes one: not an array, Date, Map or class instance. */ +function isPlainObject(value: unknown): value is Record { + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + const proto: unknown = Object.getPrototypeOf(value); + return proto === Object.prototype || proto === null; +} + +/** + * The first part of `value` that JSON.parse cannot produce (a sparse-array + * hole, `undefined`, a non-finite number, a function, a Date or other + * non-plain object, a cycle), as `[path, problem]`. Undefined for plain JSON. + */ +function jsonProblem( + value: unknown, + path: string, + ancestors: readonly object[] = [], +): [string, string] | undefined { + if (value === null || typeof value === "string" || typeof value === "boolean") return undefined; + if (typeof value === "number") { + return Number.isFinite(value) ? undefined : [path, "must be a finite number"]; + } + if (typeof value !== "object") return [path, `must be a JSON value; got ${typeof value}`]; + if (ancestors.includes(value)) return [path, "must not contain itself"]; + const inner = [...ancestors, value]; + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) { + if (!(i in value)) return [`${path}[${i}]`, "must not be a hole in a sparse array"]; + const problem = jsonProblem(value[i], `${path}[${i}]`, inner); + if (problem) return problem; + } + return undefined; + } + if (!isPlainObject(value)) return [path, "must be a plain JSON object"]; + for (const [key, item] of Object.entries(value)) { + const problem = jsonProblem(item, `${path}[${JSON.stringify(key)}]`, inner); + if (problem) return problem; + } + return undefined; +} + +/** What the caller knows about the fixture around `raw`. */ +export interface RawCheckContext { + /** + * Response-type keys written next to `raw`, such as `customToolCalls` or + * `responsesBlocks` (a legacy server drops these two before serving). + */ + otherResponseKeys?: readonly string[]; + /** Whether the fixture sets its own `misbehavior`. */ + fixtureMisbehavior?: boolean; +} + +/** + * Check a raw response's `raw` value and the fixture facts in `context`. + * Returns every finding. The fixture is servable when no finding has severity + * `"error"`: an empty array or one with only warnings. Pure: no logging, no + * throwing. Load validation and serve time both call it, so they agree on + * what a valid raw fixture is. + */ +export function checkRawResponse(raw: unknown, context: RawCheckContext = {}): RawFinding[] { + const findings: RawFinding[] = []; + const error = (message: string) => findings.push({ severity: "error", message }); + + for (const key of context.otherResponseKeys ?? []) { + error( + `raw cannot be combined with ${JSON.stringify(key)}: a raw fixture has no other response type`, + ); + } + if (context.fixtureMisbehavior) { + error("misbehavior does not apply to raw fixtures"); + } + + if (!isPlainObject(raw)) { + error("raw must be an object"); + return findings; + } + + for (const key of Object.keys(raw)) { + if (!RAW_FIELDS.includes(key)) { + findings.push({ + severity: "warning", + message: `raw[${JSON.stringify(key)}] is ignored: not a raw field (${RAW_FIELDS.join(", ")})`, + }); + } + } + + const wire = isRawWireId(raw.wire) ? raw.wire : undefined; + if (wire === undefined) { + error(`raw.wire must be one of ${RAW_WIRE_IDS.join(", ")}; got ${JSON.stringify(raw.wire)}`); + } + + const events = raw.events; + if (events === undefined && raw.json === undefined) { + error('raw needs "events" (streaming), "json" (non-streaming), or both'); + } + if (events !== undefined) { + if (!Array.isArray(events)) { + error("raw.events must be an array"); + } else { + if (events.length === 0) { + findings.push({ + severity: "warning", + message: "raw.events is empty; a streaming request gets no frames", + }); + } + // An index loop, not forEach: forEach skips the holes of a sparse array. + for (let i = 0; i < events.length; i++) { + const event: unknown = events[i]; + if (!isPlainObject(event)) { + error(`raw.events[${i}] must be a JSON object`); + continue; + } + const problem = jsonProblem(event, `raw.events[${i}]`); + if (problem) { + error(`${problem[0]} ${problem[1]}`); + continue; + } + if (wire !== undefined && TYPE_NAMED_WIRES.has(wire)) { + if (typeof event.type !== "string") { + error(`raw.events[${i}] needs a string "type" on ${wire}`); + } else if (/[\r\n]/.test(event.type)) { + // The type is written as the SSE `event:` line; a line break + // would start another SSE field or frame. + error(`raw.events[${i}].type must not contain a line break (CR or LF) on ${wire}`); + } + } + if (wire === "bedrock-converse") { + const keys = Object.keys(event); + if (keys.length !== 1) { + error( + `raw.events[${i}] must have exactly one top-level key (the event type) on bedrock-converse`, + ); + } else if (Buffer.byteLength(keys[0], "utf8") > EVENT_STREAM_HEADER_VALUE_MAX_BYTES) { + error( + `raw.events[${i}] has an event type key over ${EVENT_STREAM_HEADER_VALUE_MAX_BYTES} bytes, the event-stream header limit, on bedrock-converse`, + ); + } else if (!isPlainObject(event[keys[0]])) { + error( + `raw.events[${i}].${keys[0]} must be a JSON object (the frame payload) on bedrock-converse`, + ); + } + } + } + } + } + + if (raw.status !== undefined) { + const status = raw.status; + if (typeof status !== "number" || !Number.isInteger(status) || status < 200 || status > 599) { + error(`raw.status must be an integer from 200 to 599; got ${JSON.stringify(status)}`); + } else if (BODYLESS_STATUSES.has(status)) { + error( + `raw.status ${status} cannot carry a body, so "events" and "json" would be dropped; use a status that allows a body`, + ); + } + } + + if (raw.headers !== undefined) { + if (!isPlainObject(raw.headers)) { + error("raw.headers must be an object of string values"); + } else { + for (const [name, value] of Object.entries(raw.headers)) { + const label = `raw.headers[${JSON.stringify(name)}]`; + if (typeof value !== "string") { + error(`${label} must be a string`); + continue; + } + try { + validateHeaderName(name); + } catch { + error(`${label} is not a valid header name`); + continue; + } + try { + validateHeaderValue(name, value); + } catch { + error(`${label} has an invalid header value`); + continue; + } + if (FORBIDDEN_HEADERS.has(name.toLowerCase())) { + error(`${label} is not allowed: aimock frames the body and sets this header itself`); + } + } + } + } + + if (raw.done !== undefined && typeof raw.done !== "boolean") { + error(`raw.done must be a boolean; got ${JSON.stringify(raw.done)}`); + } else if (raw.done !== undefined && wire !== undefined && wire !== "openai-chat") { + error( + `raw.done applies only to openai-chat (the only wire with a [DONE] terminal), not ${wire}`, + ); + } + + return findings; +} From 4267deb393f33f289c9fb4b5f0ae788c6704504c Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sat, 24 Oct 2026 11:32:47 -0700 Subject: [PATCH 4/7] feat: per-wire framing and serveRawResponse for raw fixtures (#549) --- src/__tests__/raw-framing.test.ts | 273 +++++++++++++++++++ src/__tests__/raw-serve.test.ts | 424 ++++++++++++++++++++++++++++++ src/response/raw.ts | 346 ++++++++++++++++++++++++ 3 files changed, 1043 insertions(+) create mode 100644 src/__tests__/raw-framing.test.ts create mode 100644 src/__tests__/raw-serve.test.ts create mode 100644 src/response/raw.ts diff --git a/src/__tests__/raw-framing.test.ts b/src/__tests__/raw-framing.test.ts new file mode 100644 index 00000000..713dd27c --- /dev/null +++ b/src/__tests__/raw-framing.test.ts @@ -0,0 +1,273 @@ +import { crc32 } from "node:zlib"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { LLMock } from "../llmock.js"; +import { encodeRawFrame, rawContentType, rawTrailer } from "../response/raw.js"; +import type { RawWireId } from "../types.js"; + +// Per-wire framing for raw passthrough fixtures (#549 T6): exact bytes, and +// the same bytes the typed handler writes for the same event objects. + +interface DecodedFrame { + headers: Record; + payload: Buffer; +} + +/** Decode AWS event-stream frames, checking both CRCs (as aws-event-stream.test.ts does). */ +function decodeFrames(data: Buffer): DecodedFrame[] { + const frames: DecodedFrame[] = []; + let offset = 0; + while (offset < data.length) { + const total = data.readUInt32BE(offset); + const headersLength = data.readUInt32BE(offset + 4); + const frame = data.subarray(offset, offset + total); + expect(frame.readUInt32BE(8)).toBe(crc32(frame.subarray(0, 8))); + expect(frame.readUInt32BE(total - 4)).toBe(crc32(frame.subarray(0, total - 4))); + const headers: Record = {}; + let h = 12; + while (h < 12 + headersLength) { + const nameLength = frame.readUInt8(h); + const name = frame.subarray(h + 1, h + 1 + nameLength).toString("utf8"); + h += 1 + nameLength; + expect(frame.readUInt8(h)).toBe(7); // string header + const valueLength = frame.readUInt16BE(h + 1); + headers[name] = frame.subarray(h + 3, h + 3 + valueLength).toString("utf8"); + h += 3 + valueLength; + } + frames.push({ headers, payload: frame.subarray(12 + headersLength, total - 4) }); + offset += total; + } + return frames; +} + +describe("rawContentType", () => { + it.each([ + ["openai-responses", "text/event-stream"], + ["openai-chat", "text/event-stream"], + ["anthropic", "text/event-stream"], + ["gemini", "text/event-stream"], + ["gemini-interactions", "text/event-stream"], + ["cohere", "text/event-stream"], + ["ollama", "application/x-ndjson"], + ["ollama-generate", "application/x-ndjson"], + ["bedrock-invoke", "application/vnd.amazon.eventstream"], + ["bedrock-converse", "application/vnd.amazon.eventstream"], + ] as const)("%s -> %s", (wire, type) => { + expect(rawContentType(wire)).toBe(type); + }); +}); + +describe("encodeRawFrame: exact bytes", () => { + it.each(["openai-responses", "anthropic", "cohere"] as const)("%s: named SSE", (wire) => { + expect(encodeRawFrame(wire, { type: "x.y", n: 1 })).toBe( + 'event: x.y\ndata: {"type":"x.y","n":1}\n\n', + ); + }); + + it.each(["openai-chat", "gemini", "gemini-interactions"] as const)("%s: data SSE", (wire) => { + expect(encodeRawFrame(wire, { a: [1, "b"] })).toBe('data: {"a":[1,"b"]}\n\n'); + }); + + it.each(["ollama", "ollama-generate"] as const)("%s: NDJSON", (wire) => { + expect(encodeRawFrame(wire, { done: true })).toBe('{"done":true}\n'); + }); + + it("bedrock-invoke: a chunk frame whose base64 bytes are the event JSON", () => { + const event = { type: "content_block_delta", delta: { text: "é" } }; + const frame = encodeRawFrame("bedrock-invoke", event); + expect(Buffer.isBuffer(frame)).toBe(true); + const [decoded] = decodeFrames(Buffer.from(frame)); + expect(decoded.headers).toEqual({ + ":content-type": "application/json", + ":event-type": "chunk", + ":message-type": "event", + }); + const payload = JSON.parse(decoded.payload.toString("utf8")) as { bytes: string }; + expect(Buffer.from(payload.bytes, "base64").toString("utf8")).toBe(JSON.stringify(event)); + }); + + it("bedrock-converse: the single key is the event type, its value the payload", () => { + const frame = encodeRawFrame("bedrock-converse", { + contentBlockDelta: { contentBlockIndex: 0, delta: { text: "hi" } }, + }); + const [decoded] = decodeFrames(Buffer.from(frame)); + expect(decoded.headers[":event-type"]).toBe("contentBlockDelta"); + expect(decoded.headers[":message-type"]).toBe("event"); + expect(decoded.payload.toString("utf8")).toBe('{"contentBlockIndex":0,"delta":{"text":"hi"}}'); + }); +}); + +describe("rawTrailer", () => { + it("openai-chat sends [DONE] by default and when done is true", () => { + expect(rawTrailer("openai-chat", { wire: "openai-chat" }, false)).toBe("data: [DONE]\n\n"); + expect(rawTrailer("openai-chat", { wire: "openai-chat", done: true }, false)).toBe( + "data: [DONE]\n\n", + ); + }); + + it("openai-chat omits [DONE] when done is false or the stream was aborted", () => { + expect(rawTrailer("openai-chat", { wire: "openai-chat", done: false }, false)).toBeUndefined(); + expect(rawTrailer("openai-chat", { wire: "openai-chat" }, true)).toBeUndefined(); + }); + + it.each([ + "openai-responses", + "anthropic", + "gemini", + "gemini-interactions", + "cohere", + "ollama", + "ollama-generate", + "bedrock-invoke", + "bedrock-converse", + ] as const)("%s has no terminal", (wire) => { + expect(rawTrailer(wire, { wire }, false)).toBeUndefined(); + }); +}); + +// Typed-framing equality (m4): stream a typed fixture from the real server, +// split the dump into the event objects it carries, re-encode those objects +// with encodeRawFrame, and require the identical bytes. The typed writers stay +// module-private; only their wire output is compared. +describe("encodeRawFrame matches the typed handler's bytes", () => { + let mock: LLMock; + beforeAll(async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent", chunkSize: 4 }); + mock.on({ userMessage: "typed" }, { content: "Typed framing check." }); + await mock.start(); + }); + afterAll(async () => { + await mock.stop(); + }); + + const MODEL = "anthropic.claude-3-5-sonnet-20241022-v2:0"; + const requests: Record = { + "openai-responses": { + path: "/v1/responses", + body: { model: "gpt-4.1", input: "typed", stream: true }, + }, + "openai-chat": { + path: "/v1/chat/completions", + body: { model: "gpt-4.1", messages: [{ role: "user", content: "typed" }], stream: true }, + }, + anthropic: { + path: "/v1/messages", + body: { + model: "claude-sonnet-4-5", + max_tokens: 64, + messages: [{ role: "user", content: "typed" }], + stream: true, + }, + }, + gemini: { + path: "/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse", + body: { contents: [{ role: "user", parts: [{ text: "typed" }] }] }, + }, + "gemini-interactions": { + path: "/v1beta/interactions", + body: { model: "gemini-2.5-flash", input: "typed", stream: true }, + }, + cohere: { + path: "/v2/chat", + body: { + model: "command-r-plus", + messages: [{ role: "user", content: "typed" }], + stream: true, + }, + }, + ollama: { + path: "/api/chat", + body: { model: "llama3.1", messages: [{ role: "user", content: "typed" }], stream: true }, + }, + "ollama-generate": { + path: "/api/generate", + body: { model: "llama3.1", prompt: "typed", stream: true }, + }, + "bedrock-invoke": { + path: `/model/${MODEL}/invoke-with-response-stream`, + body: { + anthropic_version: "bedrock-2023-05-31", + max_tokens: 64, + messages: [{ role: "user", content: "typed" }], + }, + }, + "bedrock-converse": { + path: `/model/${MODEL}/converse-stream`, + body: { messages: [{ role: "user", content: [{ text: "typed" }] }] }, + }, + }; + + async function dump(wire: RawWireId): Promise<{ contentType: string | null; bytes: Buffer }> { + const { path, body } = requests[wire]; + const res = await fetch(`${mock.url}${path}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + expect(res.status).toBe(200); + return { + contentType: res.headers.get("content-type"), + bytes: Buffer.from(await res.arrayBuffer()), + }; + } + + /** The event objects a typed dump carries, in order, plus any terminal bytes. */ + function eventsIn( + wire: RawWireId, + bytes: Buffer, + ): { events: Record[]; tail: string } { + if (wire === "bedrock-invoke") { + return { + events: decodeFrames(bytes).map((f) => { + const { bytes: b64 } = JSON.parse(f.payload.toString("utf8")) as { bytes: string }; + return JSON.parse(Buffer.from(b64, "base64").toString("utf8")); + }), + tail: "", + }; + } + if (wire === "bedrock-converse") { + return { + events: decodeFrames(bytes).map((f) => ({ + [f.headers[":event-type"]]: JSON.parse(f.payload.toString("utf8")), + })), + tail: "", + }; + } + const text = bytes.toString("utf8"); + if (wire === "ollama" || wire === "ollama-generate") { + return { + events: text + .trimEnd() + .split("\n") + .map((line) => JSON.parse(line)), + tail: "", + }; + } + const blocks = text.split("\n\n").filter((b) => b.length > 0); + const done = blocks.at(-1) === "data: [DONE]"; + const events = (done ? blocks.slice(0, -1) : blocks).map((block) => + JSON.parse(block.slice(block.indexOf("data: ") + "data: ".length)), + ); + return { events, tail: done ? "data: [DONE]\n\n" : "" }; + } + + it.each(Object.keys(requests) as RawWireId[])("%s", async (wire) => { + const { contentType, bytes } = await dump(wire); + expect(contentType?.split(";")[0]).toBe(rawContentType(wire)); + const { events, tail } = eventsIn(wire, bytes); + expect(events.length).toBeGreaterThan(1); + const reencoded = Buffer.concat([ + ...events.map((event) => Buffer.from(encodeRawFrame(wire, event))), + Buffer.from(rawTrailer(wire, { wire }, false) ?? ""), + ]); + expect(tail).toBe(rawTrailer(wire, { wire }, false) ?? ""); + expect(reencoded.equals(bytes)).toBe(true); + }); +}); + +describe("encodeRawFrame: a checker/framer mismatch is an internal error", () => { + it("bedrock-converse with a non-object payload throws instead of framing {}", () => { + expect(() => encodeRawFrame("bedrock-converse", { messageStart: "not an object" })).toThrow( + /internal error/, + ); + }); +}); diff --git a/src/__tests__/raw-serve.test.ts b/src/__tests__/raw-serve.test.ts new file mode 100644 index 00000000..54032674 --- /dev/null +++ b/src/__tests__/raw-serve.test.ts @@ -0,0 +1,424 @@ +import * as http from "node:http"; +import Anthropic from "@anthropic-ai/sdk"; +import OpenAI from "openai"; +import type { AddressInfo } from "node:net"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { Journal } from "../journal.js"; +import { Logger } from "../logger.js"; +import { serveRawResponse } from "../response/raw.js"; +import { LLMock as PlainLLMock } from "../llmock.js"; +import { LLMock } from "./helpers/misbehavior-enabled.js"; +import type { + Fixture, + FixtureFileResponse, + FixtureOpts, + HandlerDefaults, + RawResponse, + ResponseFactory, +} from "../types.js"; + +// serveRawResponse (#549), driven through the OpenAI Responses wire: serve-time +// checks for fixtures that skip load validation, status and header passthrough, +// timing rules and misbehavior journaling. + +const json = { id: "resp_serve", object: "response", status: "completed", output: [] }; +const events = [ + { type: "response.created", response: { id: "resp_serve" } }, + { type: "response.completed", response: { id: "resp_serve" } }, +]; + +let mock: PlainLLMock | undefined; + +afterEach(async () => { + await mock?.stop(); + mock = undefined; +}); + +async function start( + response: FixtureFileResponse | ResponseFactory, + opts?: FixtureOpts, + server: PlainLLMock = new PlainLLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }), +): Promise { + mock = server; + server.on({ userMessage: "serve" }, response, opts); + await server.start(); + return server; +} + +/** A raw HTTP request, so every byte and header the server wrote is visible. */ +function post( + url: string, + stream: boolean, +): Promise<{ status: number; headers: http.IncomingHttpHeaders; body: string; ms: number }> { + const started = Date.now(); + return new Promise((resolve, reject) => { + const req = http.request( + `${url}/v1/responses`, + { method: "POST", headers: { "content-type": "application/json" } }, + (res) => { + let body = ""; + res.setEncoding("utf8"); + res.on("data", (chunk: string) => (body += chunk)); + res.on("end", () => + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body, + ms: Date.now() - started, + }), + ); + }, + ); + req.on("error", reject); + req.end(JSON.stringify({ model: "gpt-4.1", input: "serve", stream })); + }); +} + +const forbidden = { + raw: { + wire: "openai-responses", + json, + headers: { "content-length": "5", "x-raw-fixture": "leaked" }, + }, +} as RawResponse; + +describe("serveRawResponse: serve-time checks", () => { + it("a factory with a forbidden content-length header: 500 with the checker message only", async () => { + const server = await start(() => forbidden); + const got = await post(server.url, false); + expect(got.status).toBe(500); + expect(got.headers["content-type"]).toBe("application/json"); + const body = JSON.parse(got.body) as { error: { message: string; code: string } }; + expect(body.error.code).toBe("aimock_raw_fixture_invalid"); + expect(body.error.message).toMatch(/^invalid raw fixture: .*content-length/); + // Nothing of the fixture's own body or headers was written. + expect(got.body).not.toContain("resp_serve"); + expect(got.headers["x-raw-fixture"]).toBeUndefined(); + // Any Content-Length on the reply is the error body's own length, never the fixture's. + const bodyLength = String(Buffer.byteLength(got.body)); + expect(got.headers["content-length"] ?? bodyLength).toBe(bodyLength); + const [entry] = server.getRequests(); + expect(entry.response.status).toBe(500); + expect(entry.response.error).toBe(body.error.message); + }); + + it("a factory with status 99: 500", async () => { + const server = await start(() => ({ raw: { wire: "openai-responses", json, status: 99 } })); + const got = await post(server.url, false); + expect(got.status).toBe(500); + expect(JSON.parse(got.body)).toMatchObject({ + error: { code: "aimock_raw_fixture_invalid" }, + }); + }); + + it("on() with the same invalid object does not throw, and the request gets the same 500", async () => { + const server = new PlainLLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + expect(() => server.on({ userMessage: "serve" }, forbidden)).not.toThrow(); + mock = server; + await server.start(); + const got = await post(server.url, false); + expect(got.status).toBe(500); + const body = JSON.parse(got.body) as { error: { message: string; code: string } }; + expect(body.error.code).toBe("aimock_raw_fixture_invalid"); + expect(body.error.message).toMatch(/content-length/); + }); +}); + +describe("serveRawResponse: status and headers", () => { + it("status 429 with json returns 429 untimed, even with the timing opt-in set", async () => { + const server = await start( + { raw: { wire: "openai-responses", json: { error: { message: "slow down" } }, status: 429 } }, + { streamingProfile: { ttft: 1500, tps: 1 }, nonStreamingTimingAllProviders: true }, + ); + const got = await post(server.url, false); + expect(got.status).toBe(429); + expect(JSON.parse(got.body)).toEqual({ error: { message: "slow down" } }); + expect(got.ms).toBeLessThan(1000); + expect(server.getRequests()[0].response.status).toBe(429); + }); + + it("a custom status and header reach a stream", async () => { + const server = await start({ + raw: { wire: "openai-responses", events, status: 201, headers: { "x-request-id": "r1" } }, + }); + const got = await post(server.url, true); + expect(got.status).toBe(201); + expect(got.headers["x-request-id"]).toBe("r1"); + expect(got.headers["content-type"]).toBe("text/event-stream"); + }); +}); + +describe("serveRawResponse: non-streaming timing (surface rule)", () => { + const profile = { streamingProfile: { ttft: 1200, tps: 100 } }; + + it("nonStreamingTimingAllProviders delays a json reply on a non-chat wire", async () => { + const server = await start( + { raw: { wire: "openai-responses", json } }, + { ...profile, nonStreamingTimingAllProviders: true }, + ); + const got = await post(server.url, false); + expect(got.status).toBe(200); + expect(got.ms).toBeGreaterThanOrEqual(1000); + }); + + it("nonStreamingTiming alone does not delay a non-chat wire", async () => { + const server = await start( + { raw: { wire: "openai-responses", json } }, + { ...profile, nonStreamingTiming: true }, + ); + const got = await post(server.url, false); + expect(got.status).toBe(200); + expect(JSON.parse(got.body)).toEqual(json); + expect(got.ms).toBeLessThan(1000); + }); +}); + +describe("serveRawResponse: misbehavior journaling", () => { + it("records the skipped server-level fault on the journal entry", async () => { + const server = await start( + { raw: { wire: "openai-responses", json } }, + undefined, + new LLMock({ + port: 0, + host: "127.0.0.1", + logLevel: "silent", + misbehavior: { faults: [{ fault: "empty-response", rate: 1 }] }, + }), + ); + const got = await post(server.url, false); + expect(got.status).toBe(200); + expect(JSON.parse(got.body)).toEqual(json); + const [entry] = server.getRequests(); + expect(entry.response.misbehavior).toMatchObject({ + applied: false, + source: "server", + wire: "openai-responses", + evaluations: [{ fault: "empty-response", outcome: "skipped", reason: "not-applicable" }], + }); + }); + + it("records it on a raw error 500 too", async () => { + const server = await start( + () => forbidden, + undefined, + new LLMock({ + port: 0, + host: "127.0.0.1", + logLevel: "silent", + misbehavior: { faults: [{ fault: "empty-response", rate: 1 }] }, + }), + ); + const got = await post(server.url, false); + expect(got.status).toBe(500); + expect(JSON.parse(got.body)).toMatchObject({ + error: { code: "aimock_raw_fixture_invalid" }, + }); + const [entry] = server.getRequests(); + expect(entry.response.misbehavior).toMatchObject({ applied: false, source: "server" }); + }); +}); + +// ─── Error replies (CR round 1) ───────────────────────────────────────────── + +describe("serveRawResponse: error replies are never delayed", () => { + it("a stream with status 429 is not delayed, even with a streaming profile", async () => { + const server = await start( + { raw: { wire: "openai-responses", events, status: 429 } }, + { streamingProfile: { ttft: 1500, tps: 1 } }, + ); + const got = await post(server.url, true); + expect(got.status).toBe(429); + expect(got.body).toContain("response.completed"); + expect(got.ms).toBeLessThan(1000); + expect(server.getRequests()[0].response.status).toBe(429); + }); + + it("a non-streaming 429 after the client left is journaled as client aborted and not written", async () => { + const journal = new Journal(); + const response: RawResponse = { + raw: { wire: "openai-responses", json: { error: "slow down" }, status: 429 }, + }; + const fixture: Fixture = { match: { userMessage: "serve" }, response }; + const defaults: HandlerDefaults = { + latency: 0, + chunkSize: 20, + replaySpeed: 1, + logger: new Logger("silent"), + }; + let headersSent: boolean | undefined; + let done: Promise = Promise.resolve(); + const server = http.createServer((req, res) => { + // Serve only once the client has gone. + req.socket.once("close", () => { + done = serveRawResponse({ + wire: "openai-responses", + stream: false, + req, + res, + fixture, + response, + body: null, + journal, + defaults, + testId: "default", + errorWire: "openai", + }).then(() => { + headersSent = res.headersSent; + }); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve())); + try { + const port = (server.address() as AddressInfo).port; + await new Promise((resolve) => { + const req = http.request({ host: "127.0.0.1", port, method: "POST", agent: false }); + req.on("error", () => resolve()); + req.on("close", () => resolve()); + req.flushHeaders(); + setTimeout(() => req.destroy(), 50); + }); + await expect.poll(() => journal.getAll().length, { timeout: 3000 }).toBe(1); + await done; + expect(headersSent).toBe(false); + expect(journal.getAll()[0].response).toMatchObject({ + interrupted: true, + interruptReason: "client aborted", + }); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } + }); +}); + +describe("serveRawResponse: the non-streaming wait is sized from the served JSON", () => { + it("a large JSON body waits longer than a small one", async () => { + const opts: FixtureOpts = { + streamingProfile: { ttft: 0, tps: 10 }, + nonStreamingTimingAllProviders: true, + }; + const small = await start({ raw: { wire: "openai-responses", json: { a: 1 } } }, opts); + const smallReply = await post(small.url, false); + await small.stop(); + mock = undefined; + const large = await start( + { raw: { wire: "openai-responses", json: { text: "x".repeat(200) } } }, + opts, + ); + const largeReply = await post(large.url, false); + expect(smallReply.status).toBe(200); + expect(largeReply.status).toBe(200); + // 9 bytes: one chunk, no gap. 213 bytes at chunkSize 20: 11 chunks, 10 gaps of 100ms. + expect(smallReply.ms).toBeLessThan(400); + expect(largeReply.ms).toBeGreaterThanOrEqual(800); + }); +}); + +describe("serveRawResponse: invalid fixture logging", () => { + it("an invalid raw fixture's 500 is logged at error level", async () => { + const spy = vi.spyOn(console, "error").mockImplementation(() => undefined); + try { + const server = await start( + () => forbidden, + undefined, + new PlainLLMock({ port: 0, host: "127.0.0.1", logLevel: "warn" }), + ); + const got = await post(server.url, false); + expect(got.status).toBe(500); + const lines = spy.mock.calls.map((args) => args.map(String).join(" ")); + expect(lines).toContainEqual( + expect.stringMatching( + /^\[aimock\] POST \/v1\/responses: invalid raw fixture: .*content-length/, + ), + ); + } finally { + spy.mockRestore(); + } + }); +}); + +describe("serveRawResponse: an error status with json answers a streaming request with JSON", () => { + const openAIError = { + error: { message: "slow down", type: "rate_limit_error", param: null, code: "rate_limited" }, + }; + const anthropicError = { + type: "error", + error: { type: "rate_limit_error", message: "slow down" }, + }; + + async function serveOn(response: FixtureFileResponse, opts?: FixtureOpts): Promise { + return start(response, opts); + } + + it("openai-chat: the SDK throws a typed RateLimitError on stream:true", async () => { + const server = await serveOn( + { raw: { wire: "openai-chat", json: openAIError, status: 429 } }, + { streamingProfile: { ttft: 1500 } }, + ); + const client = new OpenAI({ apiKey: "k", baseURL: `${server.url}/v1`, maxRetries: 0 }); + const started = Date.now(); + const err = await client.chat.completions + .create({ model: "gpt-4.1", stream: true, messages: [{ role: "user", content: "serve" }] }) + .then( + () => undefined, + (e: unknown) => e, + ); + expect(err).toBeInstanceOf(OpenAI.RateLimitError); + expect(err instanceof OpenAI.RateLimitError && err.error).toMatchObject({ + message: "slow down", + }); + expect(Date.now() - started).toBeLessThan(1000); + expect(server.getRequests()[0].response.status).toBe(429); + }); + + it("openai-responses: json wins over events for an error status on stream:true", async () => { + const server = await serveOn({ + raw: { + wire: "openai-responses", + json: openAIError, + events, + status: 429, + headers: { "x-r": "1" }, + }, + }); + const client = new OpenAI({ apiKey: "k", baseURL: `${server.url}/v1`, maxRetries: 0 }); + const err = await client.responses + .create({ model: "gpt-4.1", input: "serve", stream: true }) + .then( + () => undefined, + (e: unknown) => e, + ); + expect(err).toBeInstanceOf(OpenAI.RateLimitError); + const got = await post(server.url, true); + expect(got.status).toBe(429); + expect(got.headers["content-type"]).toBe("application/json"); + expect(got.headers["x-r"]).toBe("1"); + expect(JSON.parse(got.body)).toEqual(openAIError); + }); + + it("anthropic: the SDK throws a typed RateLimitError on stream:true", async () => { + const server = await serveOn({ + raw: { wire: "anthropic", json: anthropicError, status: 429 }, + }); + const client = new Anthropic({ apiKey: "k", baseURL: server.url, maxRetries: 0 }); + const err = await client.messages + .create({ + model: "claude-sonnet-4-5", + max_tokens: 16, + stream: true, + messages: [{ role: "user", content: "serve" }], + }) + .then( + () => undefined, + (e: unknown) => e, + ); + expect(err).toBeInstanceOf(Anthropic.RateLimitError); + }); + + it("a 2xx json-only fixture on a streaming request keeps the mode-mismatch 500", async () => { + const server = await serveOn({ raw: { wire: "openai-responses", json } }); + const got = await post(server.url, true); + expect(got.status).toBe(500); + expect(JSON.parse(got.body)).toMatchObject({ error: { code: "aimock_raw_mode_unavailable" } }); + }); +}); diff --git a/src/response/raw.ts b/src/response/raw.ts new file mode 100644 index 00000000..03bf6b3c --- /dev/null +++ b/src/response/raw.ts @@ -0,0 +1,346 @@ +/** + * @internal Raw passthrough fixtures (#549): per-wire framing and serving. + * + * A raw fixture supplies event objects; aimock adds only the named wire's + * framing, using exactly the bytes the typed handlers write for that wire: + * + * - named SSE (`openai-responses`, `anthropic`, `cohere`): + * `event: \ndata: \n\n` + * - data SSE (`openai-chat`, `gemini`, `gemini-interactions`): `data: \n\n`; + * `openai-chat` ends with `data: [DONE]\n\n` unless `raw.done === false` + * - NDJSON (`ollama`, `ollama-generate`): `\n` + * - AWS event stream: `bedrock-invoke` wraps each event as a `chunk` frame with + * base64 `bytes`; `bedrock-converse` takes the event's single key as the + * frame's event type and its value as the payload. + * + * The framing functions are pure. `serveRawResponse` checks and journals the + * fixture. It writes one reply itself: the untimed 500 for a fixture that + * fails a check, targets another wire or has no body for the request mode. + * Every other reply goes through the shared delivery layer (`deliverStream`, + * `deliverJson`), which owns the delay loop, writability checks and abort + * handling. A reply with a status of 400 or more is never delayed. + */ +import type * as http from "node:http"; +import { encodeEventStreamMessage, eventStreamCodec } from "../aws-event-stream.js"; +import { rawFixtureContext } from "../fixture-loader.js"; +import { flattenHeaders } from "../helpers.js"; +import type { Journal } from "../journal.js"; +import { recordMisbehaviorOutcome, type MisbehaviorSummary } from "../misbehavior.js"; +import { serializeOpenRouterError } from "../openrouter-chat.js"; +import { ndjsonCodec } from "../ndjson-writer.js"; +import { checkRawResponse } from "../raw-validation.js"; +import { sseCodec, writeErrorResponse } from "../sse-writer.js"; +import type { Fixture, HandlerDefaults, JournalEntry, RawResponse, RawWireId } from "../types.js"; +import { deliverJson, deliverStream } from "./deliver.js"; +import { renderError } from "./errors.js"; +import type { FrameCodec } from "./frame-loop.js"; +import type { ErrorWire } from "./profiles.js"; + +/** The streaming Content-Type each wire's typed handler sends. */ +export function rawContentType(wire: RawWireId): string { + switch (wire) { + case "openai-responses": + case "openai-chat": + case "anthropic": + case "gemini": + case "gemini-interactions": + case "cohere": + return "text/event-stream"; + case "ollama": + case "ollama-generate": + return "application/x-ndjson"; + case "bedrock-invoke": + case "bedrock-converse": + return "application/vnd.amazon.eventstream"; + } +} + +/** + * The default headers of the typed stream on `wire`, read from the typed + * codec: SSE and NDJSON add `Cache-Control: no-cache` and + * `Connection: keep-alive`, the AWS event stream `Transfer-Encoding: chunked`. + */ +function rawStreamHeaders(wire: RawWireId): Record { + switch (wire) { + case "openai-responses": + case "openai-chat": + case "anthropic": + case "gemini": + case "gemini-interactions": + case "cohere": + return sseCodec({ done: false }).headers; + case "ollama": + case "ollama-generate": + return ndjsonCodec().headers; + case "bedrock-invoke": + case "bedrock-converse": + return eventStreamCodec().headers; + } +} + +/** The non-streaming JSON default, as `deliverJson` sends it. */ +const JSON_HEADERS: Readonly> = { "Content-Type": "application/json" }; + +/** + * `defaults` with each `authored` header laid over it. Header names are + * case-insensitive: an authored header replaces the default of the same name + * in any letter case and takes the default's spelling, so a later spread over + * the same defaults (as `deliverJson` does) still has one key per name. + */ +function overrideHeaders( + defaults: Readonly>, + authored: Readonly> | undefined, +): Record { + const merged: Record = { ...defaults }; + const spelling = new Map(Object.keys(defaults).map((name) => [name.toLowerCase(), name])); + for (const [name, value] of Object.entries(authored ?? {})) { + merged[spelling.get(name.toLowerCase()) ?? name] = value; + } + return merged; +} + +/** One raw event, framed for `wire`. Events are checked before framing (raw-validation.ts). */ +export function encodeRawFrame(wire: RawWireId, event: Record): string | Buffer { + const json = JSON.stringify(event); + switch (wire) { + case "openai-responses": + case "anthropic": + case "cohere": + return `event: ${String(event.type)}\ndata: ${json}\n\n`; + case "openai-chat": + case "gemini": + case "gemini-interactions": + return `data: ${json}\n\n`; + case "ollama": + case "ollama-generate": + return `${json}\n`; + case "bedrock-invoke": + return encodeEventStreamMessage("chunk", { + bytes: Buffer.from(json, "utf8").toString("base64"), + }); + case "bedrock-converse": { + // The checker guarantees exactly one key whose value is a JSON object. + const [eventType] = Object.keys(event); + const payload = event[eventType]; + if (payload === null || typeof payload !== "object" || Array.isArray(payload)) { + throw new Error( + `raw framing internal error: bedrock-converse event "${eventType}" has no object payload`, + ); + } + return encodeEventStreamMessage(eventType, payload); + } + } +} + +/** + * Bytes written after the last frame, if any. Only `openai-chat` has a + * terminal (`data: [DONE]`), sent unless `raw.done === false` and never after + * an interrupted stream. + */ +export function rawTrailer( + wire: RawWireId, + raw: RawResponse["raw"], + aborted: boolean, +): string | undefined { + if (wire === "openai-chat" && !aborted && raw.done !== false) return "data: [DONE]\n\n"; + return undefined; +} + +// ─── Serving ────────────────────────────────────────────────────────────────── + +/** The error code for a request the raw fixture has no body for. */ +export const RAW_MODE_UNAVAILABLE = "aimock_raw_mode_unavailable"; +/** The error code for a raw fixture that fails its checks or targets another wire. */ +export const RAW_FIXTURE_INVALID = "aimock_raw_fixture_invalid"; + +/** Everything {@link serveRawResponse} needs from the calling handler. */ +export interface RawServeContext { + wire: RawWireId; + /** Whether the request asked for a stream, as the handler decides it. */ + stream: boolean; + req: http.IncomingMessage; + res: http.ServerResponse; + fixture: Fixture; + response: RawResponse; + /** The journaled request body, as the handler journals it. */ + body: JournalEntry["body"]; + journal: Journal; + defaults: HandlerDefaults; + testId: string; + misbehaviorSummary?: MisbehaviorSummary; + /** The dialect of the handler's own `renderError(…, "misbehavior", …)` call. */ + errorWire: ErrorWire; + /** OpenAI chat requests that arrived on the OpenRouter route. */ + openRouter?: boolean; +} + +/** + * The raw events as a frame codec for the shared frame loop. Its headers are + * the typed stream's defaults for `wire` with `raw.headers` laid over them. + */ +export function rawCodec( + wire: RawWireId, + raw: RawResponse["raw"], +): FrameCodec> { + return { + headers: overrideHeaders(rawStreamHeaders(wire), raw.headers), + encode: (event) => encodeRawFrame(wire, event), + trailer: (aborted) => { + const trailer = rawTrailer(wire, raw, aborted); + return trailer === undefined ? [] : [trailer]; + }, + }; +} + +/** + * The error body for a raw 500, in the dialect of the handler's misbehavior + * errors. Dialects with no code slot (Gemini, Ollama, OpenRouter) carry the + * code as a message prefix. + */ +function renderRawError( + ctx: RawServeContext, + code: string, + message: string, +): { body: string; headers?: Record } { + if (ctx.openRouter) return { body: serializeOpenRouterError(500, `${code}: ${message}`) }; + switch (ctx.errorWire) { + case "openai": + return renderError("openai", "misbehavior", { + message, + type: "server_error", + ...(ctx.wire === "openai-chat" ? { param: null, includeParam: true as const } : {}), + code, + }); + case "messages": + return renderError("messages", "misbehavior", { type: "api_error", code, message }); + case "gemini": + return renderError("gemini", "misbehavior", { + bodyCode: 500, + message: `${code}: ${message}`, + status: "INTERNAL", + }); + case "bedrock": + case "bedrock-converse": + return renderError(ctx.errorWire, "misbehavior", { + type: "InternalServerException", + code, + message, + }); + default: + return renderError(ctx.errorWire, "misbehavior", { code, message }); + } +} + +/** + * Serve a raw passthrough fixture on the handler's wire: check it, journal it, + * and send its events or JSON through the shared delivery layer. A fixture + * that fails a check, targets another wire, or has no body for the requested + * mode gets a 500 in the handler's misbehavior error dialect, and an error log + * line. A reply with status 400 or more is never delayed, streaming or not, + * and one with `json` answers in JSON even to a streaming request. + */ +export async function serveRawResponse(ctx: RawServeContext): Promise { + const { wire, stream, req, res, fixture, journal, defaults, testId } = ctx; + const raw = ctx.response.raw; + const journalBase = { + method: req.method ?? "POST", + path: req.url ?? "/", + headers: flattenHeaders(req.headers), + body: ctx.body, + }; + const record = (entry: JournalEntry): void => + recordMisbehaviorOutcome({ entry, summary: ctx.misbehaviorSummary, defaults, testId }); + + const status = raw.status ?? 200; + // An error reply (status >= 400) is never delayed, streaming or not, as + // typed error replies are not. Fixture interruptions still apply. + const errorReply = status >= 400; + // An error status with a JSON body answers in JSON even to a streaming + // request, as provider APIs do; the JSON wins over any events. + const servesStream = stream && !(errorReply && raw.json !== undefined); + + let failure: { code: string; message: string } | undefined; + const problems = checkRawResponse(raw, rawFixtureContext(fixture, ctx.response)).filter( + (finding) => finding.severity === "error", + ); + if (problems.length > 0) { + failure = { + code: RAW_FIXTURE_INVALID, + message: `invalid raw fixture: ${problems.map((p) => p.message).join("; ")}`, + }; + } else if (raw.wire !== wire) { + failure = { code: RAW_FIXTURE_INVALID, message: `raw fixture is for ${raw.wire}, not ${wire}` }; + } else if (servesStream ? raw.events === undefined : raw.json === undefined) { + failure = { + code: RAW_MODE_UNAVAILABLE, + message: `raw fixture has no "${servesStream ? "events" : "json"}" body for a ${ + servesStream ? "streaming" : "non-streaming" + } request on ${wire}`, + }; + } + if (failure) { + defaults.logger.error(`${journalBase.method} ${journalBase.path}: ${failure.message}`); + const entry = journal.add({ ...journalBase, response: { status: 500, fixture } }); + entry.response.error = failure.message; + record(entry); + const rendered = renderRawError(ctx, failure.code, failure.message); + for (const [name, value] of Object.entries(rendered.headers ?? {})) res.setHeader(name, value); + writeErrorResponse(res, 500, rendered.body); + return; + } + + const entry = journal.add({ ...journalBase, response: { status, fixture } }); + record(entry); + + if (servesStream) { + const events = raw.events; + if (events === undefined) { + throw new Error(`raw serve internal error: a streamed reply on ${wire} has no events`); + } + await deliverStream( + res, + entry, + fixture, + rawCodec(wire, raw), + events, + errorReply + ? { latency: 0 } + : { + latency: fixture.latency ?? defaults.latency, + streamingProfile: fixture.streamingProfile, + recordedTimings: fixture.recordedTimings, + replaySpeed: fixture.replaySpeed ?? defaults.replaySpeed, + }, + // The codec already carries raw.headers over its defaults. + { status: raw.status }, + ); + return; + } + + // The wait is sized from the served JSON, as typed replies size theirs from + // the served content, in chunks of the fixture's or server's chunk size. + const json = raw.json; + if (json === undefined) { + throw new Error(`raw serve internal error: a JSON reply on ${wire} has no json`); + } + const servedLength = JSON.stringify(json).length; + // deliverJson reads the timing opt-ins from the fixture. With them off, an + // error reply still gets its writability check and journaling, but no wait. + await deliverJson( + res, + entry, + errorReply + ? { ...fixture, nonStreamingTiming: false, nonStreamingTimingAllProviders: false } + : fixture, + () => json, + { + surface: wire === "openai-chat" ? "openai-chat" : "other", + status: raw.status, + payloadLength: servedLength, + chunkSize: Math.max(1, fixture.chunkSize ?? defaults.chunkSize), + recordedTimings: fixture.recordedTimings, + replaySpeed: fixture.replaySpeed ?? defaults.replaySpeed, + headers: overrideHeaders(JSON_HEADERS, raw.headers), + }, + ); +} From 32e8d921414a3496db99862076c15e90930ef1a3 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sat, 24 Oct 2026 11:33:09 -0700 Subject: [PATCH 5/7] feat: serve raw fixtures on all HTTP LLM wires; WS Responses rejects them (#549) Raw fixtures with an error status fail over on OpenRouter like typed error fixtures. A raw fixture that sets its own misbehavior is skipped by the misbehavior planner. --- src/__tests__/openrouter-chat.test.ts | 81 +++++++++++++++++++ src/__tests__/raw-ws-responses.test.ts | 106 +++++++++++++++++++++++++ src/bedrock-converse.ts | 44 +++++++++- src/bedrock.ts | 44 +++++++++- src/cohere.ts | 24 +++++- src/gemini-interactions.ts | 23 ++++++ src/gemini.ts | 24 +++++- src/messages.ts | 23 +++++- src/misbehavior.ts | 4 + src/ollama.ts | 46 ++++++++++- src/responses.ts | 23 +++++- src/server.ts | 64 ++++++++++----- src/ws-responses.ts | 23 ++++++ 13 files changed, 501 insertions(+), 28 deletions(-) create mode 100644 src/__tests__/raw-ws-responses.test.ts diff --git a/src/__tests__/openrouter-chat.test.ts b/src/__tests__/openrouter-chat.test.ts index 68d0f263..9464a8fb 100644 --- a/src/__tests__/openrouter-chat.test.ts +++ b/src/__tests__/openrouter-chat.test.ts @@ -1368,3 +1368,84 @@ describe("OpenRouter chat: malformed blocks never leave a spurious 200 (round-3 expect(twoHundreds).toHaveLength(0); }); }); + +describe("OpenRouter chat: raw error fixtures fail over like typed errors (#549)", () => { + const typedFallback: Fixture = { + match: { model: "b", userMessage: "route" }, + response: { content: "served by b" }, + }; + const rawPrimary = (status: number): Fixture => ({ + match: { model: "a", userMessage: "route" }, + response: { + raw: { wire: "openai-chat", status, json: { error: { message: `raw ${status}` } } }, + }, + }); + const typedPrimary = (status: number): Fixture => ({ + match: { model: "a", userMessage: "route" }, + response: { error: { message: `typed ${status}` }, status }, + }); + const chainBody = { + model: "a", + models: ["a", "b"], + messages: [{ role: "user", content: "route" }], + }; + + for (const status of [429, 503]) { + it(`control: a typed ${status} on "a" fails over to "b"`, async () => { + instance = await createServer([typedPrimary(status), typedFallback]); + const res = await httpPost(`${instance.url}${OR}`, chainBody); + expect(res.status).toBe(200); + const json = JSON.parse(res.body); + expect(json.model).toBe("b"); + expect(json.choices[0].message.content).toBe("served by b"); + }); + + it(`a raw ${status} on "a" fails over to "b"`, async () => { + instance = await createServer([rawPrimary(status), typedFallback]); + const res = await httpPost(`${instance.url}${OR}`, chainBody); + expect(res.status).toBe(200); + const json = JSON.parse(res.body); + expect(json.model).toBe("b"); + expect(json.choices[0].message.content).toBe("served by b"); + }); + } + + it("a raw 429 on every candidate serves the last raw error", async () => { + const rawB: Fixture = { + match: { model: "b", userMessage: "route" }, + response: { + raw: { wire: "openai-chat", status: 503, json: { error: { message: "b down" } } }, + }, + }; + instance = await createServer([rawPrimary(429), rawB]); + const res = await httpPost(`${instance.url}${OR}`, chainBody); + expect(res.status).toBe(503); + expect(JSON.parse(res.body)).toEqual({ error: { message: "b down" } }); + }); + + it("allow_fallbacks:false serves a raw 429 primary as terminal", async () => { + instance = await createServer([rawPrimary(429), typedFallback]); + const res = await httpPost(`${instance.url}${OR}`, { + ...chainBody, + provider: { allow_fallbacks: false }, + }); + expect(res.status).toBe(429); + expect(JSON.parse(res.body)).toEqual({ error: { message: "raw 429" } }); + }); + + it("a raw 200 on 'a' is still served as success", async () => { + const rawOk: Fixture = { + match: { model: "a", userMessage: "route" }, + response: { + raw: { + wire: "openai-chat", + json: { id: "x", object: "chat.completion", model: "a", choices: [] }, + }, + }, + }; + instance = await createServer([rawOk, typedFallback]); + const res = await httpPost(`${instance.url}${OR}`, chainBody); + expect(res.status).toBe(200); + expect(JSON.parse(res.body).id).toBe("x"); + }); +}); diff --git a/src/__tests__/raw-ws-responses.test.ts b/src/__tests__/raw-ws-responses.test.ts new file mode 100644 index 00000000..55e9d075 --- /dev/null +++ b/src/__tests__/raw-ws-responses.test.ts @@ -0,0 +1,106 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { LLMock as PlainLLMock } from "../llmock.js"; +import { LLMock } from "./helpers/misbehavior-enabled.js"; +import { connectWebSocket, type WSTestClient } from "./ws-test-client.js"; + +// WebSocket Responses (#549 T7, spec Decision 12): raw fixtures are not served +// over WS yet. A raw openai-responses fixture gets a clear error event and a +// 500 journal entry instead of the generic unknown-type error. + +const RAW_WS_ERROR = "raw fixtures are not yet supported on the WebSocket transport"; +const RAW_MODE_UNAVAILABLE = "aimock_raw_mode_unavailable"; + +let mock: PlainLLMock | undefined; +let ws: WSTestClient | undefined; +afterEach(async () => { + if (ws) { + ws.close(); + await ws.waitForClose(); + ws.destroy(); + } + ws = undefined; + await mock?.stop(); + mock = undefined; + vi.restoreAllMocks(); +}); + +async function firstEvent(input: string): Promise> { + if (!mock) throw new Error("mock not started"); + ws = await connectWebSocket(mock.url, "/v1/responses"); + ws.send(JSON.stringify({ type: "response.create", model: "gpt-4.1", input })); + const [message] = await ws.waitForMessages(1); + return JSON.parse(message); +} + +function addRaw(target: PlainLLMock) { + target.addFixture({ + match: { userMessage: "raw-ws" }, + response: { raw: { wire: "openai-responses", events: [{ type: "response.created" }] } }, + }); +} + +describe("raw fixtures over WebSocket Responses", () => { + it("sends a clear server_error event and journals a 500", async () => { + mock = new PlainLLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + addRaw(mock); + await mock.start(); + const event = await firstEvent("raw-ws"); + expect(event).toEqual({ + type: "error", + error: { message: RAW_WS_ERROR, type: "server_error", code: RAW_MODE_UNAVAILABLE }, + }); + const entries = mock.getRequests(); + expect(entries).toHaveLength(1); + expect(entries[0].response.status).toBe(500); + expect(entries[0].response.fixture).toBe(mock.getFixtures()[0]); + expect(entries[0].response.error).toBe(RAW_WS_ERROR); + }); + + it("logs the raw WS error with its code", async () => { + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + mock = new PlainLLMock({ port: 0, host: "127.0.0.1", logLevel: "warn" }); + addRaw(mock); + await mock.start(); + const event = await firstEvent("raw-ws"); + expect(event).toMatchObject({ error: { code: RAW_MODE_UNAVAILABLE } }); + const logged = error.mock.calls.map((call) => call.join(" ")).join("\n"); + expect(logged).toContain(RAW_MODE_UNAVAILABLE); + expect(logged).toContain(RAW_WS_ERROR); + }); + + it("records the skipped misbehavior outcome under a server-level fault", async () => { + const withFaults = new LLMock({ + port: 0, + host: "127.0.0.1", + logLevel: "silent", + misbehavior: { faults: [{ fault: "empty-response", rate: 1 }] }, + }); + mock = withFaults; + addRaw(withFaults); + await withFaults.start(); + const event = await firstEvent("raw-ws"); + expect(event).toMatchObject({ type: "error", error: { message: RAW_WS_ERROR } }); + const [entry] = withFaults.getRequests(); + expect(entry.response.status).toBe(500); + expect(entry.response.misbehavior).toMatchObject({ + applied: false, + source: "server", + wire: "openai-responses", + evaluations: [{ fault: "empty-response", outcome: "skipped", reason: "not-applicable" }], + }); + }); + + it("a turnIndex fixture still matches as before (no record match options on WS)", async () => { + const warn = vi.spyOn(console, "warn"); + mock = new PlainLLMock({ port: 0, host: "127.0.0.1", logLevel: "warn" }); + mock.addFixture({ + match: { userMessage: "turn", turnIndex: 3 }, + response: { content: "turn reply" }, + }); + await mock.start(); + const event = await firstEvent("turn"); + expect(event.type).toBe("response.created"); + expect(mock.getRequests()[0].response.status).toBe(200); + expect(warn.mock.calls.flat().join(" ")).not.toMatch(/turnIndex/); + }); +}); diff --git a/src/bedrock-converse.ts b/src/bedrock-converse.ts index 1ecdc2d9..39096dde 100644 --- a/src/bedrock-converse.ts +++ b/src/bedrock-converse.ts @@ -30,6 +30,7 @@ import { isCombinedFixtureResponse, resolveFixtureBlockOutcome, isErrorResponse, + isRawResponse, flattenHeaders, isJsonObject, getContext, @@ -54,6 +55,7 @@ import { mapAnthropicFinish } from "./response/finish.js"; import { render as renderConverseUsage } from "./response/usage/codecs/converse.js"; import { resolveRequestDecisions, strictOverrideField } from "./response/policy.js"; import { renderError } from "./response/errors.js"; +import { serveRawResponse } from "./response/raw.js"; import { applyChaosAsync } from "./chaos.js"; import { proxyAndRecord } from "./recorder.js"; import { planMisbehavior, recordMisbehaviorOutcome, type MisbehaviorPlan } from "./misbehavior.js"; @@ -968,7 +970,7 @@ export async function handleConverse( // Record mode proxies on a miss to capture a fresh turn (see record gate // below), so keep turnIndex strict to prevent an earlier-turn fixture from // shadowing a longer request and skipping the new turn's recording. - recordMatchOptions(!!defaults.record, defaults.logger), + { ...recordMatchOptions(!!defaults.record, defaults.logger), wire: "bedrock-converse" }, ); if (fixture) { @@ -1155,6 +1157,25 @@ export async function handleConverse( return; } + // Raw passthrough (#549): the authored events or JSON, framed for this wire. + if (isRawResponse(response)) { + await serveRawResponse({ + wire: "bedrock-converse", + stream: false, + req, + res, + fixture, + response, + body: completionReq, + journal, + defaults, + testId, + misbehaviorSummary: misbehavior.summary, + errorWire: "bedrock-converse", + }); + return; + } + // Error response if (isErrorResponse(response)) { const status = response.status ?? 500; @@ -1396,7 +1417,7 @@ export async function handleConverseStream( // Record mode proxies on a miss to capture a fresh turn (see record gate // below), so keep turnIndex strict to prevent an earlier-turn fixture from // shadowing a longer request and skipping the new turn's recording. - recordMatchOptions(!!defaults.record, defaults.logger), + { ...recordMatchOptions(!!defaults.record, defaults.logger), wire: "bedrock-converse" }, ); if (fixture) { @@ -1568,6 +1589,25 @@ export async function handleConverseStream( return; } + // Raw passthrough (#549): the authored events or JSON, framed for this wire. + if (isRawResponse(response)) { + await serveRawResponse({ + wire: "bedrock-converse", + stream: true, + req, + res, + fixture, + response, + body: completionReq, + journal, + defaults, + testId, + misbehaviorSummary: misbehavior.summary, + errorWire: "bedrock-converse", + }); + return; + } + // Error response if (isErrorResponse(response)) { const status = response.status ?? 500; diff --git a/src/bedrock.ts b/src/bedrock.ts index b18db8ee..e9768ac2 100644 --- a/src/bedrock.ts +++ b/src/bedrock.ts @@ -19,6 +19,7 @@ import type * as http from "node:http"; import { renderError } from "./response/errors.js"; +import { serveRawResponse } from "./response/raw.js"; import { resolveRequestDecisions, strictOverrideField } from "./response/policy.js"; import type { ChatCompletionRequest, @@ -39,6 +40,7 @@ import { isToolCallResponse, isCombinedFixtureResponse, isErrorResponse, + isRawResponse, flattenHeaders, isJsonObject, getContext, @@ -487,7 +489,7 @@ export async function handleBedrock( // Record mode proxies on a miss to capture a fresh turn (see record gate // below), so keep turnIndex strict to prevent an earlier-turn fixture from // shadowing a longer request and skipping the new turn's recording. - recordMatchOptions(!!defaults.record, defaults.logger), + { ...recordMatchOptions(!!defaults.record, defaults.logger), wire: "bedrock-invoke" }, ); if (fixture) { @@ -675,6 +677,25 @@ export async function handleBedrock( return; } + // Raw passthrough (#549): the authored events or JSON, framed for this wire. + if (isRawResponse(response)) { + await serveRawResponse({ + wire: "bedrock-invoke", + stream: false, + req, + res, + fixture, + response, + body: completionReq, + journal, + defaults, + testId, + misbehaviorSummary: misbehavior.summary, + errorWire: "bedrock", + }); + return; + } + // Error response if (isErrorResponse(response)) { const status = response.status ?? 500; @@ -1417,7 +1438,7 @@ export async function handleBedrockStream( // Record mode proxies on a miss to capture a fresh turn (see record gate // below), so keep turnIndex strict to prevent an earlier-turn fixture from // shadowing a longer request and skipping the new turn's recording. - recordMatchOptions(!!defaults.record, defaults.logger), + { ...recordMatchOptions(!!defaults.record, defaults.logger), wire: "bedrock-invoke" }, ); if (fixture) { @@ -1594,6 +1615,25 @@ export async function handleBedrockStream( return; } + // Raw passthrough (#549): the authored events or JSON, framed for this wire. + if (isRawResponse(response)) { + await serveRawResponse({ + wire: "bedrock-invoke", + stream: true, + req, + res, + fixture, + response, + body: completionReq, + journal, + defaults, + testId, + misbehaviorSummary: misbehavior.summary, + errorWire: "bedrock", + }); + return; + } + // Error response if (isErrorResponse(response)) { const status = response.status ?? 500; diff --git a/src/cohere.ts b/src/cohere.ts index 3ae5afd8..6e07656c 100644 --- a/src/cohere.ts +++ b/src/cohere.ts @@ -33,6 +33,7 @@ import { isCombinedFixtureResponse, isEmbeddingResponse, isErrorResponse, + isRawResponse, validateChatMessages, validateToolsField, flattenHeaders, @@ -64,6 +65,7 @@ import { mapCohereFinish } from "./response/finish.js"; import { render as renderCohereUsage } from "./response/usage/codecs/cohere.js"; import { resolveRequestDecisions, strictOverrideField } from "./response/policy.js"; import { renderError } from "./response/errors.js"; +import { serveRawResponse } from "./response/raw.js"; // ─── Cohere v2 Chat request types ─────────────────────────────────────────── @@ -942,7 +944,7 @@ export async function handleCohere( // Record mode proxies on a miss to capture a fresh turn (see record gate // below), so keep turnIndex strict to prevent an earlier-turn fixture from // shadowing a longer request and skipping the new turn's recording. - recordMatchOptions(!!defaults.record, defaults.logger), + { ...recordMatchOptions(!!defaults.record, defaults.logger), wire: "cohere" }, ); if (fixture) { @@ -1086,6 +1088,26 @@ export async function handleCohere( ); return; } + + // Raw passthrough (#549): the authored events or JSON, framed for this wire. + if (isRawResponse(resolvedResponse)) { + await serveRawResponse({ + wire: "cohere", + stream: cohereReq.stream === true, + req, + res, + fixture, + response: resolvedResponse, + body: completionReq, + journal, + defaults, + testId, + misbehaviorSummary: misbehavior.summary, + errorWire: "cohere", + }); + return; + } + // Cohere carries the same raw argument strings and call identities as Chat. // Reuse the accepted preparation so generated/duplicate IDs match metadata. const prepared = diff --git a/src/gemini-interactions.ts b/src/gemini-interactions.ts index 709bb623..6103a780 100644 --- a/src/gemini-interactions.ts +++ b/src/gemini-interactions.ts @@ -8,6 +8,7 @@ */ import { resolveRequestDecisions, strictOverrideField } from "./response/policy.js"; import { renderError } from "./response/errors.js"; +import { serveRawResponse } from "./response/raw.js"; import type * as http from "node:http"; import type { @@ -28,6 +29,7 @@ import { isToolCallResponse, isCombinedFixtureResponse, isErrorResponse, + isRawResponse, extractOverrides, generateToolCallId, flattenHeaders, @@ -958,6 +960,7 @@ export async function handleGeminiInteractions( completionReq, journal.getFixtureMatchCountsForTest(testId), defaults.requestTransform, + { wire: "gemini-interactions" }, ); if (fixture) { @@ -1133,6 +1136,26 @@ export async function handleGeminiInteractions( const markIncomplete = (body: T): T | (T & { status: "incomplete" }) => applied?.summary.fault === "stop-length-mid-tool" ? { ...body, status: "incomplete" } : body; + // Raw passthrough (#549): the authored events or JSON, framed for this wire. + // `streaming` is the handler's own rule: a stream unless `stream` is false. + if (isRawResponse(response)) { + await serveRawResponse({ + wire: "gemini-interactions", + stream: streaming, + req, + res, + fixture, + response, + body: completionReq, + journal, + defaults, + testId, + misbehaviorSummary: misbehavior.summary, + errorWire: "interactions", + }); + return; + } + // Error response if (isErrorResponse(response)) { const status = response.status ?? 500; diff --git a/src/gemini.ts b/src/gemini.ts index 45458f53..1c6fbd05 100644 --- a/src/gemini.ts +++ b/src/gemini.ts @@ -7,6 +7,7 @@ */ import { resolveRequestDecisions, strictOverrideField } from "./response/policy.js"; import { renderError } from "./response/errors.js"; +import { serveRawResponse } from "./response/raw.js"; import type * as http from "node:http"; import type { @@ -26,6 +27,7 @@ import { isToolCallResponse, isCombinedFixtureResponse, isErrorResponse, + isRawResponse, isAudioResponse, extractOverrides, requireServedFunctionToolCalls, @@ -1263,7 +1265,7 @@ export async function handleGemini( // Record mode proxies on a miss to capture a fresh turn (see record gate // below), so keep turnIndex strict to prevent an earlier-turn fixture from // shadowing a longer request and skipping the new turn's recording. - recordMatchOptions(!!defaults.record, defaults.logger), + { ...recordMatchOptions(!!defaults.record, defaults.logger), wire: "gemini" }, ); const path = req.url ?? `/v1beta/models/${model}:generateContent`; @@ -1469,6 +1471,26 @@ export async function handleGemini( return; } + // Raw passthrough (#549): the authored chunks or JSON, framed for this wire. + // Covers the Vertex AI route, which shares this handler. + if (isRawResponse(response)) { + await serveRawResponse({ + wire: "gemini", + stream: streaming, + req, + res, + fixture, + response, + body: completionReq, + journal, + defaults, + testId, + misbehaviorSummary: misbehavior.summary, + errorWire: "gemini", + }); + return; + } + // Error response if (isErrorResponse(response)) { const status = response.status ?? 500; diff --git a/src/messages.ts b/src/messages.ts index b4321133..928e97a5 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -8,6 +8,7 @@ import type * as http from "node:http"; import { renderError } from "./response/errors.js"; +import { serveRawResponse } from "./response/raw.js"; import { resolvePolicy, resolveRequestDecisions, strictOverrideField } from "./response/policy.js"; import { missingRequiredHeader } from "./response/profiles.js"; import type { @@ -36,6 +37,7 @@ import { isToolCallResponse, isCombinedFixtureResponse, isErrorResponse, + isRawResponse, resolveFixtureBlockOutcome, flattenHeaders, resolveRequestId, @@ -1691,7 +1693,7 @@ export async function handleMessages( // Record mode proxies on a miss to capture a fresh turn (see record gate // below), so keep turnIndex strict to prevent an earlier-turn fixture from // shadowing a longer request and skipping the new turn's recording. - recordMatchOptions(!!defaults.record, defaults.logger), + { ...recordMatchOptions(!!defaults.record, defaults.logger), wire: "anthropic" }, ); if (fixture) { @@ -1910,6 +1912,25 @@ export async function handleMessages( return; } + // Raw passthrough (#549): the authored events or JSON, framed for this wire. + if (isRawResponse(response)) { + await serveRawResponse({ + wire: "anthropic", + stream: claudeReq.stream === true, + req, + res, + fixture, + response, + body: completionReq, + journal, + defaults, + testId, + misbehaviorSummary: misbehavior.summary, + errorWire: "messages", + }); + return; + } + // Error response if (isErrorResponse(response)) { const status = response.status ?? 500; diff --git a/src/misbehavior.ts b/src/misbehavior.ts index 251f53a8..3dc338d1 100644 --- a/src/misbehavior.ts +++ b/src/misbehavior.ts @@ -5,6 +5,7 @@ import { fixtureFiles, isCombinedFixtureResponse, isErrorResponse, + isRawResponse, isTextResponse, isToolCallResponse, rebuildOrderedBlocks, @@ -1721,6 +1722,9 @@ export function planMisbehavior(input: { source = "server"; parsed = parseMisbehavior(defaults.misbehavior.baseline); } else return { kind: "skipped" }; + // A raw fixture that sets its own misbehavior is a fixture error, not a + // fault to plan: the raw path answers it with its coded 500 (#549). + if (source === "fixture" && isRawResponse(input.response)) return { kind: "skipped" }; if (!parsed.ok) return { kind: "error", diff --git a/src/ollama.ts b/src/ollama.ts index fd7b0cfe..1003bac6 100644 --- a/src/ollama.ts +++ b/src/ollama.ts @@ -30,6 +30,7 @@ import { isToolCallResponse, isCombinedFixtureResponse, isErrorResponse, + isRawResponse, isEmbeddingResponse, isJSONResponse, validateChatMessages, @@ -69,6 +70,7 @@ import { proxyAndRecord } from "./recorder.js"; import { render as renderUsage } from "./response/usage/codecs/ollama.js"; import { resolveRequestDecisions, strictOverrideField } from "./response/policy.js"; import { renderError } from "./response/errors.js"; +import { serveRawResponse } from "./response/raw.js"; // ─── Ollama request types ──────────────────────────────────────────────────── @@ -817,7 +819,7 @@ export async function handleOllama( // Record mode proxies on a miss to capture a fresh turn (see record gate // below), so keep turnIndex strict to prevent an earlier-turn fixture from // shadowing a longer request and skipping the new turn's recording. - recordMatchOptions(!!defaults.record, defaults.logger), + { ...recordMatchOptions(!!defaults.record, defaults.logger), wire: "ollama" }, ); if (fixture) { @@ -950,6 +952,26 @@ export async function handleOllama( ); return; } + + // Raw passthrough (#549): the authored events or JSON, framed for this wire. + if (isRawResponse(response)) { + await serveRawResponse({ + wire: "ollama", + stream: ollamaReq.stream !== false, + req, + res, + fixture, + response, + body: completionReq, + journal, + defaults, + testId, + misbehaviorSummary: misbehavior.summary, + errorWire: "ollama", + }); + return; + } + const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); const replaySpeed = fixture.replaySpeed ?? defaults.replaySpeed; @@ -1308,7 +1330,7 @@ export async function handleOllamaGenerate( // Record mode proxies on a miss to capture a fresh turn (see record gate // below), so keep turnIndex strict to prevent an earlier-turn fixture from // shadowing a longer request and skipping the new turn's recording. - recordMatchOptions(!!defaults.record, defaults.logger), + { ...recordMatchOptions(!!defaults.record, defaults.logger), wire: "ollama-generate" }, ); if (fixture) { @@ -1444,6 +1466,26 @@ export async function handleOllamaGenerate( ); return; } + + // Raw passthrough (#549): the authored events or JSON, framed for this wire. + if (isRawResponse(response)) { + await serveRawResponse({ + wire: "ollama-generate", + stream: generateReq.stream !== false, + req, + res, + fixture, + response, + body: completionReq, + journal, + defaults, + testId, + misbehaviorSummary: misbehavior.summary, + errorWire: "ollama", + }); + return; + } + const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); const replaySpeed = fixture.replaySpeed ?? defaults.replaySpeed; diff --git a/src/responses.ts b/src/responses.ts index d342f3cc..caad43ee 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -36,6 +36,7 @@ import { isToolCallResponse, isCombinedFixtureResponse, isErrorResponse, + isRawResponse, serializeErrorResponse, flattenHeaders, isJsonObject, @@ -78,6 +79,7 @@ import { render as renderResponsesUsage } from "./response/usage/codecs/response import { mapResponsesFinish } from "./response/finish.js"; import { resolveRequestDecisions, strictOverrideField } from "./response/policy.js"; import { renderError } from "./response/errors.js"; +import { serveRawResponse } from "./response/raw.js"; // ─── Responses API request types ──────────────────────────────────────────── @@ -2015,7 +2017,7 @@ export async function handleResponses( // Record mode proxies on a miss to capture a fresh turn (see record gate // below), so keep turnIndex strict to prevent an earlier-turn fixture from // shadowing a longer request and skipping the new turn's recording. - recordMatchOptions(!!defaults.record, defaults.logger), + { ...recordMatchOptions(!!defaults.record, defaults.logger), wire: "openai-responses" }, ); if (fixture) { @@ -2260,6 +2262,25 @@ export async function handleResponses( return; } + // Raw passthrough (#549): the authored events or JSON, framed for this wire. + if (isRawResponse(response)) { + await serveRawResponse({ + wire: "openai-responses", + stream: responsesReq.stream === true, + req, + res, + fixture, + response, + body: completionReq, + journal, + defaults, + testId, + misbehaviorSummary: misbehavior.summary, + errorWire: "openai", + }); + return; + } + // Error response if (isErrorResponse(response)) { const status = response.status ?? 500; diff --git a/src/server.ts b/src/server.ts index 41670c07..0dfb6e6e 100644 --- a/src/server.ts +++ b/src/server.ts @@ -34,7 +34,7 @@ import { resolveMisbehaviorShortCircuit, } from "./misbehavior.js"; import { Journal } from "./journal.js"; -import { matchFixtureDiagnostic, recordMatchOptions } from "./router.js"; +import { matchFixtureDiagnostic, recordMatchOptions, type MatchOptions } from "./router.js"; import { validateFixtures, entryToFixture, @@ -56,6 +56,7 @@ import { writeErrorResponse, sseCodec } from "./sse-writer.js"; import { deliverJson, deliverStream } from "./response/deliver.js"; import { proxyInterruption, withHangUpDetection } from "./response/proxy.js"; import { CLIENT_ABORTED } from "./response/lifecycle.js"; +import { serveRawResponse } from "./response/raw.js"; import { extendRecordedTimings, misbehaviorPayloadLength } from "./response/timing.js"; import { prepareOpenAIChatMisbehavior, @@ -88,6 +89,7 @@ import { isImageResponse, isEmbeddingResponse, isJSONResponse, + isRawResponse, flattenHeaders, isJsonObject, validateToolsField, @@ -144,13 +146,7 @@ import { handleOpenRouterKey, handleOpenRouterCredits, } from "./openrouter-chat.js"; -import type { - FixtureResponse, - ErrorResponse, - ChatCompletion, - SSEChunk, - ResponseOverrides, -} from "./types.js"; +import type { FixtureResponse, ChatCompletion, SSEChunk, ResponseOverrides } from "./types.js"; import { handleResponses } from "./responses.js"; import { handleMessages } from "./messages.js"; import { handleCountTokens } from "./count-tokens.js"; @@ -623,6 +619,7 @@ function fixtureResponseKind(response: Fixture["response"]): string { if (isCombinedFixtureResponse(response)) return "contentWithToolCalls"; if (isToolCallResponse(response)) return "toolCalls"; if (isTextResponse(response)) return "text"; + if (isRawResponse(response)) return "raw"; return "unknown"; } @@ -1831,14 +1828,17 @@ async function handleCompletions( // (headers > fixture.chaos > server defaults), so the fixture has to be // known before we can roll with the right config. const testId = getTestId(req); - const matchOptions = recordMatchOptions( - // In record mode a miss proxies upstream to capture a fresh turn, so an - // earlier-turn capture must not shadow a longer request via the relaxed - // turnIndex disambiguator — keep turnIndex a strict gate while recording. - // This handler's record gate (below) is `defaults.record && providerKey`. - !!(defaults.record && providerKey), - defaults.logger, - ); + const matchOptions: MatchOptions = { + ...recordMatchOptions( + // In record mode a miss proxies upstream to capture a fresh turn, so an + // earlier-turn capture must not shadow a longer request via the relaxed + // turnIndex disambiguator — keep turnIndex a strict gate while recording. + // This handler's record gate (below) is `defaults.record && providerKey`. + !!(defaults.record && providerKey), + defaults.logger, + ), + wire: "openai-chat", + }; // OpenRouter `models[]` fallback simulation. When the request arrived on the // OpenRouter base AND carries a fallback array, attempt the candidate list @@ -1927,7 +1927,12 @@ async function handleCompletions( skippedBySequenceOrTurn = Math.max(skippedBySequenceOrTurn, attempt.skippedBySequenceOrTurn); if (!attempt.fixture) return null; const response = await resolveResponse(attempt.fixture, probe); - return { fixture: attempt.fixture, response, isError: isErrorResponse(response) }; + // A raw fixture with an error status is a provider failure, like a typed + // error fixture with the same status, so it fails over the same way. + const isError = + isErrorResponse(response) || + (isRawResponse(response) && (response.raw.status ?? 200) >= 400); + return { fixture: attempt.fixture, response, isError }; }; for (const candidate of candidates) { const outcome = await resolveCandidate(candidate); @@ -1955,7 +1960,9 @@ async function handleCompletions( // (openclaw #60191). Absent / `true` keeps the default fall-through. // Composes with the request-level `allowFallbacks` gate: fail over only // when BOTH allow it (if EITHER says don't, this candidate is terminal). - const errorFallsThrough = (outcome.response as ErrorResponse).fallthrough !== false; + // A raw fixture has no `fallthrough` field, so it keeps the default. + const errorFallsThrough = + !isErrorResponse(outcome.response) || outcome.response.fallthrough !== false; if (allowFallbacks && errorFallsThrough) { // Runtime provider failure — remember it and fail over to the next. lastErrorFixture = outcome.fixture; @@ -2480,6 +2487,27 @@ async function handleCompletions( // Opt-in `: OPENROUTER PROCESSING` keepalive comment lines (default off). const openRouterProcessing = !!(openRouter && fixture.openRouterProcessing); + // Raw passthrough (#549): the authored chunks or JSON, framed for this wire. + // OpenRouter requests get OpenRouter's error dialect. + if (isRawResponse(response)) { + await serveRawResponse({ + wire: "openai-chat", + stream: body.stream === true, + req, + res, + fixture, + response, + body, + journal, + defaults, + testId, + misbehaviorSummary: evaluation.summary, + errorWire: "openai", + openRouter, + }); + return; + } + // Error response if (isErrorResponse(response)) { const status = response.status ?? 500; diff --git a/src/ws-responses.ts b/src/ws-responses.ts index 5976f207..fee0cff1 100644 --- a/src/ws-responses.ts +++ b/src/ws-responses.ts @@ -30,6 +30,7 @@ import { isToolCallResponse, isCombinedFixtureResponse, isErrorResponse, + isRawResponse, extractOverrides, resolveResponse, resolveReasoningForModel, @@ -48,6 +49,7 @@ import { buildOpenAIError } from "./response/errors.js"; import { markWsTurnStopped, watchWsClose, wsLifecycle } from "./response/lifecycle.js"; import { runFrameLoop, wsSink, type FrameCodec } from "./response/frame-loop.js"; import { extendRecordedTimings } from "./response/timing.js"; +import { RAW_MODE_UNAVAILABLE } from "./response/raw.js"; type ResponsesWebSocketDefaults = Omit & { replaySpeed?: number; @@ -196,6 +198,9 @@ async function processMessage( completionReq, journal.getFixtureMatchCountsForTest(testId), defaults.requestTransform, + // Wire only: recordMatchOptions here would turn on strict turnIndex and the + // relaxed-turnIndex warning, which this transport has never had. + { wire: "openai-responses" }, ); if (fixture) { @@ -334,6 +339,24 @@ async function processMessage( defaults.upgradeHeaders, ).reasoningSuppression; + // Raw passthrough fixtures are served over HTTP only for now. Fail loudly + // instead of the generic unknown-type error so the author sees why. + if (isRawResponse(response)) { + const message = "raw fixtures are not yet supported on the WebSocket transport"; + const entry = journal.add({ + method: "WS", + path: "/v1/responses", + headers: flattenHeaders(defaults.upgradeHeaders ?? {}), + body: completionReq, + response: { status: 500, fixture }, + }); + entry.response.error = message; + recordOutcome(entry); + defaults.logger.error(`WebSocket responses error: ${RAW_MODE_UNAVAILABLE}: ${message}`); + ws.send(JSON.stringify(buildErrorEvent(message, "server_error", RAW_MODE_UNAVAILABLE))); + return; + } + // Error response if (isErrorResponse(response)) { const status = response.status ?? 500; From 361cc4413c67869984b0ed6bd1c9eaa47feda6b5 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sat, 24 Oct 2026 11:33:09 -0700 Subject: [PATCH 6/7] test: raw passthrough per-wire, header, feature and no-regression coverage (#549) --- .../raw-no-regression.test.ts.snap | 139 ++++ src/__tests__/raw-anthropic.test.ts | 227 +++++++ src/__tests__/raw-bedrock-converse.test.ts | 168 +++++ src/__tests__/raw-bedrock-invoke.test.ts | 171 +++++ src/__tests__/raw-cohere.test.ts | 152 +++++ src/__tests__/raw-features.test.ts | 638 ++++++++++++++++++ src/__tests__/raw-gemini-interactions.test.ts | 198 ++++++ src/__tests__/raw-gemini.test.ts | 183 +++++ src/__tests__/raw-headers.test.ts | 304 +++++++++ src/__tests__/raw-no-regression.test.ts | 204 ++++++ src/__tests__/raw-ollama.test.ts | 234 +++++++ src/__tests__/raw-openai-chat.test.ts | 270 ++++++++ src/__tests__/raw-openai-responses.test.ts | 180 +++++ src/__tests__/raw-wire-plumbing.test.ts | 204 ++++++ 14 files changed, 3272 insertions(+) create mode 100644 src/__tests__/__snapshots__/raw-no-regression.test.ts.snap create mode 100644 src/__tests__/raw-anthropic.test.ts create mode 100644 src/__tests__/raw-bedrock-converse.test.ts create mode 100644 src/__tests__/raw-bedrock-invoke.test.ts create mode 100644 src/__tests__/raw-cohere.test.ts create mode 100644 src/__tests__/raw-features.test.ts create mode 100644 src/__tests__/raw-gemini-interactions.test.ts create mode 100644 src/__tests__/raw-gemini.test.ts create mode 100644 src/__tests__/raw-headers.test.ts create mode 100644 src/__tests__/raw-no-regression.test.ts create mode 100644 src/__tests__/raw-ollama.test.ts create mode 100644 src/__tests__/raw-openai-chat.test.ts create mode 100644 src/__tests__/raw-openai-responses.test.ts create mode 100644 src/__tests__/raw-wire-plumbing.test.ts diff --git a/src/__tests__/__snapshots__/raw-no-regression.test.ts.snap b/src/__tests__/__snapshots__/raw-no-regression.test.ts.snap new file mode 100644 index 00000000..d5f385a3 --- /dev/null +++ b/src/__tests__/__snapshots__/raw-no-regression.test.ts.snap @@ -0,0 +1,139 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`raw no-regression: unscoped json and journal bodies > the journaled request body of a typed fixture is unchanged on every wire 1`] = ` +{ + "anthropic": { + "_context": undefined, + "_endpointType": "chat", + "max_tokens": 64, + "messages": [ + { + "content": "journal", + "role": "user", + }, + ], + "model": "claude-sonnet-4-5", + "stream": undefined, + "temperature": undefined, + "tools": undefined, + }, + "bedrock-converse": { + "_context": undefined, + "_endpointType": "chat", + "max_tokens": undefined, + "messages": [ + { + "content": "journal", + "role": "user", + }, + ], + "model": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "stream": false, + "temperature": undefined, + "tools": undefined, + }, + "bedrock-invoke": { + "_context": undefined, + "_endpointType": "chat", + "max_tokens": 64, + "messages": [ + { + "content": "journal", + "role": "user", + }, + ], + "model": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "stream": false, + "temperature": undefined, + "tools": undefined, + }, + "cohere": { + "_context": undefined, + "_endpointType": "chat", + "messages": [ + { + "content": "journal", + "role": "user", + }, + ], + "model": "command-r-plus", + "stream": undefined, + "tools": undefined, + }, + "gemini": { + "_context": undefined, + "_endpointType": "chat", + "max_tokens": undefined, + "messages": [ + { + "content": "journal", + "role": "user", + }, + ], + "model": "gemini-2.5-flash", + "stream": false, + "temperature": undefined, + "tools": undefined, + "top_k": undefined, + "top_p": undefined, + }, + "gemini-interactions": { + "_context": undefined, + "_endpointType": "chat", + "max_tokens": undefined, + "messages": [ + { + "content": "journal", + "role": "user", + }, + ], + "model": "gemini-2.5-flash", + "stream": false, + "temperature": undefined, + "tools": undefined, + }, + "ollama": { + "_context": undefined, + "_endpointType": "chat", + "max_tokens": undefined, + "messages": [ + { + "content": "journal", + "role": "user", + }, + ], + "model": "llama3.1", + "stream": false, + "temperature": undefined, + "tools": undefined, + }, + "openai-chat": { + "_context": undefined, + "_endpointType": "chat", + "messages": [ + { + "content": "journal", + "role": "user", + }, + ], + "model": "gpt-4.1", + }, + "openai-responses": { + "_context": undefined, + "_endpointType": "chat", + "max_tokens": undefined, + "messages": [ + { + "content": "journal", + "role": "user", + }, + ], + "model": "gpt-4.1", + "response_format": undefined, + "stream": false, + "temperature": undefined, + "tool_choice": undefined, + "tools": undefined, + }, +} +`; diff --git a/src/__tests__/raw-anthropic.test.ts b/src/__tests__/raw-anthropic.test.ts new file mode 100644 index 00000000..4f864297 --- /dev/null +++ b/src/__tests__/raw-anthropic.test.ts @@ -0,0 +1,227 @@ +import Anthropic from "@anthropic-ai/sdk"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { LLMock } from "../llmock.js"; +import type { RawResponse } from "../types.js"; + +// Raw passthrough on the Anthropic Messages wire (#549). + +const events: Record[] = [ + { + type: "message_start", + message: { + id: "msg_raw", + type: "message", + role: "assistant", + model: "claude-sonnet-4-5", + content: [], + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 0 }, + }, + }, + { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, + { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Raw stream." } }, + { type: "content_block_stop", index: 0 }, + { + type: "message_delta", + delta: { stop_reason: "end_turn", stop_sequence: null }, + usage: { output_tokens: 3 }, + }, + { type: "message_stop" }, +]; + +const json = { + id: "msg_raw_json", + type: "message", + role: "assistant", + model: "claude-sonnet-4-5", + content: [{ type: "text", text: "Raw JSON reply." }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 3 }, +}; + +const raw: RawResponse = { raw: { wire: "anthropic", events, json } }; + +// MessageStream accumulates its snapshot into the `message_start` event object +// as later events arrive, so each event is cloned when the SDK emits it. +async function collectEvents(stream: ReturnType) { + const received: unknown[] = []; + stream.on("streamEvent", (event: unknown) => received.push(structuredClone(event))); + await stream.done(); + return received; +} + +let mock: LLMock; +let client: Anthropic; + +beforeEach(async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.on({ userMessage: "raw-proof" }, () => raw); + await mock.start(); + client = new Anthropic({ apiKey: "local-proof", baseURL: mock.url, maxRetries: 0 }); +}); + +afterEach(async () => { + await mock.stop(); +}); + +describe("raw passthrough: anthropic", () => { + it("streams exactly the authored events in order", async () => { + const stream = client.messages.stream({ + model: "claude-sonnet-4-5", + max_tokens: 64, + messages: [{ role: "user", content: "raw-proof" }], + }); + const received = await collectEvents(stream); + expect(received).toEqual(events); + }); + + it("returns the authored json for a non-streaming request", async () => { + const reply = await client.messages.create({ + model: "claude-sonnet-4-5", + max_tokens: 64, + messages: [{ role: "user", content: "raw-proof" }], + }); + expect(reply).toEqual(json); + }); +}); + +describe("raw passthrough: anthropic wire details", () => { + it("passes an unmodeled delta type through messages.stream", async () => { + const future: Record[] = [ + events[0], + { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, + { type: "content_block_delta", index: 0, delta: { type: "future_delta", payload: [1, 2] } }, + { type: "content_block_stop", index: 0 }, + events[4], + events[5], + ]; + mock.on({ userMessage: "raw-future" }, () => ({ raw: { wire: "anthropic", events: future } })); + const stream = client.messages.stream({ + model: "claude-sonnet-4-5", + max_tokens: 64, + messages: [{ role: "user", content: "raw-future" }], + }); + const received = await collectEvents(stream); + expect(received).toEqual(future); + }); + + it("frames each event as a named SSE event", async () => { + const res = await fetch(`${mock.url}/v1/messages`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "claude-sonnet-4-5", + max_tokens: 64, + stream: true, + messages: [{ role: "user", content: "raw-proof" }], + }), + }); + expect(res.headers.get("content-type")).toBe("text/event-stream"); + expect(await res.text()).toBe( + events.map((e) => `event: ${String(e.type)}\ndata: ${JSON.stringify(e)}\n\n`).join(""), + ); + }); + + it("journals a client abort mid-stream as client aborted", async () => { + const slow = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent", latency: 2000 }); + slow.on({ userMessage: "raw-abort" }, () => raw); + await slow.start(); + try { + const controller = new AbortController(); + const res = await fetch(`${slow.url}/v1/messages`, { + method: "POST", + signal: controller.signal, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "claude-sonnet-4-5", + max_tokens: 64, + stream: true, + messages: [{ role: "user", content: "raw-abort" }], + }), + }); + expect(res.status).toBe(200); + const reader = res.body!.getReader(); + expect((await reader.read()).done).toBe(false); + controller.abort(); + await expect(reader.read()).rejects.toMatchObject({ name: "AbortError" }); + await expect + .poll(() => slow.getRequests()[0]?.response.interrupted, { timeout: 1000 }) + .toBe(true); + expect(slow.getRequests()[0].response.interruptReason).toBe("client aborted"); + expect(slow.getRequests()[0].response.status).toBe(200); + } finally { + await slow.stop(); + } + }, 10000); +}); + +describe("raw passthrough: anthropic error bodies", () => { + async function post(content: string, stream: boolean): Promise<{ status: number; body: string }> { + const res = await fetch(`${mock.url}/v1/messages`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "claude-sonnet-4-5", + max_tokens: 64, + stream, + messages: [{ role: "user", content }], + }), + }); + return { status: res.status, body: await res.text() }; + } + + const expected = (code: string, message: string): string => + JSON.stringify({ type: "error", error: { type: "api_error", code, message } }); + + it("a streaming request on a json-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "json-only" }, () => ({ raw: { wire: "anthropic", json } })); + const got = await post("json-only", true); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "events" body for a streaming request on anthropic`, + ), + ); + }); + + it("a non-streaming request on an events-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "events-only" }, () => ({ raw: { wire: "anthropic", events } })); + const got = await post("events-only", false); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "json" body for a non-streaming request on anthropic`, + ), + ); + }); + + it("a factory for another wire: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "other-wire" }, () => ({ raw: { wire: "openai-responses", json } })); + const got = await post("other-wire", false); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected("aimock_raw_fixture_invalid", "raw fixture is for openai-responses, not anthropic"), + ); + }); + + it("a factory that fails the checker: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "bad-status" }, () => ({ + raw: { wire: "anthropic", json, status: 99 }, + })); + const got = await post("bad-status", false); + expect(got.status).toBe(500); + const parsed = JSON.parse(got.body) as { + type: string; + error: { type: string; code: string; message: string }; + }; + expect(Object.keys(parsed)).toEqual(["type", "error"]); + expect(Object.keys(parsed.error)).toEqual(["type", "code", "message"]); + expect(parsed.error.type).toBe("api_error"); + expect(parsed.error.code).toBe("aimock_raw_fixture_invalid"); + expect(parsed.error.message).toMatch(/^invalid raw fixture: .*status/); + }); +}); diff --git a/src/__tests__/raw-bedrock-converse.test.ts b/src/__tests__/raw-bedrock-converse.test.ts new file mode 100644 index 00000000..01c987dd --- /dev/null +++ b/src/__tests__/raw-bedrock-converse.test.ts @@ -0,0 +1,168 @@ +import { + BedrockRuntimeClient, + ConverseCommand, + ConverseStreamCommand, +} from "@aws-sdk/client-bedrock-runtime"; +import { NodeHttpHandler } from "@smithy/node-http-handler"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { LLMock } from "../llmock.js"; +import type { RawResponse } from "../types.js"; + +// Raw passthrough on the Bedrock Converse wire (#549). + +const events: Record[] = [ + { messageStart: { role: "assistant" } }, + { contentBlockDelta: { contentBlockIndex: 0, delta: { text: "Raw converse." } } }, + { contentBlockStop: { contentBlockIndex: 0 } }, + { messageStop: { stopReason: "end_turn" } }, + { + metadata: { + usage: { inputTokens: 1, outputTokens: 3, totalTokens: 4 }, + metrics: { latencyMs: 1 }, + }, + }, +]; + +const json = { + output: { message: { role: "assistant", content: [{ text: "Raw JSON reply." }] } }, + stopReason: "end_turn", + usage: { inputTokens: 1, outputTokens: 3, totalTokens: 4 }, + metrics: { latencyMs: 1 }, +}; + +const raw: RawResponse = { raw: { wire: "bedrock-converse", events, json } }; + +const converse = { + modelId: "anthropic.claude-3-5-sonnet-20241022-v2:0", + messages: [{ role: "user" as const, content: [{ text: "raw-proof" }] }], +}; + +let mock: LLMock; +let client: BedrockRuntimeClient; + +beforeEach(async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.on({ userMessage: "raw-proof" }, () => raw); + await mock.start(); + client = new BedrockRuntimeClient({ + endpoint: mock.url, + region: "us-east-1", + credentials: { accessKeyId: "mock", secretAccessKey: "mock" }, + maxAttempts: 1, + requestHandler: new NodeHttpHandler(), + }); +}); + +afterEach(async () => { + client.destroy(); + await mock.stop(); +}); + +describe("raw passthrough: bedrock-converse", () => { + it("streams exactly the authored events in order", async () => { + const stream = await client.send(new ConverseStreamCommand(converse)); + const received: unknown[] = []; + for await (const event of stream.stream ?? []) received.push(event); + expect(received).toEqual(events); + }); + + it("returns the authored json for Converse", async () => { + const reply = await client.send(new ConverseCommand(converse)); + // $metadata is added by the SDK, not by the wire; every other field is exact. + expect({ ...reply, $metadata: undefined }).toEqual(json); + }); +}); + +describe("raw passthrough: bedrock-converse error bodies", () => { + const model = "anthropic.claude-3-5-sonnet-20241022-v2:0"; + + async function post( + userMessage: string, + stream: boolean, + ): Promise<{ status: number; errorType: string | null; body: string }> { + const path = stream ? "converse-stream" : "converse"; + const res = await fetch(`${mock.url}/model/${model}/${path}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ messages: [{ role: "user", content: [{ text: userMessage }] }] }), + }); + return { + status: res.status, + errorType: res.headers.get("x-amzn-errortype"), + body: await res.text(), + }; + } + + const expected = (code: string, message: string): string => + JSON.stringify({ __type: "InternalServerException", message, code }); + + it("a streaming request on a json-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "json-only" }, () => ({ raw: { wire: "bedrock-converse", json } })); + const got = await post("json-only", true); + expect(got.status).toBe(500); + expect(got.errorType).toBe("InternalServerException"); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "events" body for a streaming request on bedrock-converse`, + ), + ); + }); + + it("a non-streaming request on an events-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "events-only" }, () => ({ + raw: { wire: "bedrock-converse", events }, + })); + const got = await post("events-only", false); + expect(got.status).toBe(500); + expect(got.errorType).toBe("InternalServerException"); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "json" body for a non-streaming request on bedrock-converse`, + ), + ); + }); + + it("a factory for another wire: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "other-wire" }, () => ({ raw: { wire: "bedrock-invoke", json } })); + const got = await post("other-wire", false); + expect(got.status).toBe(500); + expect(got.errorType).toBe("InternalServerException"); + expect(got.body).toBe( + expected( + "aimock_raw_fixture_invalid", + "raw fixture is for bedrock-invoke, not bedrock-converse", + ), + ); + }); + + it("a factory that fails the checker: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "bad-status" }, () => ({ + raw: { wire: "bedrock-converse", json, status: 99 }, + })); + const got = await post("bad-status", false); + expect(got.status).toBe(500); + expect(got.errorType).toBe("InternalServerException"); + const parsed = JSON.parse(got.body) as Record; + expect(Object.keys(parsed)).toEqual(["__type", "message", "code"]); + expect(parsed.code).toBe("aimock_raw_fixture_invalid"); + expect(parsed.message).toMatch(/^invalid raw fixture: .*status/); + }); + + it("an event whose payload is not a JSON object: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "bad-event" }, () => ({ + raw: { wire: "bedrock-converse", events: [{ contentBlockDelta: "text" }] }, + })); + const got = await post("bad-event", true); + expect(got.status).toBe(500); + expect(got.errorType).toBe("InternalServerException"); + expect(got.body).toBe( + expected( + "aimock_raw_fixture_invalid", + "invalid raw fixture: raw.events[0].contentBlockDelta must be a JSON object " + + "(the frame payload) on bedrock-converse", + ), + ); + }); +}); diff --git a/src/__tests__/raw-bedrock-invoke.test.ts b/src/__tests__/raw-bedrock-invoke.test.ts new file mode 100644 index 00000000..e9afafe3 --- /dev/null +++ b/src/__tests__/raw-bedrock-invoke.test.ts @@ -0,0 +1,171 @@ +import { + BedrockRuntimeClient, + InvokeModelCommand, + InvokeModelWithResponseStreamCommand, +} from "@aws-sdk/client-bedrock-runtime"; +import { NodeHttpHandler } from "@smithy/node-http-handler"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { LLMock } from "../llmock.js"; +import type { RawResponse } from "../types.js"; + +// Raw passthrough on the Bedrock InvokeModel wire (#549). + +const events: Record[] = [ + { + type: "message_start", + message: { + id: "msg_raw", + type: "message", + role: "assistant", + model: "claude-sonnet-4-5", + content: [], + stop_reason: null, + usage: { input_tokens: 1, output_tokens: 0 }, + }, + }, + { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, + { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Raw bedrock." } }, + { type: "content_block_stop", index: 0 }, + { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 3 } }, + { type: "message_stop" }, +]; + +const json = { + id: "msg_raw_json", + type: "message", + role: "assistant", + model: "claude-sonnet-4-5", + content: [{ type: "text", text: "Raw JSON reply." }], + stop_reason: "end_turn", + usage: { input_tokens: 1, output_tokens: 3 }, +}; + +const raw: RawResponse = { raw: { wire: "bedrock-invoke", events, json } }; + +const invoke = { + modelId: "anthropic.claude-3-5-sonnet-20241022-v2:0", + contentType: "application/json", + body: JSON.stringify({ + anthropic_version: "bedrock-2023-05-31", + max_tokens: 64, + messages: [{ role: "user", content: "raw-proof" }], + }), +}; + +let mock: LLMock; +let client: BedrockRuntimeClient; + +beforeEach(async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.on({ userMessage: "raw-proof" }, () => raw); + await mock.start(); + client = new BedrockRuntimeClient({ + endpoint: mock.url, + region: "us-east-1", + credentials: { accessKeyId: "mock", secretAccessKey: "mock" }, + maxAttempts: 1, + requestHandler: new NodeHttpHandler(), + }); +}); + +afterEach(async () => { + client.destroy(); + await mock.stop(); +}); + +describe("raw passthrough: bedrock-invoke", () => { + it("streams exactly the authored events as chunk.bytes", async () => { + const streamed = await client.send(new InvokeModelWithResponseStreamCommand(invoke)); + const received: unknown[] = []; + const unexpected: unknown[] = []; + for await (const event of streamed.body ?? []) { + if (event.chunk?.bytes) + received.push(JSON.parse(new TextDecoder().decode(event.chunk.bytes))); + else unexpected.push(event); + } + expect(unexpected).toEqual([]); + expect(received).toEqual(events); + }); + + it("returns the authored json for InvokeModel", async () => { + const reply = await client.send(new InvokeModelCommand(invoke)); + expect(JSON.parse(new TextDecoder().decode(reply.body))).toEqual(json); + }); +}); + +describe("raw passthrough: bedrock-invoke error bodies", () => { + const model = "anthropic.claude-3-5-sonnet-20241022-v2:0"; + + async function post( + userMessage: string, + stream: boolean, + ): Promise<{ status: number; errorType: string | null; body: string }> { + const path = stream ? "invoke-with-response-stream" : "invoke"; + const res = await fetch(`${mock.url}/model/${model}/${path}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + anthropic_version: "bedrock-2023-05-31", + max_tokens: 64, + messages: [{ role: "user", content: userMessage }], + }), + }); + return { + status: res.status, + errorType: res.headers.get("x-amzn-errortype"), + body: await res.text(), + }; + } + + const expected = (code: string, message: string): string => + JSON.stringify({ __type: "InternalServerException", code, message }); + + it("a streaming request on a json-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "json-only" }, () => ({ raw: { wire: "bedrock-invoke", json } })); + const got = await post("json-only", true); + expect(got.status).toBe(500); + expect(got.errorType).toBe("InternalServerException"); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "events" body for a streaming request on bedrock-invoke`, + ), + ); + }); + + it("a non-streaming request on an events-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "events-only" }, () => ({ raw: { wire: "bedrock-invoke", events } })); + const got = await post("events-only", false); + expect(got.status).toBe(500); + expect(got.errorType).toBe("InternalServerException"); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "json" body for a non-streaming request on bedrock-invoke`, + ), + ); + }); + + it("a factory for another wire: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "other-wire" }, () => ({ raw: { wire: "anthropic", json } })); + const got = await post("other-wire", false); + expect(got.status).toBe(500); + expect(got.errorType).toBe("InternalServerException"); + expect(got.body).toBe( + expected("aimock_raw_fixture_invalid", "raw fixture is for anthropic, not bedrock-invoke"), + ); + }); + + it("a factory that fails the checker: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "bad-status" }, () => ({ + raw: { wire: "bedrock-invoke", events, status: 99 }, + })); + const got = await post("bad-status", true); + expect(got.status).toBe(500); + expect(got.errorType).toBe("InternalServerException"); + const parsed = JSON.parse(got.body) as Record; + expect(Object.keys(parsed)).toEqual(["__type", "code", "message"]); + expect(parsed.code).toBe("aimock_raw_fixture_invalid"); + expect(parsed.message).toMatch(/^invalid raw fixture: .*status/); + }); +}); diff --git a/src/__tests__/raw-cohere.test.ts b/src/__tests__/raw-cohere.test.ts new file mode 100644 index 00000000..c588cfc8 --- /dev/null +++ b/src/__tests__/raw-cohere.test.ts @@ -0,0 +1,152 @@ +import { CohereClientV2 } from "cohere-ai"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { LLMock } from "../llmock.js"; +import type { RawResponse } from "../types.js"; + +// Raw passthrough on the Cohere v2 chat wire (#549). + +const events: Record[] = [ + { type: "message-start", id: "raw-cohere", delta: { message: { role: "assistant" } } }, + { type: "content-start", index: 0, delta: { message: { content: { type: "text", text: "" } } } }, + { type: "content-delta", index: 0, delta: { message: { content: { text: "Raw cohere." } } } }, + { type: "content-end", index: 0 }, + { type: "message-end", delta: { finish_reason: "COMPLETE" } }, +]; + +const json = { + id: "raw-cohere-json", + finish_reason: "COMPLETE", + message: { role: "assistant", content: [{ type: "text", text: "Raw JSON reply." }] }, +}; + +const raw: RawResponse = { raw: { wire: "cohere", events, json } }; + +let mock: LLMock; +let client: CohereClientV2; + +beforeEach(async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.on({ userMessage: "raw-proof" }, () => raw); + await mock.start(); + client = new CohereClientV2({ + token: "local-proof", + baseUrl: mock.url, + maxRetries: 0, + timeoutInSeconds: 5, + }); +}); + +afterEach(async () => { + await mock.stop(); +}); + +const request = { + model: "command-r-plus", + messages: [{ role: "user" as const, content: "raw-proof" }], +}; + +describe("raw passthrough: cohere", () => { + it("streams exactly the authored events in order", async () => { + const received: { type?: string }[] = []; + for await (const event of await client.chatStream(request)) received.push(event); + // The SDK camel-cases keys (finish_reason -> finishReason); the event + // sequence and authored text must survive unchanged. + expect(received.map((event) => event.type)).toEqual(events.map((event) => event.type)); + expect(received[2]).toMatchObject(events[2]); + expect(received[4]).toMatchObject({ delta: { finishReason: "COMPLETE" } }); + }); + + it("returns the authored json for a non-streaming request", async () => { + const reply = await client.chat(request); + expect(reply).toMatchObject({ + id: "raw-cohere-json", + finishReason: "COMPLETE", + message: { role: "assistant", content: [{ type: "text", text: "Raw JSON reply." }] }, + }); + }); + + it("frames each event as a named SSE event with no [DONE] terminal", async () => { + const res = await fetch(`${mock.url}/v2/chat`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ...request, stream: true }), + }); + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toBe("text/event-stream"); + const text = await res.text(); + expect(text).toBe( + events.map((e) => `event: ${String(e.type)}\ndata: ${JSON.stringify(e)}\n\n`).join(""), + ); + expect(text).not.toContain("[DONE]"); + }); +}); + +describe("raw passthrough: cohere error bodies", () => { + async function post(content: string, stream: boolean): Promise<{ status: number; body: string }> { + const res = await fetch(`${mock.url}/v2/chat`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "command-r-plus", + messages: [{ role: "user", content }], + stream, + }), + }); + return { status: res.status, body: await res.text() }; + } + + const expected = (code: string, message: string): string => JSON.stringify({ message, code }); + + it("a streaming request on a json-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "json-only" }, () => ({ raw: { wire: "cohere", json } })); + const got = await post("json-only", true); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "events" body for a streaming request on cohere`, + ), + ); + }); + + it("a non-streaming request on an events-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "events-only" }, () => ({ raw: { wire: "cohere", events } })); + const got = await post("events-only", false); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "json" body for a non-streaming request on cohere`, + ), + ); + }); + + it("a factory for another wire: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "other-wire" }, () => ({ raw: { wire: "ollama", json } })); + const got = await post("other-wire", false); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected("aimock_raw_fixture_invalid", "raw fixture is for ollama, not cohere"), + ); + }); + + it("a factory that sets done (openai-chat only): aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "bad-done" }, () => ({ raw: { wire: "cohere", events, done: true } })); + const got = await post("bad-done", true); + expect(got.status).toBe(500); + const parsed = JSON.parse(got.body) as { message: string; code: string }; + expect(Object.keys(parsed)).toEqual(["message", "code"]); + expect(parsed.code).toBe("aimock_raw_fixture_invalid"); + expect(parsed.message).toMatch(/^invalid raw fixture: .*done/); + }); + + it("a factory that fails the checker: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "bad-status" }, () => ({ raw: { wire: "cohere", json, status: 99 } })); + const got = await post("bad-status", false); + expect(got.status).toBe(500); + const parsed = JSON.parse(got.body) as { message: string; code: string }; + expect(Object.keys(parsed)).toEqual(["message", "code"]); + expect(parsed.code).toBe("aimock_raw_fixture_invalid"); + expect(parsed.message).toMatch(/^invalid raw fixture: .*status/); + }); +}); diff --git a/src/__tests__/raw-features.test.ts b/src/__tests__/raw-features.test.ts new file mode 100644 index 00000000..2e7b4a5a --- /dev/null +++ b/src/__tests__/raw-features.test.ts @@ -0,0 +1,638 @@ +import * as http from "node:http"; +import { afterEach, describe, expect, it } from "vitest"; +import { LLMock as PlainLLMock } from "../llmock.js"; +import { LLMock } from "./helpers/misbehavior-enabled.js"; +import type { FixtureFileResponse, FixtureOpts, MockServerOptions, RawWireId } from "../types.js"; + +// Raw passthrough fixtures (#549) combined with the features every typed +// fixture already has: stream interruption, client hang-ups, streaming and +// non-streaming timing, chaos, misbehavior, strict mode and sequence/turn +// matching. Raw goes through the shared delivery layer, so each case expects +// what a typed fixture gets on the same wire. + +const MODEL = "anthropic.claude-3-5-sonnet-20241022-v2:0"; + +type Framing = "sse" | "ndjson" | "eventstream"; + +interface StreamSite { + wire: RawWireId; + framing: Framing; + path: string; + body: (text: string) => unknown; + /** Five authored events: `truncateAfterChunks: 2` stops before the end. */ + events: Record[]; +} + +/** One streaming site per framing family, plus both OpenAI wires. */ +const STREAM_SITES: StreamSite[] = [ + { + wire: "openai-responses", + framing: "sse", + path: "/v1/responses", + body: (text) => ({ model: "gpt-4.1", input: text, stream: true }), + events: [ + { type: "response.created", response: { id: "resp_f" } }, + { type: "response.output_text.delta", delta: "a" }, + { type: "response.output_text.delta", delta: "b" }, + { type: "response.output_text.delta", delta: "c" }, + { type: "response.completed", response: { id: "resp_f" } }, + ], + }, + { + wire: "openai-chat", + framing: "sse", + path: "/v1/chat/completions", + body: (text) => ({ + model: "gpt-4.1", + stream: true, + messages: [{ role: "user", content: text }], + }), + events: ["a", "b", "c", "d", "e"].map((content) => ({ + id: "chatcmpl_f", + object: "chat.completion.chunk", + choices: [{ index: 0, delta: { content }, finish_reason: null }], + })), + }, + { + wire: "ollama", + framing: "ndjson", + path: "/api/chat", + body: (text) => ({ model: "llama3.1", messages: [{ role: "user", content: text }] }), + events: ["a", "b", "c", "d", ""].map((content, i) => ({ + model: "llama3.1", + message: { role: "assistant", content }, + done: i === 4, + })), + }, + { + wire: "bedrock-converse", + framing: "eventstream", + path: `/model/${MODEL}/converse-stream`, + body: (text) => ({ messages: [{ role: "user", content: [{ text }] }] }), + events: [ + { messageStart: { role: "assistant" } }, + { contentBlockDelta: { contentBlockIndex: 0, delta: { text: "a" } } }, + { contentBlockDelta: { contentBlockIndex: 0, delta: { text: "b" } } }, + { contentBlockStop: { contentBlockIndex: 0 } }, + { messageStop: { stopReason: "end_turn" } }, + ], + }, +]; + +interface JsonSite { + wire: RawWireId; + path: string; + body: (text: string) => unknown; +} + +/** One non-streaming request per wire (all 9). */ +const JSON_SITES: JsonSite[] = [ + { + wire: "openai-responses", + path: "/v1/responses", + body: (text) => ({ model: "gpt-4.1", input: text, stream: false }), + }, + { + wire: "openai-chat", + path: "/v1/chat/completions", + body: (text) => ({ model: "gpt-4.1", messages: [{ role: "user", content: text }] }), + }, + { + wire: "anthropic", + path: "/v1/messages", + body: (text) => ({ + model: "claude-sonnet-4-5", + max_tokens: 64, + messages: [{ role: "user", content: text }], + }), + }, + { + wire: "gemini", + path: "/v1beta/models/gemini-2.5-flash:generateContent", + body: (text) => ({ contents: [{ role: "user", parts: [{ text }] }] }), + }, + { + wire: "gemini-interactions", + path: "/v1beta/interactions", + body: (text) => ({ model: "gemini-2.5-flash", input: text, stream: false }), + }, + { + wire: "cohere", + path: "/v2/chat", + body: (text) => ({ model: "command-r-plus", messages: [{ role: "user", content: text }] }), + }, + { + wire: "ollama", + path: "/api/chat", + body: (text) => ({ + model: "llama3.1", + messages: [{ role: "user", content: text }], + stream: false, + }), + }, + { + wire: "bedrock-invoke", + path: `/model/${MODEL}/invoke`, + body: (text) => ({ + anthropic_version: "bedrock-2023-05-31", + max_tokens: 64, + messages: [{ role: "user", content: text }], + }), + }, + { + wire: "bedrock-converse", + path: `/model/${MODEL}/converse`, + body: (text) => ({ messages: [{ role: "user", content: [{ text }] }] }), + }, +]; + +interface Reply { + status: number; + headers: http.IncomingHttpHeaders; + bytes: Buffer; + /** Milliseconds from the request to the first body byte (or the end). */ + firstByteMs: number; + ms: number; + /** Whether the body ended cleanly (false: the socket was torn down). */ + ended: boolean; + error?: string; +} + +interface SendOptions { + headers?: Record; + /** Destroy the request once the first body byte arrives. */ + hangUpAfterFirstByte?: boolean; + /** Destroy the request this many ms after sending it. */ + hangUpAfterMs?: number; +} + +/** A raw HTTP request that never rejects, so torn sockets can be inspected. */ +function send(url: string, path: string, body: unknown, opts: SendOptions = {}): Promise { + const started = Date.now(); + return new Promise((resolve) => { + const chunks: Buffer[] = []; + let status = 0; + let headers: http.IncomingHttpHeaders = {}; + let firstByteMs: number | undefined; + let settled = false; + const done = (ended: boolean, error?: string): void => { + if (settled) return; + settled = true; + const ms = Date.now() - started; + resolve({ + status, + headers, + bytes: Buffer.concat(chunks), + firstByteMs: firstByteMs ?? ms, + ms, + ended, + error, + }); + }; + const req = http.request( + `${url}${path}`, + { method: "POST", headers: { "content-type": "application/json", ...opts.headers } }, + (res) => { + status = res.statusCode ?? 0; + headers = res.headers; + res.on("data", (chunk: Buffer) => { + firstByteMs ??= Date.now() - started; + chunks.push(chunk); + if (opts.hangUpAfterFirstByte) { + req.destroy(); + done(false, "hung up"); + } + }); + res.on("end", () => done(true)); + res.on("aborted", () => done(false, "aborted")); + res.on("error", (err) => done(false, err.message)); + res.on("close", () => done(res.complete)); + }, + ); + req.on("error", (err) => done(false, err.message)); + if (opts.hangUpAfterMs !== undefined) { + setTimeout(() => { + req.destroy(); + done(false, "hung up"); + }, opts.hangUpAfterMs); + } + req.end(JSON.stringify(body)); + }); +} + +/** The authored frames in a streamed body, without the chat `[DONE]` terminal. */ +function frames(framing: Framing, bytes: Buffer): string[] { + if (framing === "eventstream") { + const out: string[] = []; + for (let offset = 0; offset + 12 <= bytes.length; ) { + const length = bytes.readUInt32BE(offset); + if (offset + length > bytes.length) break; + const headerLength = bytes.readUInt32BE(offset + 4); + out.push(bytes.subarray(offset + 12 + headerLength, offset + length - 4).toString("utf8")); + offset += length; + } + return out; + } + const text = bytes.toString("utf8"); + if (framing === "ndjson") return text.split("\n").filter((line) => line.length > 0); + return text + .split("\n\n") + .filter((frame) => frame.length > 0 && frame !== "data: [DONE]" && frame.endsWith("}")); +} + +let servers: PlainLLMock[] = []; + +afterEach(async () => { + await Promise.all(servers.map((s) => s.stop())); + servers = []; +}); + +async function start(options: MockServerOptions = {}, misbehavior = false): Promise { + const merged = { port: 0, host: "127.0.0.1", logLevel: "silent" as const, ...options }; + const server = misbehavior ? new LLMock(merged) : new PlainLLMock(merged); + servers.push(server); + return server; +} + +/** A server with one raw fixture on `text`, started. */ +async function serve( + text: string, + response: FixtureFileResponse, + opts?: FixtureOpts, + options?: MockServerOptions, +): Promise { + const server = await start(options); + server.on({ userMessage: text }, response, opts); + await server.start(); + return server; +} + +/** The first journal entry once it is marked interrupted. */ +async function interruptedEntry(server: PlainLLMock) { + await expect + .poll(() => server.getRequests()[0]?.response.interrupted, { timeout: 3000 }) + .toBe(true); + return server.getRequests()[0]; +} + +// ─── Stream interruption, hang-ups and streaming timing ───────────────────── + +describe.each(STREAM_SITES)("raw stream features: $wire", (site) => { + const raw = (): FixtureFileResponse => ({ raw: { wire: site.wire, events: site.events } }); + + it("streams all five authored frames by default (baseline)", async () => { + const server = await serve("f", raw()); + const got = await send(server.url, site.path, site.body("f")); + expect(got.status).toBe(200); + expect(got.ended).toBe(true); + expect(frames(site.framing, got.bytes)).toHaveLength(5); + }); + + it("truncateAfterChunks: 2 stops after the second frame and tears the stream down", async () => { + const server = await serve("f", raw(), { latency: 10, truncateAfterChunks: 2 }); + const got = await send(server.url, site.path, site.body("f")); + expect(got.status).toBe(200); + expect(got.ended).toBe(false); + // The socket is destroyed right after the second frame is written, so that + // frame may or may not arrive; nothing after it ever does. + const received = frames(site.framing, got.bytes); + expect(received.length).toBeGreaterThanOrEqual(1); + expect(received.length).toBeLessThanOrEqual(2); + expect(got.bytes.toString("utf8")).not.toContain("[DONE]"); + const entry = await interruptedEntry(server); + expect(entry.response.interruptReason).toBe("truncateAfterChunks"); + expect(entry.response.status).toBe(200); + }); + + it("disconnectAfterMs tears a slow stream down before it ends", async () => { + const server = await serve("f", raw(), { + streamingProfile: { ttft: 0, tps: 5 }, + disconnectAfterMs: 300, + }); + const got = await send(server.url, site.path, site.body("f")); + expect(got.status).toBe(200); + expect(got.ended).toBe(false); + expect(frames(site.framing, got.bytes).length).toBeLessThan(5); + const entry = await interruptedEntry(server); + expect(entry.response.interruptReason).toBe("disconnectAfterMs"); + expect(entry.response.status).toBe(200); + }); + + it("a client hang-up mid-stream is journaled as client aborted, as for a typed fixture", async () => { + const opts: FixtureOpts = { streamingProfile: { ttft: 0, tps: 2 } }; + const server = await serve("f", raw(), opts); + server.on({ userMessage: "typed" }, { content: "a b c d e f g h" }, opts); + const got = await send(server.url, site.path, site.body("f"), { hangUpAfterFirstByte: true }); + expect(got.status).toBe(200); + const entry = await interruptedEntry(server); + expect(entry.response.interruptReason).toBe("client aborted"); + expect(entry.response.status).toBe(200); + + // Parity: the same hang-up on a typed fixture on the same wire. + await send(server.url, site.path, site.body("typed"), { hangUpAfterFirstByte: true }); + await expect + .poll(() => server.getRequests()[1]?.response.interrupted, { timeout: 3000 }) + .toBe(true); + const typed = server.getRequests()[1].response; + expect({ + status: entry.response.status, + interrupted: entry.response.interrupted, + interruptReason: entry.response.interruptReason, + }).toEqual({ + status: typed.status, + interrupted: typed.interrupted, + interruptReason: typed.interruptReason, + }); + }); + + it("streamingProfile.ttft delays the first frame", async () => { + const server = await serve("f", raw(), { streamingProfile: { ttft: 600, tps: 1000 } }); + const got = await send(server.url, site.path, site.body("f")); + expect(got.status).toBe(200); + expect(got.firstByteMs).toBeGreaterThanOrEqual(500); + expect(frames(site.framing, got.bytes)).toHaveLength(5); + }); + + it("recordedTimings gaps space the frames", async () => { + const server = await serve("f", raw(), { + recordedTimings: { + ttftMs: 0, + interChunkDelaysMs: [150, 150, 150, 150], + totalDurationMs: 600, + }, + }); + const got = await send(server.url, site.path, site.body("f")); + expect(got.status).toBe(200); + expect(got.ms).toBeGreaterThanOrEqual(500); + expect(frames(site.framing, got.bytes)).toHaveLength(5); + }); + + it("chaos drop answers 500 and journals the matched raw fixture", async () => { + const server = await serve("f", raw(), { chaos: { dropRate: 1 } }); + const got = await send(server.url, site.path, site.body("f")); + expect(got.status).toBe(500); + expect(got.bytes.toString("utf8")).toContain("chaos_drop"); + const [entry] = server.getRequests(); + expect(entry.response.chaosAction).toBe("drop"); + expect(entry.response.fixture).toBe(server.getFixtures()[0]); + }); + + it("chaos malformed answers unparseable JSON instead of the raw frames", async () => { + const server = await serve("f", raw(), { chaos: { malformedRate: 1 } }); + const got = await send(server.url, site.path, site.body("f")); + expect(got.status).toBe(200); + expect(() => JSON.parse(got.bytes.toString("utf8"))).toThrow(); + expect(got.headers["content-type"]).not.toMatch(/event-stream|ndjson|eventstream/); + expect(server.getRequests()[0].response.chaosAction).toBe("malformed"); + }); + + it("chaos disconnect tears the socket down with status 0", async () => { + const server = await serve("f", raw(), { chaos: { disconnectRate: 1 } }); + const got = await send(server.url, site.path, site.body("f")); + expect(got.ended).toBe(false); + expect(got.bytes.length).toBe(0); + const [entry] = server.getRequests(); + expect(entry.response.chaosAction).toBe("disconnect"); + expect(entry.response.status).toBe(0); + }); +}); + +// ─── Misbehavior, strict mode, sequence and turn matching ─────────────────── + +describe.each(STREAM_SITES)("raw fixture matching features: $wire", (site) => { + const raw = (events = site.events): FixtureFileResponse => ({ + raw: { wire: site.wire, events }, + }); + + it("an explicit misbehavior header is not applicable: 501", async () => { + const server = await start({}, true); + server.on({ userMessage: "f" }, raw()); + await server.start(); + const got = await send(server.url, site.path, site.body("f"), { + headers: { "x-aimock-misbehavior": "empty-response" }, + }); + expect(got.status).toBe(501); + expect(got.bytes.toString("utf8")).toContain("aimock_misbehavior_not_applicable"); + expect(got.headers["content-type"]).not.toMatch(/event-stream|ndjson|eventstream/); + expect(server.getRequests()[0].response.misbehavior).toMatchObject({ + applied: false, + source: "header", + evaluations: [{ fault: "empty-response", outcome: "error", reason: "not-applicable" }], + }); + }); + + it("a server-level fault is skipped: the raw stream is served and the skip journaled", async () => { + const server = await start( + { misbehavior: { faults: [{ fault: "empty-response", rate: 1 }] } }, + true, + ); + server.on({ userMessage: "f" }, raw()); + await server.start(); + const got = await send(server.url, site.path, site.body("f")); + expect(got.status).toBe(200); + expect(frames(site.framing, got.bytes)).toHaveLength(5); + expect(server.getRequests()[0].response.misbehavior).toMatchObject({ + applied: false, + source: "server", + wire: site.wire, + evaluations: [{ fault: "empty-response", outcome: "skipped", reason: "not-applicable" }], + }); + }); + + it("strict mode serves a matched raw fixture and answers a miss with 503", async () => { + const server = await serve("f", raw(), undefined, { strict: true }); + const hit = await send(server.url, site.path, site.body("f")); + expect(hit.status).toBe(200); + expect(frames(site.framing, hit.bytes)).toHaveLength(5); + const miss = await send(server.url, site.path, site.body("nothing matches this")); + expect(miss.status).toBe(503); + }); + + it("sequenceIndex picks the raw fixture for each call in turn", async () => { + const server = await start(); + server.on({ userMessage: "f", sequenceIndex: 0 }, raw(site.events.slice(0, 2))); + server.on({ userMessage: "f", sequenceIndex: 1 }, raw(site.events.slice(0, 3))); + await server.start(); + const first = await send(server.url, site.path, site.body("f")); + const second = await send(server.url, site.path, site.body("f")); + expect(frames(site.framing, first.bytes)).toHaveLength(2); + expect(frames(site.framing, second.bytes)).toHaveLength(3); + }); +}); + +describe("raw fixture matching features: turnIndex (openai-chat)", () => { + const site = STREAM_SITES[1]; + it("turnIndex picks the raw fixture for the conversation turn", async () => { + const server = await start(); + server.on( + { userMessage: "f", turnIndex: 0 }, + { raw: { wire: site.wire, events: site.events.slice(0, 2) } }, + ); + server.on( + { userMessage: "f", turnIndex: 1 }, + { raw: { wire: site.wire, events: site.events.slice(0, 3) } }, + ); + await server.start(); + const first = await send(server.url, site.path, site.body("f")); + const second = await send(server.url, site.path, { + model: "gpt-4.1", + stream: true, + messages: [ + { role: "user", content: "f" }, + { role: "assistant", content: "ok" }, + { role: "user", content: "f" }, + ], + }); + expect(frames("sse", first.bytes)).toHaveLength(2); + expect(frames("sse", second.bytes)).toHaveLength(3); + }); +}); + +// ─── Non-streaming timing matrix (every wire) ─────────────────────────────── + +const TTFT = 2000; + +interface Case { + name: string; + opts: FixtureOpts; + /** Whether the reply waits on `openai-chat` and on every other wire. */ + waits: { chat: boolean; other: boolean }; +} + +const PROFILE: FixtureOpts = { streamingProfile: { ttft: TTFT } }; + +const CASES: Case[] = [ + { name: "neither flag", opts: PROFILE, waits: { chat: false, other: false } }, + { + name: "nonStreamingTiming only", + opts: { ...PROFILE, nonStreamingTiming: true }, + waits: { chat: true, other: false }, + }, + { + name: "nonStreamingTimingAllProviders only", + opts: { ...PROFILE, nonStreamingTimingAllProviders: true }, + waits: { chat: true, other: true }, + }, + { + name: "both flags", + opts: { ...PROFILE, nonStreamingTiming: true, nonStreamingTimingAllProviders: true }, + waits: { chat: true, other: true }, + }, + { + name: "bare latency with both flags", + opts: { latency: TTFT, nonStreamingTiming: true, nonStreamingTimingAllProviders: true }, + waits: { chat: false, other: false }, + }, + { + name: "recordedTimings (no profile) with nonStreamingTimingAllProviders", + opts: { + recordedTimings: { ttftMs: TTFT, interChunkDelaysMs: [], totalDurationMs: TTFT }, + nonStreamingTimingAllProviders: true, + }, + waits: { chat: true, other: true }, + }, +]; + +describe.each(JSON_SITES)("raw non-streaming timing: $wire", (site) => { + const json = { served: site.wire }; + const chat = site.wire === "openai-chat"; + + it("applies the typed timing rule for each flag combination", async () => { + const server = await start(); + CASES.forEach((c, i) => + server.on({ userMessage: `case-${i}` }, { raw: { wire: site.wire, json } }, c.opts), + ); + await server.start(); + const replies = await Promise.all( + CASES.map((_, i) => send(server.url, site.path, site.body(`case-${i}`))), + ); + const observed = CASES.map((c, i) => ({ + name: c.name, + status: replies[i].status, + body: replies[i].bytes.toString("utf8"), + waited: replies[i].ms >= TTFT - 500, + fast: replies[i].ms < 1000, + })); + const expected = CASES.map((c) => { + const waits = chat ? c.waits.chat : c.waits.other; + return { + name: c.name, + status: 200, + body: JSON.stringify(json), + waited: waits, + fast: !waits, + }; + }); + expect(observed).toEqual(expected); + // Both flags together wait once, not twice. + const both = replies[CASES.findIndex((c) => c.name === "both flags")]; + expect(both.ms).toBeLessThan(TTFT + 1000); + }, 15000); + + it("a tps profile sizes the wait by the payload: a bigger payload waits longer", async () => { + // Relative only: the exact payload sizing is not pinned here. The wait is + // sized from the served JSON, so the two fixtures differ only in the + // length of their json (neither has events): about 1 chunk against 20+. + const opts: FixtureOpts = { + streamingProfile: { tps: 20 }, + nonStreamingTimingAllProviders: true, + }; + const small = { raw: { wire: site.wire, json: { s: 1 } } }; + const big = { + raw: { + wire: site.wire, + json: { served: site.wire, text: "a bigger payload than the small one ".repeat(12) }, + }, + }; + const server = await start(); + server.on({ userMessage: "small" }, small, opts); + server.on({ userMessage: "big" }, big, opts); + await server.start(); + const [smallReply, bigReply] = await Promise.all([ + send(server.url, site.path, site.body("small")), + send(server.url, site.path, site.body("big")), + ]); + expect([smallReply.status, bigReply.status]).toEqual([200, 200]); + expect(JSON.parse(bigReply.bytes.toString("utf8"))).toEqual(big.raw.json); + expect(bigReply.ms).toBeGreaterThan(smallReply.ms + 300); + }, 15000); + + it("json with status 429 is never delayed, even with both flags", async () => { + const server = await serve( + "f", + { raw: { wire: site.wire, json: { error: "slow down" }, status: 429 } }, + { ...PROFILE, nonStreamingTiming: true, nonStreamingTimingAllProviders: true }, + ); + const got = await send(server.url, site.path, site.body("f")); + expect(got.status).toBe(429); + expect(JSON.parse(got.bytes.toString("utf8"))).toEqual({ error: "slow down" }); + expect(got.ms).toBeLessThan(1000); + expect(server.getRequests()[0].response.status).toBe(429); + }); + + it("a client hang-up during an opted-in wait writes nothing and is journaled as client aborted, as for a typed fixture", async () => { + const opts: FixtureOpts = { ...PROFILE, nonStreamingTimingAllProviders: true }; + const server = await serve("f", { raw: { wire: site.wire, json } }, opts); + server.on({ userMessage: "typed" }, { content: "typed reply" }, opts); + const got = await send(server.url, site.path, site.body("f"), { hangUpAfterMs: 300 }); + expect(got.status).toBe(0); + expect(got.bytes.length).toBe(0); + const entry = await interruptedEntry(server); + expect(entry.response.interruptReason).toBe("client aborted"); + + await send(server.url, site.path, site.body("typed"), { hangUpAfterMs: 300 }); + await expect + .poll(() => server.getRequests()[1]?.response.interrupted, { timeout: 3000 }) + .toBe(true); + const typed = server.getRequests()[1].response; + expect({ + status: entry.response.status, + interrupted: entry.response.interrupted, + interruptReason: entry.response.interruptReason, + }).toEqual({ + status: typed.status, + interrupted: typed.interrupted, + interruptReason: typed.interruptReason, + }); + }, 10000); +}); diff --git a/src/__tests__/raw-gemini-interactions.test.ts b/src/__tests__/raw-gemini-interactions.test.ts new file mode 100644 index 00000000..77906bb9 --- /dev/null +++ b/src/__tests__/raw-gemini-interactions.test.ts @@ -0,0 +1,198 @@ +import * as http from "node:http"; +import { GoogleGenAI } from "@google/genai"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { LLMock } from "../llmock.js"; +import type { RawResponse } from "../types.js"; + +// Raw passthrough on the Gemini Interactions wire (#549). + +const events: Record[] = [ + { + event_type: "interaction.created", + interaction: { id: "int_raw", status: "in_progress" }, + event_id: "evt_1", + }, + { event_type: "step.start", index: 0, step: { type: "model_output" }, event_id: "evt_2" }, + { + event_type: "step.delta", + index: 0, + delta: { type: "text", text: "Raw interactions stream." }, + event_id: "evt_3", + }, + { event_type: "step.stop", index: 0, event_id: "evt_4" }, + { + event_type: "interaction.completed", + interaction: { + id: "int_raw", + status: "completed", + usage: { total_input_tokens: 1, total_output_tokens: 3, total_tokens: 4 }, + }, + event_id: "evt_5", + }, +]; + +const json = { + id: "int_raw_json", + status: "completed", + steps: [{ type: "model_output", content: [{ type: "text", text: "Raw JSON reply." }] }], + usage: { total_input_tokens: 1, total_output_tokens: 3, total_tokens: 4 }, +}; + +const raw: RawResponse = { raw: { wire: "gemini-interactions", events, json } }; + +let mock: LLMock; +let client: GoogleGenAI; + +beforeEach(async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.on({ userMessage: "raw-proof" }, () => raw); + await mock.start(); + client = new GoogleGenAI({ apiKey: "local-proof", httpOptions: { baseUrl: mock.url } }); +}); + +afterEach(async () => { + await mock.stop(); +}); + +describe("raw passthrough: gemini-interactions", () => { + it("streams exactly the authored events in order", async () => { + const stream = await client.interactions.create( + { model: "gemini-2.5-flash", input: "raw-proof", stream: true }, + { maxRetries: 0, timeout: 5000 }, + ); + const received: unknown[] = []; + for await (const event of stream) received.push(event); + expect(received).toEqual(events); + }); + + it("returns the authored json for a non-streaming request", async () => { + const reply = await client.interactions.create( + { model: "gemini-2.5-flash", input: "raw-proof", stream: false }, + { maxRetries: 0, timeout: 5000 }, + ); + expect(reply).toMatchObject(json); + }); +}); + +describe("raw passthrough: gemini-interactions framing", () => { + async function post(input: string, stream?: boolean): Promise { + return fetch(`${mock.url}/v1beta/interactions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gemini-2.5-flash", + input, + ...(stream === undefined ? {} : { stream }), + }), + }); + } + + // Collects every byte that arrives before the server destroys the socket. + function postPartial(input: string): Promise { + return new Promise((resolve) => { + const chunks: Buffer[] = []; + const done = () => resolve(Buffer.concat(chunks).toString("utf8")); + const req = http.request( + `${mock.url}/v1beta/interactions`, + { method: "POST", headers: { "content-type": "application/json" } }, + (res) => { + res.on("data", (chunk: Buffer) => chunks.push(chunk)); + res.on("end", done); + res.on("error", done); + res.on("aborted", done); + }, + ); + req.on("error", done); + req.end(JSON.stringify({ model: "gemini-2.5-flash", input, stream: true })); + }); + } + + it("streams when stream is omitted, as bare SSE data lines", async () => { + const res = await post("raw-proof"); + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toBe("text/event-stream"); + expect(await res.text()).toBe(events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join("")); + }); + + it("truncateAfterChunks: 2 counts every raw frame", async () => { + // Typed Interactions streams count only step.delta toward the budget; raw + // counts every frame, so this stream (one step.delta) stops after step.start. + mock.addFixture({ + match: { userMessage: "raw-truncate" }, + response: raw, + latency: 10, + truncateAfterChunks: 2, + }); + const frame = (e: Record) => `data: ${JSON.stringify(e)}\n\n`; + const text = await postPartial("raw-truncate"); + // The socket is destroyed right after the second frame is written, so that + // frame may or may not arrive; nothing after it ever does. + expect(text.startsWith(frame(events[0]))).toBe(true); + expect((frame(events[0]) + frame(events[1])).startsWith(text)).toBe(true); + await expect + .poll(() => mock.getLastRequest()?.response.interrupted, { timeout: 1000 }) + .toBe(true); + expect(mock.getLastRequest()?.response.interruptReason).toBe("truncateAfterChunks"); + }); +}); + +describe("raw passthrough: gemini-interactions error bodies", () => { + async function post(input: string, stream: boolean): Promise<{ status: number; body: string }> { + const res = await fetch(`${mock.url}/v1beta/interactions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gemini-2.5-flash", input, stream }), + }); + return { status: res.status, body: await res.text() }; + } + + const expected = (code: string, message: string): string => + JSON.stringify({ error: { code, message } }); + + it("a streaming request on a json-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "json-only" }, () => ({ raw: { wire: "gemini-interactions", json } })); + const got = await post("json-only", true); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "events" body for a streaming request on gemini-interactions`, + ), + ); + }); + + it("a non-streaming request on an events-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "events-only" }, () => ({ + raw: { wire: "gemini-interactions", events }, + })); + const got = await post("events-only", false); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "json" body for a non-streaming request on gemini-interactions`, + ), + ); + }); + + it("a factory for another wire: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "other-wire" }, () => ({ raw: { wire: "gemini", json } })); + const got = await post("other-wire", false); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected("aimock_raw_fixture_invalid", "raw fixture is for gemini, not gemini-interactions"), + ); + }); + + it("a factory that fails the checker: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "bad-status" }, () => ({ + raw: { wire: "gemini-interactions", json, status: 99 }, + })); + const got = await post("bad-status", false); + expect(got.status).toBe(500); + const parsed = JSON.parse(got.body) as { error: { code: string; message: string } }; + expect(Object.keys(parsed.error)).toEqual(["code", "message"]); + expect(parsed.error.code).toBe("aimock_raw_fixture_invalid"); + expect(parsed.error.message).toMatch(/^invalid raw fixture: .*status/); + }); +}); diff --git a/src/__tests__/raw-gemini.test.ts b/src/__tests__/raw-gemini.test.ts new file mode 100644 index 00000000..fc272ddd --- /dev/null +++ b/src/__tests__/raw-gemini.test.ts @@ -0,0 +1,183 @@ +import { GoogleGenAI } from "@google/genai"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { LLMock } from "../llmock.js"; +import type { RawResponse } from "../types.js"; + +// Raw passthrough on the Gemini generateContent wire (#549). + +const events: Record[] = [ + { + candidates: [{ content: { role: "model", parts: [{ text: "Raw " }] }, index: 0 }], + modelVersion: "gemini-2.5-flash", + }, + { + candidates: [ + { + content: { + role: "model", + parts: [{ inlineData: { mimeType: "image/png", data: "iVBORw0KGgo=" } }], + }, + index: 0, + }, + ], + modelVersion: "gemini-2.5-flash", + }, + { + candidates: [ + { content: { role: "model", parts: [{ text: "stream." }] }, finishReason: "STOP", index: 0 }, + ], + usageMetadata: { promptTokenCount: 1, candidatesTokenCount: 3, totalTokenCount: 4 }, + modelVersion: "gemini-2.5-flash", + }, +]; + +const json = { + candidates: [ + { + content: { role: "model", parts: [{ text: "Raw JSON reply." }] }, + finishReason: "STOP", + index: 0, + }, + ], + usageMetadata: { promptTokenCount: 1, candidatesTokenCount: 3, totalTokenCount: 4 }, + modelVersion: "gemini-2.5-flash", +}; + +const raw: RawResponse = { raw: { wire: "gemini", events, json } }; + +let mock: LLMock; +let client: GoogleGenAI; + +beforeEach(async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.on({ userMessage: "raw-proof" }, () => raw); + await mock.start(); + client = new GoogleGenAI({ + apiKey: "local-proof", + httpOptions: { baseUrl: mock.url, timeout: 5000, retryOptions: { attempts: 1 } }, + }); +}); + +afterEach(async () => { + await mock.stop(); +}); + +describe("raw passthrough: gemini", () => { + it("streams exactly the authored chunks in order", async () => { + const received: unknown[] = []; + for await (const chunk of await client.models.generateContentStream({ + model: "gemini-2.5-flash", + contents: "raw-proof", + })) + received.push(chunk); + expect(received).toHaveLength(events.length); + received.forEach((chunk, index) => expect(chunk).toMatchObject(events[index])); + }); + + it("returns the authored json for a non-streaming request", async () => { + const reply = await client.models.generateContent({ + model: "gemini-2.5-flash", + contents: "raw-proof", + }); + expect(reply).toMatchObject(json); + }); +}); + +describe("raw passthrough: gemini framing", () => { + const sse = events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join(""); + const body = JSON.stringify({ contents: [{ role: "user", parts: [{ text: "raw-proof" }] }] }); + + it("frames each chunk as a bare SSE data line, inlineData included", async () => { + const res = await fetch( + `${mock.url}/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse`, + { method: "POST", headers: { "content-type": "application/json" }, body }, + ); + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toBe("text/event-stream"); + expect(await res.text()).toBe(sse); + }); + + it("serves the same chunks and json on the Vertex AI path", async () => { + const base = `${mock.url}/v1/projects/p/locations/us-central1/publishers/google/models/gemini-2.5-flash`; + const stream = await fetch(`${base}:streamGenerateContent?alt=sse`, { + method: "POST", + headers: { "content-type": "application/json" }, + body, + }); + expect(stream.status).toBe(200); + expect(await stream.text()).toBe(sse); + const nonStream = await fetch(`${base}:generateContent`, { + method: "POST", + headers: { "content-type": "application/json" }, + body, + }); + expect(nonStream.status).toBe(200); + expect(await nonStream.text()).toBe(JSON.stringify(json)); + }); +}); + +describe("raw passthrough: gemini error bodies", () => { + async function post(text: string, stream: boolean): Promise<{ status: number; body: string }> { + const method = stream ? "streamGenerateContent?alt=sse" : "generateContent"; + const res = await fetch(`${mock.url}/v1beta/models/gemini-2.5-flash:${method}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ contents: [{ role: "user", parts: [{ text }] }] }), + }); + return { status: res.status, body: await res.text() }; + } + + // Gemini's misbehavior template has no code slot: the code leads the message. + const expected = (code: string, message: string): string => + JSON.stringify({ error: { code: 500, message: `${code}: ${message}`, status: "INTERNAL" } }); + + it("a streaming request on a json-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "json-only" }, () => ({ raw: { wire: "gemini", json } })); + const got = await post("json-only", true); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "events" body for a streaming request on gemini`, + ), + ); + }); + + it("a non-streaming request on an events-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "events-only" }, () => ({ raw: { wire: "gemini", events } })); + const got = await post("events-only", false); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "json" body for a non-streaming request on gemini`, + ), + ); + }); + + it("a factory for another wire: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "other-wire" }, () => ({ raw: { wire: "anthropic", json } })); + const got = await post("other-wire", false); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected("aimock_raw_fixture_invalid", "raw fixture is for anthropic, not gemini"), + ); + }); + + it("a factory that fails the checker: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "bad-status" }, () => ({ + raw: { wire: "gemini", json, status: 99 }, + })); + const got = await post("bad-status", false); + expect(got.status).toBe(500); + const parsed = JSON.parse(got.body) as { + error: { code: number; message: string; status: string }; + }; + expect(Object.keys(parsed.error)).toEqual(["code", "message", "status"]); + expect(parsed.error.code).toBe(500); + expect(parsed.error.status).toBe("INTERNAL"); + expect(parsed.error.message).toMatch( + /^aimock_raw_fixture_invalid: invalid raw fixture: .*status/, + ); + }); +}); diff --git a/src/__tests__/raw-headers.test.ts b/src/__tests__/raw-headers.test.ts new file mode 100644 index 00000000..13b60539 --- /dev/null +++ b/src/__tests__/raw-headers.test.ts @@ -0,0 +1,304 @@ +import * as http from "node:http"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { Journal } from "../journal.js"; +import { LLMock } from "../llmock.js"; +import { Logger } from "../logger.js"; +import { serveRawResponse } from "../response/raw.js"; +import type { HandlerDefaults, RawResponse, RawWireId } from "../types.js"; + +// Raw response headers (#549): an authored header replaces a default of the +// same name in any letter case (one header on the wire, never two), and a raw +// stream sends the same default headers as the typed stream on its wire. + +const MODEL = "anthropic.claude-3-5-sonnet-20241022-v2:0"; + +/** A streaming request per wire whose user message is `text`. */ +function streamRequest(wire: RawWireId, text: string): { path: string; body: unknown } { + switch (wire) { + case "openai-responses": + return { path: "/v1/responses", body: { model: "gpt-4.1", input: text, stream: true } }; + case "openai-chat": + return { + path: "/v1/chat/completions", + body: { model: "gpt-4.1", messages: [{ role: "user", content: text }], stream: true }, + }; + case "anthropic": + return { + path: "/v1/messages", + body: { + model: "claude-sonnet-4-5", + max_tokens: 64, + messages: [{ role: "user", content: text }], + stream: true, + }, + }; + case "gemini": + return { + path: "/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse", + body: { contents: [{ role: "user", parts: [{ text }] }] }, + }; + case "gemini-interactions": + return { + path: "/v1beta/interactions", + body: { model: "gemini-2.5-flash", input: text, stream: true }, + }; + case "cohere": + return { + path: "/v2/chat", + body: { + model: "command-r-plus", + messages: [{ role: "user", content: text }], + stream: true, + }, + }; + case "ollama": + return { + path: "/api/chat", + body: { model: "llama3.1", messages: [{ role: "user", content: text }], stream: true }, + }; + case "ollama-generate": + return { path: "/api/generate", body: { model: "llama3.1", prompt: text, stream: true } }; + case "bedrock-invoke": + return { + path: `/model/${MODEL}/invoke-with-response-stream`, + body: { + anthropic_version: "bedrock-2023-05-31", + max_tokens: 64, + messages: [{ role: "user", content: text }], + }, + }; + case "bedrock-converse": + return { + path: `/model/${MODEL}/converse-stream`, + body: { messages: [{ role: "user", content: [{ text }] }] }, + }; + } +} + +/** One valid raw event for each wire. */ +function rawEvent(wire: RawWireId): Record { + return wire === "bedrock-converse" ? { messageStop: { stopReason: "end_turn" } } : { type: "x" }; +} + +const WIRES: RawWireId[] = [ + "openai-responses", + "openai-chat", + "anthropic", + "gemini", + "gemini-interactions", + "cohere", + "ollama", + "ollama-generate", + "bedrock-invoke", + "bedrock-converse", +]; + +interface Reply { + status: number; + /** The header lines exactly as written, as [name, value] pairs. */ + rawHeaders: [string, string][]; +} + +function send(url: string, path: string, body: unknown): Promise { + return new Promise((resolve, reject) => { + const req = http.request( + `${url}${path}`, + { method: "POST", headers: { "content-type": "application/json" } }, + (res) => { + res.resume(); + res.on("end", () => { + const pairs: [string, string][] = []; + for (let i = 0; i < res.rawHeaders.length; i += 2) { + pairs.push([res.rawHeaders[i], res.rawHeaders[i + 1]]); + } + resolve({ status: res.statusCode ?? 0, rawHeaders: pairs }); + }); + }, + ); + req.on("error", reject); + req.end(JSON.stringify(body)); + }); +} + +/** Headers whose value differs per response; only their presence is compared. */ +const PER_RESPONSE = new Set(["date", "x-request-id"]); + +/** Header lines, names lowercased, sorted: the comparable header set. */ +function comparable(reply: Reply): string[] { + return reply.rawHeaders + .map(([name, value]) => { + const lower = name.toLowerCase(); + return PER_RESPONSE.has(lower) ? `${lower}: *` : `${lower}: ${value}`; + }) + .sort(); +} + +/** Every value sent under `name`, in any letter case. */ +function valuesOf(reply: Reply, name: string): string[] { + return reply.rawHeaders.filter(([n]) => n.toLowerCase() === name).map(([, v]) => v); +} + +describe("raw streams send the typed stream's default headers", () => { + let mock: LLMock; + beforeAll(async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.on({ userMessage: "typed" }, { content: "Typed headers check." }); + for (const wire of WIRES) { + mock.on({ userMessage: "raw" }, { raw: { wire, events: [rawEvent(wire)] } }); + } + await mock.start(); + }); + afterAll(async () => { + await mock.stop(); + }); + + it.each(WIRES)("%s", async (wire) => { + const typedReq = streamRequest(wire, "typed"); + const rawReq = streamRequest(wire, "raw"); + const typed = await send(mock.url, typedReq.path, typedReq.body); + const raw = await send(mock.url, rawReq.path, rawReq.body); + expect(typed.status).toBe(200); + expect(raw.status).toBe(200); + expect(comparable(raw)).toEqual(comparable(typed)); + }); +}); + +describe("an authored header replaces the default in any letter case", () => { + let mock: LLMock; + const fixtures: Record = { + "json 200": { + wire: "openai-responses", + json: { id: "r" }, + headers: { "content-type": "application/vnd.test+json" }, + }, + "json 4xx": { + wire: "openai-responses", + json: { error: "x" }, + status: 418, + headers: { "content-type": "application/vnd.test+json" }, + }, + "stream, default status": { + wire: "openai-responses", + events: [{ type: "x" }], + headers: { "content-type": "text/plain", "cache-control": "max-age=5" }, + }, + "stream, authored status": { + wire: "openai-responses", + events: [{ type: "x" }], + status: 200, + headers: { "content-type": "text/plain", "cache-control": "max-age=5" }, + }, + }; + + beforeAll(async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + for (const [name, raw] of Object.entries(fixtures)) { + mock.on({ userMessage: name }, { raw }); + } + await mock.start(); + }); + afterAll(async () => { + await mock.stop(); + }); + + it.each(["json 200", "json 4xx"])("%s: one content-type, the authored value", async (name) => { + const reply = await send(mock.url, "/v1/responses", { + model: "gpt-4.1", + input: name, + stream: false, + }); + expect(reply.status).toBe(fixtures[name].status ?? 200); + expect(valuesOf(reply, "content-type")).toEqual(["application/vnd.test+json"]); + }); + + it.each(["stream, default status", "stream, authored status"])( + "%s: one content-type and one cache-control, the authored values", + async (name) => { + const reply = await send(mock.url, "/v1/responses", { + model: "gpt-4.1", + input: name, + stream: true, + }); + expect(reply.status).toBe(200); + expect(valuesOf(reply, "content-type")).toEqual(["text/plain"]); + expect(valuesOf(reply, "cache-control")).toEqual(["max-age=5"]); + // The other stream defaults stay. + expect(valuesOf(reply, "connection")).toEqual(["keep-alive"]); + }, + ); +}); + +// A server that sets no header before the raw serve (an exported handler on a +// bare http.Server): Node then writes `writeHead`s header object as given, so +// two spellings of one name would go out as two header lines. +describe("an authored header replaces the default on a bare server", () => { + let server: http.Server; + let url = ""; + const fixtures: Record = { + "/json-200": { + wire: "ollama", + json: { done: true }, + headers: { "content-type": "application/vnd.test+json" }, + }, + "/json-4xx": { + wire: "ollama", + json: { error: "x" }, + status: 418, + headers: { "content-type": "application/vnd.test+json" }, + }, + "/stream-status": { + wire: "ollama", + events: [{ done: true }], + status: 200, + headers: { "content-type": "text/plain", "cache-control": "max-age=5" }, + }, + }; + + beforeAll(async () => { + const defaults: HandlerDefaults = { + latency: 0, + chunkSize: 10, + replaySpeed: 1, + logger: new Logger("silent"), + }; + const journal = new Journal(); + server = http.createServer((req, res) => { + const raw = fixtures[req.url ?? ""]; + req.resume(); + void serveRawResponse({ + wire: "ollama", + stream: raw.events !== undefined, + req, + res, + fixture: { match: {}, response: { raw } }, + response: { raw }, + body: null, + journal, + defaults, + testId: "default", + errorWire: "ollama", + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (address === null || typeof address === "string") throw new Error("no port"); + url = `http://127.0.0.1:${address.port}`; + }); + afterAll(async () => { + await new Promise((resolve) => server.close(() => resolve())); + }); + + it.each(["/json-200", "/json-4xx"])("%s: one content-type, the authored value", async (path) => { + const reply = await send(url, path, {}); + expect(reply.status).toBe(fixtures[path].status ?? 200); + expect(valuesOf(reply, "content-type")).toEqual(["application/vnd.test+json"]); + }); + + it("/stream-status: the authored content-type and cache-control, the other defaults kept", async () => { + const reply = await send(url, "/stream-status", {}); + expect(reply.status).toBe(200); + expect(valuesOf(reply, "content-type")).toEqual(["text/plain"]); + expect(valuesOf(reply, "cache-control")).toEqual(["max-age=5"]); + expect(valuesOf(reply, "connection")).toEqual(["keep-alive"]); + }); +}); diff --git a/src/__tests__/raw-no-regression.test.ts b/src/__tests__/raw-no-regression.test.ts new file mode 100644 index 00000000..42d83eaf --- /dev/null +++ b/src/__tests__/raw-no-regression.test.ts @@ -0,0 +1,204 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { LLMock } from "../llmock.js"; +import { entryToFixture, validateFixtures } from "../fixture-loader.js"; +import type { FixtureFileEntry, FixtureFileResponse } from "../types.js"; + +// No-regression guard for raw passthrough fixtures (#549). Every test here is +// GREEN on main and must stay GREEN: the feature is opt-in, so a fixture that +// already has a type marker plus a stray `raw` key, an unscoped `{ json }` +// fixture and every typed request's journal body behave exactly as before. + +const WIRES = [ + "openai-responses", + "openai-chat", + "anthropic", + "gemini", + "gemini-interactions", + "cohere", + "ollama", + "bedrock-invoke", + "bedrock-converse", +] as const; +type Wire = (typeof WIRES)[number]; + +const BEDROCK_MODEL = "anthropic.claude-3-5-sonnet-20241022-v2:0"; + +/** One non-streaming request on `wire` whose last user message is `text`. */ +function request(wire: Wire, text: string): { path: string; body: unknown } { + switch (wire) { + case "openai-responses": + return { path: "/v1/responses", body: { model: "gpt-4.1", input: text, stream: false } }; + case "openai-chat": + return { + path: "/v1/chat/completions", + body: { model: "gpt-4.1", messages: [{ role: "user", content: text }] }, + }; + case "anthropic": + return { + path: "/v1/messages", + body: { + model: "claude-sonnet-4-5", + max_tokens: 64, + messages: [{ role: "user", content: text }], + }, + }; + case "gemini": + return { + path: "/v1beta/models/gemini-2.5-flash:generateContent", + body: { contents: [{ role: "user", parts: [{ text }] }] }, + }; + case "gemini-interactions": + return { + path: "/v1beta/interactions", + body: { model: "gemini-2.5-flash", input: text, stream: false }, + }; + case "cohere": + return { + path: "/v2/chat", + body: { model: "command-r-plus", messages: [{ role: "user", content: text }] }, + }; + case "ollama": + return { + path: "/api/chat", + body: { model: "llama3.1", messages: [{ role: "user", content: text }], stream: false }, + }; + case "bedrock-invoke": + return { + path: `/model/${BEDROCK_MODEL}/invoke`, + body: { + anthropic_version: "bedrock-2023-05-31", + max_tokens: 64, + messages: [{ role: "user", content: text }], + }, + }; + case "bedrock-converse": + return { + path: `/model/${BEDROCK_MODEL}/converse`, + body: { messages: [{ role: "user", content: [{ text }] }] }, + }; + } +} + +async function send(mock: LLMock, wire: Wire, text: string) { + const { path, body } = request(wire, text); + const res = await fetch(`${mock.url}${path}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + return { status: res.status, text: await res.text() }; +} + +// Ids and timestamps are generated per request; everything else must match. +const VOLATILE = /^(id|created|created_at|createdAt|responseId|call_id|toolUseId|event_id)$/; +function stable(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stable); + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([k, v]) => [k, VOLATILE.test(k) ? "" : stable(v)]), + ); + } + return value; +} +function stableBody(text: string): unknown { + try { + return stable(JSON.parse(text)); + } catch { + return text; + } +} + +const strayRaw = { wire: "openai-chat", events: [{ stray: true }], json: { stray: true } }; +const typed: { kind: string; response: FixtureFileResponse }[] = [ + { kind: "text", response: { content: "hi" } }, + { + kind: "toolCalls", + response: { toolCalls: [{ name: "lookup", arguments: '{"city":"Paris"}' }] }, + }, + { kind: "error", response: { error: { message: "typed boom", type: "server_error" } } }, +]; +/** The same typed response with a stray `raw` key, built from JSON (the file-load shape). */ +function withStrayRaw(response: FixtureFileResponse): FixtureFileResponse { + return JSON.parse(JSON.stringify({ ...response, raw: strayRaw })); +} + +let mock: LLMock | undefined; +afterEach(async () => { + await mock?.stop(); + mock = undefined; +}); + +describe("raw no-regression: typed fixtures with a stray raw key", () => { + for (const { kind, response } of typed) { + it(`${kind} + raw: zero load findings`, () => { + const entry: FixtureFileEntry = { + match: { userMessage: "x" }, + response: withStrayRaw(response), + }; + expect(validateFixtures([entryToFixture(entry)])).toEqual([]); + }); + + it(`${kind} + raw: /__aimock/fixtures reports kind "${kind}"`, async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.addFixturesFromJSON([{ match: { userMessage: "x" }, response: withStrayRaw(response) }]); + await mock.start(); + const res = await fetch(`${mock.url}/__aimock/fixtures?include=fixtures`); + const body = (await res.json()) as { fixtures: { responseKind: string }[] }; + expect(body.fixtures.map((f) => f.responseKind)).toEqual([kind]); + }); + + it(`${kind} + raw: matched on all 9 wires and served exactly like the plain fixture`, async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.addFixturesFromJSON([ + { match: { userMessage: "stray" }, response: withStrayRaw(response) }, + { match: { userMessage: "plain" }, response }, + ]); + await mock.start(); + const [strayFixture] = mock.getFixtures(); + for (const wire of WIRES) { + mock.clearRequests(); + const stray = await send(mock, wire, "stray"); + const entry = mock.getLastRequest(); + expect(entry?.response.fixture, wire).toBe(strayFixture); + const plain = await send(mock, wire, "plain"); + expect(stray.status, wire).toBe(plain.status); + expect(stableBody(stray.text), wire).toEqual(stableBody(plain.text)); + } + }); + } +}); + +describe("raw no-regression: unscoped json and journal bodies", () => { + it("an unscoped { json } (fal-style) fixture still returns 500 on every LLM wire", async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.addFixturesFromJSON([{ match: { userMessage: "fal" }, response: { json: { ok: 1 } } }]); + await mock.start(); + for (const wire of WIRES) { + const reply = await send(mock, wire, "fal"); + expect(reply.status, wire).toBe(500); + expect(reply.text, wire).not.toContain('"ok"'); + } + }); + + it("the journaled request body of a typed fixture is unchanged on every wire", async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.addFixturesFromJSON([{ match: { userMessage: "journal" }, response: { content: "hi" } }]); + await mock.start(); + const bodies: Record = {}; + for (const wire of WIRES) { + mock.clearRequests(); + expect((await send(mock, wire, "journal")).status, wire).toBe(200); + bodies[wire] = mock.getLastRequest()?.body; + } + for (const body of Object.values(bodies)) expect(JSON.stringify(body)).not.toContain("_wire"); + expect(bodies).toMatchSnapshot(); + }); + + it("on() with an invalid raw object does not throw", () => { + const local = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + const invalid: FixtureFileResponse = JSON.parse( + '{"raw":{"wire":"not-a-wire","status":99,"headers":{"content-length":"5"}}}', + ); + expect(() => local.on({ userMessage: "bad" }, invalid)).not.toThrow(); + }); +}); diff --git a/src/__tests__/raw-ollama.test.ts b/src/__tests__/raw-ollama.test.ts new file mode 100644 index 00000000..ad004b8d --- /dev/null +++ b/src/__tests__/raw-ollama.test.ts @@ -0,0 +1,234 @@ +import { Ollama } from "ollama"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { LLMock } from "../llmock.js"; +import type { RawResponse } from "../types.js"; + +// Raw passthrough on the Ollama wire, /api/chat and /api/generate (#549). +// The ollama SDK has no retry loop, so a 500 fails once. + +const chatEvents: Record[] = [ + { + model: "llama3.1", + created_at: "2026-01-01T00:00:00Z", + message: { role: "assistant", content: "Raw " }, + done: false, + }, + { + model: "llama3.1", + created_at: "2026-01-01T00:00:00Z", + message: { role: "assistant", content: "ollama." }, + done: true, + done_reason: "stop", + }, +]; +const chatJson = { + model: "llama3.1", + created_at: "2026-01-01T00:00:00Z", + message: { role: "assistant", content: "Raw JSON reply." }, + done: true, + done_reason: "stop", +}; +const generateEvents: Record[] = [ + { model: "llama3.1", created_at: "2026-01-01T00:00:00Z", response: "Raw ", done: false }, + { + model: "llama3.1", + created_at: "2026-01-01T00:00:00Z", + response: "generate.", + done: true, + done_reason: "stop", + }, +]; +const generateJson = { + model: "llama3.1", + created_at: "2026-01-01T00:00:00Z", + response: "Raw JSON reply.", + done: true, + done_reason: "stop", +}; + +const chatRaw: RawResponse = { raw: { wire: "ollama", events: chatEvents, json: chatJson } }; +const generateRaw: RawResponse = { + raw: { wire: "ollama-generate", events: generateEvents, json: generateJson }, +}; + +let mock: LLMock; +let client: Ollama; + +beforeEach(async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.on({ userMessage: "raw-proof" }, () => chatRaw); + mock.on({ userMessage: "raw-generate" }, () => generateRaw); + await mock.start(); + client = new Ollama({ host: mock.url }); +}); + +afterEach(async () => { + await mock.stop(); +}); + +const chatRequest = { model: "llama3.1", messages: [{ role: "user", content: "raw-proof" }] }; + +describe("raw passthrough: ollama", () => { + it("chat streams exactly the authored objects in order", async () => { + const received: unknown[] = []; + for await (const chunk of await client.chat({ ...chatRequest, stream: true })) + received.push(chunk); + expect(received).toEqual(chatEvents); + }); + + it("chat returns the authored json with stream:false", async () => { + expect(await client.chat({ ...chatRequest, stream: false })).toEqual(chatJson); + }); + + it("generate streams exactly the authored objects in order", async () => { + const received: unknown[] = []; + for await (const chunk of await client.generate({ + model: "llama3.1", + prompt: "raw-generate", + stream: true, + })) + received.push(chunk); + expect(received).toEqual(generateEvents); + }); + + it("generate returns the authored json with stream:false", async () => { + expect( + await client.generate({ model: "llama3.1", prompt: "raw-generate", stream: false }), + ).toEqual(generateJson); + }); + + it("writes NDJSON, one authored object per line, when stream is absent", async () => { + for (const [path, body, expected] of [ + ["/api/chat", chatRequest, chatEvents], + ["/api/generate", { model: "llama3.1", prompt: "raw-generate" }, generateEvents], + ] as const) { + const res = await fetch(`${mock.url}${path}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toBe("application/x-ndjson"); + expect(await res.text()).toBe(expected.map((e) => `${JSON.stringify(e)}\n`).join("")); + } + }); +}); + +describe("raw passthrough: ollama error bodies", () => { + async function post( + path: "/api/chat" | "/api/generate", + content: string, + stream: boolean, + ): Promise<{ status: number; body: string }> { + const body = + path === "/api/chat" + ? { model: "llama3.1", messages: [{ role: "user", content }], stream } + : { model: "llama3.1", prompt: content, stream }; + const res = await fetch(`${mock.url}${path}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + return { status: res.status, body: await res.text() }; + } + + const expected = (code: string, message: string): string => + JSON.stringify({ error: `${code}: ${message}` }); + + for (const path of ["/api/chat", "/api/generate"] as const) { + const wire = path === "/api/chat" ? "ollama" : "ollama-generate"; + + it(`${path}: a streaming request on a json-only fixture: aimock_raw_mode_unavailable`, async () => { + mock.on({ userMessage: "json-only" }, () => ({ raw: { wire, json: chatJson } })); + const got = await post(path, "json-only", true); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "events" body for a streaming request on ${wire}`, + ), + ); + }); + + it(`${path}: a non-streaming request on an events-only fixture: aimock_raw_mode_unavailable`, async () => { + mock.on({ userMessage: "events-only" }, () => ({ + raw: { wire, events: chatEvents }, + })); + const got = await post(path, "events-only", false); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "json" body for a non-streaming request on ${wire}`, + ), + ); + }); + + it(`${path}: a factory for another wire: aimock_raw_fixture_invalid`, async () => { + mock.on({ userMessage: "other-wire" }, () => ({ raw: { wire: "cohere", json: chatJson } })); + const got = await post(path, "other-wire", false); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected("aimock_raw_fixture_invalid", `raw fixture is for cohere, not ${wire}`), + ); + }); + + it(`${path}: a factory that fails the checker: aimock_raw_fixture_invalid`, async () => { + mock.on({ userMessage: "bad-status" }, () => ({ + raw: { wire, json: chatJson, status: 99 }, + })); + const got = await post(path, "bad-status", false); + expect(got.status).toBe(500); + const parsed = JSON.parse(got.body) as { error: string }; + expect(Object.keys(parsed)).toEqual(["error"]); + expect(parsed.error).toMatch(/^aimock_raw_fixture_invalid: invalid raw fixture: .*status/); + }); + } +}); + +describe("raw passthrough: ollama chat and generate are separate wires", () => { + // Both fixtures match the same text, so only the wire can pick between them. + // The chat fixture comes first: before the fix it also served /api/generate. + let shared: LLMock; + + beforeEach(async () => { + shared = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + shared.addFixture({ + match: { userMessage: "shared" }, + response: { raw: { wire: "ollama", events: chatEvents, json: chatJson } }, + }); + shared.addFixture({ + match: { userMessage: "shared" }, + response: { raw: { wire: "ollama-generate", events: generateEvents, json: generateJson } }, + }); + await shared.start(); + }); + + afterEach(async () => { + await shared.stop(); + }); + + it("/api/generate gets the ollama-generate fixture in generate shape", async () => { + const sdk = new Ollama({ host: shared.url }); + const received: unknown[] = []; + for await (const chunk of await sdk.generate({ + model: "llama3.1", + prompt: "shared", + stream: true, + })) + received.push(chunk); + expect(received).toEqual(generateEvents); + expect(await sdk.generate({ model: "llama3.1", prompt: "shared", stream: false })).toEqual( + generateJson, + ); + }); + + it("/api/chat gets the ollama fixture in chat shape", async () => { + const sdk = new Ollama({ host: shared.url }); + const req = { model: "llama3.1", messages: [{ role: "user", content: "shared" }] }; + const received: unknown[] = []; + for await (const chunk of await sdk.chat({ ...req, stream: true })) received.push(chunk); + expect(received).toEqual(chatEvents); + expect(await sdk.chat({ ...req, stream: false })).toEqual(chatJson); + }); +}); diff --git a/src/__tests__/raw-openai-chat.test.ts b/src/__tests__/raw-openai-chat.test.ts new file mode 100644 index 00000000..3bc5faac --- /dev/null +++ b/src/__tests__/raw-openai-chat.test.ts @@ -0,0 +1,270 @@ +import OpenAI from "openai"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { LLMock } from "../llmock.js"; +import type { RawResponse } from "../types.js"; + +// Raw passthrough on the OpenAI Chat Completions wire (#549), including the +// OpenRouter and Azure routes that share handleCompletions. + +const events: Record[] = [ + { + id: "chatcmpl-raw", + object: "chat.completion.chunk", + created: 1, + model: "gpt-4.1", + choices: [{ index: 0, delta: { role: "assistant", content: "Raw " }, finish_reason: null }], + }, + { + id: "chatcmpl-raw", + object: "chat.completion.chunk", + created: 1, + model: "gpt-4.1", + choices: [{ index: 0, delta: { content: "chunk." }, finish_reason: null }], + unmodeled_extension: { kept: true }, + }, + { + id: "chatcmpl-raw", + object: "chat.completion.chunk", + created: 1, + model: "gpt-4.1", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + }, +]; + +const json = { + id: "chatcmpl-raw-json", + object: "chat.completion", + created: 1, + model: "gpt-4.1", + choices: [ + { + index: 0, + message: { role: "assistant", content: "Raw JSON reply.", refusal: null }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 3, total_tokens: 4 }, +}; + +const raw: RawResponse = { raw: { wire: "openai-chat", events, json } }; + +let mock: LLMock; +let client: OpenAI; + +beforeEach(async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.on({ userMessage: "raw-proof" }, () => raw); + await mock.start(); + client = new OpenAI({ apiKey: "local-proof", baseURL: `${mock.url}/v1`, maxRetries: 0 }); +}); + +afterEach(async () => { + await mock.stop(); +}); + +describe("raw passthrough: openai-chat", () => { + it("streams exactly the authored chunks in order", async () => { + const stream = await client.chat.completions.create({ + model: "gpt-4.1", + messages: [{ role: "user", content: "raw-proof" }], + stream: true, + }); + const received: unknown[] = []; + for await (const chunk of stream) received.push(chunk); + expect(received).toEqual(events); + }); + + it("returns the authored json for a non-streaming request", async () => { + const reply = await client.chat.completions.create({ + model: "gpt-4.1", + messages: [{ role: "user", content: "raw-proof" }], + }); + expect(reply).toEqual(json); + }); +}); + +describe("raw passthrough: openai-chat framing", () => { + async function streamText(content: string): Promise<{ type: string | null; text: string }> { + const res = await fetch(`${mock.url}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4.1", + messages: [{ role: "user", content }], + stream: true, + }), + }); + return { type: res.headers.get("content-type"), text: await res.text() }; + } + + const frames = events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join(""); + + it("frames each chunk as a data line and ends with data: [DONE] by default", async () => { + const got = await streamText("raw-proof"); + expect(got.type).toBe("text/event-stream"); + expect(got.text).toBe(`${frames}data: [DONE]\n\n`); + }); + + it("omits data: [DONE] when done is false", async () => { + mock.on({ userMessage: "raw-no-done" }, () => ({ raw: { ...raw.raw, done: false } })); + const got = await streamText("raw-no-done"); + expect(got.text).toBe(frames); + }); + + it("passes the authored status and headers to the SDK", async () => { + mock.on({ userMessage: "raw-headers" }, () => ({ + raw: { ...raw.raw, status: 201, headers: { "x-request-id": "req_raw_chat" } }, + })); + const { response } = await client.chat.completions + .create({ + model: "gpt-4.1", + messages: [{ role: "user", content: "raw-headers" }], + stream: true, + }) + .withResponse(); + expect(response.status).toBe(201); + expect(response.headers.get("x-request-id")).toBe("req_raw_chat"); + }); +}); + +describe("raw passthrough: openai-chat via OpenRouter and Azure", () => { + it("serves the same raw chunks and json on the OpenRouter route", async () => { + const openRouter = new OpenAI({ + apiKey: "local-proof", + baseURL: `${mock.url}/api/v1`, + maxRetries: 0, + }); + const stream = await openRouter.chat.completions.create({ + model: "openai/gpt-4.1", + messages: [{ role: "user", content: "raw-proof" }], + stream: true, + }); + const received: unknown[] = []; + for await (const chunk of stream) received.push(chunk); + expect(received).toEqual(events); + + const reply = await openRouter.chat.completions.create({ + model: "openai/gpt-4.1", + messages: [{ role: "user", content: "raw-proof" }], + }); + expect(reply).toEqual(json); + }); + + it("sends the same bytes on the OpenRouter route as on /v1", async () => { + const send = async (path: string): Promise => { + const res = await fetch(`${mock.url}${path}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "openai/gpt-4.1", + messages: [{ role: "user", content: "raw-proof" }], + stream: true, + }), + }); + return res.text(); + }; + expect(await send("/api/v1/chat/completions")).toBe(await send("/v1/chat/completions")); + }); + + it("serves the raw json on the Azure deployment route", async () => { + const res = await fetch( + `${mock.url}/openai/deployments/my-gpt/chat/completions?api-version=2024-10-21`, + { + method: "POST", + headers: { "content-type": "application/json", "api-key": "local-proof" }, + body: JSON.stringify({ messages: [{ role: "user", content: "raw-proof" }] }), + }, + ); + expect(res.status).toBe(200); + expect(await res.json()).toEqual(json); + }); +}); + +describe("raw passthrough: openai-chat error bodies", () => { + async function post( + content: string, + stream: boolean, + path = "/v1/chat/completions", + ): Promise<{ status: number; body: string }> { + const res = await fetch(`${mock.url}${path}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-4.1", messages: [{ role: "user", content }], stream }), + }); + return { status: res.status, body: await res.text() }; + } + + const expected = (code: string, message: string): string => + JSON.stringify({ error: { message, type: "server_error", param: null, code } }); + const expectedOpenRouter = (code: string, message: string): string => + JSON.stringify({ error: { code: 500, message: `${code}: ${message}` } }); + + it("a streaming request on a json-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "json-only" }, () => ({ raw: { wire: "openai-chat", json } })); + const got = await post("json-only", true); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "events" body for a streaming request on openai-chat`, + ), + ); + }); + + it("a non-streaming request on an events-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "events-only" }, () => ({ raw: { wire: "openai-chat", events } })); + const got = await post("events-only", false); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "json" body for a non-streaming request on openai-chat`, + ), + ); + }); + + it("a factory for another wire: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "other-wire" }, () => ({ raw: { wire: "anthropic", json } })); + const got = await post("other-wire", false); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected("aimock_raw_fixture_invalid", "raw fixture is for anthropic, not openai-chat"), + ); + }); + + it("a factory that fails the checker: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "bad-status" }, () => ({ + raw: { wire: "openai-chat", json, status: 99 }, + })); + const got = await post("bad-status", false); + expect(got.status).toBe(500); + const parsed = JSON.parse(got.body) as { error: { code: string; message: string } }; + expect(Object.keys(parsed.error)).toEqual(["message", "type", "param", "code"]); + expect(parsed.error.code).toBe("aimock_raw_fixture_invalid"); + expect(parsed.error.message).toMatch(/^invalid raw fixture: .*status/); + }); + + it("OpenRouter, mode mismatch: aimock_raw_mode_unavailable in the OpenRouter dialect", async () => { + mock.on({ userMessage: "json-only" }, () => ({ raw: { wire: "openai-chat", json } })); + const got = await post("json-only", true, "/api/v1/chat/completions"); + expect(got.status).toBe(500); + expect(got.body).toBe( + expectedOpenRouter( + "aimock_raw_mode_unavailable", + `raw fixture has no "events" body for a streaming request on openai-chat`, + ), + ); + }); + + it("OpenRouter, wrong wire: aimock_raw_fixture_invalid in the OpenRouter dialect", async () => { + mock.on({ userMessage: "other-wire" }, () => ({ raw: { wire: "anthropic", json } })); + const got = await post("other-wire", false, "/api/v1/chat/completions"); + expect(got.status).toBe(500); + expect(got.body).toBe( + expectedOpenRouter( + "aimock_raw_fixture_invalid", + "raw fixture is for anthropic, not openai-chat", + ), + ); + }); +}); diff --git a/src/__tests__/raw-openai-responses.test.ts b/src/__tests__/raw-openai-responses.test.ts new file mode 100644 index 00000000..c097a909 --- /dev/null +++ b/src/__tests__/raw-openai-responses.test.ts @@ -0,0 +1,180 @@ +import OpenAI from "openai"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { LLMock } from "../llmock.js"; +import type { RawResponse } from "../types.js"; + +// Raw passthrough on the OpenAI Responses wire (#549). + +const events: Record[] = [ + { + type: "response.created", + sequence_number: 0, + response: { id: "resp_raw", object: "response", status: "in_progress", output: [] }, + }, + { + type: "response.output_text.delta", + sequence_number: 1, + item_id: "msg_raw", + output_index: 0, + content_index: 0, + delta: "Paris is the capital.", + }, + { + type: "response.output_text.annotation.added", + sequence_number: 2, + item_id: "msg_raw", + output_index: 0, + content_index: 0, + annotation_index: 0, + annotation: { + type: "url_citation", + url: "https://example.com/paris", + title: "Paris", + start_index: 0, + end_index: 5, + }, + }, + { + type: "response.completed", + sequence_number: 3, + response: { id: "resp_raw", object: "response", status: "completed", output: [] }, + }, +]; + +const json = { + id: "resp_raw_json", + object: "response", + created_at: 1, + status: "completed", + model: "gpt-4.1", + output: [ + { + type: "message", + id: "msg_raw_json", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "Raw JSON reply.", annotations: [] }], + }, + ], +}; + +const raw: RawResponse = { raw: { wire: "openai-responses", events, json } }; + +let mock: LLMock; +let client: OpenAI; + +beforeEach(async () => { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.on({ userMessage: "raw-proof" }, () => raw); + await mock.start(); + client = new OpenAI({ apiKey: "local-proof", baseURL: `${mock.url}/v1`, maxRetries: 0 }); +}); + +afterEach(async () => { + await mock.stop(); +}); + +describe("raw passthrough: openai-responses", () => { + it("streams exactly the authored events in order", async () => { + const stream = await client.responses.create({ + model: "gpt-4.1", + input: "raw-proof", + stream: true, + }); + const received: unknown[] = []; + for await (const event of stream) received.push(event); + expect(received).toEqual(events); + }); + + it("returns the authored json for a non-streaming request", async () => { + const reply = await client.responses.create({ model: "gpt-4.1", input: "raw-proof" }); + expect(reply).toMatchObject(json); + }); + + it("passes the authored status and headers to the SDK", async () => { + mock.on({ userMessage: "raw-headers" }, () => ({ + raw: { ...raw.raw, status: 201, headers: { "x-request-id": "req_raw_proof" } }, + })); + const { response } = await client.responses + .create({ model: "gpt-4.1", input: "raw-headers", stream: true }) + .withResponse(); + expect(response.status).toBe(201); + expect(response.headers.get("x-request-id")).toBe("req_raw_proof"); + expect(response.headers.get("content-type")).toBe("text/event-stream"); + }); + + it("frames each event as a named SSE event with no [DONE]", async () => { + const res = await fetch(`${mock.url}/v1/responses`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-4.1", input: "raw-proof", stream: true }), + }); + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toBe("text/event-stream"); + const text = await res.text(); + expect(text).toBe( + events.map((e) => `event: ${String(e.type)}\ndata: ${JSON.stringify(e)}\n\n`).join(""), + ); + }); +}); + +describe("raw passthrough: openai-responses error bodies", () => { + async function post(input: string, stream: boolean): Promise<{ status: number; body: string }> { + const res = await fetch(`${mock.url}/v1/responses`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-4.1", input, stream }), + }); + return { status: res.status, body: await res.text() }; + } + + const expected = (code: string, message: string): string => + JSON.stringify({ error: { message, type: "server_error", code } }); + + it("a streaming request on a json-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "json-only" }, () => ({ raw: { wire: "openai-responses", json } })); + const got = await post("json-only", true); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "events" body for a streaming request on openai-responses`, + ), + ); + }); + + it("a non-streaming request on an events-only fixture: aimock_raw_mode_unavailable", async () => { + mock.on({ userMessage: "events-only" }, () => ({ + raw: { wire: "openai-responses", events }, + })); + const got = await post("events-only", false); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected( + "aimock_raw_mode_unavailable", + `raw fixture has no "json" body for a non-streaming request on openai-responses`, + ), + ); + }); + + it("a factory for another wire: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "other-wire" }, () => ({ raw: { wire: "anthropic", json } })); + const got = await post("other-wire", false); + expect(got.status).toBe(500); + expect(got.body).toBe( + expected("aimock_raw_fixture_invalid", "raw fixture is for anthropic, not openai-responses"), + ); + }); + + it("a factory that fails the checker: aimock_raw_fixture_invalid", async () => { + mock.on({ userMessage: "bad-status" }, () => ({ + raw: { wire: "openai-responses", json, status: 99 }, + })); + const got = await post("bad-status", false); + expect(got.status).toBe(500); + const parsed = JSON.parse(got.body) as { error: { code: string; message: string } }; + expect(Object.keys(parsed.error)).toEqual(["message", "type", "code"]); + expect(parsed.error.code).toBe("aimock_raw_fixture_invalid"); + expect(parsed.error.message).toMatch(/^invalid raw fixture: .*status/); + }); +}); diff --git a/src/__tests__/raw-wire-plumbing.test.ts b/src/__tests__/raw-wire-plumbing.test.ts new file mode 100644 index 00000000..fce12643 --- /dev/null +++ b/src/__tests__/raw-wire-plumbing.test.ts @@ -0,0 +1,204 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { LLMock } from "../llmock.js"; +import type { RawWireId } from "../types.js"; + +// Every LLM handler tells the router its wire (#549 T7), so a raw fixture +// reaches its own wire's handler and no other. Until the HTTP raw branches +// land (T8-T13) the handler answers a matched raw fixture with its existing +// "did not match any known type" 500; the served-body assertions come then. + +const MODEL = "anthropic.claude-3-5-sonnet-20241022-v2:0"; + +interface Site { + label: string; + wire: RawWireId; + path: string; + body: (text: string) => unknown; +} + +/** One request per handler that gains a wire (11 HTTP handler sites, 9 wires). */ +const SITES: Site[] = [ + { + label: "openai-responses (handleResponses)", + wire: "openai-responses", + path: "/v1/responses", + body: (text) => ({ model: "gpt-4.1", input: text, stream: false }), + }, + { + label: "openai-chat (handleCompletions)", + wire: "openai-chat", + path: "/v1/chat/completions", + body: (text) => ({ model: "gpt-4.1", messages: [{ role: "user", content: text }] }), + }, + { + label: "anthropic (handleMessages)", + wire: "anthropic", + path: "/v1/messages", + body: (text) => ({ + model: "claude-sonnet-4-5", + max_tokens: 64, + messages: [{ role: "user", content: text }], + }), + }, + { + label: "gemini (handleGemini)", + wire: "gemini", + path: "/v1beta/models/gemini-2.5-flash:generateContent", + body: (text) => ({ contents: [{ role: "user", parts: [{ text }] }] }), + }, + { + label: "gemini-interactions (handleGeminiInteractions)", + wire: "gemini-interactions", + path: "/v1beta/interactions", + body: (text) => ({ model: "gemini-2.5-flash", input: text, stream: false }), + }, + { + label: "cohere (handleCohere)", + wire: "cohere", + path: "/v2/chat", + body: (text) => ({ model: "command-r-plus", messages: [{ role: "user", content: text }] }), + }, + { + label: "ollama (handleOllama)", + wire: "ollama", + path: "/api/chat", + body: (text) => ({ + model: "llama3.1", + messages: [{ role: "user", content: text }], + stream: false, + }), + }, + { + label: "ollama (handleOllamaGenerate)", + wire: "ollama-generate", + path: "/api/generate", + body: (text) => ({ model: "llama3.1", prompt: text, stream: false }), + }, + { + label: "bedrock-invoke (handleBedrock)", + wire: "bedrock-invoke", + path: `/model/${MODEL}/invoke`, + body: (text) => ({ + anthropic_version: "bedrock-2023-05-31", + max_tokens: 64, + messages: [{ role: "user", content: text }], + }), + }, + { + label: "bedrock-invoke (handleBedrockStream)", + wire: "bedrock-invoke", + path: `/model/${MODEL}/invoke-with-response-stream`, + body: (text) => ({ + anthropic_version: "bedrock-2023-05-31", + max_tokens: 64, + messages: [{ role: "user", content: text }], + }), + }, + { + label: "bedrock-converse (handleConverse)", + wire: "bedrock-converse", + path: `/model/${MODEL}/converse`, + body: (text) => ({ messages: [{ role: "user", content: [{ text }] }] }), + }, + { + label: "bedrock-converse (handleConverseStream)", + wire: "bedrock-converse", + path: `/model/${MODEL}/converse-stream`, + body: (text) => ({ messages: [{ role: "user", content: [{ text }] }] }), + }, +]; + +/** A wire other than `wire`, for the cross-wire case. */ +function otherWire(wire: RawWireId): RawWireId { + return wire === "openai-responses" ? "anthropic" : "openai-responses"; +} + +let mock: LLMock | undefined; +afterEach(async () => { + await mock?.stop(); + mock = undefined; + vi.restoreAllMocks(); +}); + +async function serve(rawWire: RawWireId, site: Site) { + mock = new LLMock({ port: 0, host: "127.0.0.1", logLevel: "silent" }); + mock.addFixture({ + match: { userMessage: "plumbing" }, + response: { raw: { wire: rawWire, events: [{ type: "x" }], json: { raw: true } } }, + }); + mock.addFixture({ match: { userMessage: "plumbing" }, response: { content: "typed fallback" } }); + await mock.start(); + const [raw, typed] = mock.getFixtures(); + const res = await fetch(`${mock.url}${site.path}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(site.body("plumbing")), + }); + const text = Buffer.from(await res.arrayBuffer()).toString("latin1"); + const entries = mock.getRequests(); + return { + status: res.status, + text, + entries, + rawCount: mock.journal.getFixtureMatchCount(raw), + typedCount: mock.journal.getFixtureMatchCount(typed), + raw, + typed, + }; +} + +describe("same wire: the raw fixture reaches its own wire's handler", () => { + it.each(SITES)("$label", async (site) => { + const result = await serve(site.wire, site); + expect(result.entries).toHaveLength(1); + expect(result.entries[0].response.fixture).toBe(result.raw); + expect(result.rawCount).toBe(1); + expect(result.typedCount).toBe(0); + expect(result.text).not.toContain("typed fallback"); + }); +}); + +describe("cross wire: a raw fixture for another wire never matches", () => { + it.each(SITES)("$label", async (site) => { + const result = await serve(otherWire(site.wire), site); + expect(result.status).toBe(200); + expect(result.entries).toHaveLength(1); + expect(result.entries[0].response.fixture).toBe(result.typed); + expect(result.rawCount).toBe(0); + expect(result.typedCount).toBe(1); + expect(result.entries[0].response.status).toBe(200); + }); +}); + +describe("gemini-interactions passes only the wire (no record match options)", () => { + it("a turnIndex fixture matches exactly as before, with record configured and no relaxed-turnIndex warning", async () => { + const dir = mkdtempSync(join(tmpdir(), "aimock-raw-turn-")); + const warn = vi.spyOn(console, "warn"); + try { + mock = new LLMock({ + port: 0, + host: "127.0.0.1", + logLevel: "warn", + record: { providers: { "gemini-interactions": "http://127.0.0.1:9" }, fixturePath: dir }, + }); + mock.addFixture({ + match: { userMessage: "turn", turnIndex: 3 }, + response: { content: "turn reply" }, + }); + await mock.start(); + const res = await fetch(`${mock.url}/v1beta/interactions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gemini-2.5-flash", input: "turn", stream: false }), + }); + expect(res.status).toBe(200); + expect(await res.text()).toContain("turn reply"); + expect(warn.mock.calls.flat().join(" ")).not.toMatch(/turnIndex/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); From 0a1fe0ebe9c2554c438a26c5ac39174cc29d151e Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sat, 24 Oct 2026 11:33:09 -0700 Subject: [PATCH 7/7] docs: raw passthrough fixtures guide, skill, changelog and API report (#549) --- CHANGELOG.md | 1 + docs/aws-bedrock/index.html | 5 + docs/chat-completions/index.html | 5 + docs/claude-messages/index.html | 5 + docs/cohere/index.html | 4 + docs/fixtures/index.html | 8 + docs/gemini-interactions/index.html | 5 + docs/gemini/index.html | 4 + docs/ollama/index.html | 6 + docs/openrouter-chat/index.html | 5 + docs/raw-passthrough/index.html | 402 ++++++++++++++++++++++++++++ docs/responses-api/index.html | 5 + docs/sidebar.js | 1 + docs/streaming-physics/index.html | 5 + docs/vertex-ai/index.html | 4 + etc/aimock-a2a.import.api.md | 17 +- etc/aimock-a2a.require.api.md | 17 +- etc/aimock-agui.import.api.md | 17 +- etc/aimock-agui.require.api.md | 17 +- etc/aimock-index.import.api.md | 23 +- etc/aimock-index.require.api.md | 23 +- etc/aimock-jest.import.api.md | 19 +- etc/aimock-jest.require.api.md | 19 +- etc/aimock-mcp.import.api.md | 17 +- etc/aimock-mcp.require.api.md | 17 +- etc/aimock-vector.import.api.md | 17 +- etc/aimock-vector.require.api.md | 17 +- etc/aimock-vitest.import.api.md | 19 +- etc/aimock-vitest.require.api.md | 19 +- skills/write-fixtures/SKILL.md | 32 +++ 30 files changed, 733 insertions(+), 22 deletions(-) create mode 100644 docs/raw-passthrough/index.html diff --git a/CHANGELOG.md b/CHANGELOG.md index b254ec70..bc926330 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Fixture load validation warns about malformed `citations` and `files`, with the same problem description the 500 carries (the 500 message adds an `aimock: invalid fixture` prefix; the warning adds which 500 the requests answer). The fixture still loads. - `InvalidFixtureOutputError` (the error behind `aimock_invalid_fixture_citation` and `aimock_invalid_fixture_file`) is exported from `@copilotkit/aimock`. - Opt-in fixture `nonStreamingTimingAllProviders: true` delays non-streaming replies on OpenAI chat, Anthropic, Responses, Gemini/Vertex, Gemini Interactions, Cohere, Bedrock invoke/converse and Ollama; `nonStreamingTiming` is unchanged (OpenAI chat only); error replies are never delayed. On the OpenAI Responses API the wait also counts the stream frames that fixture `citations` and generated `files` add +- Raw passthrough fixtures: a `raw` response (`wire`, `events`, `json`, optional `status`, `headers`, and `done` on `openai-chat`) sends provider events or JSON exactly as written, with only the named wire's framing, on the HTTP LLM wires (OpenAI Responses and Chat Completions, Anthropic, Gemini/Vertex, Gemini Interactions, Cohere, Ollama `/api/chat` as `ollama` and `/api/generate` as `ollama-generate`, Bedrock invoke and converse). A raw fixture matches only its own wire. A request whose mode has no body answers that wire's 500 with `aimock_raw_mode_unavailable`; an invalid raw fixture added in code answers a 500 with `aimock_raw_fixture_invalid`, and `--validate-on-load` reports it at load. Streaming timing, truncation, disconnects and chaos apply; misbehavior faults skip raw fixtures. A non-streaming raw reply follows `nonStreamingTiming` like a typed fixture: `nonStreamingTiming: true` delays it on OpenAI Chat Completions only, `nonStreamingTimingAllProviders: true` on every provider; a raw reply with status 400 or above is never delayed, and with a `json` it answers JSON even to a streaming request. WebSocket Responses answers a raw fixture with an error event with the code `aimock_raw_mode_unavailable`. New exported types `RawResponse`, `RawWireId` and `WireId` ([#549](https://github.com/CopilotKit/aimock/issues/549)) ### Fixed diff --git a/docs/aws-bedrock/index.html b/docs/aws-bedrock/index.html index 4b84bd61..52ba7154 100644 --- a/docs/aws-bedrock/index.html +++ b/docs/aws-bedrock/index.html @@ -55,6 +55,11 @@

AWS Bedrock

against the Bedrock-format requests, returning responses in the authentic Bedrock format including AWS Event Stream binary framing for streaming.

+

+ To send events or fields aimock does not model yet, use a + raw passthrough fixture with + wire: "bedrock-invoke" or wire: "bedrock-converse". +

How It Works

diff --git a/docs/chat-completions/index.html b/docs/chat-completions/index.html index d337fa28..f9782f6e 100644 --- a/docs/chat-completions/index.html +++ b/docs/chat-completions/index.html @@ -54,6 +54,11 @@

OpenAI Chat Completions

non-streaming JSON responses, including text content and tool calls. This is the most commonly used endpoint.

+

+ To send events or fields aimock does not model yet, use a + raw passthrough fixture with + wire: "openai-chat". +

Endpoint

diff --git a/docs/claude-messages/index.html b/docs/claude-messages/index.html index b324555f..3e66e343 100644 --- a/docs/claude-messages/index.html +++ b/docs/claude-messages/index.html @@ -54,6 +54,11 @@

Anthropic Claude Messages API

streaming SSE using event: + data: format, including content blocks for text and tool use.

+

+ To send events or fields aimock does not model yet, use a + raw passthrough fixture with + wire: "anthropic". +

Endpoint

diff --git a/docs/cohere/index.html b/docs/cohere/index.html index ec1c4f76..3cd56804 100644 --- a/docs/cohere/index.html +++ b/docs/cohere/index.html @@ -54,6 +54,10 @@

Cohere v2 Chat API

streaming events and dual usage tracking (billed_units and tokens).

+

+ To send events or fields aimock does not model yet, use a + raw passthrough fixture with wire: "cohere". +

Endpoint

diff --git a/docs/fixtures/index.html b/docs/fixtures/index.html index c6331240..ca25fced 100644 --- a/docs/fixtures/index.html +++ b/docs/fixtures/index.html @@ -451,6 +451,14 @@

Response Types

usage envelopes + + + + +
Rawraw.wire, raw.events?, raw.json?, raw.status?, raw.headers?, raw.done? + Provider events or JSON sent as written, framed for one wire. See + Raw Passthrough Fixtures. +
diff --git a/docs/gemini-interactions/index.html b/docs/gemini-interactions/index.html index 2995a865..a7897b79 100644 --- a/docs/gemini-interactions/index.html +++ b/docs/gemini-interactions/index.html @@ -56,6 +56,11 @@

Gemini Interactions API

outputs[] instead of nested candidates[].content.parts[], and typed SSE events with an event_type field inside each JSON payload.

+

+ To send events or fields aimock does not model yet, use a + raw passthrough fixture with + wire: "gemini-interactions". +

Endpoint

diff --git a/docs/gemini/index.html b/docs/gemini/index.html index c5ed266b..852a1ecb 100644 --- a/docs/gemini/index.html +++ b/docs/gemini/index.html @@ -54,6 +54,10 @@

Google Gemini

streamGenerateContent (SSE) endpoints, plus Gemini Live over WebSocket. The same fixtures drive all three transports.

+

+ To send events or fields aimock does not model yet, use a + raw passthrough fixture with wire: "gemini". +

Endpoints

diff --git a/docs/ollama/index.html b/docs/ollama/index.html index ae3b5089..8af29fb7 100644 --- a/docs/ollama/index.html +++ b/docs/ollama/index.html @@ -54,6 +54,12 @@

Ollama

/api/tags endpoints with NDJSON streaming, matching Ollama's wire format including its key differences from OpenAI.

+

+ To send events or fields aimock does not model yet, use a + raw passthrough fixture with + wire: "ollama" for /api/chat or + wire: "ollama-generate" for /api/generate. +

Endpoints

diff --git a/docs/openrouter-chat/index.html b/docs/openrouter-chat/index.html index 645e414f..468e4bbb 100644 --- a/docs/openrouter-chat/index.html +++ b/docs/openrouter-chat/index.html @@ -59,6 +59,11 @@

OpenRouter Chat & Router

cost. Callers on the plain OpenAI base (/v1/…) are left byte-for-byte unchanged.

+

+ To send events or fields aimock does not model yet, use a + raw passthrough fixture with + wire: "openai-chat". +

diff --git a/docs/raw-passthrough/index.html b/docs/raw-passthrough/index.html new file mode 100644 index 00000000..f6e9dafe --- /dev/null +++ b/docs/raw-passthrough/index.html @@ -0,0 +1,402 @@ + + + + + + Raw Passthrough Fixtures — aimock + + + + + + + + +

+ +
+ + +
+

Raw Passthrough Fixtures

+

+ Send provider events exactly as you write them. A raw fixture lists the events (streaming) + or the JSON body (non-streaming) of one provider wire, and aimock adds only that wire's + framing. Use it to test an SDK against a new event type, field or error shape before + aimock models it. +

+ +
+

+ Prefer typed fixtures. content, toolCalls, + blocks and the other typed responses work on every provider, get usage, + reasoning and override handling, and are checked for drift against the real APIs. Raw + fixtures are wire-specific and are not checked for drift: when the + provider changes, your raw events do not. Use raw only for what typed fixtures cannot + express yet. +

+
+ +

Shape

+
+
raw.json json
+ 
{
+  "fixtures": [
+    {
+      "match": { "userMessage": "cite paris" },
+      "response": {
+        "raw": {
+          "wire": "openai-responses",
+          "events": [
+            { "type": "response.created", "response": { "id": "resp_1", "status": "in_progress", "output": [] } },
+            { "type": "response.output_text.delta", "delta": "Paris.", "output_index": 0, "content_index": 0 },
+            { "type": "response.output_text.annotation.added", "annotation": { "type": "url_citation", "url": "https://example.com" } },
+            { "type": "response.completed", "response": { "id": "resp_1", "status": "completed", "output": [] } }
+          ],
+          "json": { "id": "resp_1", "object": "response", "status": "completed", "output": [] },
+          "headers": { "x-request-id": "req_123" }
+        }
+      }
+    }
+  ]
+}
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldTypeDescription
wirestring, required + The wire the events belong to (see the framing table). A raw fixture matches only + requests on that wire; every other handler skips it and keeps looking. A factory + is the exception: its result is checked when it runs, and a raw result for another + wire gets a 500 (see below). +
eventsobject[] + Streamed body. Each entry is sent as one frame, framed per wire. +
jsonanyNon-streaming body, sent as JSON.stringify(json).
statusinteger 200-599 + HTTP status for both streaming and non-streaming replies. Default 200; 204 and 304 + are rejected (they cannot carry a body). A reply with status 400 or above is never + delayed. A 4xx/5xx with json sends that JSON error body as written, + even to a streaming request. +
headersobject of strings + Extra response headers, for example x-request-id or + anthropic-ratelimit-*. Unless you set Content-Type here, + a stream gets the same default headers as a typed stream on its wire + (text/event-stream, application/x-ndjson on Ollama, + application/vnd.amazon.eventstream on Bedrock) and a non-streaming + reply gets application/json on every wire. A header you set replaces + the default of the same name in any letter case, so it is sent once. + content-length, transfer-encoding, + connection and content-encoding are rejected. +
doneboolean + openai-chat only: send the terminal data: [DONE]. + Default true. It must be a boolean; any other wire rejects it. +
+ +

+ raw is recognized only when the response has no other type's key. A fixture + such as { "content": "hi", "raw": { … } } stays a text fixture, as before. + The exceptions are customToolCalls and responsesBlocks: a + fixture that writes either next to raw is rejected. A key inside + raw that is not one of the fields above is ignored with a warning. +

+ +

Framing per wire

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
wireEndpointsEach event is sent as
openai-responsesPOST /v1/responsesevent: <type>\ndata: <json>\n\n (no terminal)
openai-chatPOST /v1/chat/completions (also Azure and OpenRouter routes)data: <json>\n\n, then data: [DONE]\n\n
anthropicPOST /v1/messagesevent: <type>\ndata: <json>\n\n
gemini + generateContent, streamGenerateContent (also Vertex AI) + data: <json>\n\n
gemini-interactionsPOST /v1beta/interactionsdata: <json>\n\n
coherePOST /v2/chatevent: <type>\ndata: <json>\n\n (no terminal)
ollamaPOST /api/chat<json>\n (NDJSON)
ollama-generatePOST /api/generate<json>\n (NDJSON)
bedrock-invoke/invoke, /invoke-with-response-streamAn AWS event-stream chunk frame with base64 bytes
bedrock-converse/converse, /converse-stream + An AWS event-stream frame: the event's single key is the event type, its value the + payload ({ "contentBlockDelta": { … } }) +
+
+

+ On openai-responses, anthropic and cohere every + event needs a string type; it names the SSE event. Each raw event is one + frame on every wire, including Gemini Interactions (where typed fixtures split text by + chunkSize). +

+ +

Streaming and non-streaming requests

+

+ A streaming request gets events; a non-streaming request gets + json. The exception is an error reply: with a status of 400 or + above and a json, a streaming request also gets the JSON, as provider APIs + answer errors. If the one the request needs is missing, aimock answers with that wire's + native 500 error and the code aimock_raw_mode_unavailable. An invalid raw + fixture (for example a factory that returns status: 99) gets the same 500 + with the code aimock_raw_fixture_invalid, and so does a factory that returns + a raw fixture for another wire. A raw fixture added as an object for another wire is never + served there: the router skips it, and when no other fixture matches, the server log warns + that a raw fixture matched everything except its wire. The error body uses each provider's + own error shape (on Gemini, Ollama and OpenRouter the code is prefixed to the message). +

+

+ In an OpenRouter models fallback chain, an openai-chat raw + fixture with a status of 400 or above fails over to the next model, as a + typed error fixture with the same status does. +

+ +

Timing

+

+ latency, streamingProfile, recordedTimings and + replaySpeed pace streamed raw events exactly as they pace typed frames (see + Streaming Physics). +

+

+ A non-streaming raw reply follows nonStreamingTiming like a typed fixture: + nonStreamingTiming: true delays a raw stream: false reply on + OpenAI Chat Completions only, and nonStreamingTimingAllProviders: true + delays it on every raw wire (including OpenAI Chat Completions). Either flag needs a + streamingProfile or recordedTimings; a bare + latency never delays a non-streaming reply. The wait is sized from the served + JSON, as a typed reply's is sized from its content. A raw reply with + status 400 or above is never delayed, streaming or not. +

+ +

Which features apply

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FeatureRaw fixtures
+ Matching (all match keys, sequenceIndex, turnIndex, + predicate, factories) + Yes. Plus the wire filter above.
truncateAfterChunks, disconnectAfterMs, chaosYes. Every raw event counts as one chunk.
Streaming timingYes (see Timing).
chunkSizeNo. Each event is one frame. Setting it warns at load.
Model misbehavior + No: faults skip raw fixtures. A server-level fault is skipped, the raw reply is + served, and the journal records the fault as skipped. An + X-AIMock-Misbehavior header returns 501 + aimock_misbehavior_not_applicable. A raw fixture that sets its own + misbehavior is an invalid fixture: load validation reports it (with + --misbehavior, its file fails to load), and when served it gets the + wire's 500 with aimock_raw_fixture_invalid. +
Overrides, reasoning gating, usage synthesis, drift checksNo. The events are sent as written.
Record modeA matched raw fixture never proxies. Recording raw fixtures is a follow-up.
+
+ +

Validation

+

+ --validate-on-load, addFixturesFromJSON and + POST /__aimock/fixtures check raw fixtures the same way: a known + wire, events or json present, events that are plain + JSON objects (no sparse-array holes, undefined, non-finite numbers, + functions, Dates or cycles), the per-wire type / single-key rules (no CR or + LF in a type; a bedrock-converse key of at most 65535 bytes), + status 200-599 except 204 and 304, valid header names and values, a boolean + done only on openai-chat, no customToolCalls or + responsesBlocks next to raw, and no fixture + misbehavior. Unknown keys inside raw and an empty + events are warnings. on(), addFixture() and + factories are not validated when added; an invalid raw fixture registered that way returns + the wire's 500 at request time. The same check runs again on every raw fixture when it is + served. +

+ +

Programmatic use

+
+
raw.test.ts ts
+ 
const mock = new LLMock();
+mock.on({ userMessage: "weather" }, {
+  raw: {
+    wire: "anthropic",
+    events: [
+      { type: "message_start", message: { /* … */ } },
+      { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Sunny." } },
+      { type: "message_stop" },
+    ],
+  },
+});
+
+ +

Limits

+
    +
  • + WebSocket Responses (/v1/responses over WebSocket) answers a raw fixture + with an error event with the code aimock_raw_mode_unavailable: "raw + fixtures are not yet supported on the WebSocket transport". Serving raw over WebSocket + is a follow-up. +
    • +
  • + Realtime, Gemini Live and non-LLM endpoints (embeddings, images, audio, video) never + match raw fixtures. For a transport aimock does not cover, use + mount(). +
    • +
+ + + +
+
+
$ aimock · MIT License
+ +
+
+ + + + diff --git a/docs/responses-api/index.html b/docs/responses-api/index.html index 1d68aa42..41b26874 100644 --- a/docs/responses-api/index.html +++ b/docs/responses-api/index.html @@ -53,6 +53,11 @@

OpenAI Responses API

The Responses API uses event: + data: SSE format over HTTP, and is also available over WebSocket. aimock supports both transports with the same fixtures.

+

+ To send events or fields aimock does not model yet, use a + raw passthrough fixture with + wire: "openai-responses". +

Endpoints

diff --git a/docs/sidebar.js b/docs/sidebar.js index 1b7c684a..706f31f9 100644 --- a/docs/sidebar.js +++ b/docs/sidebar.js @@ -62,6 +62,7 @@ { label: "Chaos Testing", href: "/chaos-testing" }, { label: "Model Misbehavior", href: "/model-misbehavior" }, { label: "Streaming Physics", href: "/streaming-physics" }, + { label: "Raw Passthrough", href: "/raw-passthrough" }, { label: "WebSocket APIs", href: "/websocket" }, { label: "Prometheus Metrics", href: "/metrics" }, { label: "Mount & Composition", href: "/mount" }, diff --git a/docs/streaming-physics/index.html b/docs/streaming-physics/index.html index 82733cd9..dd8cbed0 100644 --- a/docs/streaming-physics/index.html +++ b/docs/streaming-physics/index.html @@ -54,6 +54,11 @@

Streaming Physics

tokens-per-second (TPS), and random jitter. Perfect for testing loading states, progress indicators, and streaming UX under realistic conditions.

+

+ Streaming timing also paces raw passthrough fixtures, and a + non-streaming raw reply follows nonStreamingTiming like a typed fixture (see + Raw Passthrough: Timing). +

StreamingProfile

diff --git a/docs/vertex-ai/index.html b/docs/vertex-ai/index.html index 779b221c..3490bb9b 100644 --- a/docs/vertex-ai/index.html +++ b/docs/vertex-ai/index.html @@ -54,6 +54,10 @@

Vertex AI

different URL routing pattern. Vertex AI requests are handled by the same Gemini handler internally.

+

+ To send events or fields aimock does not model yet, use a + raw passthrough fixture with wire: "gemini". +

Endpoints

diff --git a/etc/aimock-a2a.import.api.md b/etc/aimock-a2a.import.api.md index 7c4306d9..7071f057 100644 --- a/etc/aimock-a2a.import.api.md +++ b/etc/aimock-a2a.import.api.md @@ -415,7 +415,7 @@ interface FixtureMatch { } // @public (undocumented) -type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public interface FixtureUrlCitation { @@ -839,6 +839,21 @@ interface RawJSONResponse extends ResponseOverrides { status?: number; } +// @public +interface RawResponse { + raw: { + wire: RawWireId; + events?: Record[]; + json?: unknown; + status?: number; + headers?: Record; + done?: boolean; + }; +} + +// @public +type RawWireId = Exclude | "ollama-generate"; + // @public interface RecordedTimings { // (undocumented) diff --git a/etc/aimock-a2a.require.api.md b/etc/aimock-a2a.require.api.md index 7c4306d9..7071f057 100644 --- a/etc/aimock-a2a.require.api.md +++ b/etc/aimock-a2a.require.api.md @@ -415,7 +415,7 @@ interface FixtureMatch { } // @public (undocumented) -type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public interface FixtureUrlCitation { @@ -839,6 +839,21 @@ interface RawJSONResponse extends ResponseOverrides { status?: number; } +// @public +interface RawResponse { + raw: { + wire: RawWireId; + events?: Record[]; + json?: unknown; + status?: number; + headers?: Record; + done?: boolean; + }; +} + +// @public +type RawWireId = Exclude | "ollama-generate"; + // @public interface RecordedTimings { // (undocumented) diff --git a/etc/aimock-agui.import.api.md b/etc/aimock-agui.import.api.md index d3a8449b..6de35a82 100644 --- a/etc/aimock-agui.import.api.md +++ b/etc/aimock-agui.import.api.md @@ -1044,7 +1044,7 @@ interface FixtureMatch { } // @public (undocumented) -type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public interface FixtureUrlCitation { @@ -1477,6 +1477,21 @@ interface RawJSONResponse extends ResponseOverrides { status?: number; } +// @public +interface RawResponse { + raw: { + wire: RawWireId; + events?: Record[]; + json?: unknown; + status?: number; + headers?: Record; + done?: boolean; + }; +} + +// @public +type RawWireId = Exclude | "ollama-generate"; + // @public interface RecordedTimings { // (undocumented) diff --git a/etc/aimock-agui.require.api.md b/etc/aimock-agui.require.api.md index d3a8449b..6de35a82 100644 --- a/etc/aimock-agui.require.api.md +++ b/etc/aimock-agui.require.api.md @@ -1044,7 +1044,7 @@ interface FixtureMatch { } // @public (undocumented) -type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public interface FixtureUrlCitation { @@ -1477,6 +1477,21 @@ interface RawJSONResponse extends ResponseOverrides { status?: number; } +// @public +interface RawResponse { + raw: { + wire: RawWireId; + events?: Record[]; + json?: unknown; + status?: number; + headers?: Record; + done?: boolean; + }; +} + +// @public +type RawWireId = Exclude | "ollama-generate"; + // @public interface RecordedTimings { // (undocumented) diff --git a/etc/aimock-index.import.api.md b/etc/aimock-index.import.api.md index 242db462..e843e40f 100644 --- a/etc/aimock-index.import.api.md +++ b/etc/aimock-index.import.api.md @@ -2239,7 +2239,7 @@ export interface FixtureFileEntry { } // @public (undocumented) -export type FixtureFileResponse = FixtureFileTextResponse | FixtureFileToolCallResponse | FixtureFileContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +export type FixtureFileResponse = FixtureFileTextResponse | FixtureFileToolCallResponse | FixtureFileContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public export type FixtureFileResponsesBlock = FixtureFileBlock | { @@ -2378,7 +2378,7 @@ export interface FixtureMatch { export type FixtureOpts = Omit; // @public (undocumented) -export type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +export type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public export interface FixturesWithServices { @@ -3351,6 +3351,8 @@ interface MatchOptions { }; // (undocumented) strictTurnIndex?: boolean; + // @internal + wire?: WireId | RawWireId; } // @public @@ -4595,6 +4597,21 @@ export interface RawJSONResponse extends ResponseOverrides { status?: number; } +// @public +export interface RawResponse { + raw: { + wire: RawWireId; + events?: Record[]; + json?: unknown; + status?: number; + headers?: Record; + done?: boolean; + }; +} + +// @public +export type RawWireId = Exclude | "ollama-generate"; + // @public interface ReasoningTextDetail { // (undocumented) @@ -5549,7 +5566,7 @@ export interface WebSocketLimits { } // @public -type WireId = "openai-chat" | "openai-responses" | "openai-realtime" | "anthropic" | "bedrock-invoke" | "bedrock-converse" | "gemini" | "gemini-live" | "gemini-interactions" | "cohere" | "ollama"; +export type WireId = "openai-chat" | "openai-responses" | "openai-realtime" | "anthropic" | "bedrock-invoke" | "bedrock-converse" | "gemini" | "gemini-live" | "gemini-interactions" | "cohere" | "ollama"; // @public export function writeAGUIEventStream(res: http$1.ServerResponse, events: AGUIEvent[], opts?: { diff --git a/etc/aimock-index.require.api.md b/etc/aimock-index.require.api.md index 242db462..e843e40f 100644 --- a/etc/aimock-index.require.api.md +++ b/etc/aimock-index.require.api.md @@ -2239,7 +2239,7 @@ export interface FixtureFileEntry { } // @public (undocumented) -export type FixtureFileResponse = FixtureFileTextResponse | FixtureFileToolCallResponse | FixtureFileContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +export type FixtureFileResponse = FixtureFileTextResponse | FixtureFileToolCallResponse | FixtureFileContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public export type FixtureFileResponsesBlock = FixtureFileBlock | { @@ -2378,7 +2378,7 @@ export interface FixtureMatch { export type FixtureOpts = Omit; // @public (undocumented) -export type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +export type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public export interface FixturesWithServices { @@ -3351,6 +3351,8 @@ interface MatchOptions { }; // (undocumented) strictTurnIndex?: boolean; + // @internal + wire?: WireId | RawWireId; } // @public @@ -4595,6 +4597,21 @@ export interface RawJSONResponse extends ResponseOverrides { status?: number; } +// @public +export interface RawResponse { + raw: { + wire: RawWireId; + events?: Record[]; + json?: unknown; + status?: number; + headers?: Record; + done?: boolean; + }; +} + +// @public +export type RawWireId = Exclude | "ollama-generate"; + // @public interface ReasoningTextDetail { // (undocumented) @@ -5549,7 +5566,7 @@ export interface WebSocketLimits { } // @public -type WireId = "openai-chat" | "openai-responses" | "openai-realtime" | "anthropic" | "bedrock-invoke" | "bedrock-converse" | "gemini" | "gemini-live" | "gemini-interactions" | "cohere" | "ollama"; +export type WireId = "openai-chat" | "openai-responses" | "openai-realtime" | "anthropic" | "bedrock-invoke" | "bedrock-converse" | "gemini" | "gemini-live" | "gemini-interactions" | "cohere" | "ollama"; // @public export function writeAGUIEventStream(res: http$1.ServerResponse, events: AGUIEvent[], opts?: { diff --git a/etc/aimock-jest.import.api.md b/etc/aimock-jest.import.api.md index add7814e..7fb501e2 100644 --- a/etc/aimock-jest.import.api.md +++ b/etc/aimock-jest.import.api.md @@ -399,7 +399,7 @@ interface FixtureFileEntry { } // @public (undocumented) -type FixtureFileResponse = FixtureFileTextResponse | FixtureFileToolCallResponse | FixtureFileContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureFileResponse = FixtureFileTextResponse | FixtureFileToolCallResponse | FixtureFileContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public type FixtureFileResponsesBlock = FixtureFileBlock | { @@ -489,7 +489,7 @@ interface FixtureMatch { type FixtureOpts = Omit; // @public (undocumented) -type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public interface FixtureUrlCitation { @@ -1157,6 +1157,21 @@ interface RawJSONResponse extends ResponseOverrides { status?: number; } +// @public +interface RawResponse { + raw: { + wire: RawWireId; + events?: Record[]; + json?: unknown; + status?: number; + headers?: Record; + done?: boolean; + }; +} + +// @public +type RawWireId = Exclude | "ollama-generate"; + // @public (undocumented) interface RecordConfig { bodyTimeoutMs?: number; diff --git a/etc/aimock-jest.require.api.md b/etc/aimock-jest.require.api.md index add7814e..7fb501e2 100644 --- a/etc/aimock-jest.require.api.md +++ b/etc/aimock-jest.require.api.md @@ -399,7 +399,7 @@ interface FixtureFileEntry { } // @public (undocumented) -type FixtureFileResponse = FixtureFileTextResponse | FixtureFileToolCallResponse | FixtureFileContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureFileResponse = FixtureFileTextResponse | FixtureFileToolCallResponse | FixtureFileContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public type FixtureFileResponsesBlock = FixtureFileBlock | { @@ -489,7 +489,7 @@ interface FixtureMatch { type FixtureOpts = Omit; // @public (undocumented) -type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public interface FixtureUrlCitation { @@ -1157,6 +1157,21 @@ interface RawJSONResponse extends ResponseOverrides { status?: number; } +// @public +interface RawResponse { + raw: { + wire: RawWireId; + events?: Record[]; + json?: unknown; + status?: number; + headers?: Record; + done?: boolean; + }; +} + +// @public +type RawWireId = Exclude | "ollama-generate"; + // @public (undocumented) interface RecordConfig { bodyTimeoutMs?: number; diff --git a/etc/aimock-mcp.import.api.md b/etc/aimock-mcp.import.api.md index 8d4b6a43..cb263512 100644 --- a/etc/aimock-mcp.import.api.md +++ b/etc/aimock-mcp.import.api.md @@ -332,7 +332,7 @@ interface FixtureMatch { } // @public (undocumented) -type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public interface FixtureUrlCitation { @@ -1387,6 +1387,21 @@ interface RawJSONResponse extends ResponseOverrides { status?: number; } +// @public +interface RawResponse { + raw: { + wire: RawWireId; + events?: Record[]; + json?: unknown; + status?: number; + headers?: Record; + done?: boolean; + }; +} + +// @public +type RawWireId = Exclude | "ollama-generate"; + // @public interface RecordedTimings { // (undocumented) diff --git a/etc/aimock-mcp.require.api.md b/etc/aimock-mcp.require.api.md index 8d4b6a43..cb263512 100644 --- a/etc/aimock-mcp.require.api.md +++ b/etc/aimock-mcp.require.api.md @@ -332,7 +332,7 @@ interface FixtureMatch { } // @public (undocumented) -type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public interface FixtureUrlCitation { @@ -1387,6 +1387,21 @@ interface RawJSONResponse extends ResponseOverrides { status?: number; } +// @public +interface RawResponse { + raw: { + wire: RawWireId; + events?: Record[]; + json?: unknown; + status?: number; + headers?: Record; + done?: boolean; + }; +} + +// @public +type RawWireId = Exclude | "ollama-generate"; + // @public interface RecordedTimings { // (undocumented) diff --git a/etc/aimock-vector.import.api.md b/etc/aimock-vector.import.api.md index 1a6ad453..03a0a5d3 100644 --- a/etc/aimock-vector.import.api.md +++ b/etc/aimock-vector.import.api.md @@ -277,7 +277,7 @@ interface FixtureMatch { } // @public (undocumented) -type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public interface FixtureUrlCitation { @@ -716,6 +716,21 @@ interface RawJSONResponse extends ResponseOverrides { status?: number; } +// @public +interface RawResponse { + raw: { + wire: RawWireId; + events?: Record[]; + json?: unknown; + status?: number; + headers?: Record; + done?: boolean; + }; +} + +// @public +type RawWireId = Exclude | "ollama-generate"; + // @public interface RecordedTimings { // (undocumented) diff --git a/etc/aimock-vector.require.api.md b/etc/aimock-vector.require.api.md index 1a6ad453..03a0a5d3 100644 --- a/etc/aimock-vector.require.api.md +++ b/etc/aimock-vector.require.api.md @@ -277,7 +277,7 @@ interface FixtureMatch { } // @public (undocumented) -type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public interface FixtureUrlCitation { @@ -716,6 +716,21 @@ interface RawJSONResponse extends ResponseOverrides { status?: number; } +// @public +interface RawResponse { + raw: { + wire: RawWireId; + events?: Record[]; + json?: unknown; + status?: number; + headers?: Record; + done?: boolean; + }; +} + +// @public +type RawWireId = Exclude | "ollama-generate"; + // @public interface RecordedTimings { // (undocumented) diff --git a/etc/aimock-vitest.import.api.md b/etc/aimock-vitest.import.api.md index de442b65..3f65963c 100644 --- a/etc/aimock-vitest.import.api.md +++ b/etc/aimock-vitest.import.api.md @@ -399,7 +399,7 @@ interface FixtureFileEntry { } // @public (undocumented) -type FixtureFileResponse = FixtureFileTextResponse | FixtureFileToolCallResponse | FixtureFileContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureFileResponse = FixtureFileTextResponse | FixtureFileToolCallResponse | FixtureFileContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public type FixtureFileResponsesBlock = FixtureFileBlock | { @@ -489,7 +489,7 @@ interface FixtureMatch { type FixtureOpts = Omit; // @public (undocumented) -type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public interface FixtureUrlCitation { @@ -1157,6 +1157,21 @@ interface RawJSONResponse extends ResponseOverrides { status?: number; } +// @public +interface RawResponse { + raw: { + wire: RawWireId; + events?: Record[]; + json?: unknown; + status?: number; + headers?: Record; + done?: boolean; + }; +} + +// @public +type RawWireId = Exclude | "ollama-generate"; + // @public (undocumented) interface RecordConfig { bodyTimeoutMs?: number; diff --git a/etc/aimock-vitest.require.api.md b/etc/aimock-vitest.require.api.md index de442b65..3f65963c 100644 --- a/etc/aimock-vitest.require.api.md +++ b/etc/aimock-vitest.require.api.md @@ -399,7 +399,7 @@ interface FixtureFileEntry { } // @public (undocumented) -type FixtureFileResponse = FixtureFileTextResponse | FixtureFileToolCallResponse | FixtureFileContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureFileResponse = FixtureFileTextResponse | FixtureFileToolCallResponse | FixtureFileContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public type FixtureFileResponsesBlock = FixtureFileBlock | { @@ -489,7 +489,7 @@ interface FixtureMatch { type FixtureOpts = Omit; // @public (undocumented) -type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | RawResponse | LiveFixtureResponse; // @public interface FixtureUrlCitation { @@ -1157,6 +1157,21 @@ interface RawJSONResponse extends ResponseOverrides { status?: number; } +// @public +interface RawResponse { + raw: { + wire: RawWireId; + events?: Record[]; + json?: unknown; + status?: number; + headers?: Record; + done?: boolean; + }; +} + +// @public +type RawWireId = Exclude | "ollama-generate"; + // @public (undocumented) interface RecordConfig { bodyTimeoutMs?: number; diff --git a/skills/write-fixtures/SKILL.md b/skills/write-fixtures/SKILL.md index 436e4696..891f8729 100644 --- a/skills/write-fixtures/SKILL.md +++ b/skills/write-fixtures/SKILL.md @@ -198,6 +198,38 @@ Video uses async polling — `POST /v1/videos` creates, `GET /v1/videos/{id}` ch { error: { message: "Rate limited", type: "rate_limit_error" }, status: 429 } ``` +### Raw passthrough (provider events as written) + +Prefer the typed responses above: they work on every provider and are drift-checked. Use `raw` only for an event, field or error shape aimock does not model yet (for example a new Responses event type). aimock sends `events` (streaming) or `json` (non-streaming) exactly as written and adds only the named wire's framing. + +```typescript +{ + raw: { + wire: "openai-responses", // required; the fixture matches only this wire + events: [ + { + type: "response.output_text.annotation.added", + annotation: { type: "url_citation", url: "https://example.com" }, + }, + ], + json: { id: "resp_1", object: "response", status: "completed", output: [] }, + status: 200, // optional, 200-599, not 204/304 + headers: { "x-request-id": "req_1" }, // optional; no content-length/transfer-encoding/connection/content-encoding + }, +} +``` + +- `wire` is one of `openai-responses`, `openai-chat`, `anthropic`, `gemini`, `gemini-interactions`, `cohere`, `ollama` (`/api/chat`), `ollama-generate` (`/api/generate`), `bedrock-invoke`, `bedrock-converse`. The router skips a raw fixture on every other wire and keeps matching; only a factory that returns a raw fixture for another wire gets a 500. +- Events on `openai-responses`, `anthropic` and `cohere` need a string `type`; a `bedrock-converse` event is `{ "": { …payload } }`. +- `done: false` (openai-chat only, a boolean) omits `data: [DONE]`. +- Unless `headers` sets `Content-Type`, a stream gets the typed stream's default headers for its wire (`text/event-stream`, `application/x-ndjson` on Ollama, `application/vnd.amazon.eventstream` on Bedrock) and a non-streaming reply gets `application/json` on every wire. An authored header replaces the default of the same name in any letter case. +- An error reply (`status` >= 400) is never delayed, streaming or not. With `json`, it answers JSON even to a streaming request. +- A request whose mode has no body (`events` for streaming, `json` for non-streaming) gets the wire's 500 with `aimock_raw_mode_unavailable`. An invalid raw fixture registered with `on()` or a factory is not checked when added; it gets the same 500 with `aimock_raw_fixture_invalid`, as does a factory result for another wire. Every raw fixture is checked again when it is served. `--validate-on-load` reports invalid fixtures at load. Also invalid: events that are not plain JSON (holes, `undefined`, non-finite numbers, Dates, cycles), a CR or LF in a `type`, `customToolCalls` or `responsesBlocks` next to `raw`, and a raw fixture with its own `misbehavior`. Unknown keys inside `raw` are a warning. +- `chunkSize` does not apply (each event is one frame). Misbehavior faults skip raw fixtures: a server-level fault is journaled as skipped, an `X-AIMock-Misbehavior` header returns 501 `aimock_misbehavior_not_applicable`, and a fixture's own `misbehavior` makes it invalid (load error, 500 `aimock_raw_fixture_invalid`). Streaming timing, `truncateAfterChunks`, `disconnectAfterMs` and chaos apply. +- Non-streaming timing: `nonStreamingTiming: true` delays a raw `stream: false` reply on `openai-chat` only; `nonStreamingTimingAllProviders: true` on every raw wire. Either needs `streamingProfile` or `recordedTimings`; the wait is sized from the served JSON. +- OpenRouter `models` fallback: an `openai-chat` raw fixture with `status` >= 400 fails over like a typed error fixture. +- Not served over WebSocket Responses yet (error event with code `aimock_raw_mode_unavailable`). Docs: `/raw-passthrough`. + ### Chaos (Failure Injection) The optional `chaos` field on a fixture enables probabilistic failure injection: