From a9d1c1c4c528847447f7bc2ca0f2b5a63dde015e Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 23 Oct 2026 15:11:44 -0700 Subject: [PATCH 1/6] feat: add fixture citation and generated-file types, errors and validators Add the types the citations and files work builds on: FixtureUrlCitation and FixtureFileCitation (FixtureCitation is now their union), the Chat Completions annotation type, and the fixture-output errors UnsupportedGeneratedFileError and InvalidFixtureOutputError, all exported from the package root. Add the shared validators every wire uses: fixtureCitationsProblem and fixtureFilesProblem with the null, surrogate-pair and mixed URL/file field rules, fixtureCitations, fixtureFiles, resolveCitations, rejectGeneratedFiles and surrogateSafeChunks. The fixture errors join the coded-500 path. Responses skips file citations here until it builds file_citation annotations. --- ...nvalid-fixture-output-error-export.test.ts | 16 + src/helpers.ts | 465 +++++++++++++++++- src/index.ts | 5 + src/responses.ts | 17 +- src/types.ts | 107 +++- 5 files changed, 583 insertions(+), 27 deletions(-) create mode 100644 src/__tests__/invalid-fixture-output-error-export.test.ts diff --git a/src/__tests__/invalid-fixture-output-error-export.test.ts b/src/__tests__/invalid-fixture-output-error-export.test.ts new file mode 100644 index 00000000..da6eda72 --- /dev/null +++ b/src/__tests__/invalid-fixture-output-error-export.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from "vitest"; +import * as aimock from "../index.js"; +import { InvalidFixtureOutputError as FromHelpers, fixtureFiles } from "../helpers.js"; + +describe("InvalidFixtureOutputError export", () => { + it("is exported from the package entry point as the class the helpers throw", () => { + expect(aimock.InvalidFixtureOutputError).toBe(FromHelpers); + let thrown: unknown; + try { + fixtureFiles("not-an-array"); + } catch (err) { + thrown = err; + } + expect(thrown).toBeInstanceOf(aimock.InvalidFixtureOutputError); + }); +}); diff --git a/src/helpers.ts b/src/helpers.ts index abe95146..8f8d4c4f 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -49,6 +49,8 @@ import type { FixtureFileResponsesBlock, ResponsesFixtureBlock, ChatCompletion, + FixtureCitation, + FixtureGeneratedFile, ResponseOverrides, RecordConfig, RecordProviderKey, @@ -317,18 +319,459 @@ export function requireEmittedFunctionToolCalls( return requireFunctionToolCalls(response.toolCalls ?? []); } -/** A fixture tool-call error that a handler surfaces as a coded 500. */ +/** + * A fixture's model-generated `files` reached a wire that has no faithful + * projection for them. Gemini (`inlineData` parts) and the OpenAI Responses + * API (`image_generation_call` items, `image/*` only) serve generated files. + * Chat Completions (OpenAI, Azure OpenAI, OpenRouter and BytePlus ModelArk) + * and Anthropic Messages raise this through {@link rejectGeneratedFiles}, and + * the Responses API raises a subclass for a non-`image/*` file, before any + * content is written. The handler surfaces it as a coded 500 + * (`aimock_unsupported_generated_file`) like {@link UnsupportedToolCallError}. + */ +export class UnsupportedGeneratedFileError extends Error { + readonly wire: string; + + constructor(wire: string) { + super( + `aimock: fixture "files" (model-generated files) are only supported on Gemini and the OpenAI Responses API; ` + + `this request arrived on ${wire}`, + ); + this.name = "UnsupportedGeneratedFileError"; + this.wire = wire; + } +} + +/** + * Reject a fixture that sets a non-empty `files` on a wire that cannot carry + * model-generated files. Throws {@link UnsupportedGeneratedFileError} naming + * `wire`. A malformed `files` value (see {@link fixtureFiles}) throws + * {@link InvalidFixtureOutputError} (`aimock_invalid_fixture_file`) first. + * Chat Completions and Anthropic Messages call this. Gemini and OpenAI + * Responses serve `files`; other wires ignore the field. + */ +export function rejectGeneratedFiles(response: object, wire: string): void { + const files = fixtureFiles((response as { files?: unknown }).files); + if (files.length > 0) throw new UnsupportedGeneratedFileError(wire); +} + +/** + * A fixture's `citations` or `files` is malformed: not an array, an entry + * missing a required field, a non-integer offset, citation offsets outside + * `0 <= startIndex <= endIndex <= text.length`, a file citation `index` + * outside `0 <= index <= text.length`, an offset or `index` that splits a + * UTF-16 surrogate pair, a citation that mixes URL and file citation fields, + * or any `citations` but an empty array on a turn with no text (tool calls + * only, `blocks` with no text block, or empty content). The wires that read + * these keys (Chat Completions, Anthropic Messages, Gemini and OpenAI + * Responses) read them + * through {@link fixtureFiles} and {@link fixtureCitations}, which throw this + * at serve time, before any byte is written; the handler surfaces it as a + * coded 500 like {@link UnsupportedGeneratedFileError}. Other wires ignore + * `citations` and `files`, so they never throw this. The message names the + * offending entry and how to fix the fixture. + */ +export class InvalidFixtureOutputError extends Error { + readonly code: "aimock_invalid_fixture_citation" | "aimock_invalid_fixture_file"; + + constructor( + code: "aimock_invalid_fixture_citation" | "aimock_invalid_fixture_file", + problem: string, + ) { + super( + `aimock: invalid fixture "${code === "aimock_invalid_fixture_file" ? "files" : "citations"}": ${problem}`, + ); + this.name = "InvalidFixtureOutputError"; + this.code = code; + } +} + +/** + * A short description of a fixture value for an error message. Never throws: + * a programmatic fixture can hold values JSON cannot carry (BigInt, Symbol, + * function, NaN, Infinity), and each gets a readable name. + */ +function describeFixtureValue(value: unknown): string { + if (value === undefined) return "undefined"; + if (Array.isArray(value)) return "an array"; + if (value !== null && typeof value === "object") return "an object"; + if (typeof value === "function") return "a function"; + if (typeof value === "symbol") return value.toString(); + if (typeof value === "bigint") return `${value}n`; + if (typeof value === "number" && !Number.isFinite(value)) return String(value); + const json = JSON.stringify(value); + return json.length > 40 ? `${json.slice(0, 37)}...` : json; +} + +/** + * The UTF-16 index `at` falls between the two halves of a surrogate pair, so + * a span that starts or ends there cuts a character in half. + */ +function splitsSurrogatePair(text: string, at: number): boolean { + if (at <= 0 || at >= text.length) return false; + const before = text.charCodeAt(at - 1); + const after = text.charCodeAt(at); + return before >= 0xd800 && before <= 0xdbff && after >= 0xdc00 && after <= 0xdfff; +} + +/** + * `text` cut into slices of at most `chunkSize` UTF-16 units, never between + * the two halves of a surrogate pair: a cut that would land inside a pair + * moves back before it, or, when that would leave the slice empty + * (`chunkSize` 1), forward past it. The slices rejoin to `text` exactly. + */ +export function surrogateSafeChunks(text: string, chunkSize: number): string[] { + const chunks: string[] = []; + for (let i = 0; i < text.length; ) { + let end = Math.min(i + chunkSize, text.length); + const high = text.charCodeAt(end - 1); + const low = text.charCodeAt(end); + if (high >= 0xd800 && high <= 0xdbff && low >= 0xdc00 && low <= 0xdfff) { + end = end - 1 > i ? end - 1 : end + 1; + } + chunks.push(text.slice(i, end)); + i = end; + } + return chunks; +} + +/** The message for a citation offset that splits a surrogate pair. */ +function surrogateSplitProblem(where: string, at: number): string { + return ( + `${where} ${at} splits a UTF-16 surrogate pair (one character, e.g. an emoji); ` + + `fix the offset to fall before or after the whole character` + ); +} + +/** Fields only a URL citation has. */ +const URL_CITATION_FIELDS = ["url", "title", "startIndex", "endIndex"] as const; +/** Fields only a file citation has. */ +const FILE_CITATION_FIELDS = ["fileId", "filename", "index"] as const; + +/** + * A citation that mixes URL and file citation fields, or a file-shaped one + * (`fileId` / `filename` / `index`) with no `type`, which would otherwise be + * read as a URL citation and lose its file data. Undefined when the fields + * agree. + */ +function mixedCitationProblem(citation: Record, i: number): string | undefined { + const present = (names: readonly string[]) => names.filter((n) => citation[n] != null); + const fileFields = present(FILE_CITATION_FIELDS); + if (fileFields.length === 0) return undefined; + const urlFields = present(URL_CITATION_FIELDS); + // The fix depends on what the citation says it is: a typed citation keeps + // its type and drops the other kind of fields. + const fix = + citation.type === "file" + ? `remove the URL citation fields (${urlFields.join(", ")}) from this file citation` + : citation.type === "url" + ? `remove the file citation fields (${fileFields.join(", ")}) from this URL citation, ` + + `or set type: "file" for a file citation` + : `set type: "file" for a file citation, or remove the stray fields`; + if (urlFields.length > 0) { + return ( + `citations[${i}] mixes URL citation fields (${urlFields.join(", ")}) with file citation ` + + `fields (${fileFields.join(", ")}); ${fix}` + ); + } + if (citation.type == null) { + return `citations[${i}] has file citation fields (${fileFields.join(", ")}) but no type; ${fix}`; + } + if (citation.type === "url") { + return `citations[${i}] has type "url" but file citation fields (${fileFields.join(", ")}); ${fix}`; + } + return undefined; +} + +/** "an object", "a string", ... for a value that should have been an array. */ +function describeFixtureKind(value: unknown): string { + if (value === null) return "null"; + if (value !== null && typeof value === "object") return "an object"; + return typeof value === "string" ? "a string" : `a ${typeof value}`; +} + +/** + * What is wrong with a fixture `files` value, or undefined when it is absent + * or an array of `{ mimeType, data }` string pairs. Shared by the serve-time + * check ({@link fixtureFiles}) and fixture load validation. + */ +export function fixtureFilesProblem(files: unknown): string | undefined { + // JSON fixtures commonly carry null for an unset key: treat it as absent. + if (files === undefined || files === null) return undefined; + if (!Array.isArray(files)) { + return ( + `"files" must be an array of { mimeType, data } objects, got ${describeFixtureKind(files)}; ` + + `wrap a single file in [ ]` + ); + } + for (let i = 0; i < files.length; i++) { + const file: unknown = files[i]; + if (file === null || typeof file !== "object" || Array.isArray(file)) { + return `files[${i}] must be a { mimeType, data } object, got ${describeFixtureValue(file)}`; + } + const { mimeType, data } = file as { mimeType?: unknown; data?: unknown }; + if (typeof mimeType !== "string" || mimeType.length === 0) { + return ( + `files[${i}].mimeType must be a non-empty string, got ${describeFixtureValue(mimeType)}; ` + + `set the file's MIME type, e.g. "image/png"` + ); + } + if (typeof data !== "string") { + return ( + `files[${i}].data must be a string, got ${describeFixtureValue(data)}; ` + + `set the base64-encoded file bytes` + ); + } + } + return undefined; +} + +/** + * What is wrong with a fixture `citations` value, or undefined when it is + * absent or valid: an array of citation objects. A URL citation (no `type`, + * or `type: "url"`) has a non-empty string `url`, an optional string `title` + * and optional integer offsets. A file citation (`type: "file"`) has a + * non-empty string `fileId` and `filename` and an optional integer `index`; + * it has no `url`. When `text` is given, the resolved offsets must also + * satisfy `0 <= startIndex <= endIndex <= text.length`, and a file citation's + * `index` must satisfy `0 <= index <= text.length`, and no offset may split a + * UTF-16 surrogate pair. A citation must not mix URL and file citation + * fields, and one with file fields needs `type: "file"`. A `null` value, or a + * `null` optional field, is treated as absent. An empty `text` is a turn with + * no text (tool calls only, `blocks` with no text block, or empty content): a + * well-formed non-empty array is then a problem, since there is nothing to + * cite; a malformed value is reported by its shape first. Shared by the + * serve-time check ({@link fixtureCitations}) and fixture load validation. + */ +export function fixtureCitationsProblem(citations: unknown, text?: string): string | undefined { + // JSON fixtures commonly carry null for an unset key: treat it as absent. + if (citations === undefined || citations === null) return undefined; + if (!Array.isArray(citations)) { + return ( + `"citations" must be an array of { url, title?, startIndex?, endIndex? } or ` + + `{ type: "file", fileId, filename, index? } objects, ` + + `got ${describeFixtureKind(citations)}; wrap a single citation in [ ]` + ); + } + // The shape comes first: a malformed entry is named even on a turn with no text. + const shape = citationEntriesProblem(citations, undefined); + if (shape !== undefined || text === undefined) return shape; + if (text === "" && citations.length > 0) { + return ( + `"citations" has nothing to cite: this turn has no text (only tool calls, or empty content); ` + + `add text to the response or remove the citations` + ); + } + return citationEntriesProblem(citations, text); +} + +/** + * What is wrong with the entries of a `citations` array, or undefined when + * all are valid. Offsets are checked against `text` only when it is given. A + * `null` optional field (`type`, `title`, `startIndex`, `endIndex`, `index`) + * is treated as absent, as a `null` top-level key is. + */ +function citationEntriesProblem( + citations: readonly unknown[], + text: string | undefined, +): string | undefined { + for (let i = 0; i < citations.length; i++) { + const citation: unknown = citations[i]; + if (citation === null || typeof citation !== "object" || Array.isArray(citation)) { + return ( + `citations[${i}] must be a { url, title?, startIndex?, endIndex? } or ` + + `{ type: "file", fileId, filename, index? } object, ` + + `got ${describeFixtureValue(citation)}` + ); + } + const fields = citation as Record; + const url = fields.url; + const type = fields.type ?? undefined; + const title = fields.title ?? undefined; + const startIndex = fields.startIndex ?? undefined; + const endIndex = fields.endIndex ?? undefined; + if (type === undefined || type === "url" || type === "file") { + const mixed = mixedCitationProblem(citation as Record, i); + if (mixed !== undefined) return mixed; + } + if (type === "file") { + const problem = fixtureFileCitationProblem(citation as Record, i, text); + if (problem !== undefined) return problem; + continue; + } + if (type !== undefined && type !== "url") { + return ( + `citations[${i}].type must be "url" or "file" when set, got ${describeFixtureValue(type)}; ` + + `omit it for a URL citation` + ); + } + if (typeof url !== "string" || url.length === 0) { + return `citations[${i}].url must be a non-empty string, got ${describeFixtureValue(url)}; set the cited source URL`; + } + if (title !== undefined && typeof title !== "string") { + return `citations[${i}].title must be a string when set, got ${describeFixtureValue(title)}`; + } + for (const [name, value] of [ + ["startIndex", startIndex], + ["endIndex", endIndex], + ] as const) { + if (value !== undefined && !Number.isInteger(value)) { + return ( + `citations[${i}].${name} must be an integer when set, got ${describeFixtureValue(value)}; ` + + `give a character offset into the text, or omit it` + ); + } + } + if (text === undefined) continue; + const start = (startIndex as number | undefined) ?? 0; + const end = (endIndex as number | undefined) ?? text.length; + if (start < 0 || start > end || end > text.length) { + return ( + `citations[${i}] offsets must satisfy 0 <= startIndex <= endIndex <= text length (${text.length}), ` + + `got startIndex ${start}, endIndex ${end}; fix the offsets, or omit them to cite the whole text` + ); + } + if (splitsSurrogatePair(text, start)) { + return surrogateSplitProblem(`citations[${i}].startIndex`, start); + } + if (splitsSurrogatePair(text, end)) { + return surrogateSplitProblem(`citations[${i}].endIndex`, end); + } + } + return undefined; +} + +/** What is wrong with the file citation `citations[i]`, or undefined when valid. */ +function fixtureFileCitationProblem( + citation: Record, + i: number, + text: string | undefined, +): string | undefined { + const { fileId, filename } = citation; + const index = citation.index ?? undefined; + if (typeof fileId !== "string" || fileId.length === 0) { + return ( + `citations[${i}].fileId must be a non-empty string, got ${describeFixtureValue(fileId)}; ` + + `set the cited file's ID, e.g. "file-abc123"` + ); + } + if (typeof filename !== "string" || filename.length === 0) { + return ( + `citations[${i}].filename must be a non-empty string, got ${describeFixtureValue(filename)}; ` + + `set the cited file's name` + ); + } + if (index !== undefined && !Number.isInteger(index)) { + return ( + `citations[${i}].index must be an integer when set, got ${describeFixtureValue(index)}; ` + + `give a character position in the text, or omit it` + ); + } + if (text === undefined) return undefined; + const at = (index as number | undefined) ?? text.length; + if (at < 0 || at > text.length) { + return ( + `citations[${i}].index must satisfy 0 <= index <= text length (${text.length}), ` + + `got ${at}; fix the index, or omit it to cite the end of the text` + ); + } + if (splitsSurrogatePair(text, at)) return surrogateSplitProblem(`citations[${i}].index`, at); + return undefined; +} + +/** + * Fixture `citations`, checked against the text they cite: the array itself, + * or empty when absent. Throws {@link InvalidFixtureOutputError} + * (`aimock_invalid_fixture_citation`) for every problem + * {@link fixtureCitationsProblem} reports: a malformed value, an offset + * outside the text or one that splits a surrogate pair, mixed URL and file + * citation fields, or citations on a turn with no text (`content` is empty: + * tool calls only, `blocks` with no text block, or empty content). Every + * provider that reads `citations` goes through this. + */ +export function fixtureCitations(content: string, citations: unknown): FixtureCitation[] { + const problem = fixtureCitationsProblem(citations, content); + if (problem !== undefined) { + throw new InvalidFixtureOutputError("aimock_invalid_fixture_citation", problem); + } + return (citations as FixtureCitation[] | undefined) ?? []; +} + +/** + * Fixture `files`, checked: the array itself, or empty when absent. Throws + * {@link InvalidFixtureOutputError} (`aimock_invalid_fixture_file`) for a + * malformed value. Every provider that reads `files` goes through this. + */ +export function fixtureFiles(files: unknown): FixtureGeneratedFile[] { + const problem = fixtureFilesProblem(files); + if (problem !== undefined) + throw new InvalidFixtureOutputError("aimock_invalid_fixture_file", problem); + return (files as FixtureGeneratedFile[] | undefined) ?? []; +} + +/** A fixture citation with its defaults applied. */ +export interface ResolvedCitation { + url: string; + /** The fixture title, or the URL when the fixture sets none. */ + title: string; + /** + * UTF-16 string indices into the joined text of all text blocks, the same + * on Chat Completions, Anthropic, Gemini and Responses; default to the + * whole text. + */ + startIndex: number; + endIndex: number; +} + +/** + * Fixture URL `citations` against the text they cite, with defaults applied: + * the title falls back to the URL and the offsets to the whole text. Empty + * when the fixture sets none. File citations (`type: "file"`) are skipped: + * only OpenAI Responses has a wire shape for them (`file_citation`), and it + * builds those itself. Chat Completions, Anthropic Messages and Gemini, which + * resolve through here, serve URL citations only. This reads `citations` + * through {@link fixtureCitations}, so it is the serve-time check for those + * three wires: every problem {@link fixtureCitationsProblem} reports against + * `content`, including one in a file citation it then skips, throws + * {@link InvalidFixtureOutputError} (`aimock_invalid_fixture_citation`). + * OpenAI Responses calls {@link fixtureCitations} directly. Other wires + * ignore `citations`. + */ +export function resolveCitations( + content: string, + citations?: ResponseOverrides["citations"], +): ResolvedCitation[] { + return fixtureCitations(content, citations) + .flatMap((c) => (c.type === "file" ? [] : [c])) + .map((c) => ({ + url: c.url, + title: c.title ?? c.url, + startIndex: c.startIndex ?? 0, + endIndex: c.endIndex ?? content.length, + })); +} + +/** + * A fixture error that a handler surfaces as a coded 500. Despite the name, + * it is not limited to tool calls: it also covers the fixture `files` and + * `citations` errors ({@link UnsupportedGeneratedFileError} and + * {@link InvalidFixtureOutputError}). + */ export type FixtureToolCallError = | InvalidToolArgumentsError | UnsupportedToolCallError - | InvalidFixtureToolCallError; + | InvalidFixtureToolCallError + | UnsupportedGeneratedFileError + | InvalidFixtureOutputError; -/** True for the fixture tool-call errors a handler surfaces as a coded 500. */ +/** True for the fixture errors a handler surfaces as a coded 500. */ export function isFixtureToolCallError(error: unknown): error is FixtureToolCallError { return ( error instanceof InvalidToolArgumentsError || error instanceof UnsupportedToolCallError || - error instanceof InvalidFixtureToolCallError + error instanceof InvalidFixtureToolCallError || + error instanceof UnsupportedGeneratedFileError || + error instanceof InvalidFixtureOutputError ); } @@ -338,7 +781,12 @@ export function fixtureToolCallErrorCode( ): | "aimock_unsupported_tool_call" | "aimock_invalid_tool_arguments" - | "aimock_invalid_fixture_tool_call" { + | "aimock_invalid_fixture_tool_call" + | "aimock_unsupported_generated_file" + | "aimock_invalid_fixture_citation" + | "aimock_invalid_fixture_file" { + if (error instanceof UnsupportedGeneratedFileError) return "aimock_unsupported_generated_file"; + if (error instanceof InvalidFixtureOutputError) return error.code; if (error instanceof UnsupportedToolCallError) return "aimock_unsupported_tool_call"; if (error instanceof InvalidFixtureToolCallError) return error.code; return "aimock_invalid_tool_arguments"; @@ -361,9 +809,10 @@ export function googleFixtureToolCallErrorDetails(code: string) { } /** - * Run a WebSocket preflight `check` that may raise a fixture tool-call error - * ({@link InvalidToolArgumentsError}, {@link UnsupportedToolCallError} or - * {@link InvalidFixtureToolCallError}). + * Run a WebSocket preflight `check` that may raise a fixture error + * ({@link InvalidToolArgumentsError}, {@link UnsupportedToolCallError}, + * {@link InvalidFixtureToolCallError}, {@link UnsupportedGeneratedFileError} + * or {@link InvalidFixtureOutputError}). * Such an error marks the already-journaled entry as a 500 carrying the * message, then propagates to the transport's error handler. */ diff --git a/src/index.ts b/src/index.ts index dba505a3..91f65e31 100644 --- a/src/index.ts +++ b/src/index.ts @@ -231,6 +231,8 @@ export { resolveResponsesBlocks, UnsupportedToolCallError, InvalidFixtureToolCallError, + UnsupportedGeneratedFileError, + InvalidFixtureOutputError, requireFunctionToolCalls, isErrorResponse, isEmbeddingResponse, @@ -536,6 +538,7 @@ export type { ChatCompletion, ChatCompletionChoice, ChatCompletionMessage, + ChatCompletionAnnotation, ImageItem, ImageResponse, AudioResponse, @@ -546,6 +549,8 @@ export type { RawJSONResponse, ResponseOverrides, FixtureCitation, + FixtureUrlCitation, + FixtureFileCitation, FixtureGeneratedFile, ContentWithToolCallsResponse, FixtureFileResponse, diff --git a/src/responses.ts b/src/responses.ts index c3c2e727..132fa1b0 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -1116,13 +1116,16 @@ interface MessageBlockResult { } function buildAnnotations(content: string, citations?: ResponseOverrides["citations"]): object[] { - return (citations ?? []).map((c) => ({ - type: "url_citation", - url: c.url, - title: c.title ?? c.url, - start_index: c.startIndex ?? 0, - end_index: c.endIndex ?? content.length, - })); + // Only URL citations have a wire shape here so far; file citations are skipped. + return (citations ?? []) + .flatMap((c) => (c.type === "file" ? [] : [c])) + .map((c) => ({ + type: "url_citation", + url: c.url, + title: c.title ?? c.url, + start_index: c.startIndex ?? 0, + end_index: c.endIndex ?? content.length, + })); } function buildMessageOutputEvents( diff --git a/src/types.ts b/src/types.ts index 85b3b4c0..e60cb4f8 100644 --- a/src/types.ts +++ b/src/types.ts @@ -263,16 +263,70 @@ export interface FixtureMatch { // Fixture response types -/** A source citation attached to a text response. Served on the OpenAI Responses API as `url_citation` annotations. */ -export interface FixtureCitation { +/** + * A web source citation attached to a text response. Served as `url_citation` + * annotations on OpenAI Responses (`output_text.annotations`) and Chat + * Completions (`message.annotations` / `delta.annotations`), as + * `web_search_result_location` citations on Anthropic text blocks, and as + * Gemini `groundingMetadata` (`groundingChunks` + `groundingSupports`). + */ +export interface FixtureUrlCitation { + /** + * Optional discriminator; a citation without `type` is a URL citation. A + * citation that also sets file citation fields (`fileId`, `filename`, + * `index`) is rejected: set `type: "file"` or remove the stray fields. + */ + type?: "url"; url: string; title?: string; - /** Character offsets into the text; default to the whole text. */ + /** + * Offsets of the cited span, as UTF-16 string indices (JavaScript string + * positions) into the joined text of all the response's text blocks. Chat + * Completions, Anthropic, Gemini and Responses read them the same way; + * Gemini converts them to the UTF-8 byte offsets its wire carries. Default + * to the whole text. + */ startIndex?: number; endIndex?: number; } -/** A model-generated file (e.g. an inline image). Served on Gemini as `inlineData` parts. */ +/** + * A citation of an uploaded file, served as a `file_citation` annotation on + * OpenAI Responses (`output_text.annotations`, streamed through + * `response.output_text.annotation.added`). Chat Completions, Anthropic + * Messages and Gemini have no file-citation wire shape, so they skip it and + * serve only the fixture's URL citations. + */ +export interface FixtureFileCitation { + type: "file"; + /** Wire `file_id`, e.g. `file-abc123`. */ + fileId: string; + /** Wire `filename` of the cited file. */ + filename: string; + /** + * Character position cited: a UTF-16 string index into the joined text of + * all the response's text blocks; defaults to the end of the text. The wire + * `index` is not this value: the annotation goes on the `output_text` whose + * text holds the position, and its `index` counts from the start of that + * `output_text`. + */ + index?: number; +} + +/** A source citation attached to a text response: a web page or an uploaded file. */ +export type FixtureCitation = FixtureUrlCitation | FixtureFileCitation; + +/** + * A model-generated file (e.g. an inline image). Served on Gemini as + * `inlineData` parts and on the OpenAI Responses API as + * `image_generation_call` output items (`image/*` files only; another + * `mimeType` is rejected). Chat Completions (OpenAI, Azure OpenAI, OpenRouter + * and BytePlus ModelArk) and Anthropic Messages cannot carry one and reject + * the request. Every rejection is a coded 500, + * `aimock_unsupported_generated_file`, sent before any content. Every other + * wire (Bedrock, Cohere, Ollama, Gemini Interactions and the rest) drops + * `files` without an error. + */ export interface FixtureGeneratedFile { mimeType: string; /** Base64-encoded file bytes. */ @@ -290,17 +344,30 @@ export interface FixtureGeneratedFile { * When total_tokens (or provider equivalent) is omitted, it is auto-computed * from the component fields. * - * Provider support: seven fields are common (id, created, model, usage, - * systemFingerprint, finishReason, role); provider and nativeFinishReason are - * OpenRouter-only and ignored elsewhere. OpenAI Chat honors the seven common - * fields; OpenRouter additionally honors provider/nativeFinishReason; Responses - * API honors five common fields (no role, systemFingerprint) plus optional phase; - * Claude (5: no created, systemFingerprint); Gemini (2: only finishReason, usage). + * Provider support: seven envelope fields are common (id, created, model, + * usage, systemFingerprint, finishReason, role); provider and + * nativeFinishReason are OpenRouter-only and ignored elsewhere; `citations` + * and `files` are content fields (see their docs). OpenAI Chat honors the + * seven common fields plus `citations` (it rejects `files`); OpenRouter + * additionally honors provider/nativeFinishReason; Responses API honors five + * common fields (no role, systemFingerprint) plus optional phase, `citations` + * and `files`; Claude honors five (no created, systemFingerprint) plus + * `citations` (it rejects `files`); Gemini honors two (only finishReason, + * usage) plus `citations` and `files`. */ export interface ResponseOverrides { - /** Model-generated files emitted before the response text and tool calls (Gemini only). */ + /** + * Model-generated files emitted before the response text and tool calls. + * Served on Gemini and the OpenAI Responses API (`image/*` only); Chat + * Completions and Anthropic Messages reject them; every other wire drops + * them without an error (see {@link FixtureGeneratedFile}). + */ files?: FixtureGeneratedFile[]; - /** Source citations for the response text (OpenAI Responses API only). */ + /** + * Source citations for the response text (OpenAI Responses, Chat Completions, Anthropic, Gemini). + * File citations are served on OpenAI Responses only; the other three skip them. + * Every other wire ignores `citations`. + */ citations?: FixtureCitation[]; id?: string; created?: number; @@ -1308,6 +1375,17 @@ export interface SSEDelta { reasoning_details?: ReasoningTextDetail[]; reasoning_content?: string; tool_calls?: SSEToolCallDelta[]; + /** + * Fixture URL `citations`, sent once on their own chunk just before the + * finish chunk: after the content chunks and after any tool-call chunks. + */ + annotations?: ChatCompletionAnnotation[]; +} + +/** A Chat Completions `url_citation` annotation (`message.annotations` / `delta.annotations`). */ +export interface ChatCompletionAnnotation { + type: "url_citation"; + url_citation: { url: string; title: string; start_index: number; end_index: number }; } export interface SSEToolCallDelta { @@ -1364,6 +1442,11 @@ export interface ChatCompletionMessage { */ reasoning?: string | null; tool_calls?: ToolCallMessage[]; + /** + * Fixture URL `citations`; omitted when the fixture sets none, or only file + * citations (Chat Completions has no `file_citation` annotation). + */ + annotations?: ChatCompletionAnnotation[]; } // Server options From 450f85f88654859caa2239804796206131145862 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 23 Oct 2026 15:12:50 -0700 Subject: [PATCH 2/6] feat: serve fixture citations on Chat Completions, Anthropic, Gemini and Responses Fixture citations index the joined text of all text blocks, in UTF-16 string offsets, the same way on every wire that reads them. - Chat Completions: url_citation annotations on the message, and one delta.annotations chunk before the finish chunk. Every builder cites the text it serves; refusal, content-filter and reasoning-only turns carry none. - Anthropic Messages: web_search_result_location citations on the owning text block, streamed as citations_delta events. Bedrock InvokeModel shares the builders and serves no citations. - Gemini: groundingMetadata with UTF-8 byte offsets into the cited part; a streamed chunk grounds its own text part, and text chunks never cut a surrogate pair. - Responses: url_citation and file_citation annotations on the owning output_text. A file citation on a block boundary stays at the end of the block before it. A citation on a turn with no text is rejected on every wire. --- .../chat-builder-citation-source-text.test.ts | 111 +++ src/__tests__/gemini.test.ts | 745 ++++++++++++++++++ src/__tests__/messages.test.ts | 258 ++++++ .../misbehavior-bedrock-invoke.test.ts | 54 ++ src/__tests__/misbehavior-openai.test.ts | 59 ++ src/__tests__/responses.test.ts | 367 +++++++++ src/__tests__/server.test.ts | 204 +++++ src/bedrock.ts | 2 + src/gemini.ts | 229 +++++- src/helpers.ts | 78 +- src/messages.ts | 119 ++- src/responses.ts | 152 +++- 12 files changed, 2313 insertions(+), 65 deletions(-) create mode 100644 src/__tests__/chat-builder-citation-source-text.test.ts diff --git a/src/__tests__/chat-builder-citation-source-text.test.ts b/src/__tests__/chat-builder-citation-source-text.test.ts new file mode 100644 index 00000000..9452e700 --- /dev/null +++ b/src/__tests__/chat-builder-citation-source-text.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, test } from "vitest"; +import { + buildContentWithToolCallsChunks, + buildContentWithToolCallsCompletion, + buildOpenAIContentFilterChunks, + buildOpenAIContentFilterCompletion, + buildOpenAIReasoningChunks, + buildOpenAIReasoningCompletion, + buildOpenAIRefusalChunks, + buildOpenAIRefusalCompletion, +} from "../helpers.js"; +import type { FixtureBlock, ResponseOverrides } from "../types.js"; + +/** + * Every exported Chat Completions builder validates and emits `citations` + * against the exact text it serves, and the fault builders drop `citations` + * in both the streaming and the non-streaming form. + */ + +const CITED_URL = "https://example.com/cited"; + +// Served block text: "Sunny skies today. More." (24 UTF-16 units). The +// citation covers " More." at 18..24, which is outside the 5-unit `content`. +const blocks: FixtureBlock[] = [ + { type: "text", text: "Sunny skies today." }, + { type: "toolCall", id: "call_weather", name: "weather", arguments: '{"city":"Paris"}' }, + { type: "text", text: " More." }, +]; +const staleContent = "Stale"; +const citedOverrides: ResponseOverrides = { + citations: [{ url: CITED_URL, title: "Forecast", startIndex: 18, endIndex: 24 }], +}; +const expectedAnnotations = [ + { + type: "url_citation", + url_citation: { url: CITED_URL, title: "Forecast", start_index: 18, end_index: 24 }, + }, +]; + +describe("content-plus-tool-calls builders cite the text they serve", () => { + test("streaming: blocks text, not the content argument, bounds the citations", () => { + const chunks = buildContentWithToolCallsChunks( + staleContent, + [], + "gpt-4o", + 4, + undefined, + citedOverrides, + blocks, + ); + const streamed = chunks + .map((c) => c.choices[0]?.delta.content) + .filter((s): s is string => typeof s === "string") + .join(""); + expect(streamed).toBe("Sunny skies today. More."); + const annotations = chunks.flatMap((c) => c.choices[0]?.delta.annotations ?? []); + expect(annotations).toEqual(expectedAnnotations); + }); + + test("non-streaming: the served content bounds the citations", () => { + const served = "Sunny skies today. More."; + const completion = buildContentWithToolCallsCompletion( + served, + [{ name: "weather", arguments: '{"city":"Paris"}' }], + "gpt-4o", + undefined, + citedOverrides, + ); + expect(completion.choices[0].message.content).toBe(served); + expect(completion.choices[0].message.annotations).toEqual(expectedAnnotations); + }); +}); + +describe("non-streaming fault builders drop citations like the streaming ones", () => { + // Valid for the fixture's own reply text, which the fault replaces. + const overrides: ResponseOverrides = { + citations: [{ url: CITED_URL, startIndex: 0, endIndex: 20 }], + }; + + test("refusal", () => { + expect(() => buildOpenAIRefusalChunks("No.", "gpt-4o", 4, overrides)).not.toThrow(); + const completion = buildOpenAIRefusalCompletion("No.", "gpt-4o", overrides); + expect(completion.choices[0].message).toEqual({ + role: "assistant", + content: null, + refusal: "No.", + }); + }); + + test("content filter", () => { + expect(() => buildOpenAIContentFilterChunks("gpt-4o", overrides)).not.toThrow(); + const completion = buildOpenAIContentFilterCompletion("gpt-4o", overrides); + expect(completion.choices[0].finish_reason).toBe("content_filter"); + expect(completion.choices[0].message).toEqual({ + role: "assistant", + content: null, + refusal: null, + }); + }); + + test("reasoning, hidden and exposed", () => { + for (const expose of [false, true]) { + expect(() => + buildOpenAIReasoningChunks("Think.", "gpt-4o", 4, expose, overrides), + ).not.toThrow(); + const completion = buildOpenAIReasoningCompletion("Think.", "gpt-4o", expose, overrides); + expect(completion.choices[0].finish_reason).toBe("length"); + expect(completion.choices[0].message.content).toBeNull(); + } + }); +}); diff --git a/src/__tests__/gemini.test.ts b/src/__tests__/gemini.test.ts index 4ace5ef9..c420a9fb 100644 --- a/src/__tests__/gemini.test.ts +++ b/src/__tests__/gemini.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, afterEach } from "vitest"; +import { GoogleGenAI } from "@google/genai"; import * as http from "node:http"; import type { Fixture } from "../types.js"; import { createServer, type ServerInstance } from "../server.js"; @@ -1449,3 +1450,747 @@ describe("Gemini model-generated files", () => { } }); }); + +describe("Gemini citations (groundingMetadata)", () => { + const citeFixture: Fixture = { + match: { userMessage: "cite-me" }, + response: { + // "é" is two UTF-8 bytes, so byte offsets differ from character offsets. + content: "Café Mastra is a framework.", + citations: [ + { url: "https://mastra.ai", title: "Mastra", startIndex: 5, endIndex: 11 }, + { url: "https://mastra.ai/docs" }, + ], + }, + }; + const expected = { + groundingChunks: [ + { web: { uri: "https://mastra.ai", title: "Mastra" } }, + { web: { uri: "https://mastra.ai/docs", title: "https://mastra.ai/docs" } }, + ], + groundingSupports: [ + { + segment: { partIndex: 0, startIndex: 6, endIndex: 12, text: "Mastra" }, + groundingChunkIndices: [0], + }, + { + segment: { partIndex: 0, startIndex: 0, endIndex: 28, text: "Café Mastra is a framework." }, + groundingChunkIndices: [1], + }, + ], + }; + type GroundedChunk = { + candidates: { + content?: { parts: { text?: string }[] }; + finishReason?: string; + groundingMetadata?: unknown; + }[]; + }; + + it("non-streaming candidate carries groundingMetadata with byte offsets", async () => { + instance = await createServer([citeFixture]); + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:generateContent`, { + contents: [{ role: "user", parts: [{ text: "cite-me" }] }], + }); + expect(JSON.parse(res.body).candidates[0].groundingMetadata).toEqual(expected); + }); + + it("streaming grounds each chunk with byte offsets into its own text part", async () => { + instance = await createServer([citeFixture]); + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:streamGenerateContent`, { + contents: [{ role: "user", parts: [{ text: "cite-me" }] }], + }); + const chunks = parseGeminiSSEChunks(res.body) as GroundedChunk[]; + // The default chunkSize (20) streams "Café Mastra is a fra" then "mework.". + expect(chunks.map((c) => c.candidates[0].content?.parts[0].text)).toEqual([ + "Café Mastra is a fra", + "mework.", + ]); + expect(chunks[1].candidates[0].finishReason).toBe("STOP"); + const { groundingChunks } = expected; + expect(chunks.map((c) => c.candidates[0].groundingMetadata)).toEqual([ + { + groundingChunks, + groundingSupports: [ + { + segment: { partIndex: 0, startIndex: 6, endIndex: 12, text: "Mastra" }, + groundingChunkIndices: [0], + }, + { + segment: { partIndex: 0, startIndex: 0, endIndex: 21, text: "Café Mastra is a fra" }, + groundingChunkIndices: [1], + }, + ], + }, + { + groundingChunks, + groundingSupports: [ + { + segment: { partIndex: 0, startIndex: 0, endIndex: 7, text: "mework." }, + groundingChunkIndices: [1], + }, + ], + }, + ]); + }); + + it.each([ + [ + "legacy", + 0, + { + content: "Checking.", + toolCalls: [{ name: "lookup", arguments: "{}" }], + citations: [{ url: "https://example.com" }], + }, + "FUNCTION_CALL", + ], + [ + "blocks", + 1, + { + blocks: [ + { type: "toolCall" as const, name: "lookup", arguments: "{}" }, + { type: "text" as const, text: "Checking." }, + ], + citations: [{ url: "https://example.com" }], + }, + "FUNCTION_CALL", + ], + ])( + "content+toolCalls (%s) carries groundingMetadata on both modes", + async (_name, textPart, response, finishReason) => { + instance = await createServer([{ match: { userMessage: "cite-tools" }, response }]); + const want = (partIndex: number) => ({ + groundingChunks: [{ web: { uri: "https://example.com", title: "https://example.com" } }], + groundingSupports: [ + { + segment: { partIndex, startIndex: 0, endIndex: 9, text: "Checking." }, + groundingChunkIndices: [0], + }, + ], + }); + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:generateContent`, { + contents: [{ role: "user", parts: [{ text: "cite-tools" }] }], + }); + expect(JSON.parse(res.body).candidates[0].groundingMetadata).toEqual(want(textPart)); + expect(JSON.parse(res.body).candidates[0].finishReason).toBe(finishReason); + const streamed = await post( + `${instance.url}/v1beta/models/gemini-2.0-flash:streamGenerateContent`, + { contents: [{ role: "user", parts: [{ text: "cite-tools" }] }] }, + ); + const chunks = parseGeminiSSEChunks(streamed.body) as GroundedChunk[]; + // The stream finishes with the fixture's finish reason. The grounded + // chunk is the text chunk, which is not always the finishing chunk. + expect(chunks[chunks.length - 1].candidates[0].finishReason).toBe(finishReason); + const grounded = chunks.filter((c) => c.candidates[0].groundingMetadata !== undefined); + expect(grounded).toHaveLength(1); + // The grounded chunk carries the text the offsets refer to, as part 0. + expect(grounded[0].candidates[0].content?.parts[0].text).toBe("Checking."); + expect(grounded[0].candidates[0].groundingMetadata).toEqual(want(0)); + }, + ); + + it("skips file citations, which grounding metadata has no form for (both modes)", async () => { + instance = await createServer([ + { + match: { userMessage: "cite-file" }, + response: { + content: "Hello.", + citations: [ + { type: "file", fileId: "file-abc123", filename: "plan.pdf" }, + { url: "https://example.com" }, + ], + }, + }, + { + match: { userMessage: "file-only" }, + response: { + content: "Hello.", + citations: [{ type: "file", fileId: "file-abc123", filename: "plan.pdf" }], + }, + }, + ]); + const want = { + groundingChunks: [{ web: { uri: "https://example.com", title: "https://example.com" } }], + groundingSupports: [ + { + segment: { partIndex: 0, startIndex: 0, endIndex: 6, text: "Hello." }, + groundingChunkIndices: [0], + }, + ], + }; + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:generateContent`, { + contents: [{ role: "user", parts: [{ text: "cite-file" }] }], + }); + expect(JSON.parse(res.body).candidates[0].groundingMetadata).toEqual(want); + const streamed = await post( + `${instance.url}/v1beta/models/gemini-2.0-flash:streamGenerateContent`, + { contents: [{ role: "user", parts: [{ text: "cite-file" }] }] }, + ); + const grounded = (parseGeminiSSEChunks(streamed.body) as GroundedChunk[]).filter( + (c) => c.candidates[0].groundingMetadata !== undefined, + ); + expect(grounded.map((c) => c.candidates[0].groundingMetadata)).toEqual([want]); + + for (const path of ["generateContent", "streamGenerateContent"]) { + const fileOnly = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:${path}`, { + contents: [{ role: "user", parts: [{ text: "file-only" }] }], + }); + expect(fileOnly.body).not.toContain("groundingMetadata"); + } + }); + + it.each([ + [ + "generateContent", + '{"candidates":[{"content":{"role":"model","parts":[{"text":"hi"}]},"finishReason":"STOP","index":0}],"usageMetadata":{"promptTokenCount":0,"candidatesTokenCount":0,"totalTokenCount":0}}', + ], + [ + "streamGenerateContent", + 'data: {"candidates":[{"content":{"role":"model","parts":[{"text":"hi"}]},"index":0,"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":0,"candidatesTokenCount":0,"totalTokenCount":0}}\n\n', + ], + ])( + "omitting citations leaves :%s byte-identical to the no-citations baseline", + async (method, baseline) => { + instance = await createServer([ + { match: { userMessage: "plain" }, response: { content: "hi" } }, + ]); + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:${method}`, { + contents: [{ role: "user", parts: [{ text: "plain" }] }], + }); + expect(res.body).toBe(baseline); + }, + ); +}); + +// `groundingSupports[].segment.partIndex` must name the text part the byte +// offsets refer to, within the parts of the candidate that carries the +// grounding. Leading thought and inlineData parts shift that index, and a +// trailing functionCall part must never be the one it names. +describe("Gemini citations — segment.partIndex", () => { + type Part = { text?: string; thought?: boolean; inlineData?: unknown; functionCall?: unknown }; + type Support = { + segment: { partIndex?: number; startIndex: number; endIndex: number; text: string }; + groundingChunkIndices: number[]; + }; + type Candidate = { + content?: { parts: Part[] }; + finishReason?: string; + groundingMetadata?: { groundingSupports: Support[] }; + }; + type Body = { candidates: Candidate[] }; + + const png = "iVBORw0KGgo="; + const citations = [ + { url: "https://mastra.ai", title: "Mastra", startIndex: 5, endIndex: 11 }, + { url: "https://mastra.ai/docs" }, + ]; + // Byte offsets of "Mastra" (after the two-byte "é") and of the whole text. + const segments = [ + { startIndex: 6, endIndex: 12, text: "Mastra" }, + { startIndex: 0, endIndex: 28, text: "Café Mastra is a framework." }, + ]; + const content = "Café Mastra is a framework."; + + async function nonStreaming(userMessage: string): Promise { + const res = await post(`${instance!.url}/v1beta/models/gemini-2.5-flash:generateContent`, { + contents: [{ role: "user", parts: [{ text: userMessage }] }], + }); + expect(res.status).toBe(200); + return (JSON.parse(res.body) as Body).candidates[0]; + } + + // The streamed candidates carrying groundingMetadata, and the streamed + // (non-thought) text. + async function streaming(userMessage: string): Promise<{ grounded: Candidate[]; text: string }> { + const res = await post( + `${instance!.url}/v1beta/models/gemini-2.5-flash:streamGenerateContent`, + { contents: [{ role: "user", parts: [{ text: userMessage }] }] }, + ); + expect(res.status).toBe(200); + const chunks = parseGeminiSSEChunks(res.body) as Body[]; + const grounded = chunks + .map((c) => c.candidates[0]) + .filter((c) => c.groundingMetadata !== undefined); + const text = chunks + .flatMap((c) => c.candidates[0].content?.parts ?? []) + .filter((p) => p.text !== undefined && !p.thought) + .map((p) => p.text) + .join(""); + return { grounded, text }; + } + + // Every streamed support names part 0 of its chunk, and its byte offsets + // slice that part's text to `segment.text`. Returns the cited text per + // grounding chunk index, joined across chunks in stream order. + function streamedCitedText(grounded: Candidate[]): string[] { + const cited: string[] = []; + for (const candidate of grounded) { + for (const { segment, part } of citedParts(candidate)) { + expect(segment.partIndex).toBe(0); + expect(bytes(part.text!, segment.startIndex, segment.endIndex)).toBe(segment.text); + } + for (const s of candidate.groundingMetadata!.groundingSupports) { + for (const i of s.groundingChunkIndices) cited[i] = (cited[i] ?? "") + s.segment.text; + } + } + return cited; + } + + // Every support names a non-thought text part of `candidate`; returns the + // segments with that part's text for offset checks. + function citedParts(candidate: Candidate): { segment: Support["segment"]; part: Part }[] { + const parts = candidate.content?.parts ?? []; + return (candidate.groundingMetadata?.groundingSupports ?? []).map((s) => { + expect(s.segment.partIndex).toBeTypeOf("number"); + const part = parts[s.segment.partIndex ?? 0]; + expect(part?.text).toBeTypeOf("string"); + expect(part?.thought).toBeUndefined(); + return { segment: s.segment, part }; + }); + } + + function bytes(text: string, start: number, end: number): string { + return Buffer.from(text, "utf8").subarray(start, end).toString("utf8"); + } + + it.each([ + ["reasoning", { content, reasoning: "Thinking it through.", citations }, 1], + ["files", { content, files: [{ mimeType: "image/png", data: png }], citations }, 1], + [ + "reasoning + files", + { + content, + reasoning: "Thinking it through.", + files: [{ mimeType: "image/png", data: png }], + citations, + }, + 2, + ], + ])("%s: non-streaming partIndex names the text part", async (_n, response, want) => { + instance = await createServer([{ match: { userMessage: "pi-text" }, response }]); + const candidate = await nonStreaming("pi-text"); + const cited = citedParts(candidate); + expect(cited.map((c) => c.segment)).toEqual(segments.map((s) => ({ ...s, partIndex: want }))); + for (const { segment, part } of cited) { + expect(bytes(part.text!, segment.startIndex, segment.endIndex)).toBe(segment.text); + } + }); + + it.each([ + ["reasoning", { content, reasoning: "Thinking it through.", citations }], + ["files", { content, files: [{ mimeType: "image/png", data: png }], citations }], + ])("%s: streaming partIndex names the text part of each grounded chunk", async (_n, response) => { + instance = await createServer([{ match: { userMessage: "pi-text" }, response }]); + const { grounded, text } = await streaming("pi-text"); + expect(grounded[grounded.length - 1].finishReason).toBe("STOP"); + expect(text).toBe(content); + expect(streamedCitedText(grounded)).toEqual(segments.map((s) => s.text)); + }); + + const toolLast = { + legacy: { + content, + toolCalls: [{ name: "lookup", arguments: "{}" }], + files: [{ mimeType: "image/png", data: png }], + reasoning: "Thinking it through.", + citations, + }, + blocks: { + blocks: [ + { type: "text" as const, text: content }, + { type: "toolCall" as const, name: "lookup", arguments: "{}" }, + ], + files: [{ mimeType: "image/png", data: png }], + reasoning: "Thinking it through.", + citations, + }, + }; + + it.each(Object.entries(toolLast))( + "content + trailing toolCall (%s): non-streaming partIndex skips thought/files and names the text", + async (_n, response) => { + instance = await createServer([{ match: { userMessage: "pi-tool" }, response }]); + const candidate = await nonStreaming("pi-tool"); + expect(candidate.content?.parts.map((p) => Object.keys(p).sort().join(","))).toEqual([ + "text,thought", + "inlineData", + "text", + "functionCall", + ]); + const cited = citedParts(candidate); + expect(cited.map((c) => c.segment)).toEqual(segments.map((s) => ({ ...s, partIndex: 2 }))); + for (const { segment, part } of cited) { + expect(bytes(part.text!, segment.startIndex, segment.endIndex)).toBe(segment.text); + } + }, + ); + + it.each(Object.entries(toolLast))( + "content + trailing toolCall (%s): streaming grounds the text chunk, not the functionCall chunk", + async (_n, response) => { + instance = await createServer([{ match: { userMessage: "pi-tool" }, response }]); + const { grounded, text } = await streaming("pi-tool"); + for (const candidate of grounded) { + expect(candidate.content?.parts.some((p) => p.functionCall !== undefined)).toBe(false); + } + expect(text).toBe(content); + expect(streamedCitedText(grounded)).toEqual(segments.map((s) => s.text)); + }, + ); + + it.each(Object.entries(toolLast))( + "content + trailing toolCall (%s): @google/genai parses partIndex onto the cited text part", + async (_n, response) => { + instance = await createServer([{ match: { userMessage: "pi-tool" }, response }]); + // Loopback only: the SDK talks to the local aimock instance. + const client = new GoogleGenAI({ + apiKey: "local", + httpOptions: { baseUrl: instance.url, timeout: 5000, retryOptions: { attempts: 1 } }, + }); + const request = { model: "gemini-2.5-flash", contents: "pi-tool" }; + const whole = await client.models.generateContent(request); + const candidate = whole.candidates![0]; + const support = candidate.groundingMetadata!.groundingSupports![0]; + expect(support.segment).toEqual({ ...segments[0], partIndex: 2 }); + expect(candidate.content!.parts![support.segment!.partIndex!].text).toBe(content); + const grounded = []; + for await (const chunk of await client.models.generateContentStream(request)) { + if (chunk.candidates?.[0].groundingMetadata) grounded.push(chunk.candidates[0]); + } + // The default chunkSize streams "Café Mastra is a fra" then "mework."; + // "Mastra" lies in the first chunk, at the same byte offsets. + expect(grounded).toHaveLength(2); + const streamed = grounded[0].groundingMetadata!.groundingSupports![0].segment!; + expect(streamed).toEqual({ ...segments[0], partIndex: 0 }); + // Every streamed segment slices its own chunk's part, as a client reads it. + for (const chunk of grounded) { + for (const { segment } of chunk.groundingMetadata!.groundingSupports!) { + const part = chunk.content!.parts![segment!.partIndex!]; + expect(part.functionCall).toBeUndefined(); + expect(bytes(part.text!, segment!.startIndex!, segment!.endIndex!)).toBe(segment!.text); + } + } + }, + ); + + it("blocks with two text parts: non-streaming names the part holding the citation", async () => { + instance = await createServer([ + { + match: { userMessage: "pi-two" }, + response: { + blocks: [ + { type: "text", text: "Hi. " }, + { type: "toolCall", name: "lookup", arguments: "{}" }, + { type: "text", text: content }, + ], + // "Mastra" in the concatenated text "Hi. Café Mastra is a framework." + citations: [{ url: "https://mastra.ai", title: "Mastra", startIndex: 9, endIndex: 15 }], + }, + }, + ]); + const candidate = await nonStreaming("pi-two"); + const cited = citedParts(candidate); + expect(cited.map((c) => c.segment)).toEqual([{ ...segments[0], partIndex: 2 }]); + expect(bytes(cited[0].part.text!, 6, 12)).toBe("Mastra"); + }); + + it("blocks with two text parts: streaming grounds every text chunk, through the last text block", async () => { + instance = await createServer([ + { + match: { userMessage: "pi-two-stream" }, + response: { + blocks: [ + { type: "text", text: "Hi. " }, + { type: "toolCall", name: "lookup", arguments: "{}" }, + { type: "text", text: content }, + ], + citations: [{ url: "https://mastra.ai" }], + }, + }, + ]); + const res = await post(`${instance.url}/v1beta/models/gemini-2.5-flash:streamGenerateContent`, { + contents: [{ role: "user", parts: [{ text: "pi-two-stream" }] }], + }); + expect(res.status).toBe(200); + const candidates = (parseGeminiSSEChunks(res.body) as Body[]).map((c) => c.candidates[0]); + // Per-chunk grounding: the offset-less citation covers the joined text, so + // every chunk carrying text is grounded within its own part, and the + // functionCall chunk is not. + const groundedAt = candidates.flatMap((c, i) => (c.groundingMetadata ? [i] : [])); + const textAt = candidates.flatMap((c, i) => + c.content?.parts.some((p) => typeof p.text === "string" && p.text !== "" && !p.thought) + ? [i] + : [], + ); + expect(groundedAt).toEqual(textAt); + const callAt = candidates.findIndex((c) => + c.content?.parts.some((p) => p.functionCall !== undefined), + ); + expect(callAt).toBeGreaterThanOrEqual(0); + expect(groundedAt).not.toContain(callAt); + // The last grounded chunk comes after the functionCall, and ends the last text block. + const last = groundedAt[groundedAt.length - 1]; + expect(last).toBeGreaterThan(callAt); + expect(content.endsWith(candidates[last].content!.parts[0].text!)).toBe(true); + expect(streamedCitedText(candidates.filter((c) => c.groundingMetadata))).toEqual([ + "Hi. " + content, + ]); + }); + + it.each([ + ["text", { content: "", citations: [{ url: "https://example.com" }] }], + [ + "legacy content + toolCall", + { + content: "", + toolCalls: [{ name: "lookup", arguments: "{}" }], + citations: [{ url: "https://example.com" }], + }, + ], + [ + "blocks empty text + toolCall", + { + blocks: [ + { type: "text" as const, text: "" }, + { type: "toolCall" as const, name: "lookup", arguments: "{}" }, + ], + citations: [{ url: "https://example.com" }], + }, + ], + ])("empty text (%s): both modes answer the no-text citation 500", async (_n, response) => { + // Citations on a turn with no text have nothing to cite: a fixture error. + instance = await createServer([{ match: { userMessage: "pi-empty" }, response }]); + for (const path of [ + "/v1beta/models/gemini-2.5-flash:generateContent", + "/v1beta/models/gemini-2.5-flash:streamGenerateContent", + ]) { + const res = await post(`${instance!.url}${path}`, { + contents: [{ role: "user", parts: [{ text: "pi-empty" }] }], + }); + expect(res.status, res.body).toBe(500); + const body = JSON.parse(res.body) as { + error: { message: string; details?: { metadata?: { code?: string } }[] }; + }; + expect(body.error.details?.[0]?.metadata?.code).toBe("aimock_invalid_fixture_citation"); + expect(body.error.message).toContain("add text to the response or remove the citations"); + } + }); +}); + +// A client reads each support the way the @google/genai `Segment` type +// documents it: UTF-8 byte offsets into `parts[partIndex]` of the Content that +// carries the grounding. In every candidate (non-streaming) or streamed chunk +// carrying groundingMetadata, that slice must equal `segment.text`, and the +// slices for one grounding source, joined in order, must equal the cited text. +describe("Gemini citations — client-side segment slicing", () => { + type Part = { text?: string; thought?: boolean }; + type Candidate = { + content?: { parts: Part[] }; + groundingMetadata?: { + groundingChunks: { web: { uri: string } }[]; + groundingSupports: { + segment: { partIndex: number; startIndex: number; endIndex: number; text: string }; + groundingChunkIndices: number[]; + }[]; + }; + }; + + // The cited text per grounding source URI, as a client rebuilds it from the + // candidates that carry groundingMetadata. + function clientCitedText(candidates: Candidate[]): Record { + const cited: Record = {}; + for (const candidate of candidates) { + const meta = candidate.groundingMetadata; + if (!meta) continue; + for (const support of meta.groundingSupports) { + const part = candidate.content?.parts[support.segment.partIndex]; + expect(part?.text).toBeTypeOf("string"); + expect(part?.thought).toBeUndefined(); + const sliced = Buffer.from(part?.text ?? "", "utf8") + .subarray(support.segment.startIndex, support.segment.endIndex) + .toString("utf8"); + expect(sliced).toBe(support.segment.text); + for (const idx of support.groundingChunkIndices) { + const uri = meta.groundingChunks[idx].web.uri; + cited[uri] = (cited[uri] ?? "") + sliced; + } + } + } + return cited; + } + + const textContent = "Café Mastra is a framework."; + const blockA = "Hi é. "; + const fullBlocks = blockA + textContent; + const cases: [string, Fixture["response"], number | undefined, Record][] = [ + [ + "single text, default chunkSize", + { + content: textContent, + citations: [ + { url: "https://mastra.ai", title: "Mastra", startIndex: 5, endIndex: 11 }, + { url: "https://mastra.ai/docs" }, + ], + }, + undefined, + { "https://mastra.ai": "Mastra", "https://mastra.ai/docs": textContent }, + ], + [ + "single text, chunkSize 4", + { + content: textContent, + citations: [ + { url: "https://mastra.ai", title: "Mastra", startIndex: 2, endIndex: 11 }, + { url: "https://mastra.ai/docs" }, + ], + }, + 4, + { "https://mastra.ai": "fé Mastra", "https://mastra.ai/docs": textContent }, + ], + [ + "multi-block, a citation spanning blocks", + { + blocks: [ + { type: "text", text: blockA }, + { type: "toolCall", name: "lookup", arguments: "{}" }, + { type: "text", text: textContent }, + ], + citations: [ + { url: "https://span.example", startIndex: 3, endIndex: 15 }, + { url: "https://mastra.ai", startIndex: 11, endIndex: 17 }, + ], + }, + 5, + { + "https://span.example": fullBlocks.slice(3, 15), + "https://mastra.ai": fullBlocks.slice(11, 17), + }, + ], + ]; + + async function serve(response: Fixture["response"], chunkSize: number | undefined) { + instance = await createServer([ + { match: { userMessage: "slice" }, response, ...(chunkSize ? { chunkSize } : {}) }, + ]); + } + + it.each(cases)( + "%s: non-streaming slices equal the cited text", + async (_n, response, cs, want) => { + await serve(response, cs); + const res = await post(`${instance!.url}/v1beta/models/gemini-2.5-flash:generateContent`, { + contents: [{ role: "user", parts: [{ text: "slice" }] }], + }); + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as { candidates: Candidate[] }; + expect(clientCitedText(body.candidates)).toEqual(want); + }, + ); + + it.each(cases)("%s: streaming slices equal the cited text", async (_n, response, cs, want) => { + await serve(response, cs); + const res = await post( + `${instance!.url}/v1beta/models/gemini-2.5-flash:streamGenerateContent`, + { + contents: [{ role: "user", parts: [{ text: "slice" }] }], + }, + ); + expect(res.status).toBe(200); + const chunks = parseGeminiSSEChunks(res.body) as { candidates: Candidate[] }[]; + expect(clientCitedText(chunks.map((c) => c.candidates[0]))).toEqual(want); + }); +}); + +// A chunk boundary that falls inside an emoji must not split its surrogate +// pair: each streamed part must be valid UTF-16, and the byte offsets of the +// citations after the emoji must slice the part to the cited text. +describe("Gemini citations — emoji at a stream chunk boundary", () => { + type Part = { text?: string }; + type Candidate = { + content?: { parts: Part[] }; + groundingMetadata?: { + groundingChunks: { web: { uri: string } }[]; + groundingSupports: { + segment: { partIndex: number; startIndex: number; endIndex: number; text: string }; + groundingChunkIndices: number[]; + }[]; + }; + }; + const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?][] = [ + [ + "text", + { content: text, citations: cites }, + { "https://mastra.ai": "Mastra", "https://mastra.ai/docs": text }, + ], + [ + "content with tool calls", + { + content: text, + toolCalls: [{ name: "lookup", arguments: "{}" }], + citations: cites, + }, + { "https://mastra.ai": "Mastra", "https://mastra.ai/docs": text }, + ], + [ + "blocks", + { + blocks: [ + { type: "toolCall", name: "lookup", arguments: "{}" }, + { type: "text", text }, + ], + citations: cites, + }, + { "https://mastra.ai": "Mastra", "https://mastra.ai/docs": text }, + ], + ]; + + it.each(cases)( + "%s: parts stay valid UTF-16 and offsets slice the cited text", + async (_n, response, want) => { + instance = await createServer([{ match: { userMessage: "emoji" }, response, chunkSize: 4 }]); + const res = await post( + `${instance!.url}/v1beta/models/gemini-2.5-flash:streamGenerateContent`, + { contents: [{ role: "user", parts: [{ text: "emoji" }] }] }, + ); + expect(res.status).toBe(200); + const chunks = parseGeminiSSEChunks(res.body) as { candidates: Candidate[] }[]; + const candidates = chunks.map((c) => c.candidates[0]); + let joined = ""; + for (const candidate of candidates) { + for (const part of candidate.content?.parts ?? []) { + if (typeof part.text !== "string") continue; + expect(part.text).not.toMatch(LONE_SURROGATE); + joined += part.text; + } + } + expect(joined).toBe(text); + const cited: Record = {}; + for (const candidate of candidates) { + const meta = candidate.groundingMetadata; + if (!meta) continue; + for (const support of meta.groundingSupports) { + const part = candidate.content!.parts[support.segment.partIndex]; + const sliced = Buffer.from(part.text ?? "", "utf8") + .subarray(support.segment.startIndex, support.segment.endIndex) + .toString("utf8"); + expect(sliced).toBe(support.segment.text); + for (const idx of support.groundingChunkIndices) { + const uri = meta.groundingChunks[idx].web.uri; + cited[uri] = (cited[uri] ?? "") + sliced; + } + } + } + expect(cited).toEqual(want); + }, + ); +}); diff --git a/src/__tests__/messages.test.ts b/src/__tests__/messages.test.ts index 25c76fd1..cbefbe8c 100644 --- a/src/__tests__/messages.test.ts +++ b/src/__tests__/messages.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, afterEach } from "vitest"; import * as http from "node:http"; +import Anthropic from "@anthropic-ai/sdk"; import { PassThrough } from "node:stream"; import type { Fixture } from "../types.js"; import { createServer, type ServerInstance } from "../server.js"; @@ -1760,3 +1761,260 @@ describe("handleMessages (direct call — ?? fallback branches)", () => { expect(entry!.response.status).toBe(503); }); }); + +describe("POST /v1/messages (citations and files)", () => { + const citeFixture: Fixture = { + match: { userMessage: "cite-me" }, + response: { + content: "Mastra is a TypeScript framework.", + citations: [ + { url: "https://mastra.ai", title: "Mastra" }, + { url: "https://mastra.ai/docs", startIndex: 0, endIndex: 6 }, + ], + }, + }; + const messagesBody = (msg: string, stream: boolean) => ({ + model: "claude-3-5-sonnet-20241022", + max_tokens: 100, + stream, + messages: [{ role: "user", content: msg }], + }); + const citation = (cited: string, url: string, title: string) => ({ + type: "web_search_result_location", + cited_text: cited, + url, + title, + encrypted_index: expect.any(String), + }); + + it("non-streaming text block carries web_search_result_location citations", async () => { + instance = await createServer([citeFixture]); + const res = await post(`${instance.url}/v1/messages`, messagesBody("cite-me", false)); + expect(res.status).toBe(200); + expect(JSON.parse(res.body).content[0].citations).toEqual([ + citation("Mastra is a TypeScript framework.", "https://mastra.ai", "Mastra"), + citation("Mastra", "https://mastra.ai/docs", "https://mastra.ai/docs"), + ]); + }); + + it("streaming emits citations_delta after the text deltas, before content_block_stop", async () => { + instance = await createServer([citeFixture]); + const res = await post(`${instance.url}/v1/messages`, messagesBody("cite-me", true)); + const events = parseClaudeSSEEvents(res.body); + const kinds = events + .filter((e) => e.type === "content_block_delta" || e.type === "content_block_stop") + .map((e) => (e.type === "content_block_stop" ? "stop" : (e.delta as { type: string }).type)); + const firstCitation = kinds.indexOf("citations_delta"); + expect(firstCitation).toBeGreaterThan(0); + expect(kinds.slice(0, firstCitation).every((k) => k === "text_delta")).toBe(true); + expect(kinds.slice(firstCitation)).toEqual(["citations_delta", "citations_delta", "stop"]); + const citations = events + .filter((e) => (e.delta as { type?: string } | undefined)?.type === "citations_delta") + .map((e) => (e.delta as { citation: unknown }).citation); + expect(citations).toEqual([ + citation("Mastra is a TypeScript framework.", "https://mastra.ai", "Mastra"), + citation("Mastra", "https://mastra.ai/docs", "https://mastra.ai/docs"), + ]); + }); + + it("content+toolCalls turns carry citations on the text block", async () => { + instance = await createServer([ + { + match: { userMessage: "cite-tools" }, + response: { + content: "Checking.", + toolCalls: [{ name: "lookup", arguments: "{}" }], + citations: [{ url: "https://example.com" }], + }, + }, + ]); + const res = await post(`${instance.url}/v1/messages`, messagesBody("cite-tools", false)); + const content = JSON.parse(res.body).content; + expect(content[0].citations).toEqual([ + citation("Checking.", "https://example.com", "https://example.com"), + ]); + expect(content[1].type).toBe("tool_use"); + const streamed = await post(`${instance.url}/v1/messages`, messagesBody("cite-tools", true)); + const deltas = parseClaudeSSEEvents(streamed.body).filter( + (e) => (e.delta as { type?: string } | undefined)?.type === "citations_delta", + ); + expect(deltas).toHaveLength(1); + expect(deltas[0].index).toBe(0); + }); + + // `blocks` citation offsets index the joined text of all text blocks (the + // Chat Completions / Gemini convention): "Alpha one. " + "Beta two.". + const blocksCiteFixture = (reasoning?: string): Fixture => ({ + match: { userMessage: "cite-blocks" }, + response: { + content: "", + ...(reasoning !== undefined ? { reasoning } : {}), + blocks: [ + { type: "text", text: "Alpha one. " }, + { type: "toolCall", name: "lookup", arguments: "{}" }, + { type: "text", text: "Beta two." }, + ], + citations: [ + { url: "https://alpha.example", title: "Alpha", startIndex: 0, endIndex: 5 }, + { url: "https://beta.example", title: "Beta", startIndex: 11, endIndex: 15 }, + ], + }, + }); + const alphaCite = citation("Alpha", "https://alpha.example", "Alpha"); + const betaCite = citation("Beta", "https://beta.example", "Beta"); + const citationDeltas = (body: string) => + parseClaudeSSEEvents(body) + .filter((e) => (e.delta as { type?: string } | undefined)?.type === "citations_delta") + .map((e) => ({ index: e.index, citation: (e.delta as { citation: unknown }).citation })); + + it("non-streaming blocks attach each citation only to the text block it starts in", async () => { + instance = await createServer([blocksCiteFixture()]); + const res = await post(`${instance.url}/v1/messages`, messagesBody("cite-blocks", false)); + const content = JSON.parse(res.body).content; + expect(content.map((b: { type: string }) => b.type)).toEqual(["text", "tool_use", "text"]); + expect(content[0].citations).toEqual([alphaCite]); + expect(content[1].citations).toBeUndefined(); + expect(content[2].citations).toEqual([betaCite]); + }); + + it("streaming blocks emit each citations_delta once, on the owning text block", async () => { + instance = await createServer([blocksCiteFixture()]); + const res = await post(`${instance.url}/v1/messages`, messagesBody("cite-blocks", true)); + expect(citationDeltas(res.body)).toEqual([ + { index: 0, citation: alphaCite }, + { index: 2, citation: betaCite }, + ]); + }); + + it("thinking-first blocks: citations_delta indexes shift with the thinking block", async () => { + instance = await createServer([blocksCiteFixture("Let me think.")]); + const res = await post(`${instance.url}/v1/messages`, messagesBody("cite-blocks", true)); + expect(citationDeltas(res.body)).toEqual([ + { index: 1, citation: alphaCite }, + { index: 3, citation: betaCite }, + ]); + const nonStream = await post(`${instance.url}/v1/messages`, messagesBody("cite-blocks", false)); + const content = JSON.parse(nonStream.body).content; + expect(content[0].type).toBe("thinking"); + expect(content[1].citations).toEqual([alphaCite]); + expect(content[3].citations).toEqual([betaCite]); + }); + + it("a citation spanning two text blocks goes to the block it starts in, cut at its end", async () => { + // [6, 15) starts in "Alpha one. " (0-11) and ends in "Beta two." (11-20): + // the starting block owns it, and cited_text stops at that block's end. + instance = await createServer([ + { + match: { userMessage: "cite-span" }, + response: { + content: "", + blocks: [ + { type: "text", text: "Alpha one. " }, + { type: "text", text: "Beta two." }, + ], + citations: [{ url: "https://span.example", title: "Span", startIndex: 6, endIndex: 15 }], + }, + }, + ]); + const spanCite = citation("one. ", "https://span.example", "Span"); + const res = await post(`${instance.url}/v1/messages`, messagesBody("cite-span", false)); + const content = JSON.parse(res.body).content; + expect(content.map((b: { type: string; text: string }) => [b.type, b.text])).toEqual([ + ["text", "Alpha one. "], + ["text", "Beta two."], + ]); + expect(content[0].citations).toEqual([spanCite]); + expect(content[1].citations).toBeUndefined(); + const streamed = await post(`${instance.url}/v1/messages`, messagesBody("cite-span", true)); + expect(citationDeltas(streamed.body)).toEqual([{ index: 0, citation: spanCite }]); + }); + + it("@anthropic-ai/sdk stream helper assembles per-block citations like the non-streaming body", async () => { + instance = await createServer([blocksCiteFixture("Let me think.")]); + const client = new Anthropic({ apiKey: "local", baseURL: instance.url, maxRetries: 0 }); + const params = { + model: "claude-3-5-sonnet-20241022", + max_tokens: 100, + messages: [{ role: "user" as const, content: "cite-blocks" }], + }; + const assembled = await client.messages.stream(params).finalMessage(); + const direct = await client.messages.create(params); + const citesOf = (content: Anthropic.ContentBlock[]) => + content.map((b) => (b.type === "text" ? (b.citations ?? []).map((c) => c.cited_text) : null)); + expect(citesOf(assembled.content)).toEqual([null, ["Alpha"], null, ["Beta"]]); + expect(citesOf(direct.content)).toEqual(citesOf(assembled.content)); + }); + + it.each([false, true])( + "omitting citations leaves the wire byte-identical to the no-citations baseline (stream: %s)", + async (stream) => { + instance = await createServer([textFixture]); + const res = await post(`${instance.url}/v1/messages`, messagesBody("hello", stream)); + // Ids are random; everything else must match byte for byte. + const normalized = res.body.replace(/"id":"[^"]*"/g, '"id":""'); + const model = '"model":"claude-3-5-sonnet-20241022"'; + if (!stream) { + expect(normalized).toBe( + `{"id":"","type":"message","role":"assistant","content":[{"type":"text","text":"Hi there!"}],${model},"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}`, + ); + return; + } + const sse = (type: string, data: string) => + `event: ${type}\ndata: {"type":"${type}"${data}}\n\n`; + expect(normalized).toBe( + [ + sse( + "message_start", + `,"message":{"id":"","type":"message","role":"assistant","content":[],${model},"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}`, + ), + sse("content_block_start", ',"index":0,"content_block":{"type":"text","text":""}'), + sse("content_block_delta", ',"index":0,"delta":{"type":"text_delta","text":"Hi there!"}'), + sse("content_block_stop", ',"index":0'), + sse( + "message_delta", + ',"delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":0}', + ), + sse("message_stop", ""), + ].join(""), + ); + }, + ); + + it("skips file citations, which Anthropic has no URL-sourced form for (both modes)", async () => { + instance = await createServer([ + { + match: { userMessage: "cite-file" }, + response: { + content: "Hello.", + citations: [ + { type: "file", fileId: "file-abc123", filename: "plan.pdf" }, + { url: "https://example.com" }, + ], + }, + }, + { + match: { userMessage: "file-only" }, + response: { + content: "Hello.", + citations: [{ type: "file", fileId: "file-abc123", filename: "plan.pdf" }], + }, + }, + ]); + const urlOnly = [citation("Hello.", "https://example.com", "https://example.com")]; + const res = await post(`${instance.url}/v1/messages`, messagesBody("cite-file", false)); + expect(JSON.parse(res.body).content[0].citations).toEqual(urlOnly); + const streamed = await post(`${instance.url}/v1/messages`, messagesBody("cite-file", true)); + const deltas = parseClaudeSSEEvents(streamed.body) + .filter((e) => (e.delta as { type?: string } | undefined)?.type === "citations_delta") + .map((e) => (e.delta as { citation: unknown }).citation); + expect(deltas).toEqual(urlOnly); + + const fileOnly = await post(`${instance.url}/v1/messages`, messagesBody("file-only", false)); + expect("citations" in JSON.parse(fileOnly.body).content[0]).toBe(false); + const fileOnlyStream = await post( + `${instance.url}/v1/messages`, + messagesBody("file-only", true), + ); + expect(fileOnlyStream.body).not.toContain("citations_delta"); + }); +}); diff --git a/src/__tests__/misbehavior-bedrock-invoke.test.ts b/src/__tests__/misbehavior-bedrock-invoke.test.ts index 9b98d725..d44fbe43 100644 --- a/src/__tests__/misbehavior-bedrock-invoke.test.ts +++ b/src/__tests__/misbehavior-bedrock-invoke.test.ts @@ -720,3 +720,57 @@ for (const stream of [false, true]) { } }); } + +const citedResponse: FixtureFileResponse = { + content: "Before", + toolCalls: [tool], + citations: [{ url: "https://mastra.ai", title: "Mastra" }], +}; + +test.each([ + { label: "normal", stream: false, faulted: false }, + { label: "normal", stream: true, faulted: false }, + { label: "fault", stream: false, faulted: true }, + { label: "fault", stream: true, faulted: true }, +])( + "Invoke $label path (stream=$stream) never emits fixture citations", + async ({ stream, faulted }) => { + await withClaudeFault( + faulted ? { faults: [{ fault: "tool-unknown-name" }] } : undefined, + async ({ url, mock }) => { + const { wire, tools } = await read(url, stream); + expect(mock.getLastRequest()?.response.misbehavior?.applied ?? false).toBe(faulted); + expect(tools.map((t) => t.name)).toEqual([faulted ? "lookup_v2" : "lookup"]); + expect(wire).toContain(stream ? '"text_delta"' : '"text":"Before"'); + expect(wire).not.toContain("citations"); + expect(wire).not.toContain("mastra.ai"); + }, + citedResponse, + ); + }, +); + +test.each([false, true])( + "Messages fault path (stream=%s) still emits fixture citations", + async (stream) => { + await withClaudeFault( + { faults: [{ fault: "tool-unknown-name" }] }, + async ({ url, mock }) => { + const res = await fetch(`${url}/v1/messages`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ...request, stream }), + signal: AbortSignal.timeout(5000), + }); + const text = await res.text(); + expect(res.status).toBe(200); + expect(mock.getLastRequest()?.response.misbehavior?.applied).toBe(true); + expect(text).toContain("lookup_v2"); + expect(text).toContain(stream ? "citations_delta" : '"citations"'); + expect(text).toContain("web_search_result_location"); + expect(text).toContain("https://mastra.ai"); + }, + citedResponse, + ); + }, +); diff --git a/src/__tests__/misbehavior-openai.test.ts b/src/__tests__/misbehavior-openai.test.ts index ebda5e2e..3c321da2 100644 --- a/src/__tests__/misbehavior-openai.test.ts +++ b/src/__tests__/misbehavior-openai.test.ts @@ -1392,6 +1392,65 @@ for (const stream of [false, true]) { }); } +test.each([ + { + fault: { fault: "refusal", message: "No." }, + deltas: [{ role: "assistant", content: null }, { refusal: "No." }, {}], + finish: "stop", + }, + { + fault: { fault: "content-filter" }, + deltas: [{ role: "assistant", content: null }, {}], + finish: "content_filter", + }, + { + fault: { fault: "reasoning-only", reasoning: "Hm." }, + deltas: [{ role: "assistant", content: null }, {}], + finish: "length", + }, +])( + "OpenAI $fault.fault stream sends no annotation chunk for a citations fixture", + async ({ fault, deltas, finish }) => { + await withFaultFixture( + { faults: [fault] }, + async ({ url }) => { + const res = await fetch(`${url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + stream: true, + messages: [{ role: "user", content: "weather" }], + }), + }); + const text = await res.text(); + expect(res.status, text).toBe(200); + const chunks = text + .split("\n\n") + .map((frame) => frame.replace(/^data: /, "").trim()) + .filter((data) => data !== "" && data !== "[DONE]") + .map( + (data) => + JSON.parse(data) as { + choices: { delta: Record; finish_reason: string | null }[]; + }, + ); + expect(chunks.map((chunk) => chunk.choices[0].delta)).toEqual(deltas); + expect(chunks.map((chunk) => chunk.choices[0].finish_reason)).toEqual([ + ...deltas.slice(0, -1).map(() => null), + finish, + ]); + }, + { + response: { + content: "Sunny.", + citations: [{ url: "https://example.com/weather" }], + }, + }, + ); + }, +); + test.each([ undefined, "Request refused.", diff --git a/src/__tests__/responses.test.ts b/src/__tests__/responses.test.ts index 9f9b4395..b2a2d793 100644 --- a/src/__tests__/responses.test.ts +++ b/src/__tests__/responses.test.ts @@ -2746,6 +2746,373 @@ describe("POST /v1/responses (citations)", () => { expect((done!.part as { annotations: unknown[] }).annotations).toHaveLength(2); }); + const fileFixture: Fixture = { + match: { userMessage: "cite-file" }, + response: { + content: "The plan ships in Q3.", + citations: [ + { type: "file", fileId: "file-abc123", filename: "plan.pdf" }, + { url: "https://mastra.ai", title: "Mastra" }, + { type: "file", fileId: "file-def456", filename: "notes.md", index: 8 }, + ], + }, + }; + const expectedFileAnnotations = [ + { type: "file_citation", file_id: "file-abc123", filename: "plan.pdf", index: 21 }, + { + type: "url_citation", + url: "https://mastra.ai", + title: "Mastra", + start_index: 0, + end_index: 21, + }, + { type: "file_citation", file_id: "file-def456", filename: "notes.md", index: 8 }, + ]; + + it("non-streaming output_text carries file_citation annotations in fixture order", async () => { + instance = await createServer([fileFixture]); + const res = await post(`${instance.url}/v1/responses`, { + model: "gpt-4", + input: [{ role: "user", content: "cite-file" }], + stream: false, + }); + const msg = JSON.parse(res.body).output.find((o: { type: string }) => o.type === "message"); + expect(msg.content[0].annotations).toEqual(expectedFileAnnotations); + }); + + it("streaming emits file_citation through annotation.added events", async () => { + instance = await createServer([fileFixture]); + const res = await post(`${instance.url}/v1/responses`, { + model: "gpt-4", + input: [{ role: "user", content: "cite-file" }], + stream: true, + }); + const events = parseResponsesSSEEvents(res.body); + const added = events.filter((e) => e.type === "response.output_text.annotation.added"); + expect(added.map((e) => e.annotation)).toEqual(expectedFileAnnotations); + expect(added.map((e) => e.annotation_index)).toEqual([0, 1, 2]); + const done = events.find((e) => e.type === "response.content_part.done"); + expect((done!.part as { annotations: unknown[] }).annotations).toEqual(expectedFileAnnotations); + }); + + // `blocks` citation offsets index the joined text of all text blocks (the + // Chat Completions / Anthropic / Gemini convention): "Alpha one. " + + // "Beta two.". Responses offsets are per output_text, so each citation is + // rebased onto the message item it starts in. + const blocksCiteFixture: Fixture = { + match: { userMessage: "cite-blocks" }, + response: { + content: "", + blocks: [ + { type: "text", text: "Alpha one. " }, + { type: "toolCall", name: "lookup", arguments: "{}" }, + { type: "text", text: "Beta two." }, + ], + citations: [ + { url: "https://alpha.example", title: "Alpha", startIndex: 0, endIndex: 5 }, + { url: "https://beta.example", title: "Beta", startIndex: 11, endIndex: 15 }, + { url: "https://whole.example" }, + { type: "file", fileId: "file-beta", filename: "beta.md", index: 13 }, + ], + }, + }; + // The offset-less citation takes the shared defaults (start 0, end of the + // joined text), so it goes on the first message, clamped to its text. + const alphaAnnotations = [ + { + type: "url_citation", + url: "https://alpha.example", + title: "Alpha", + start_index: 0, + end_index: 5, + }, + { + type: "url_citation", + url: "https://whole.example", + title: "https://whole.example", + start_index: 0, + end_index: 11, + }, + ]; + const betaAnnotations = [ + { + type: "url_citation", + url: "https://beta.example", + title: "Beta", + start_index: 0, + end_index: 4, + }, + { type: "file_citation", file_id: "file-beta", filename: "beta.md", index: 2 }, + ]; + + it("non-streaming blocks attach each citation only to the message it starts in, rebased", async () => { + instance = await createServer([blocksCiteFixture]); + const res = await post(`${instance.url}/v1/responses`, { + model: "gpt-4", + input: [{ role: "user", content: "cite-blocks" }], + stream: false, + }); + expect(res.status).toBe(200); + const output = JSON.parse(res.body).output; + expect(output.map((o: { type: string }) => o.type)).toEqual([ + "message", + "function_call", + "message", + ]); + expect(output[0].content[0].annotations).toEqual(alphaAnnotations); + expect(output[2].content[0].annotations).toEqual(betaAnnotations); + }); + + it("streaming blocks emit each annotation.added once, on the owning message", async () => { + instance = await createServer([blocksCiteFixture]); + const res = await post(`${instance.url}/v1/responses`, { + model: "gpt-4", + input: [{ role: "user", content: "cite-blocks" }], + stream: true, + }); + expect(res.status).toBe(200); + const events = parseResponsesSSEEvents(res.body); + const added = events + .filter((e) => e.type === "response.output_text.annotation.added") + .map((e) => ({ + output_index: e.output_index, + content_index: e.content_index, + annotation_index: e.annotation_index, + annotation: e.annotation, + })); + expect(added).toEqual([ + ...alphaAnnotations.map((annotation, i) => ({ + output_index: 0, + content_index: 0, + annotation_index: i, + annotation, + })), + ...betaAnnotations.map((annotation, i) => ({ + output_index: 2, + content_index: 0, + annotation_index: i, + annotation, + })), + ]); + const parts = events + .filter((e) => e.type === "response.content_part.done") + .map((e) => (e.part as { annotations: unknown[] }).annotations); + expect(parts).toEqual([alphaAnnotations, betaAnnotations]); + const completed = events.find((e) => e.type === "response.completed"); + const output = (completed!.response as { output: { content?: { annotations: unknown[] }[] }[] }) + .output; + expect(output[0].content![0].annotations).toEqual(alphaAnnotations); + expect(output[2].content![0].annotations).toEqual(betaAnnotations); + }); + + it("blocks put an offset-less citation only on the first message, streaming and not", async () => { + instance = await createServer([ + { + match: { userMessage: "cite-whole" }, + response: { + content: "", + blocks: [ + { type: "text", text: "Alpha one. " }, + { type: "text", text: "Beta two." }, + ], + citations: [{ url: "https://whole.example" }], + }, + }, + ]); + const whole = { + type: "url_citation", + url: "https://whole.example", + title: "https://whole.example", + start_index: 0, + end_index: 11, + }; + const body = (stream: boolean) => ({ + model: "gpt-4", + input: [{ role: "user", content: "cite-whole" }], + stream, + }); + const res = await post(`${instance.url}/v1/responses`, body(false)); + const output = JSON.parse(res.body).output; + expect( + output.map((o: { content: { annotations: unknown[] }[] }) => o.content[0].annotations), + ).toEqual([[whole], []]); + const events = parseResponsesSSEEvents( + (await post(`${instance.url}/v1/responses`, body(true))).body, + ); + const added = events.filter((e) => e.type === "response.output_text.annotation.added"); + expect(added.map((e) => [e.output_index, e.annotation])).toEqual([[0, whole]]); + }); + + // A file citation is a point after the cited text, so an `index` equal to a + // block's end belongs to that block (at its end), not to index 0 of the + // next one. A URL citation is a range, owned by the block holding its start. + // "Alpha one. " is 11 long, "Beta two." 9, so the joined text is 20 long. + const boundaryFixture: Fixture = { + match: { userMessage: "cite-boundary" }, + response: { + content: "", + blocks: [ + { type: "text", text: "Alpha one. " }, + { type: "text", text: "Beta two." }, + ], + citations: [ + { type: "file", fileId: "file-boundary", filename: "boundary.md", index: 11 }, + { type: "file", fileId: "file-zero", filename: "zero.md", index: 0 }, + { type: "file", fileId: "file-end", filename: "end.md", index: 20 }, + { url: "https://beta.example", startIndex: 11, endIndex: 15 }, + ], + }, + }; + const boundaryFirst = [ + { type: "file_citation", file_id: "file-boundary", filename: "boundary.md", index: 11 }, + { type: "file_citation", file_id: "file-zero", filename: "zero.md", index: 0 }, + ]; + const boundarySecond = [ + { type: "file_citation", file_id: "file-end", filename: "end.md", index: 9 }, + { + type: "url_citation", + url: "https://beta.example", + title: "https://beta.example", + start_index: 0, + end_index: 4, + }, + ]; + const boundaryBody = (stream: boolean) => ({ + model: "gpt-4", + input: [{ role: "user", content: "cite-boundary" }], + stream, + }); + + it("non-streaming blocks attach a file citation on a block boundary to the block before it", async () => { + instance = await createServer([boundaryFixture]); + const res = await post(`${instance.url}/v1/responses`, boundaryBody(false)); + expect(res.status).toBe(200); + const output = JSON.parse(res.body).output; + expect( + output.map((o: { content: { annotations: unknown[] }[] }) => o.content[0].annotations), + ).toEqual([boundaryFirst, boundarySecond]); + }); + + it("streaming blocks attach a file citation on a block boundary to the block before it", async () => { + instance = await createServer([boundaryFixture]); + const events = parseResponsesSSEEvents( + (await post(`${instance.url}/v1/responses`, boundaryBody(true))).body, + ); + const added = events + .filter((e) => e.type === "response.output_text.annotation.added") + .map((e) => ({ + output_index: e.output_index, + content_index: e.content_index, + annotation_index: e.annotation_index, + annotation: e.annotation, + })); + expect(added).toEqual([ + ...boundaryFirst.map((annotation, i) => ({ + output_index: 0, + content_index: 0, + annotation_index: i, + annotation, + })), + ...boundarySecond.map((annotation, i) => ({ + output_index: 1, + content_index: 0, + annotation_index: i, + annotation, + })), + ]); + }); + + it("a citation spanning two messages goes to the one it starts in, end clamped to it", async () => { + // [6, 15) starts in "Alpha one. " (0-11) and ends in "Beta two." (11-20): + // the starting message owns it, rebased, with end_index clamped to its text. + instance = await createServer([ + { + match: { userMessage: "cite-span" }, + response: { + content: "", + blocks: [ + { type: "text", text: "Alpha one. " }, + { type: "text", text: "Beta two." }, + ], + citations: [{ url: "https://span.example", title: "Span", startIndex: 6, endIndex: 15 }], + }, + }, + ]); + const span = { + type: "url_citation", + url: "https://span.example", + title: "Span", + start_index: 6, + end_index: 11, + }; + const body = (stream: boolean) => ({ + model: "gpt-4", + input: [{ role: "user", content: "cite-span" }], + stream, + }); + const res = await post(`${instance.url}/v1/responses`, body(false)); + const output = JSON.parse(res.body).output as { + content: { text: string; annotations: unknown[] }[]; + }[]; + expect(output.map((o) => [o.content[0].text, o.content[0].annotations])).toEqual([ + ["Alpha one. ", [span]], + ["Beta two.", []], + ]); + const events = parseResponsesSSEEvents( + (await post(`${instance.url}/v1/responses`, body(true))).body, + ); + const added = events.filter((e) => e.type === "response.output_text.annotation.added"); + expect(added.map((e) => [e.output_index, e.annotation_index, e.annotation])).toEqual([ + [0, 0, span], + ]); + const completed = events.find((e) => e.type === "response.completed"); + const streamedOutput = ( + completed!.response as { output: { content: { annotations: unknown[] }[] }[] } + ).output; + expect(streamedOutput.map((o) => o.content[0].annotations)).toEqual([[span], []]); + }); + + it("misbehavior blocks turns attach each citation only to the owning message", async () => { + instance = await createServer([{ ...blocksCiteFixture, misbehavior: "tool-unknown-name" }], { + enableMisbehavior: true, + }); + const body = (stream: boolean) => ({ + model: "gpt-4", + input: [{ role: "user", content: "cite-blocks" }], + stream, + }); + const res = await post(`${instance.url}/v1/responses`, body(false)); + expect(res.status).toBe(200); + const output = JSON.parse(res.body).output; + // The fault was applied: the call between the messages is renamed. + expect(output.map((o: { type: string; name?: string }) => [o.type, o.name])).toEqual([ + ["message", undefined], + ["function_call", "lookup_v2"], + ["message", undefined], + ]); + expect(output[0].content[0].annotations).toEqual(alphaAnnotations); + expect(output[2].content[0].annotations).toEqual(betaAnnotations); + const streamed = await post(`${instance.url}/v1/responses`, body(true)); + expect(streamed.status).toBe(200); + const streamedEvents = parseResponsesSSEEvents(streamed.body); + const doneItems = streamedEvents + .filter((e) => e.type === "response.output_item.done") + .map((e) => e.item as { type: string; name?: string }) + .map((item) => [item.type, item.name]); + expect(doneItems).toEqual([ + ["message", undefined], + ["function_call", "lookup_v2"], + ["message", undefined], + ]); + const added = streamedEvents + .filter((e) => e.type === "response.output_text.annotation.added") + .map((e) => [e.output_index, e.annotation_index, e.annotation]); + expect(added).toEqual([ + ...alphaAnnotations.map((annotation, i) => [0, i, annotation]), + ...betaAnnotations.map((annotation, i) => [2, i, annotation]), + ]); + }); + it("omitting citations keeps annotations empty", async () => { instance = await createServer([ { match: { userMessage: "plain" }, response: { content: "hi" } }, diff --git a/src/__tests__/server.test.ts b/src/__tests__/server.test.ts index eaff221c..e4aa95cc 100644 --- a/src/__tests__/server.test.ts +++ b/src/__tests__/server.test.ts @@ -2480,3 +2480,207 @@ describe("per-request journal attribution and client-fault classification", () = expect(entries[0].path).toBe("/v1/voices/%E0%A4%A"); }); }); + +describe("POST /v1/chat/completions (citations and files)", () => { + const citeFixture: Fixture = { + match: { userMessage: "cite-me" }, + response: { + content: "Mastra is a TypeScript framework.", + citations: [ + { url: "https://mastra.ai", title: "Mastra" }, + { url: "https://mastra.ai/docs", startIndex: 0, endIndex: 6 }, + ], + }, + }; + const expectedAnnotations = [ + { + type: "url_citation", + url_citation: { url: "https://mastra.ai", title: "Mastra", start_index: 0, end_index: 33 }, + }, + { + type: "url_citation", + url_citation: { + url: "https://mastra.ai/docs", + title: "https://mastra.ai/docs", + start_index: 0, + end_index: 6, + }, + }, + ]; + const chatBody = (msg: string, stream: boolean) => ({ + model: "gpt-4", + stream, + messages: [{ role: "user", content: msg }], + }); + type ChatChunk = { + choices: { delta: { content?: string; annotations?: unknown }; finish_reason: string | null }[]; + }; + + it("non-streaming message carries url_citation annotations", async () => { + instance = await createServer([citeFixture]); + const res = await post(`${instance.url}/v1/chat/completions`, chatBody("cite-me", false)); + expect(res.status).toBe(200); + expect(JSON.parse(res.body).choices[0].message.annotations).toEqual(expectedAnnotations); + }); + + it("streaming sends one delta.annotations chunk after the content, before the finish chunk", async () => { + instance = await createServer([citeFixture]); + const res = await post(`${instance.url}/v1/chat/completions`, chatBody("cite-me", true)); + const chunks = parseSSEEvents(res.body) as ChatChunk[]; + const withAnnotations = chunks.filter((c) => c.choices[0]?.delta.annotations !== undefined); + expect(withAnnotations).toHaveLength(1); + expect(withAnnotations[0].choices[0].delta.annotations).toEqual(expectedAnnotations); + const idx = chunks.indexOf(withAnnotations[0]); + expect(chunks[idx + 1].choices[0].finish_reason).toBe("stop"); + expect(chunks.slice(0, idx).some((c) => c.choices[0].delta.content)).toBe(true); + }); + + it("content+toolCalls turns carry the annotations on both modes", async () => { + instance = await createServer([ + { + match: { userMessage: "cite-tools" }, + response: { + content: "Checking.", + toolCalls: [{ name: "lookup", arguments: "{}" }], + citations: [{ url: "https://example.com" }], + }, + }, + ]); + const res = await post(`${instance.url}/v1/chat/completions`, chatBody("cite-tools", false)); + expect(JSON.parse(res.body).choices[0].message.annotations).toEqual([ + { + type: "url_citation", + url_citation: { + url: "https://example.com", + title: "https://example.com", + start_index: 0, + end_index: 9, + }, + }, + ]); + const streamed = await post( + `${instance.url}/v1/chat/completions`, + chatBody("cite-tools", true), + ); + const chunks = parseSSEEvents(streamed.body) as ChatChunk[]; + expect(chunks.filter((c) => c.choices[0]?.delta.annotations !== undefined)).toHaveLength(1); + }); + + it("blocks turns carry the annotations on both modes", async () => { + instance = await createServer([ + { + match: { userMessage: "cite-blocks" }, + response: { + blocks: [ + { type: "text", text: "Checking." }, + { type: "toolCall", name: "lookup", arguments: "{}" }, + ], + citations: [{ url: "https://example.com" }], + }, + }, + ]); + const expected = [ + { + type: "url_citation", + url_citation: { + url: "https://example.com", + title: "https://example.com", + start_index: 0, + end_index: 9, + }, + }, + ]; + const res = await post(`${instance.url}/v1/chat/completions`, chatBody("cite-blocks", false)); + expect(res.status).toBe(200); + expect(JSON.parse(res.body).choices[0].message.annotations).toEqual(expected); + const streamed = await post( + `${instance.url}/v1/chat/completions`, + chatBody("cite-blocks", true), + ); + expect(streamed.status).toBe(200); + const chunks = parseSSEEvents(streamed.body) as ChatChunk[]; + const withAnnotations = chunks.filter((c) => c.choices[0]?.delta.annotations !== undefined); + expect(withAnnotations.map((c) => c.choices[0].delta.annotations)).toEqual([expected]); + const idx = chunks.indexOf(withAnnotations[0]); + expect(chunks[idx + 1].choices[0].finish_reason).toBe("tool_calls"); + }); + + it("skips file citations, which Chat Completions has no annotation for (both modes)", async () => { + instance = await createServer([ + { + match: { userMessage: "cite-file" }, + response: { + content: "Hello.", + citations: [ + { type: "file", fileId: "file-abc123", filename: "plan.pdf" }, + { url: "https://example.com" }, + ], + }, + }, + { + match: { userMessage: "file-only" }, + response: { + content: "Hello.", + citations: [{ type: "file", fileId: "file-abc123", filename: "plan.pdf" }], + }, + }, + ]); + const urlOnly = [ + { + type: "url_citation", + url_citation: { + url: "https://example.com", + title: "https://example.com", + start_index: 0, + end_index: 6, + }, + }, + ]; + const res = await post(`${instance.url}/v1/chat/completions`, chatBody("cite-file", false)); + expect(JSON.parse(res.body).choices[0].message.annotations).toEqual(urlOnly); + const streamed = await post(`${instance.url}/v1/chat/completions`, chatBody("cite-file", true)); + const chunks = parseSSEEvents(streamed.body) as ChatChunk[]; + const withAnnotations = chunks.filter((c) => c.choices[0]?.delta.annotations !== undefined); + expect(withAnnotations.map((c) => c.choices[0].delta.annotations)).toEqual([urlOnly]); + + const fileOnly = await post( + `${instance.url}/v1/chat/completions`, + chatBody("file-only", false), + ); + expect("annotations" in JSON.parse(fileOnly.body).choices[0].message).toBe(false); + const fileOnlyStream = await post( + `${instance.url}/v1/chat/completions`, + chatBody("file-only", true), + ); + expect(fileOnlyStream.body).not.toContain("annotations"); + }); + + it.each([false, true])( + "omitting citations leaves the wire byte-identical to the no-citations baseline (stream: %s)", + async (stream) => { + instance = await createServer([ + { match: { userMessage: "plain" }, response: { content: "hi" } }, + ]); + const res = await post(`${instance.url}/v1/chat/completions`, chatBody("plain", stream)); + // Ids and timestamps are random; everything else must match byte for byte. + const normalized = res.body + .replace(/"id":"[^"]*"/g, '"id":""') + .replace(/"created":\d+/g, '"created":0'); + const envelope = '"id":"","object":"chat.completion","created":0,"model":"gpt-4"'; + if (!stream) { + expect(normalized).toBe( + `{${envelope},"choices":[{"index":0,"message":{"role":"assistant","content":"hi","refusal":null},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}`, + ); + return; + } + const chunk = (delta: string, finish: string) => + `data: {"id":"","object":"chat.completion.chunk","created":0,"model":"gpt-4","choices":[{"index":0,"delta":${delta},"logprobs":null,"finish_reason":${finish}}]}\n\n`; + expect(normalized).toBe( + chunk('{"role":"assistant","content":""}', "null") + + chunk('{"content":"hi"}', "null") + + chunk("{}", '"stop"') + + "data: [DONE]\n\n", + ); + }, + ); +}); diff --git a/src/bedrock.ts b/src/bedrock.ts index e4cb7635..17a4fac3 100644 --- a/src/bedrock.ts +++ b/src/bedrock.ts @@ -638,6 +638,7 @@ export async function handleBedrock( chunkSize: Math.max(1, fixture.chunkSize ?? defaults.chunkSize), logger, strict: resolveRequestDecisions(defaults.strict, req.headers).reasoningSuppression, + citations: false, }); if (prepared.stream) throw new TypeError("InvokeModel requires non-stream Claude output"); const entry = journal.add({ @@ -1548,6 +1549,7 @@ export async function handleBedrockStream( chunkSize, logger, strict: resolveRequestDecisions(defaults.strict, req.headers).reasoningSuppression, + citations: false, }); if (!prepared.stream) throw new TypeError("InvokeModel stream requires Claude events"); const entry = journal.add({ diff --git a/src/gemini.ts b/src/gemini.ts index 1e18a293..99a2c957 100644 --- a/src/gemini.ts +++ b/src/gemini.ts @@ -39,11 +39,14 @@ import { getContext, getTestId, resolveFixtureBlockOutcome, + resolveCitations, resolveResponse, resolveReasoningForModel, strictNoMatchMessage, strictNoMatchLogLine, servedToolArgs, + type ResolvedCitation, + surrogateSafeChunks, } from "./helpers.js"; import { matchFixtureDiagnostic, recordMatchOptions } from "./router.js"; import { writeErrorResponse, delay, calculateDelay } from "./sse-writer.js"; @@ -297,6 +300,7 @@ interface GeminiResponseChunk { finishReason?: string; finishMessage?: string; safetyRatings?: { category: string; probability: string; blocked: boolean }[]; + groundingMetadata?: GeminiGroundingMetadata; index: number; }[]; usageMetadata?: { @@ -306,6 +310,159 @@ interface GeminiResponseChunk { }; } +interface GeminiGroundingMetadata { + groundingChunks: { web: { uri: string; title: string } }[]; + groundingSupports: { + segment: { partIndex: number; startIndex: number; endIndex: number; text: string }; + groundingChunkIndices: number[]; + }[]; +} + +// A text part of the Content that carries the grounding: its index in that +// Content's `parts`, and the character span [start, end) of the cited text +// it holds. +interface GeminiCitedTextPart { + partIndex: number; + start: number; + end: number; +} + +type GeminiGroundingSupport = GeminiGroundingMetadata["groundingSupports"][number]; + +// The whole cited text as the single text part `partIndex`. +function geminiWholeTextPart(text: string, partIndex: number): GeminiCitedTextPart[] { + return [{ partIndex, start: 0, end: text.length }]; +} + +// Fixture `citations` as Gemini grounding supports, one or more per citation, +// each pointing at grounding chunk `i` (the citation's index). The @google/genai +// `Segment` type documents `startIndex` / `endIndex` as byte offsets "in the +// given Part" — the Part at `partIndex` within its parent Content — so every +// segment holds UTF-8 byte offsets into the text of one part. A citation that +// spans several text parts becomes one support per part, in text order, all +// naming the same grounding chunk; a client joining their segments gets the +// whole cited text. An empty citation names the part holding its start. +function geminiGroundingSupports( + text: string, + citations: ResolvedCitation[], + textParts: GeminiCitedTextPart[], +): GeminiGroundingSupport[] { + const segment = ( + part: GeminiCitedTextPart, + from: number, + to: number, + ): GeminiGroundingSupport["segment"] => { + const partText = text.slice(part.start, part.end); + const start = Math.min(Math.max(from - part.start, 0), partText.length); + const end = Math.min(Math.max(to - part.start, start), partText.length); + const byteOffset = (charIndex: number) => + Buffer.byteLength(partText.slice(0, charIndex), "utf8"); + return { + partIndex: part.partIndex, + startIndex: byteOffset(start), + endIndex: byteOffset(end), + text: partText.slice(start, end), + }; + }; + return citations.flatMap((c, i) => { + const spanned = textParts.filter( + (p) => Math.max(c.startIndex, p.start) < Math.min(c.endIndex, p.end), + ); + const parts = + spanned.length > 0 + ? spanned + : [textParts.find((p) => c.startIndex < p.end) ?? textParts[textParts.length - 1]]; + return parts.map((p) => ({ + segment: segment(p, c.startIndex, c.endIndex), + groundingChunkIndices: [i], + })); + }); +} + +// Fixture `citations` as Gemini `groundingMetadata` for one Content holding +// `textParts`: one `web` grounding chunk per citation and the supports from +// {@link geminiGroundingSupports}. File citations are skipped (see +// resolveCitations): grounding chunks carry no file IDs. Nothing when the +// fixture has no URL citations or the Content has no text part to cite. +function geminiGroundingMetadata( + text: string, + overrides: ResponseOverrides | undefined, + textParts: GeminiCitedTextPart[], +): GeminiGroundingMetadata | undefined { + const citations = resolveCitations(text, overrides?.citations); + if (citations.length === 0 || textParts.length === 0) return undefined; + return { + groundingChunks: geminiGroundingChunks(citations), + groundingSupports: geminiGroundingSupports(text, citations, textParts), + }; +} + +function geminiGroundingChunks( + citations: ResolvedCitation[], +): GeminiGroundingMetadata["groundingChunks"] { + return citations.map((c) => ({ web: { uri: c.url, title: c.title } })); +} + +// `{ groundingMetadata }` for the candidate holding `textParts`, or nothing +// when {@link geminiGroundingMetadata} yields none. +function geminiGroundingField( + text: string, + overrides: ResponseOverrides | undefined, + textParts: GeminiCitedTextPart[], +): { groundingMetadata?: GeminiGroundingMetadata } { + const groundingMetadata = geminiGroundingMetadata(text, overrides, textParts); + return groundingMetadata ? { groundingMetadata } : {}; +} + +// Streaming form of {@link geminiGroundingField}. `spans` are the character +// spans of `text` that the streamed text chunks carry, in stream order, each +// as part 0 of its own chunk. Each chunk's Content holds only that chunk's +// text, so, per the `Segment` contract above, a chunk's supports must index +// that chunk's part. (Neither @google/genai nor Google's public Gemini API +// docs say streamed segments index the accumulated text instead.) Every chunk +// holding some cited text carries groundingMetadata whose supports cover just +// its part (partIndex 0, byte offsets into the chunk's text), splitting a +// citation across chunks as across parts. Each grounded chunk lists every grounding +// chunk, so `groundingChunkIndices` mean the same in every chunk and in the +// non-streaming response. Returns one field per span; empty for a span that +// holds no cited text. +function geminiStreamGroundingFields( + text: string, + overrides: ResponseOverrides | undefined, + spans: { start: number; end: number }[], +): { groundingMetadata?: GeminiGroundingMetadata }[] { + const fields: { groundingMetadata?: GeminiGroundingMetadata }[] = spans.map(() => ({})); + const citations = resolveCitations(text, overrides?.citations); + if (citations.length === 0 || spans.length === 0) return fields; + // Number the spans as parts so each support says which chunk it belongs to. + const textParts = spans.map((s, k) => ({ partIndex: k, start: s.start, end: s.end })); + const groundingChunks = geminiGroundingChunks(citations); + for (const support of geminiGroundingSupports(text, citations, textParts)) { + const k = support.segment.partIndex; + const metadata = (fields[k].groundingMetadata ??= { groundingChunks, groundingSupports: [] }); + metadata.groundingSupports.push({ ...support, segment: { ...support.segment, partIndex: 0 } }); + } + return fields; +} + +// The [start, end) spans of the chunks that streaming `text` as +// {@link surrogateSafeChunks} yields, as offsets into the joined text, where +// `text` starts at `offset`. Empty `text` yields one empty span at `offset`. +function geminiTextChunkSpans( + text: string, + chunkSize: number, + offset = 0, +): { start: number; end: number }[] { + if (text.length === 0) return [{ start: offset, end: offset }]; + const spans: { start: number; end: number }[] = []; + let start = offset; + for (const slice of surrogateSafeChunks(text, chunkSize)) { + spans.push({ start, end: start + slice.length }); + start += slice.length; + } + return spans; +} + // Fixture `files` as Gemini `inlineData` parts. Every text and tool-call builder // emits these after any thought parts and before the text / functionCall parts. function geminiFileParts(overrides?: ResponseOverrides): GeminiPart[] { @@ -349,22 +506,31 @@ function buildGeminiTextStreamChunks( chunks.push(...geminiFileChunks(overrides)); - // Content chunks - for (let i = 0; i < content.length; i += chunkSize) { - const slice = content.slice(i, i + chunkSize); - const isLast = i + chunkSize >= content.length; + // Each content chunk carries the grounding for the cited text it holds. + const grounding = geminiStreamGroundingFields( + content, + overrides, + geminiTextChunkSpans(content, chunkSize), + ); + + // Content chunks, never cutting a surrogate pair, so each chunk is valid + // UTF-16 and its grounding byte offsets count whole characters. + const contentSlices = surrogateSafeChunks(content, chunkSize); + contentSlices.forEach((slice, k) => { + const isLast = k === contentSlices.length - 1; const chunk: GeminiResponseChunk = { candidates: [ { content: { role: "model", parts: [{ text: slice }] }, index: 0, ...(isLast ? { finishReason: effectiveFinish } : {}), + ...grounding[k], }, ], ...(isLast ? { usageMetadata: usage } : {}), }; chunks.push(chunk); - } + }); // Handle empty content if (content.length === 0) { @@ -373,6 +539,7 @@ function buildGeminiTextStreamChunks( { content: { role: "model", parts: [{ text: "" }] }, finishReason: effectiveFinish, + ...grounding[0], index: 0, }, ], @@ -433,6 +600,8 @@ function buildGeminiToolCallStreamChunks( chunks.push(...geminiFileChunks(overrides)); + // A tool-only turn has no text for `citations` to point at. + resolveCitations("", overrides?.citations); const parts: GeminiPart[] = toolCalls.map((tc) => parseToolCallPart(tc, logger)); // Gemini sends all tool calls in a single response chunk @@ -462,6 +631,7 @@ function buildGeminiTextResponse( parts.push({ text: reasoning, thought: true }); } parts.push(...geminiFileParts(overrides)); + const textParts = geminiWholeTextPart(content, parts.length); parts.push({ text: content }); return { @@ -469,6 +639,7 @@ function buildGeminiTextResponse( { content: { role: "model", parts }, finishReason: geminiFinishReason(overrides?.finishReason, "STOP"), + ...geminiGroundingField(content, overrides, textParts), index: 0, }, ], @@ -482,6 +653,8 @@ function buildGeminiToolCallResponse( reasoning?: string, overrides?: ResponseOverrides, ): GeminiResponseChunk { + // A tool-only turn has no text for `citations` to point at. + resolveCitations("", overrides?.citations); const parts: GeminiPart[] = []; if (reasoning) { parts.push({ text: reasoning, thought: true }); @@ -535,8 +708,20 @@ function buildGeminiContentWithToolCallsStreamChunks( // its text part. Gemini's ordered `parts` make this fully expressible. The // terminal block carries the finishReason regardless of its type. Legacy // fixtures (no `blocks`) never enter here — see the else branch below. + // The grounding rides on the text chunks holding the cited text, never on + // a functionCall chunk: each text block streams as chunks whose part 0 is + // the text their offsets refer to. const outcome = resolveFixtureBlockOutcome(blocks); const resolved = outcome.ordered; + const spans: { start: number; end: number }[] = []; + let offset = 0; + for (const block of resolved) { + if (block.type !== "text") continue; + spans.push(...geminiTextChunkSpans(block.text, chunkSize, offset)); + offset += block.text.length; + } + const grounding = geminiStreamGroundingFields(outcome.content, overrides, spans); + let textChunk = 0; resolved.forEach((block, i) => { const isLast = i === resolved.length - 1; const finishReason = isLast @@ -568,27 +753,29 @@ function buildGeminiContentWithToolCallsStreamChunks( { content: { role: "model", parts: [{ text: "" }] }, ...(finishReason ? { finishReason } : {}), + ...grounding[textChunk++], index: 0, }, ], ...(isLast ? { usageMetadata: geminiUsageMetadata(overrides) } : {}), }); } else { - for (let j = 0; j < text.length; j += chunkSize) { - const slice = text.slice(j, j + chunkSize); - const lastSlice = j + chunkSize >= text.length; + const slices = surrogateSafeChunks(text, chunkSize); + slices.forEach((slice, j) => { + const lastSlice = j === slices.length - 1; const sliceFinish = isLast && lastSlice ? finishReason : undefined; chunks.push({ candidates: [ { content: { role: "model", parts: [{ text: slice }] }, ...(sliceFinish ? { finishReason: sliceFinish } : {}), + ...grounding[textChunk++], index: 0, }, ], ...(isLast && lastSlice ? { usageMetadata: geminiUsageMetadata(overrides) } : {}), }); - } + }); } } }); @@ -596,27 +783,35 @@ function buildGeminiContentWithToolCallsStreamChunks( return chunks; } + // The grounding rides on the text chunks holding the cited text, not on the + // functionCall chunk, so their part 0 is the text the offsets refer to. + const grounding = geminiStreamGroundingFields( + content, + overrides, + geminiTextChunkSpans(content, chunkSize), + ); if (content.length === 0) { chunks.push({ candidates: [ { content: { role: "model", parts: [{ text: "" }] }, + ...grounding[0], index: 0, }, ], }); } else { - for (let i = 0; i < content.length; i += chunkSize) { - const slice = content.slice(i, i + chunkSize); + surrogateSafeChunks(content, chunkSize).forEach((slice, k) => { chunks.push({ candidates: [ { content: { role: "model", parts: [{ text: slice }] }, + ...grounding[k], index: 0, }, ], }); - } + }); } const parts: GeminiPart[] = toolCalls.map((tc) => parseToolCallPart(tc, logger)); @@ -650,6 +845,9 @@ function buildGeminiContentWithToolCallsResponse( parts.push(...geminiFileParts(overrides)); let blockHasTools: boolean | undefined; + let groundingText = content; + // Each text part, with the span of `groundingText` it holds. + const textParts: GeminiCitedTextPart[] = []; if (blocks && blocks.length > 0) { // NEW PATH: the non-streaming `parts[]` array is positionally observable, so // emit parts in the fixture's ARRAY ORDER (after any leading thought part). @@ -658,17 +856,21 @@ function buildGeminiContentWithToolCallsResponse( const outcome = resolveFixtureBlockOutcome(blocks); const resolved = outcome.ordered; blockHasTools = outcome.hasToolCalls; + groundingText = outcome.content; for (const block of resolved) { if (block.type === "toolCall") { parts.push( parseToolCallPart({ name: block.name, arguments: block.arguments, id: block.id }, logger), ); } else { + const start = textParts.length > 0 ? textParts[textParts.length - 1].end : 0; + textParts.push({ partIndex: parts.length, start, end: start + block.text.length }); parts.push({ text: block.text }); } } } else { - // LEGACY PATH (unchanged): text part first, then functionCall parts. + // LEGACY PATH: text part first, then functionCall parts. + textParts.push(...geminiWholeTextPart(content, parts.length)); parts.push({ text: content }); parts.push(...toolCalls.map((tc) => parseToolCallPart(tc, logger))); } @@ -681,6 +883,7 @@ function buildGeminiContentWithToolCallsResponse( overrides?.finishReason, blockHasTools === false ? "STOP" : "FUNCTION_CALL", ), + ...geminiGroundingField(groundingText, overrides, textParts), index: 0, }, ], diff --git a/src/helpers.ts b/src/helpers.ts index 8f8d4c4f..796a4ac2 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -49,6 +49,7 @@ import type { FixtureFileResponsesBlock, ResponsesFixtureBlock, ChatCompletion, + ChatCompletionAnnotation, FixtureCitation, FixtureGeneratedFile, ResponseOverrides, @@ -751,6 +752,55 @@ export function resolveCitations( })); } +/** + * Fixture `citations` as Chat Completions `url_citation` annotations. File + * citations are skipped (see {@link resolveCitations}): Chat Completions has no + * `file_citation` annotation. + */ +function chatCompletionAnnotations( + content: string, + citations?: ResponseOverrides["citations"], +): ChatCompletionAnnotation[] { + return resolveCitations(content, citations).map((c) => ({ + type: "url_citation", + url_citation: { url: c.url, title: c.title, start_index: c.startIndex, end_index: c.endIndex }, + })); +} + +/** `{ annotations }` for a Chat Completions message, or nothing when the fixture has no citations. */ +function annotationsField( + content: string, + overrides?: ResponseOverrides, +): { annotations?: ChatCompletionAnnotation[] } { + const annotations = chatCompletionAnnotations(content, overrides?.citations); + return annotations.length > 0 ? { annotations } : {}; +} + +/** + * The streamed form of {@link chatCompletionAnnotations}: one chunk whose + * `delta.annotations` holds every annotation, sent after the content and + * before the finish chunk. Nothing when the fixture has no citations. + */ +function chatAnnotationChunks( + content: string, + overrides: ResponseOverrides | undefined, + envelope: { id: string; created: number; model: string }, +): SSEChunk[] { + const annotations = chatCompletionAnnotations(content, overrides?.citations); + if (annotations.length === 0) return []; + const fingerprint = overrides?.systemFingerprint; + return [ + { + id: envelope.id, + object: "chat.completion.chunk", + created: envelope.created, + model: envelope.model, + choices: [{ index: 0, delta: { annotations }, logprobs: null, finish_reason: null }], + ...(fingerprint !== undefined && { system_fingerprint: fingerprint }), + }, + ]; +} + /** * A fixture error that a handler surfaces as a coded 500. Despite the name, * it is not limited to tool calls: it also covers the fixture `files` and @@ -2087,6 +2137,8 @@ export function buildTextChunks( }); } + chunks.push(...chatAnnotationChunks(content, overrides, { id, created, model: effectiveModel })); + // Finish chunk chunks.push({ id, @@ -2112,6 +2164,7 @@ export function buildOpenAIReasoningChunks( ): SSEChunk[] { return buildTextChunks(exposeReasoning ? reasoning : "", model, chunkSize, undefined, { ...overrides, + citations: undefined, role: "assistant", finishReason: "length", }).map((chunk) => ({ @@ -2155,6 +2208,7 @@ export function buildOpenAIReasoningCompletion( undefined, { ...overrides, + citations: undefined, role: "assistant", finishReason: "length", usage: undefined, @@ -2194,6 +2248,7 @@ export function buildOpenAIContentFilterChunks( ): SSEChunk[] { return buildTextChunks("", model, 1, undefined, { ...overrides, + citations: undefined, role: "assistant", finishReason: "content_filter", }).map((chunk) => ({ @@ -2217,6 +2272,7 @@ export function buildOpenAIContentFilterCompletion( undefined, { ...overrides, + citations: undefined, role: "assistant", finishReason: "content_filter", usage: undefined, @@ -2241,6 +2297,7 @@ export function buildOpenAIRefusalChunks( ) { return buildTextChunks(refusal, model, chunkSize, undefined, { ...overrides, + citations: undefined, role: "assistant", finishReason: "stop", }).map((chunk) => ({ @@ -2270,6 +2327,7 @@ export function buildOpenAIRefusalCompletion( undefined, { ...overrides, + citations: undefined, role: "assistant", finishReason: "stop", usage: undefined, @@ -2292,6 +2350,8 @@ export function buildToolCallChunks( reasoning?: string, overrides?: ResponseOverrides, ): SSEChunk[] { + // A tool-only turn has no text for `citations` to point at. + fixtureCitations("", overrides?.citations); const id = overrides?.id ?? generateId(); const created = overrides?.created ?? Math.floor(Date.now() / 1000); const effectiveModel = overrides?.model ?? model; @@ -2440,6 +2500,7 @@ export function buildTextCompletion( content, refusal: null, ...(reasoning ? { reasoning_content: reasoning } : {}), + ...annotationsField(content, overrides), }, logprobs: null, finish_reason: overrides?.finishReason ?? "stop", @@ -2459,6 +2520,8 @@ export function buildToolCallCompletion( overrides?: ResponseOverrides, requestMessages?: ChatCompletionRequest["messages"], ): ChatCompletion { + // A tool-only turn has no text for `citations` to point at. + fixtureCitations("", overrides?.citations); const promptText = requestMessages ? requestMessages .map((m) => @@ -2641,6 +2704,15 @@ export function buildContentWithToolCallsChunks( } } + // Cite the block text this path streams, not the `content` argument. + chunks.push( + ...chatAnnotationChunks(outcome.content, overrides, { + id, + created, + model: effectiveModel, + }), + ); + // Derive the default terminal from the blocks actually emitted. chunks.push({ id, @@ -2661,7 +2733,8 @@ export function buildContentWithToolCallsChunks( return chunks; } - // EXISTING legacy code, byte-for-byte UNCHANGED. + // EXISTING legacy code. It is unchanged except for the annotations chunk + // (chatAnnotationChunks) emitted before the finish chunk. // Reasoning chunks (emitted before content, OpenRouter format) if (reasoning) { for (let i = 0; i < reasoning.length; i += chunkSize) { @@ -2772,6 +2845,8 @@ export function buildContentWithToolCallsChunks( } } + chunks.push(...chatAnnotationChunks(content, overrides, { id, created, model: effectiveModel })); + // Finish chunk chunks.push({ id, @@ -2838,6 +2913,7 @@ export function buildContentWithToolCallsCompletion( type: "function" as const, function: { name: tc.name, arguments: tc.arguments }, })), + ...annotationsField(content, overrides), }, logprobs: null, finish_reason: overrides?.finishReason ?? "tool_calls", diff --git a/src/messages.ts b/src/messages.ts index fd65aa5e..ebf3c860 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -32,6 +32,7 @@ import { generateToolUseId, extractOverrides, requireEmittedFunctionToolCalls, + resolveCitations, isTextResponse, isToolCallResponse, isCombinedFixtureResponse, @@ -455,6 +456,71 @@ function pushRedactedThinkingStreamEvents( return blockIndex; } +/** + * Fixture `citations` as Anthropic `web_search_result_location` citations, the + * URL-sourced citation type (see `CitationsWebSearchResultLocation` in + * `@anthropic-ai/sdk`), grouped per text block. `texts` are the turn's text + * blocks in order; citation offsets index their joined text, the same + * convention as Chat Completions and Gemini. Each citation goes only to the + * last text block whose start offset in the joined text is at or before the + * citation's `startIndex`, with `cited_text` sliced from that block's own text + * (cut off at the block's end). + * `web_search_result_location` carries no offsets, so nothing is rebased. + * `encrypted_index` is opaque on the real wire, so aimock fills a stable + * base64 placeholder. File citations are skipped (see {@link resolveCitations}): + * Anthropic has no citation type for an OpenAI file ID. + */ +function claudeBlockCitations(texts: string[], overrides?: ResponseOverrides): object[][] { + const starts: number[] = []; + let offset = 0; + for (const text of texts) { + starts.push(offset); + offset += text.length; + } + const perBlock: object[][] = texts.map(() => []); + if (texts.length === 0) { + // Blocks with no text block: a turn with no text for `citations` to point at. + resolveCitations("", overrides?.citations); + return perBlock; + } + resolveCitations(texts.join(""), overrides?.citations).forEach((c, i) => { + let owner = 0; + for (let k = 1; k < starts.length; k++) if (starts[k] <= c.startIndex) owner = k; + const start = starts[owner]; + perBlock[owner].push({ + type: "web_search_result_location", + cited_text: texts[owner].slice(c.startIndex - start, c.endIndex - start), + url: c.url, + title: c.title, + encrypted_index: Buffer.from(`aimock-citation:${i}:${c.url}`).toString("base64"), + }); + }); + return perBlock; +} + +/** {@link claudeBlockCitations} for a turn with a single text block. */ +function claudeCitations(text: string, overrides?: ResponseOverrides): object[] { + return claudeBlockCitations([text], overrides)[0]; +} + +/** A text content block; `citations` is added only when it has some. */ +function claudeTextBlock(text: string, citations: object[]): object { + return citations.length > 0 ? { type: "text", text, citations } : { type: "text", text }; +} + +/** + * Streamed form of a text block's citations: one `citations_delta` per + * citation, after the block's text deltas and before its `content_block_stop`, + * as in the Anthropic citations streaming example. Nothing when it has none. + */ +function claudeCitationDeltas(citations: object[], index: number): ClaudeSSEEvent[] { + return citations.map((citation) => ({ + type: "content_block_delta", + index, + delta: { type: "citations_delta", citation }, + })); +} + function buildClaudeTextStreamEvents( content: string, model: string, @@ -544,6 +610,8 @@ function buildClaudeTextStreamEvents( }); } + events.push(...claudeCitationDeltas(claudeCitations(content, overrides), blockIndex)); + // content_block_stop events.push({ type: "content_block_stop", @@ -576,6 +644,8 @@ function buildClaudeToolCallStreamEvents( reasoningSignature?: string, redactedThinking?: string[], ): ClaudeSSEEvent[] { + // A tool-only turn has no text for `citations` to point at. + resolveCitations("", overrides?.citations); const msgId = overrides?.id ?? generateMessageId(); const effectiveModel = overrides?.model ?? model; // Prefer a recorded real signature when one was captured; otherwise fall back @@ -743,7 +813,7 @@ function buildClaudeTextResponse( }); } - contentBlocks.push({ type: "text", text: content }); + contentBlocks.push(claudeTextBlock(content, claudeCitations(content, overrides))); return { id: overrides?.id ?? generateMessageId(), @@ -766,6 +836,8 @@ function buildClaudeToolCallResponse( reasoningSignature?: string, redactedThinking?: string[], ): object { + // A tool-only turn has no text for `citations` to point at. + resolveCitations("", overrides?.citations); const contentBlocks: object[] = []; // Redacted-thinking blocks lead the content array (before thinking / tool_use) @@ -890,6 +962,13 @@ function buildClaudeContentWithToolCallsStreamEvents( // encounter order, continuing from any leading thinking/redacted blocks. const outcome = resolveFixtureBlockOutcome(blocks); const ordered = outcome.ordered; + // Citation offsets index the joined text of every text block; each text + // block carries only the citations that start inside it. + const textCitations = claudeBlockCitations( + ordered.flatMap((b) => (b.type === "text" ? [b.text] : [])), + overrides, + ); + let textPos = 0; for (const block of ordered) { if (block.type === "text") { @@ -908,6 +987,8 @@ function buildClaudeContentWithToolCallsStreamEvents( }); } + events.push(...claudeCitationDeltas(textCitations[textPos++] ?? [], blockIndex)); + events.push({ type: "content_block_stop", index: blockIndex, @@ -968,10 +1049,11 @@ function buildClaudeContentWithToolCallsStreamEvents( return events; } - // LEGACY PATH (byte-for-byte unchanged): text content block, then tool_use - // blocks in `toolCalls` order. Reached only when `blocks` is absent; the - // leading redacted-thinking/thinking blocks above are shared with the new - // path and produce identical wire output here. + // LEGACY PATH: text content block, then tool_use blocks in `toolCalls` + // order. Reached only when `blocks` is absent; the leading + // redacted-thinking/thinking blocks above are shared with the new path and + // produce identical wire output here. Unchanged except that the text block + // now ends with one `citations_delta` per fixture URL citation, if any. // Text content block events.push({ type: "content_block_start", @@ -988,6 +1070,8 @@ function buildClaudeContentWithToolCallsStreamEvents( }); } + events.push(...claudeCitationDeltas(claudeCitations(content, overrides), blockIndex)); + events.push({ type: "content_block_stop", index: blockIndex, @@ -1095,10 +1179,17 @@ function buildClaudeContentWithToolCallsResponse( // streaming path for the same `blocks` fixture. const outcome = resolveFixtureBlockOutcome(blocks); const ordered = outcome.ordered; + // Citation offsets index the joined text of every text block; each text + // block carries only the citations that start inside it. + const textCitations = claudeBlockCitations( + ordered.flatMap((b) => (b.type === "text" ? [b.text] : [])), + overrides, + ); + let textPos = 0; blockHasTools = outcome.hasToolCalls; for (const block of ordered) { if (block.type === "text") { - contentBlocks.push({ type: "text", text: block.text }); + contentBlocks.push(claudeTextBlock(block.text, textCitations[textPos++] ?? [])); } else { contentBlocks.push( toolUseBlock({ name: block.name, arguments: block.arguments, id: block.id }), @@ -1106,8 +1197,10 @@ function buildClaudeContentWithToolCallsResponse( } } } else { - // LEGACY PATH (unchanged): text content block, then tool_use blocks. - contentBlocks.push({ type: "text", text: content }); + // LEGACY PATH: text content block, then tool_use blocks. Unchanged except + // that the text block now carries `citations` when the fixture has URL + // citations. + contentBlocks.push(claudeTextBlock(content, claudeCitations(content, overrides))); for (const tc of toolCalls) { contentBlocks.push(toolUseBlock(tc)); } @@ -1147,6 +1240,11 @@ export function prepareClaudeMisbehavior( chunkSize: number; logger: Logger; strict: boolean; + /** + * Emit fixture `citations`. Only the Anthropic Messages endpoint does; + * Bedrock InvokeModel shares these builders but serves no citations. + */ + citations: boolean; }, ): PreparedClaudeMisbehavior { const response = plan.response; @@ -1232,8 +1330,10 @@ export function prepareClaudeMisbehavior( ), }); const usage = renderAnthropicUsage(resolvedUsage, "messages-final", "applied-misbehavior"); + const { citations, ...rest } = extractOverrides(response); const overrides = { - ...extractOverrides(response), + ...rest, + ...(context.citations ? { citations } : {}), ...(plan.stop === "length" ? { finishReason: "length" } : {}), usage, }; @@ -1814,6 +1914,7 @@ export async function handleMessages( chunkSize, logger, strict: resolveRequestDecisions(defaults.strict, req.headers).reasoningSuppression, + citations: true, }); const entry = journal.add({ method: req.method ?? "POST", diff --git a/src/responses.ts b/src/responses.ts index 132fa1b0..b9148310 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -51,6 +51,8 @@ import { servedToolCalls, toolCallFixtureBlock, withoutLegacyNamespaces, + fixtureCitations, + surrogateSafeChunks, } from "./helpers.js"; import type { MisbehaviorPlan } from "./misbehavior.js"; import { isReasoningModel } from "./model-utils.js"; @@ -578,7 +580,7 @@ export function buildTextStreamEvents( chunkSize, nextOutputIndex, overrides?.phase, - overrides?.citations, + buildAnnotations(content, overrides?.citations), ); events.push(...msgEvents); @@ -751,6 +753,8 @@ export function toolCallStreamEvents( synthesizeSummarylessReasoning = false, ): ResponsesSSEEvent[] { assertResponsesToolCalls(toolCalls); + // A tool-only turn has no text for `citations` to point at. + fixtureCitations("", overrides?.citations); const { respId, created, events, prefixOutputItems, nextOutputIndex } = buildResponsePreamble( model, chunkSize, @@ -1115,28 +1119,105 @@ interface MessageBlockResult { msgItem: object; } -function buildAnnotations(content: string, citations?: ResponseOverrides["citations"]): object[] { - // Only URL citations have a wire shape here so far; file citations are skipped. - return (citations ?? []) - .flatMap((c) => (c.type === "file" ? [] : [c])) - .map((c) => ({ +/** + * Fixture `citations` as `output_text` annotations for each text in `texts` + * (one per message item), in fixture order: a URL citation becomes + * `url_citation`, a file citation (`type: "file"`) becomes `file_citation` + * (`file_id`, `filename`, `index`; see `ResponseOutputText.FileCitation` in + * the openai SDK). Offsets index the joined text of all texts, as on every + * other provider. A URL citation is a range and goes only to the text holding + * its `startIndex`. A file citation is a point after the cited text, so it + * goes to the last text starting before its `index`: an `index` equal to a + * text's end stays at the end of that text (not at 0 of the next text), and + * an `index` of 0 goes to the first text. Each citation is rebased onto its + * text since Responses offsets are per `output_text`, with `end_index` + * clamped to it. Anthropic and Gemini need no point rule: they have no file + * citation. Anthropic URL citations follow the same start-owns-range rule; + * Gemini does not, and splits a citation that spans several text parts into + * one grounding support per part (see geminiGroundingSupports). + * Missing offsets take the defaults `resolveCitations` applies: a URL + * citation starts at 0 and ends at the end of the joined text, so one without + * offsets goes on the first text and covers all of it; a file citation's + * `index` defaults to the end of the joined text (the last text). + */ +function blockAnnotations(texts: string[], citations?: ResponseOverrides["citations"]): object[][] { + const perText: object[][] = texts.map(() => []); + if (texts.length === 0) { + // Blocks with no text block: a turn with no text for `citations` to point at. + fixtureCitations("", citations); + return perText; + } + const starts: number[] = []; + let offset = 0; + for (const text of texts) { + starts.push(offset); + offset += text.length; + } + const joinedLength = offset; + // A range goes to the last text starting at or before its start. + const ownerOf = (start: number): number => { + let owner = 0; + for (let k = 1; k < starts.length; k++) if (starts[k] <= start) owner = k; + return owner; + }; + // A point goes to the last text starting before it, so a point on a + // boundary stays at the end of the text before the boundary. + const pointOwnerOf = (index: number): number => { + let owner = 0; + for (let k = 1; k < starts.length; k++) if (starts[k] < index) owner = k; + return owner; + }; + for (const c of fixtureCitations(texts.join(""), citations)) { + if (c.type === "file") { + const index = c.index ?? joinedLength; + const owner = pointOwnerOf(index); + perText[owner].push({ + type: "file_citation", + file_id: c.fileId, + filename: c.filename, + index: index - starts[owner], + }); + continue; + } + // The defaults `resolveCitations` applies on every other provider. + const start = c.startIndex ?? 0; + const end = c.endIndex ?? joinedLength; + const owner = ownerOf(start); + perText[owner].push({ type: "url_citation", url: c.url, title: c.title ?? c.url, - start_index: c.startIndex ?? 0, - end_index: c.endIndex ?? content.length, - })); + start_index: start - starts[owner], + end_index: Math.min(end - starts[owner], texts[owner].length), + }); + } + return perText; +} + +/** {@link blockAnnotations} for a turn with a single text. */ +function buildAnnotations(content: string, citations?: ResponseOverrides["citations"]): object[] { + return blockAnnotations([content], citations)[0]; +} + +/** The annotations of each text block in `blocks`, in block order. */ +function orderedBlockAnnotations( + blocks: ResponsesFixtureBlock[], + citations?: ResponseOverrides["citations"], +): object[][] { + return blockAnnotations( + blocks.flatMap((b) => (b.type === "text" ? [b.text] : [])), + citations, + ); } function buildMessageOutputEvents( content: string, chunkSize: number, outputIndex: number, - phase?: ResponseOverrides["phase"], - citations?: ResponseOverrides["citations"], + phase: ResponseOverrides["phase"] | undefined, + annotations: object[], ): MessageBlockResult { const msgId = itemId(); - const annotations = buildAnnotations(content, citations); const events: ResponsesSSEEvent[] = []; events.push({ @@ -1260,27 +1341,6 @@ function buildFunctionCallOutputEvents( return { events, item }; } -/** - * `text` cut into slices of at most `chunkSize` UTF-16 units, never between - * the two halves of a surrogate pair: a cut that would land inside a pair - * moves back before it, or, when that would leave the slice empty - * (`chunkSize` 1), forward past it. The slices rejoin to `text` exactly. - */ -function surrogateSafeChunks(text: string, chunkSize: number): string[] { - const chunks: string[] = []; - for (let i = 0; i < text.length; ) { - let end = Math.min(i + chunkSize, text.length); - const high = text.charCodeAt(end - 1); - const low = text.charCodeAt(end); - if (high >= 0xd800 && high <= 0xdbff && low >= 0xdc00 && low <= 0xdfff) { - end = end - 1 > i ? end - 1 : end + 1; - } - chunks.push(text.slice(i, end)); - i = end; - } - return chunks; -} - /** * Emit output_item.added → custom_tool_call_input.delta×N → * custom_tool_call_input.done → output_item.done for one `custom_tool_call` at @@ -1438,6 +1498,8 @@ function buildToolCallResponse( synthesizeSummarylessReasoning = false, ): object { assertResponsesToolCalls(toolCalls); + // A tool-only turn has no text for `citations` to point at. + fixtureCitations("", overrides?.citations); const output: object[] = []; if ( reasoning || @@ -1532,6 +1594,8 @@ export function contentWithToolCallsStreamEvents( // placed before a text block therefore yields a tool call item at // a LOWER output_index than the message — it leads the output array. let outputIndex = nextOutputIndex; + const textAnnotations = orderedBlockAnnotations(ordered, overrides?.citations); + let textPos = 0; for (const block of ordered) { if (block.type === "text") { const { events: msgEvents, msgItem } = buildMessageOutputEvents( @@ -1539,7 +1603,7 @@ export function contentWithToolCallsStreamEvents( chunkSize, outputIndex, overrides?.phase, - overrides?.citations, + textAnnotations[textPos++], ); events.push(...msgEvents); orderedOutputItems.push(msgItem); @@ -1565,7 +1629,7 @@ export function contentWithToolCallsStreamEvents( chunkSize, nextOutputIndex, overrides?.phase, - overrides?.citations, + buildAnnotations(content, overrides?.citations), ); events.push(...msgEvents); orderedOutputItems.push(msgItem); @@ -1619,8 +1683,8 @@ function buildToolCallOutputItem(tc: EmittableToolCall): object { function buildMessageOutputItem( content: string, - phase?: ResponseOverrides["phase"], - citations?: ResponseOverrides["citations"], + phase: ResponseOverrides["phase"] | undefined, + annotations: object[], ): object { return { type: "message", @@ -1628,9 +1692,7 @@ function buildMessageOutputItem( status: "completed", role: "assistant", ...(phase !== undefined ? { phase } : {}), - content: [ - { type: "output_text", text: content, annotations: buildAnnotations(content, citations) }, - ], + content: [{ type: "output_text", text: content, annotations }], }; } @@ -1673,9 +1735,13 @@ function buildContentWithToolCallsResponse( }); } } + const textAnnotations = orderedBlockAnnotations(ordered, overrides?.citations); + let textPos = 0; for (const block of ordered) { if (block.type === "text") { - output.push(buildMessageOutputItem(block.text, overrides?.phase, overrides?.citations)); + output.push( + buildMessageOutputItem(block.text, overrides?.phase, textAnnotations[textPos++]), + ); } else { output.push(buildToolCallOutputItem(block)); } @@ -2439,6 +2505,8 @@ export function buildResponsesMisbehavior( : []), ]; let callIndex = 0; + const textAnnotations = orderedBlockAnnotations(blocks, overrides.citations); + let textPos = 0; for (const block of blocks) { if (block.type === "text") { const message = buildMessageOutputEvents( @@ -2446,7 +2514,7 @@ export function buildResponsesMisbehavior( chunkSize, outputIndex, overrides.phase, - overrides.citations, + textAnnotations[textPos++], ); events.push(...message.events); output.push(message.msgItem); From f9b7e75e60bf1d4f80428a1b589b1f3d89ab6815 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 23 Oct 2026 15:14:12 -0700 Subject: [PATCH 3/6] feat: serve generated fixture files on Responses and reject them where no wire shape exists - Responses: image/* fixture files become image_generation_call output items, streamed through the image_generation_call lifecycle events before the message. A non-image file is a coded 500. Refusal, empty-response, content-filter and reasoning-only faults withhold the images with the rest of the output. - Chat Completions and Anthropic Messages have no projection for generated files, so a fixture with files is a coded 500 (aimock_unsupported_generated_file) before any byte is written, on the normal path and as a safety net on the fault path. - Gemini validates fixture files and keeps them as inlineData parts before the text under stop-length-mid-tool. --- src/__tests__/golden/harness.ts | 1 + src/__tests__/messages.test.ts | 104 +++++++ src/__tests__/misbehavior-gemini.test.ts | 57 ++++ src/__tests__/responses.test.ts | 346 +++++++++++++++++++++++ src/__tests__/server.test.ts | 126 +++++++++ src/gemini.ts | 16 +- src/messages.ts | 11 + src/responses.ts | 121 +++++++- src/server.ts | 23 +- 9 files changed, 792 insertions(+), 13 deletions(-) diff --git a/src/__tests__/golden/harness.ts b/src/__tests__/golden/harness.ts index 7bfca953..a181d925 100644 --- a/src/__tests__/golden/harness.ts +++ b/src/__tests__/golden/harness.ts @@ -202,6 +202,7 @@ export const ID_PREFIXES = [ "fc", "file", "ftjob", + "ig", "item", "modr", "msg", diff --git a/src/__tests__/messages.test.ts b/src/__tests__/messages.test.ts index cbefbe8c..89941006 100644 --- a/src/__tests__/messages.test.ts +++ b/src/__tests__/messages.test.ts @@ -1980,6 +1980,110 @@ describe("POST /v1/messages (citations and files)", () => { }, ); + it.each([false, true])("rejects fixture files with a coded 500 (stream: %s)", async (stream) => { + instance = await createServer([ + { + match: { userMessage: "draw" }, + response: { content: "Here.", files: [{ mimeType: "image/png", data: "iVBORw0KGgo=" }] }, + }, + ]); + const res = await post(`${instance.url}/v1/messages`, messagesBody("draw", stream)); + expect(res.status).toBe(500); + const body = JSON.parse(res.body); + expect(body.error.code).toBe("aimock_unsupported_generated_file"); + expect(body.error.message).toContain("Anthropic Messages"); + }); + + const generatedFile = [{ mimeType: "image/png", data: "iVBORw0KGgo=" }]; + + it.each<[string, Fixture["response"]]>([ + ["tool-only", { toolCalls: [{ name: "lookup", arguments: "{}" }], files: generatedFile }], + [ + "content + tool calls", + { + content: "Checking.", + toolCalls: [{ name: "lookup", arguments: "{}" }], + files: generatedFile, + }, + ], + ])("rejects fixture files on the %s path with a coded 500", async (_label, response) => { + instance = await createServer([{ match: { userMessage: "draw" }, response }]); + for (const stream of [false, true]) { + const res = await post(`${instance.url}/v1/messages`, messagesBody("draw", stream)); + expect(res.status).toBe(500); + const body = JSON.parse(res.body); + expect(body.error.code).toBe("aimock_unsupported_generated_file"); + expect(body.error.message).toContain("Anthropic Messages"); + } + }); + + type TurnBlock = { type: string; text?: string; thinking?: string }; + // What a turn looks like on the wire, folded the same way for both modes. + const messagesTurn = (body: string, stream: boolean) => { + if (!stream) { + const msg = JSON.parse(body) as { content: TurnBlock[]; stop_reason: string }; + return { + blocks: msg.content.map((b) => b.type), + text: msg.content.map((b) => b.text ?? b.thinking ?? "").join(""), + stop: msg.stop_reason, + }; + } + const events = body + .split("\n") + .filter((line) => line.startsWith("data: ")) + .map((line) => JSON.parse(line.slice(6))); + return { + blocks: events + .filter((e) => e.type === "content_block_start") + .map((e) => e.content_block.type), + text: events + .filter((e) => e.type === "content_block_delta") + .map((e) => e.delta.text ?? e.delta.thinking ?? "") + .join(""), + stop: events.find((e) => e.type === "message_delta").delta.stop_reason, + }; + }; + + it.each([ + { fault: { fault: "refusal" as const }, turn: { blocks: [], text: "", stop: "refusal" } }, + { + fault: { fault: "empty-response" as const }, + turn: { blocks: [], text: "", stop: "end_turn" }, + }, + { + fault: { fault: "reasoning-only" as const, reasoning: "Thinking only." }, + turn: { blocks: ["thinking"], text: "Thinking only.", stop: "max_tokens" }, + }, + ])( + "rejects fixture files under the $fault.fault misbehavior with a coded 500", + async ({ fault, turn }) => { + const misbehavior = { faults: [{ ...fault, rate: 1 }] }; + instance = await createServer( + [ + { + match: { userMessage: "draw" }, + response: { content: "Here.", files: generatedFile }, + misbehavior, + }, + // Control: the same fault on a servable fixture is applied on this wire. + { match: { userMessage: "plain" }, response: { content: "Here." }, misbehavior }, + ], + { enableMisbehavior: true }, + ); + for (const stream of [false, true]) { + const control = await post(`${instance.url}/v1/messages`, messagesBody("plain", stream)); + expect(control.status).toBe(200); + expect(messagesTurn(control.body, stream)).toEqual(turn); + + const res = await post(`${instance.url}/v1/messages`, messagesBody("draw", stream)); + expect(res.status).toBe(500); + const body = JSON.parse(res.body); + expect(body.error.code).toBe("aimock_unsupported_generated_file"); + expect(body.error.message).toContain("Anthropic Messages"); + } + }, + ); + it("skips file citations, which Anthropic has no URL-sourced form for (both modes)", async () => { instance = await createServer([ { diff --git a/src/__tests__/misbehavior-gemini.test.ts b/src/__tests__/misbehavior-gemini.test.ts index 1c7c9a65..ab074b50 100644 --- a/src/__tests__/misbehavior-gemini.test.ts +++ b/src/__tests__/misbehavior-gemini.test.ts @@ -533,3 +533,60 @@ test.each(routes.flatMap((route) => idCases.map((scenario) => ({ ...route, ...sc } }, ); + +// stop-length-mid-tool keeps the text before the cut, so it keeps the +// fixture's generated files too, as `inlineData` parts before the text part, +// and the grounding supports index that text part. +test.each([false, true])( + "stop-length-mid-tool keeps files before the text stream=%s", + async (stream) => { + const png = Buffer.from("png-bytes").toString("base64"); + const cited = "https://example.com/cited"; + await withFaultFixture( + { faults: [{ fault: "stop-length-mid-tool" }] }, + async ({ url }) => { + const res = await fetch( + `${url}/v1beta/models/gemini-2.5-flash:${stream ? "streamGenerateContent?alt=sse" : "generateContent"}`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "weather" }] }], + tools: [{ functionDeclarations: [{ name: "weather" }] }], + }), + signal: AbortSignal.timeout(5000), + }, + ); + const body = await res.text(); + expect(res.status, body).toBe(200); + const payloads: GenerateContentResponse[] = stream + ? body + .split("\n") + .filter((line) => line.startsWith("data: ")) + .map((line) => JSON.parse(line.slice("data: ".length))) + : [JSON.parse(body)]; + const candidates = payloads.flatMap((p) => p.candidates ?? []); + const parts = candidates.flatMap((c) => c.content?.parts ?? []); + expect(parts).toEqual([ + { inlineData: { mimeType: "image/png", data: png } }, + { text: "Sunny skies today." }, + ]); + const supports = candidates.flatMap((c) => c.groundingMetadata?.groundingSupports ?? []); + expect(supports).toHaveLength(1); + expect(supports[0].segment).toMatchObject({ partIndex: 1, text: "skies today" }); + expect(candidates.at(-1)?.finishReason).toBe("MAX_TOKENS"); + expect(body).toContain(cited); + }, + { + response: { + blocks: [ + { type: "text", text: "Sunny skies today." }, + { type: "toolCall", id: "call_weather", name: "weather", arguments: { city: "Paris" } }, + ], + files: [{ mimeType: "image/png", data: png }], + citations: [{ url: cited, title: "Forecast", startIndex: 6, endIndex: 17 }], + }, + }, + ); + }, +); diff --git a/src/__tests__/responses.test.ts b/src/__tests__/responses.test.ts index b2a2d793..c8696683 100644 --- a/src/__tests__/responses.test.ts +++ b/src/__tests__/responses.test.ts @@ -3126,3 +3126,349 @@ describe("POST /v1/responses (citations)", () => { expect(msg.content[0].annotations).toEqual([]); }); }); + +describe("POST /v1/responses (generated files)", () => { + const png = "iVBORw0KGgo="; + const fileFixture: Fixture = { + match: { userMessage: "draw" }, + response: { content: "Here is your image.", files: [{ mimeType: "image/png", data: png }] }, + }; + const body = (msg: string, stream: boolean) => ({ + model: "gpt-4", + input: [{ role: "user", content: msg }], + stream, + }); + + it("non-streaming output leads with a completed image_generation_call item", async () => { + instance = await createServer([fileFixture]); + const res = await post(`${instance.url}/v1/responses`, body("draw", false)); + const output = JSON.parse(res.body).output; + expect(output.map((o: { type: string }) => o.type)).toEqual([ + "image_generation_call", + "message", + ]); + expect(output[0]).toEqual({ + type: "image_generation_call", + id: expect.any(String), + status: "completed", + result: png, + }); + }); + + it("streaming emits the image_generation_call lifecycle before the message", async () => { + instance = await createServer([fileFixture]); + const res = await post(`${instance.url}/v1/responses`, body("draw", true)); + const events = parseResponsesSSEEvents(res.body); + const types = events.map((e) => e.type); + const added = types.indexOf("response.output_item.added"); + expect(types.slice(added, added + 5)).toEqual([ + "response.output_item.added", + "response.image_generation_call.in_progress", + "response.image_generation_call.generating", + "response.image_generation_call.completed", + "response.output_item.done", + ]); + expect(events[added].item).toMatchObject({ type: "image_generation_call", result: null }); + expect(events[added + 4].item).toMatchObject({ status: "completed", result: png }); + // One item: added, the three phase events and done all name the same id. + const igId = (events[added].item as { id: string }).id; + expect(igId).toEqual(expect.any(String)); + expect((events[added + 4].item as { id: string }).id).toBe(igId); + for (const phase of events.slice(added + 1, added + 4)) expect(phase.item_id).toBe(igId); + const message = events.find( + (e) => + e.type === "response.output_item.added" && (e.item as { type: string }).type === "message", + ); + expect(message!.output_index).toBe(1); + const completed = events.find((e) => e.type === "response.completed"); + const output = (completed!.response as { output: { type: string; id: string }[] }).output; + expect(output.map((o) => o.type)).toEqual(["image_generation_call", "message"]); + expect(output[0].id).toBe(igId); + }); + + it.each([ + ["tool-only", { toolCalls: [{ name: "save", arguments: "{}" }] }, ["function_call"]], + [ + "content+toolCalls", + { content: "Saving.", toolCalls: [{ name: "save", arguments: "{}" }] }, + ["message", "function_call"], + ], + [ + "blocks", + { + blocks: [ + { type: "toolCall" as const, name: "save", arguments: "{}" }, + { type: "text" as const, text: "Saved." }, + ], + }, + ["function_call", "message"], + ], + ])("%s turns lead with the image_generation_call item", async (_name, response, rest) => { + instance = await createServer([ + { + match: { userMessage: "draw-tools" }, + response: { ...response, files: [{ mimeType: "image/png", data: png }] }, + }, + ]); + for (const stream of [false, true]) { + const res = await post(`${instance.url}/v1/responses`, body("draw-tools", stream)); + const output = stream + ? ( + parseResponsesSSEEvents(res.body).find((e) => e.type === "response.completed")! + .response as { output: { type: string }[] } + ).output + : (JSON.parse(res.body).output as { type: string }[]); + expect(output.map((o) => o.type)).toEqual(["image_generation_call", ...rest]); + } + }); + + it.each([ + ["text", { content: "Here is your file." }], + ["tool-only", { toolCalls: [{ name: "save", arguments: "{}" }] }], + ["content+toolCalls", { content: "Saving.", toolCalls: [{ name: "save", arguments: "{}" }] }], + ["blocks", { blocks: [{ type: "text" as const, text: "Saved." }] }], + ])( + "%s turns reject a non-image file with a coded 500 naming its mimeType", + async (_name, response) => { + instance = await createServer([ + { + match: { userMessage: "pdf" }, + response: { + ...response, + files: [ + { mimeType: "image/png", data: png }, + { mimeType: "application/pdf", data: "JVBERi0=" }, + ], + }, + }, + ]); + for (const stream of [false, true]) { + const res = await post(`${instance.url}/v1/responses`, body("pdf", stream)); + expect(res.status).toBe(500); + expect(res.body).not.toContain('"type":"image_generation_call"'); + const err = JSON.parse(res.body).error; + expect(err.code).toBe("aimock_unsupported_generated_file"); + expect(err.message).toContain("application/pdf"); + } + }, + ); + + it("still serves image/* files on both paths", async () => { + instance = await createServer([ + { + match: { userMessage: "jpeg" }, + response: { content: "ok", files: [{ mimeType: "image/jpeg", data: png }] }, + }, + ]); + for (const stream of [false, true]) { + const res = await post(`${instance.url}/v1/responses`, body("jpeg", stream)); + expect(res.status).toBe(200); + expect(res.body).toContain("image_generation_call"); + } + }); + + it("an upper-case IMAGE/* mimeType is still an image on both paths", async () => { + instance = await createServer([ + { + match: { userMessage: "upper" }, + response: { content: "ok", files: [{ mimeType: "IMAGE/PNG", data: png }] }, + }, + ]); + for (const stream of [false, true]) { + const res = await post(`${instance.url}/v1/responses`, body("upper", stream)); + expect(res.status, res.body).toBe(200); + expect(res.body).toContain('"type":"image_generation_call"'); + } + }); + + it("two files stream as two image_generation_call items, in fixture order, before the message", async () => { + instance = await createServer([ + { + match: { userMessage: "draw-two" }, + response: { + content: "Two images.", + files: [ + { mimeType: "image/png", data: "AAAA" }, + { mimeType: "image/png", data: "BBBB" }, + ], + }, + }, + ]); + type Item = { type: string; id: string; result?: string | null }; + const nonStreamed = JSON.parse( + (await post(`${instance.url}/v1/responses`, body("draw-two", false))).body, + ).output as Item[]; + expect(nonStreamed.map((o) => [o.type, o.result])).toEqual([ + ["image_generation_call", "AAAA"], + ["image_generation_call", "BBBB"], + ["message", undefined], + ]); + expect(nonStreamed[0].id).not.toBe(nonStreamed[1].id); + + const events = parseResponsesSSEEvents( + (await post(`${instance.url}/v1/responses`, body("draw-two", true))).body, + ); + const done = events.filter((e) => e.type === "response.output_item.done"); + expect( + done.map((e) => [e.output_index, (e.item as Item).type, (e.item as Item).result]), + ).toEqual([ + [0, "image_generation_call", "AAAA"], + [1, "image_generation_call", "BBBB"], + [2, "message", undefined], + ]); + const completed = events.find((e) => e.type === "response.completed")!.response as { + output: Item[]; + }; + expect(completed.output.map((o) => o.id)).toEqual(done.map((e) => (e.item as Item).id)); + }); + + it("files with reasoning order the output reasoning, image_generation_call, message", async () => { + instance = await createServer([ + { + match: { userMessage: "draw-think" }, + response: { + content: "Here.", + reasoning: "Plan the image.", + files: [{ mimeType: "image/png", data: png }], + }, + }, + ]); + const want = ["reasoning", "image_generation_call", "message"]; + const nonStreamed = JSON.parse( + (await post(`${instance.url}/v1/responses`, body("draw-think", false))).body, + ).output as { type: string }[]; + expect(nonStreamed.map((o) => o.type)).toEqual(want); + + const events = parseResponsesSSEEvents( + (await post(`${instance.url}/v1/responses`, body("draw-think", true))).body, + ); + const done = events.filter((e) => e.type === "response.output_item.done"); + expect(done.map((e) => [e.output_index, (e.item as { type: string }).type])).toEqual( + want.map((type, i) => [i, type]), + ); + const completed = events.find((e) => e.type === "response.completed")!.response as { + output: { type: string }[]; + }; + expect(completed.output.map((o) => o.type)).toEqual(want); + }); + + it.each([false, true])( + "omitting files leaves the wire byte-identical to the no-files baseline (stream: %s)", + async (stream) => { + instance = await createServer([ + { match: { userMessage: "plain" }, response: { content: "hi" } }, + ]); + const res = await post(`${instance.url}/v1/responses`, body("plain", stream)); + // Ids and timestamps are random; everything else must match byte for byte. + const normalized = res.body + .replace(/"(id|item_id)":"[^"]*"/g, '"$1":""') + .replace(/"created_at":\d+/g, '"created_at":0'); + const envelope = '"id":"","object":"response","created_at":0,"model":"gpt-4"'; + const message = + '{"type":"message","id":"","status":"completed","role":"assistant","content":[{"type":"output_text","text":"hi","annotations":[]}]}'; + const usage = '"usage":{"input_tokens":0,"output_tokens":0,"total_tokens":0}'; + if (!stream) { + expect(normalized).toBe( + `{${envelope},"status":"completed","output":[${message}],${usage}}`, + ); + return; + } + const sse = (type: string, data: string) => + `event: ${type}\ndata: {"type":"${type}",${data}}\n\n`; + const part = (text: string) => + `"item_id":"","output_index":0,"content_index":0,"part":{"type":"output_text","text":"${text}","annotations":[]}`; + const inProgress = `"response":{${envelope},"status":"in_progress","output":[]}`; + expect(normalized).toBe( + [ + sse("response.created", inProgress), + sse("response.in_progress", inProgress), + sse( + "response.output_item.added", + '"output_index":0,"item":{"type":"message","id":"","status":"in_progress","role":"assistant","content":[]}', + ), + sse("response.content_part.added", part("")), + sse( + "response.output_text.delta", + '"item_id":"","output_index":0,"content_index":0,"delta":"hi"', + ), + sse( + "response.output_text.done", + '"item_id":"","output_index":0,"content_index":0,"text":"hi"', + ), + sse("response.content_part.done", part("hi")), + sse("response.output_item.done", `"output_index":0,"item":${message}`), + sse( + "response.completed", + `"response":{${envelope},"status":"completed","output":[${message}],${usage}}`, + ), + ].join(""), + ); + }, + ); +}); + +describe("POST /v1/responses (generated files under output-withholding faults)", () => { + const png = "iVBORw0KGgo="; + const files = [{ mimeType: "image/png", data: png }]; + const body = (stream: boolean) => ({ + model: "gpt-4", + input: [{ role: "user", content: "draw" }], + stream, + }); + const outputOf = (res: { body: string }, stream: boolean): { type: string }[] => + stream + ? ( + parseResponsesSSEEvents(res.body).find( + (e) => e.type === "response.completed" || e.type === "response.incomplete", + )!.response as { output: { type: string }[] } + ).output + : JSON.parse(res.body).output; + + it.each([ + ["empty-response", false, []], + ["empty-response", true, []], + ["refusal", false, ["message"]], + ["refusal", true, ["message"]], + ["content-filter", false, []], + ["content-filter", true, []], + ["reasoning-only", false, ["reasoning"]], + ["reasoning-only", true, ["reasoning"]], + ] as const)("%s emits no image_generation_call, stream=%s", async (fault, stream, types) => { + instance = await createServer( + [ + { + match: { userMessage: "draw" }, + response: { content: "Here.", files }, + misbehavior: fault, + }, + ], + { enableMisbehavior: true }, + ); + const res = await post(`${instance.url}/v1/responses`, body(stream)); + expect(res.status).toBe(200); + expect(res.body).not.toContain("image_generation_call"); + expect(outputOf(res, stream).map((o) => o.type)).toEqual(types); + }); + + it.each([false, true])( + "a fault that keeps the turn's output still serves the images, stream=%s", + async (stream) => { + instance = await createServer( + [ + { + match: { userMessage: "draw" }, + response: { toolCalls: [{ name: "save", arguments: '{"a":1}' }], files }, + misbehavior: "tool-unknown-name", + }, + ], + { enableMisbehavior: true }, + ); + const res = await post(`${instance.url}/v1/responses`, body(stream)); + expect(res.status).toBe(200); + const output = outputOf(res, stream) as { type: string; name?: string }[]; + expect(output.map((o) => o.type)).toEqual(["image_generation_call", "function_call"]); + // The fault was applied: the call is renamed to an undeclared tool. + expect(output[1].name).toBe("save_v2"); + }, + ); +}); diff --git a/src/__tests__/server.test.ts b/src/__tests__/server.test.ts index e4aa95cc..b3c7284f 100644 --- a/src/__tests__/server.test.ts +++ b/src/__tests__/server.test.ts @@ -2683,4 +2683,130 @@ describe("POST /v1/chat/completions (citations and files)", () => { ); }, ); + + it.each([false, true])("rejects fixture files with a coded 500 (stream: %s)", async (stream) => { + instance = await createServer([ + { + match: { userMessage: "draw" }, + response: { content: "Here.", files: [{ mimeType: "image/png", data: "iVBORw0KGgo=" }] }, + }, + ]); + const res = await post(`${instance.url}/v1/chat/completions`, chatBody("draw", stream)); + expect(res.status).toBe(500); + const body = JSON.parse(res.body); + expect(body.error.code).toBe("aimock_unsupported_generated_file"); + expect(body.error.message).toContain("OpenAI Chat Completions"); + }); + + const generatedFile = [{ mimeType: "image/png", data: "iVBORw0KGgo=" }]; + + it.each<[string, Fixture["response"]]>([ + ["tool-only", { toolCalls: [{ name: "lookup", arguments: "{}" }], files: generatedFile }], + [ + "content + tool calls", + { + content: "Checking.", + toolCalls: [{ name: "lookup", arguments: "{}" }], + files: generatedFile, + }, + ], + ])("rejects fixture files on the %s path with a coded 500", async (_label, response) => { + instance = await createServer([{ match: { userMessage: "draw" }, response }]); + for (const stream of [false, true]) { + const res = await post(`${instance.url}/v1/chat/completions`, chatBody("draw", stream)); + expect(res.status).toBe(500); + const body = JSON.parse(res.body); + expect(body.error.code).toBe("aimock_unsupported_generated_file"); + expect(body.error.message).toContain("OpenAI Chat Completions"); + } + }); + + // What a turn looks like on the wire, folded the same way for both modes. + const chatTurn = (body: string, stream: boolean) => { + if (!stream) { + const choice = JSON.parse(body).choices[0]; + return { + content: choice.message.content ?? "", + refusal: choice.message.refusal ?? "", + finish: choice.finish_reason, + }; + } + const chunks = body + .split("\n") + .filter((line) => line.startsWith("data: ") && line !== "data: [DONE]") + .map((line) => JSON.parse(line.slice(6)).choices[0]); + return { + content: chunks.map((c) => c.delta.content ?? "").join(""), + refusal: chunks.map((c) => c.delta.refusal ?? "").join(""), + finish: chunks.map((c) => c.finish_reason).filter((f) => f !== null)[0], + }; + }; + + it.each([ + { + fault: { fault: "refusal" as const }, + turn: { content: "", refusal: "I can't help with that.", finish: "stop" }, + }, + { + fault: { fault: "content-filter" as const }, + turn: { content: "", refusal: "", finish: "content_filter" }, + }, + { + fault: { fault: "reasoning-only" as const, reasoning: "Thinking only." }, + turn: { content: "", refusal: "", finish: "length" }, + }, + ])( + "rejects fixture files under the $fault.fault misbehavior with a coded 500", + async ({ fault, turn }) => { + const misbehavior = { faults: [{ ...fault, rate: 1 }] }; + instance = await createServer( + [ + { + match: { userMessage: "draw" }, + response: { content: "Here.", files: generatedFile }, + misbehavior, + }, + // Control: the same fault on a servable fixture is applied on this wire. + { match: { userMessage: "plain" }, response: { content: "Here." }, misbehavior }, + ], + { enableMisbehavior: true }, + ); + for (const stream of [false, true]) { + const control = await post( + `${instance.url}/v1/chat/completions`, + chatBody("plain", stream), + ); + expect(control.status).toBe(200); + expect(chatTurn(control.body, stream)).toEqual(turn); + + const res = await post(`${instance.url}/v1/chat/completions`, chatBody("draw", stream)); + expect(res.status).toBe(500); + const body = JSON.parse(res.body); + expect(body.error.code).toBe("aimock_unsupported_generated_file"); + expect(body.error.message).toContain("OpenAI Chat Completions"); + } + }, + ); + + it.each([false, true])( + "journals the rejected files fixture as a 500 with the error (stream: %s)", + async (stream) => { + const fixture: Fixture = { + match: { userMessage: "draw" }, + response: { content: "Here.", files: [{ mimeType: "image/png", data: "iVBORw0KGgo=" }] }, + }; + instance = await createServer([fixture]); + const res = await post(`${instance.url}/v1/chat/completions`, chatBody("draw", stream)); + const wireError = JSON.parse(res.body).error as { message: string; code: string }; + expect(wireError.code).toBe("aimock_unsupported_generated_file"); + const entries = instance.journal.getAll(); + expect(entries).toHaveLength(1); + expect(entries[0].path).toBe("/v1/chat/completions"); + expect(entries[0].response.status).toBe(500); + expect(entries[0].response.fixture).toBe(fixture); + // The journal keeps the message the client got, which names the error. + expect(entries[0].response.error).toBe(wireError.message); + expect(entries[0].response.error).toContain('fixture "files" (model-generated files)'); + }, + ); }); diff --git a/src/gemini.ts b/src/gemini.ts index 99a2c957..5914f115 100644 --- a/src/gemini.ts +++ b/src/gemini.ts @@ -45,6 +45,7 @@ import { strictNoMatchMessage, strictNoMatchLogLine, servedToolArgs, + fixtureFiles, type ResolvedCitation, surrogateSafeChunks, } from "./helpers.js"; @@ -466,7 +467,7 @@ function geminiTextChunkSpans( // Fixture `files` as Gemini `inlineData` parts. Every text and tool-call builder // emits these after any thought parts and before the text / functionCall parts. function geminiFileParts(overrides?: ResponseOverrides): GeminiPart[] { - return (overrides?.files ?? []).map((f) => ({ + return fixtureFiles(overrides?.files).map((f) => ({ inlineData: { mimeType: f.mimeType, data: f.data }, })); } @@ -1076,7 +1077,18 @@ function prepareGeminiMisbehavior( chunks = [{ candidates: [candidate] }]; } else if (fault === "stop-length-mid-tool") { // Object arguments cannot be partial; preserve only text preceding the cut. - candidate.content.parts = content ? [{ text: content }] : []; + // The fixture's files precede that text, as on the normal path. + const parts = geminiFileParts(overrides); + const textPartIndex = parts.length; + if (content) parts.push({ text: content }); + candidate.content.parts = parts; + // The planner drops `citations` whenever the fault changes the served + // text in any way, not only when the cut removes cited text. + if (content) + Object.assign( + candidate, + geminiGroundingField(content, overrides, geminiWholeTextPart(content, textPartIndex)), + ); candidate.finishReason = "MAX_TOKENS"; chunks = [{ candidates: [candidate] }]; } else if (fault === "empty-response") { diff --git a/src/messages.ts b/src/messages.ts index ebf3c860..0b85df92 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -32,6 +32,7 @@ import { generateToolUseId, extractOverrides, requireEmittedFunctionToolCalls, + rejectGeneratedFiles, resolveCitations, isTextResponse, isToolCallResponse, @@ -1908,6 +1909,10 @@ export async function handleMessages( const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); if (misbehavior.kind === "applied") { + // Safety net: the planner already skips a fixture with generated `files` + // (they have no Anthropic Messages projection), so it is never faulted. + // Reject here anyway, before any byte is written, if one gets through. + rejectGeneratedFiles(misbehavior.response, "Anthropic Messages"); const prepared = prepareClaudeMisbehavior(misbehavior, { request: completionReq, stream: claudeReq.stream === true, @@ -2012,6 +2017,8 @@ export async function handleMessages( defaults, testId, }); + // Generated `files` have no Anthropic Messages projection: reject before any byte is written. + rejectGeneratedFiles(response, "Anthropic Messages"); const functionToolCalls = requireEmittedFunctionToolCalls(response, "Anthropic Messages"); if (claudeReq.stream !== true) { const body = buildClaudeContentWithToolCallsResponse( @@ -2098,6 +2105,8 @@ export async function handleMessages( defaults, testId, }); + // Generated `files` have no Anthropic Messages projection: reject before any byte is written. + rejectGeneratedFiles(response, "Anthropic Messages"); if (claudeReq.stream !== true) { const body = buildClaudeTextResponse( response.content, @@ -2177,6 +2186,8 @@ export async function handleMessages( defaults, testId, }); + // Generated `files` have no Anthropic Messages projection: reject before any byte is written. + rejectGeneratedFiles(response, "Anthropic Messages"); const functionToolCalls = requireEmittedFunctionToolCalls(response, "Anthropic Messages"); if (claudeReq.stream !== true) { const body = buildClaudeToolCallResponse( diff --git a/src/responses.ts b/src/responses.ts index b9148310..08304ab5 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -16,6 +16,7 @@ import type { CustomToolDefinition, Fixture, FixtureBlock, + FixtureGeneratedFile, FixtureToolCall, HandlerDefaults, ResponseOverrides, @@ -51,6 +52,8 @@ import { servedToolCalls, toolCallFixtureBlock, withoutLegacyNamespaces, + UnsupportedGeneratedFileError, + fixtureFiles, fixtureCitations, surrogateSafeChunks, } from "./helpers.js"; @@ -1029,6 +1032,91 @@ function buildWebSearchStreamEvents( return events; } +/** + * A Responses `image_generation_call` output item for one fixture file (see + * `ResponseOutputItem.ImageGenerationCall` in the openai SDK): `result` holds + * the base64 bytes once completed and is null while in progress. + */ +function imageGenerationCallItem( + id: string, + file: FixtureGeneratedFile, + status: "in_progress" | "completed", +): object { + return { + type: "image_generation_call", + id, + status, + result: status === "completed" ? file.data : null, + }; +} + +/** + * A fixture file that is not `image/*` reached the Responses API, which + * carries generated files only as `image_generation_call` items. Surfaces as + * the same coded 500 (`aimock_unsupported_generated_file`), naming `mimeType`. + */ +class NonImageGeneratedFileError extends UnsupportedGeneratedFileError { + constructor(mimeType: string) { + super("the OpenAI Responses API"); + this.message = + `aimock: fixture "files" on the OpenAI Responses API carry only image/* files ` + + `(as image_generation_call items); got mimeType "${mimeType}"`; + } +} + +/** + * Throw {@link NonImageGeneratedFileError} for the first fixture file whose + * `mimeType` is not `image/*`. Called before any output item is built, so + * nothing reaches the client. + */ +function requireImageFiles(files: FixtureGeneratedFile[]): void { + const nonImage = files.find((f) => !f.mimeType.toLowerCase().startsWith("image/")); + if (nonImage) throw new NonImageGeneratedFileError(nonImage.mimeType); +} + +/** Fixture `files` as completed `image_generation_call` items (non-streaming). */ +function imageGenerationCallItems(files?: FixtureGeneratedFile[]): object[] { + const validated = fixtureFiles(files); + requireImageFiles(validated); + return validated.map((f) => imageGenerationCallItem(generateId("ig"), f, "completed")); +} + +/** + * Streamed form of {@link imageGenerationCallItems}: per file, the item's + * `output_item.added`, the `image_generation_call` `in_progress`, + * `generating` and `completed` events, then `output_item.done` carrying the + * completed item. + */ +function buildImageGenerationStreamEvents( + files: FixtureGeneratedFile[], + startOutputIndex: number, +): ResponsesSSEEvent[] { + requireImageFiles(files); + const events: ResponsesSSEEvent[] = []; + files.forEach((file, i) => { + const id = generateId("ig"); + const outputIndex = startOutputIndex + i; + events.push({ + type: "response.output_item.added", + output_index: outputIndex, + item: imageGenerationCallItem(id, file, "in_progress"), + }); + for (const phase of ["in_progress", "generating", "completed"]) { + events.push({ + type: `response.image_generation_call.${phase}`, + output_index: outputIndex, + item_id: id, + }); + } + events.push({ + type: "response.output_item.done", + output_index: outputIndex, + item: imageGenerationCallItem(id, file, "completed"), + }); + }); + return events; +} + // ─── Shared streaming helpers ──────────────────────────────────────────────── interface PreambleResult { @@ -1111,6 +1199,16 @@ function buildResponsePreamble( nextOutputIndex += webSearches.length; } + const files = fixtureFiles(overrides?.files); + if (files.length > 0) { + const fileEvents = buildImageGenerationStreamEvents(files, nextOutputIndex); + events.push(...fileEvents); + for (const e of fileEvents) { + if (e.type === "response.output_item.done") prefixOutputItems.push(e.item as object); + } + nextOutputIndex += files.length; + } + return { respId, created, events, prefixOutputItems, nextOutputIndex }; } @@ -1407,6 +1505,7 @@ function buildOutputPrefix( synthesizeSummarylessReasoning = false, phase?: ResponseOverrides["phase"], citations?: ResponseOverrides["citations"], + files?: ResponseOverrides["files"], ): object[] { const output: object[] = []; @@ -1433,6 +1532,8 @@ function buildOutputPrefix( } } + output.push(...imageGenerationCallItems(files)); + output.push({ type: "message", id: itemId(), @@ -1483,6 +1584,7 @@ export function buildTextResponse( synthesizeSummarylessReasoning, overrides?.phase, overrides?.citations, + overrides?.files, ), overrides, ); @@ -1522,6 +1624,7 @@ function buildToolCallResponse( }); } } + output.push(...imageGenerationCallItems(overrides?.files)); for (const tc of toolCalls) { output.push(buildToolCallOutputItem(tc)); } @@ -1735,6 +1838,7 @@ function buildContentWithToolCallsResponse( }); } } + output.push(...imageGenerationCallItems(overrides?.files)); const textAnnotations = orderedBlockAnnotations(ordered, overrides?.citations); let textPos = 0; for (const block of ordered) { @@ -1761,6 +1865,7 @@ function buildContentWithToolCallsResponse( synthesizeSummarylessReasoning, overrides?.phase, overrides?.citations, + overrides?.files, ); for (const tc of toolCalls) { output.push(buildToolCallOutputItem(tc)); @@ -2441,12 +2546,20 @@ export function buildResponsesMisbehavior( const overrides = extractOverrides(response); const reasoning = plan.reasoning ?? ("reasoning" in response ? response.reasoning : undefined); const webSearches = "webSearches" in response ? response.webSearches : undefined; + // Refusal, empty-response, content-filter and reasoning-only withhold the + // turn's normal output. Their candidates already drop text, tool calls and + // web searches, so the fixture's generated images are withheld too. + const outputWithheld = + plan.refusal !== undefined || + plan.summary.fault === "empty-response" || + plan.stop === "content_filter" || + plan.reasoning !== undefined; const { respId, created, events, prefixOutputItems, nextOutputIndex } = buildResponsePreamble( request.model, chunkSize, reasoning, webSearches, - overrides, + outputWithheld ? { ...overrides, files: undefined } : overrides, emitEncryptedReasoning, false, plan.reasoning !== undefined, @@ -2488,11 +2601,7 @@ export function buildResponsesMisbehavior( { type: "response.output_item.done", output_index: outputIndex, item }, ); output.push(item); - } else if ( - plan.summary.fault !== "empty-response" && - plan.stop !== "content_filter" && - plan.reasoning === undefined - ) { + } else if (!outputWithheld) { const combined = isCombinedFixtureResponse(response); // Custom tool calls and namespaces pass through a fault unchanged. const outcome = combined ? resolveServedBlockOutcome(response) : undefined; diff --git a/src/server.ts b/src/server.ts index d265a3a4..563fd465 100644 --- a/src/server.ts +++ b/src/server.ts @@ -98,6 +98,7 @@ import { isFixtureToolCallError, requireServedFunctionToolCalls, requireEmittedFunctionToolCalls, + rejectGeneratedFiles, resolveRequestId, markMintedRequestId, resolveResponse, @@ -2550,6 +2551,10 @@ async function handleCompletions( response: { status: 200, fixture }, }); recordOutcome(entry); + // Safety net: the planner already skips a fixture with generated `files` + // (they have no Chat Completions projection), so it is never faulted. + // Reject here anyway, before any byte is written, if one gets through. + rejectGeneratedFiles(response, wire); if (body.stream !== true) { const completion = appliedPlan.stop === "refusal" @@ -2656,6 +2661,8 @@ async function handleCompletions( response: { status: 200, fixture }, }); recordOutcome(journalEntry); + // Generated `files` have no Chat Completions projection: reject before any byte is written. + rejectGeneratedFiles(response, wire); // A Responses-only custom tool call (customToolCalls / responsesBlocks) is // rejected before any byte is written. It runs after journaling so the 500 // entry keeps the request body and the matched fixture (routeError amends @@ -2754,6 +2761,8 @@ async function handleCompletions( response: { status: 200, fixture }, }); recordOutcome(journalEntry); + // Generated `files` have no Chat Completions projection: reject before any byte is written. + rejectGeneratedFiles(response, wire); if (body.stream !== true) { await awaitNonStreamingTiming((effReasoning?.length ?? 0) + response.content.length); const completion = buildTextCompletion( @@ -2832,6 +2841,8 @@ async function handleCompletions( response: { status: 200, fixture }, }); recordOutcome(journalEntry); + // Generated `files` have no Chat Completions projection: reject before any byte is written. + rejectGeneratedFiles(response, wire); const toolCalls = requireServedFunctionToolCalls(response, wire); if (body.stream !== true) { const toolCallText = toolCalls.map((tc) => tc.name + tc.arguments).join(""); @@ -3291,10 +3302,12 @@ async function startServer( } /** - * Envelope for a fixture tool-call error (see isFixtureToolCallError: invalid - * JSON arguments on an object wire, a custom tool call on a non-Responses - * wire, or a malformed fixture tool call). It is the provider envelope aimock - * emits for that route, with the aimock `code` in it: Google's + * Envelope for a fixture error (see isFixtureToolCallError: invalid JSON + * arguments on an object wire, a custom tool call on a non-Responses wire, a + * malformed fixture tool call, generated `files` on a wire that cannot carry + * them, or a malformed fixture `citations` or `files` value). It is the + * provider envelope aimock emits for that route, with the aimock `code` in + * it: Google's * `google.rpc.ErrorInfo` detail on Gemini and Vertex AI, a string * `error.code` on Gemini Interactions and Anthropic Messages, a `reason` * member on Bedrock (a `code` member would replace `__type` as the AWS SDK's @@ -3421,7 +3434,7 @@ async function startServer( */ envelope?: (msg: string, status: number) => string; /** - * Provider-shaped body for a fixture tool-call error (always a 500), + * Provider-shaped body for a fixture error (always a 500), * carrying the aimock `code`. Defaults to the envelope the pathname * implies ({@link invalidToolArgumentsEnvelope}). */ From 61dd00e14cace8e65e43532ff21aafe973b87633 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 23 Oct 2026 15:14:48 -0700 Subject: [PATCH 4/6] fix: check fixture citations and files before misbehavior faults, and journal faults in order - The planner checks a fixture's citations and files on the original turn, before any fault rewrites it. A fixture the wire would answer with a coded 500 is skipped before fault selection, so it spends no times budget and no seed ordinal, and takes the normal path. - A fault that replaces, truncates or empties the reply text drops the fixture's citations; a fault that serves the text unchanged keeps them. - Anthropic Messages, Gemini, Responses and the Responses WebSocket journal an applied fault before building it, so a fixture error thrown while building leaves a 500 entry that keeps the fixture, and the fault is recorded only once the reply is built. - Chat Completions records the outcome only once the reply is built; a non-stream reply records it before its timing wait, and a client that hangs up during the wait is journaled as interrupted. --- src/__tests__/misbehavior-citations.test.ts | 197 ++++++++ .../misbehavior-fault-journal-order.test.ts | 431 ++++++++++++++++++ ...behavior-fixture-output-validation.test.ts | 351 ++++++++++++++ .../ws-responses-fault-journal-order.test.ts | 169 +++++++ src/gemini.ts | 17 +- src/messages.ts | 17 +- src/misbehavior.ts | 112 ++++- src/responses.ts | 17 +- src/server.ts | 42 +- src/ws-responses.ts | 43 +- 10 files changed, 1341 insertions(+), 55 deletions(-) create mode 100644 src/__tests__/misbehavior-citations.test.ts create mode 100644 src/__tests__/misbehavior-fault-journal-order.test.ts create mode 100644 src/__tests__/misbehavior-fixture-output-validation.test.ts create mode 100644 src/__tests__/ws-responses-fault-journal-order.test.ts diff --git a/src/__tests__/misbehavior-citations.test.ts b/src/__tests__/misbehavior-citations.test.ts new file mode 100644 index 00000000..f5c94636 --- /dev/null +++ b/src/__tests__/misbehavior-citations.test.ts @@ -0,0 +1,197 @@ +import { describe, expect, test } from "vitest"; +import type { FixtureFileResponse, MisbehaviorFault } from "../types.js"; +import { withFaultFixture } from "./helpers/misbehavior-server.js"; + +/** + * A fault that replaces, truncates or empties the reply text drops the + * fixture's `citations` for that turn: their offsets index text that is no + * longer served. A fault that keeps the text keeps its citations. + */ + +const CITED_URL = "https://example.com/cited"; + +// Joined text: "Sunny skies today. More." (24 UTF-16 units). The citation +// covers the trailing text block, which stop-length-mid-tool cuts away. +const citedResponse: FixtureFileResponse = { + blocks: [ + { type: "text", text: "Sunny skies today." }, + { type: "toolCall", id: "call_weather", name: "weather", arguments: { city: "Paris" } }, + { type: "text", text: " More." }, + ], + citations: [{ url: CITED_URL, title: "Forecast", startIndex: 19, endIndex: 24 }], +}; + +type Wire = "openai-chat" | "anthropic" | "gemini"; + +async function send(url: string, wire: Wire, stream: boolean): Promise { + switch (wire) { + case "openai-chat": + return fetch(`${url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + stream, + messages: [{ role: "user", content: "weather" }], + }), + }); + case "anthropic": + return fetch(`${url}/v1/messages`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + model: "claude-sonnet-4-20250514", + max_tokens: 100, + stream, + messages: [{ role: "user", content: "weather" }], + }), + }); + case "gemini": + return fetch( + `${url}/v1beta/models/gemini-2.0-flash:${stream ? "streamGenerateContent?alt=sse" : "generateContent"}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + // Declared tools: without them Gemini answers tool-unknown-name + // with a text-free UNEXPECTED_TOOL_CALL candidate. + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "weather" }] }], + tools: [{ functionDeclarations: [{ name: "weather" }] }], + }), + }, + ); + } +} + +const textAltering: Record = { + "openai-chat": [ + { fault: "empty-response" }, + { fault: "refusal" }, + { fault: "content-filter" }, + { fault: "reasoning-only", reasoning: "Hm." }, + { fault: "stop-length-mid-tool" }, + ], + anthropic: [ + { fault: "empty-response" }, + { fault: "refusal" }, + { fault: "reasoning-only", reasoning: "Hm." }, + { fault: "stop-length-mid-tool" }, + ], + gemini: [ + { fault: "empty-response" }, + { fault: "content-filter" }, + { fault: "reasoning-only", reasoning: "Hm." }, + { fault: "stop-length-mid-tool" }, + ], +}; + +const cells = (Object.keys(textAltering) as Wire[]).flatMap((wire) => + textAltering[wire].flatMap((fault) => + [false, true].map((stream) => ({ wire, fault, stream, name: fault.fault })), + ), +); + +describe("misbehavior faults that alter the reply text drop its citations", () => { + test.each(cells)("$wire $name stream=$stream", async ({ wire, fault, stream }) => { + await withFaultFixture( + { faults: [fault] }, + async ({ url }) => { + const res = await send(url, wire, stream); + const body = await res.text(); + expect(body).not.toContain("aimock_invalid_fixture_citation"); + expect(res.status).toBe(200); + expect(body).not.toContain(CITED_URL); + }, + { response: citedResponse }, + ); + }); +}); + +/** + * Citations on a turn with no text are a fixture error (a coded 500). A fault + * that leaves the turn with no text drops the citations first, so the fault + * reply is served, not that 500. + */ +describe("misbehavior faults that empty the reply text do not answer the no-text 500", () => { + // The only text block follows the tool call, so stop-length-mid-tool's cut + // leaves no text at all. + const trailingTextOnly: FixtureFileResponse = { + blocks: [ + { type: "toolCall", id: "call_weather", name: "weather", arguments: { city: "Paris" } }, + { type: "text", text: "Sunny." }, + ], + citations: [{ url: CITED_URL, title: "Forecast", startIndex: 0, endIndex: 6 }], + }; + const emptying = (["openai-chat", "anthropic", "gemini"] as const).flatMap((wire) => + [ + { fault: { fault: "empty-response" } as MisbehaviorFault, response: citedResponse }, + { fault: { fault: "stop-length-mid-tool" } as MisbehaviorFault, response: trailingTextOnly }, + ].flatMap(({ fault, response }) => + [false, true].map((stream) => ({ wire, fault, response, stream, name: fault.fault })), + ), + ); + test.each(emptying)("$wire $name stream=$stream", async ({ wire, fault, response, stream }) => { + await withFaultFixture( + { faults: [fault] }, + async ({ url, mock }) => { + const res = await send(url, wire, stream); + const body = await res.text(); + expect(body).not.toContain("this turn has no text"); + expect(body).not.toContain("aimock_invalid_fixture_citation"); + expect(res.status, body).toBe(200); + expect(body).not.toContain(CITED_URL); + expect(mock.getRequests().at(-1)?.response.status).toBe(200); + }, + { response }, + ); + }); +}); + +describe("misbehavior faults that keep the reply text keep its citations", () => { + const controls = (["openai-chat", "anthropic", "gemini"] as const).flatMap((wire) => + [false, true].map((stream) => ({ wire, stream })), + ); + test.each(controls)("$wire tool-unknown-name stream=$stream", async ({ wire, stream }) => { + await withFaultFixture( + { faults: [{ fault: "tool-unknown-name" }] }, + async ({ url }) => { + const res = await send(url, wire, stream); + const body = await res.text(); + expect(res.status).toBe(200); + expect(body).toContain("weather_v2"); + expect(body).toContain(CITED_URL); + }, + { response: citedResponse }, + ); + }); + + // Without a trailing text block, the cut keeps every text block whole. + test.each(controls)( + "$wire stop-length-mid-tool with untouched text stream=$stream", + async ({ wire, stream }) => { + await withFaultFixture( + { faults: [{ fault: "stop-length-mid-tool" }] }, + async ({ url }) => { + const res = await send(url, wire, stream); + const body = await res.text(); + expect(res.status).toBe(200); + expect(body).toContain(CITED_URL); + }, + { + response: { + blocks: [ + { type: "text", text: "Sunny skies today." }, + { + type: "toolCall", + id: "call_weather", + name: "weather", + arguments: { city: "Paris" }, + }, + ], + citations: [{ url: CITED_URL, title: "Forecast", startIndex: 6, endIndex: 18 }], + }, + }, + ); + }, + ); +}); diff --git a/src/__tests__/misbehavior-fault-journal-order.test.ts b/src/__tests__/misbehavior-fault-journal-order.test.ts new file mode 100644 index 00000000..6884bc98 --- /dev/null +++ b/src/__tests__/misbehavior-fault-journal-order.test.ts @@ -0,0 +1,431 @@ +/** + * A misbehavior fault applied to a fixture that cannot be served (generated + * `files` on a wire with no projection for them, or malformed `citations`) + * answers a coded 500. The journal must say what happened: the entry carries + * the matched fixture and status 500, and the fault is NOT recorded as + * applied, in the entry or in `aimock_misbehavior_total`. The request was + * never served with the fault, so counting it would lie. + * + * Real surface: a real aimock server over HTTP, reading its own journal and + * /metrics. + */ +import http from "node:http"; +import { afterEach, describe, expect, it } from "vitest"; +import type { ServerInstance } from "../server.js"; +import type { Fixture, FixtureResponse, MisbehaviorFault, TextResponse, WireId } from "../types.js"; +import { createServer } from "./helpers/misbehavior-enabled.js"; + +const png = { mimeType: "image/png", data: "iVBORw0KGgo=" }; +const badCitations = [{ url: "https://example.com", startIndex: 0, endIndex: 999 }]; + +interface Route { + name: string; + path: string; + body: Record; + code: string; + /** The `wire` label the misbehavior counter carries for this route. */ + wire: WireId; +} + +const chatBody = { model: "gpt-4o", messages: [{ role: "user", content: "go" }] }; +const messagesBody = { + model: "claude-sonnet-4-5", + max_tokens: 64, + messages: [{ role: "user", content: "go" }], +}; +// Gemini serves a renamed tool through its ordinary builders only when the +// request declares tools. +const geminiBody = { + contents: [{ role: "user", parts: [{ text: "go" }] }], + tools: [{ functionDeclarations: [{ name: "lookup", parameters: { type: "object" } }] }], +}; +const responsesBody = { model: "gpt-4o", input: "go" }; + +interface Cell { + wire: string; + routes: Route[]; + /** A servable fixture response. */ + response: FixtureResponse; + /** What makes it unservable on this wire. */ + unservable: Pick; + fault: MisbehaviorFault; +} + +function both( + name: string, + path: string, + body: Record, + code: string, + wire: WireId, +): Route[] { + const streamPath = path.includes(":generateContent") + ? path.replace(":generateContent", ":streamGenerateContent?alt=sse") + : path; + return [ + { name, path, body, code, wire }, + { + name: `${name} stream`, + path: streamPath, + body: streamPath === path ? { ...body, stream: true } : body, + code, + wire, + }, + ]; +} + +const FILES = "aimock_unsupported_generated_file"; +const CITE = "aimock_invalid_fixture_citation"; +const chatFiles = both("Chat", "/v1/chat/completions", chatBody, FILES, "openai-chat"); +const chatCite = both("Chat", "/v1/chat/completions", chatBody, CITE, "openai-chat"); +const messagesFiles = both("Anthropic", "/v1/messages", messagesBody, FILES, "anthropic"); +const messagesCite = both("Anthropic", "/v1/messages", messagesBody, CITE, "anthropic"); +const geminiCite = both( + "Gemini", + "/v1beta/models/gemini-2.0-flash:generateContent", + geminiBody, + CITE, + "gemini", +); +const responsesCite = both("Responses", "/v1/responses", responsesBody, CITE, "openai-responses"); + +const text = "Hello world."; +const toolCalls = [{ name: "lookup", arguments: '{"q":"x"}' }]; + +const cells: Cell[] = [ + // Chat fault terminals (refusal / content-filter / reasoning-only). + { + wire: "chat refusal + files", + routes: chatFiles, + response: { content: text }, + unservable: { files: [png] }, + fault: { fault: "refusal" }, + }, + { + wire: "chat reasoning-only + files", + routes: chatFiles, + response: { content: text }, + unservable: { files: [png] }, + fault: { fault: "reasoning-only", reasoning: "Thinking." }, + }, + // Chat ordinary builders under an applied fault. + { + wire: "chat tool-args-invalid-json + files (tool calls)", + routes: chatFiles, + response: { toolCalls }, + unservable: { files: [png] }, + fault: { fault: "tool-args-invalid-json" }, + }, + { + wire: "chat tool-args-invalid-json + files (content + tool calls)", + routes: chatFiles, + response: { content: text, toolCalls }, + unservable: { files: [png] }, + fault: { fault: "tool-args-invalid-json" }, + }, + { + wire: "chat tool-unknown-name + bad citations (content + tool calls)", + routes: chatCite, + response: { content: text, toolCalls }, + unservable: { citations: badCitations }, + fault: { fault: "tool-unknown-name" }, + }, + // Anthropic applied fault. + { + wire: "anthropic refusal + files", + routes: messagesFiles, + response: { content: text }, + unservable: { files: [png] }, + fault: { fault: "refusal" }, + }, + { + wire: "anthropic tool-unknown-name + bad citations", + routes: messagesCite, + response: { content: text, toolCalls }, + unservable: { citations: badCitations }, + fault: { fault: "tool-unknown-name" }, + }, + // Gemini and Responses applied faults. + { + wire: "gemini tool-unknown-name + bad citations", + routes: geminiCite, + response: { content: text, toolCalls }, + unservable: { citations: badCitations }, + fault: { fault: "tool-unknown-name" }, + }, + { + wire: "responses tool-unknown-name + bad citations", + routes: responsesCite, + response: { content: text, toolCalls }, + unservable: { citations: badCitations }, + fault: { fault: "tool-unknown-name" }, + }, +]; + +let instance: ServerInstance | undefined; + +afterEach(async () => { + if (instance) { + await new Promise((resolve) => instance!.server.close(() => resolve())); + instance = undefined; + } +}); + +function misbehaviorLines(metrics: string): string[] { + return metrics.split("\n").filter((line) => line.startsWith("aimock_misbehavior_total{")); +} + +describe("a fault on a fixture that cannot be served is journaled as a 500, not an applied fault", () => { + for (const cell of cells) { + for (const route of cell.routes) { + it(`${cell.wire} — ${route.name}`, async () => { + const fixture: Fixture = { + match: {}, + response: { ...cell.response, ...cell.unservable }, + misbehavior: { faults: [{ ...cell.fault, rate: 1 }] }, + }; + instance = await createServer([fixture], { metrics: true }); + const res = await fetch(`${instance.url}${route.path}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(route.body), + }); + const raw = await res.text(); + expect(res.status).toBe(500); + expect(raw).toContain(route.code); + + const entries = instance.journal.getAll(); + expect(entries).toHaveLength(1); + expect(entries[0].response.status).toBe(500); + expect(entries[0].response.fixture).toBe(fixture); + expect(entries[0].response.error).toContain(route.code === FILES ? "files" : "citations"); + // Exact recorded state: the entry carries no misbehavior record at all, + // so nothing claims the fault was applied. + expect(entries[0].response.misbehavior).toBeUndefined(); + + // And the counter has no series at all, applied or otherwise. + const metrics = await (await fetch(`${instance.url}/metrics`)).text(); + expect(misbehaviorLines(metrics)).toEqual([]); + }); + } + } +}); + +describe("the same fault on a servable fixture is still recorded as applied", () => { + for (const cell of cells) { + for (const route of cell.routes) { + it(`${cell.wire} — ${route.name}`, async () => { + const fixture: Fixture = { + match: {}, + response: cell.response, + misbehavior: { faults: [{ ...cell.fault, rate: 1 }] }, + }; + instance = await createServer([fixture], { metrics: true }); + const res = await fetch(`${instance.url}${route.path}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(route.body), + }); + await res.text(); + expect(res.status).toBe(200); + + const entries = instance.journal.getAll(); + expect(entries).toHaveLength(1); + expect(entries[0].response.status).toBe(200); + expect(entries[0].response.fixture).toBe(fixture); + const record = entries[0].response.misbehavior; + expect(record?.applied).toBe(true); + expect(record?.fault).toBe(cell.fault.fault); + expect(record?.evaluations.map((row) => [row.fault, row.outcome])).toEqual([ + [cell.fault.fault, "applied"], + ]); + + // The counter holds exactly one series for this fault, with value 1. + const metrics = await (await fetch(`${instance.url}/metrics`)).text(); + expect(misbehaviorLines(metrics)).toEqual([ + `aimock_misbehavior_total{fault="${cell.fault.fault}",outcome="applied",wire="${route.wire}"} 1`, + ]); + }); + } + } +}); + +/** The request body with its user text ("go") replaced, so a fixture can match it. */ +function withUserText(body: Record, userText: string): Record { + return JSON.parse(JSON.stringify(body).replace('"go"', JSON.stringify(userText))) as Record< + string, + unknown + >; +} + +async function post(url: string, body: Record): Promise { + return fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); +} + +describe("a fixture that cannot be served spends no fault budget and no seed ordinal", () => { + for (const cell of cells) { + for (const route of cell.routes) { + it(`${cell.wire} — ${route.name}`, async () => { + // A server-wide fault with `times: 1` is shared by every fixture, so + // the unservable request and the servable one draw on the same budget. + const fault = { ...cell.fault, rate: 1, times: 1 } as MisbehaviorFault; + const fixtures = (): Fixture[] => [ + { match: { userMessage: "bad" }, response: { ...cell.response, ...cell.unservable } }, + { match: { userMessage: "good" }, response: cell.response }, + ]; + + // Control: the servable request alone, on a fresh server. + instance = await createServer(fixtures(), { misbehavior: { faults: [fault] } }); + const control = await post( + `${instance.url}${route.path}`, + withUserText(route.body, "good"), + ); + await control.text(); + expect(control.status).toBe(200); + const controlRow = instance.journal.getAll()[0].response.misbehavior?.evaluations[0]; + expect(controlRow?.outcome).toBe("applied"); + await new Promise((resolve) => instance!.server.close(() => resolve())); + + instance = await createServer(fixtures(), { misbehavior: { faults: [fault] } }); + const bad = await post(`${instance.url}${route.path}`, withUserText(route.body, "bad")); + expect(await bad.text()).toContain(route.code); + expect(bad.status).toBe(500); + const good = await post(`${instance.url}${route.path}`, withUserText(route.body, "good")); + await good.text(); + expect(good.status).toBe(200); + + const entries = instance.journal.getAll(); + expect(entries).toHaveLength(2); + const rows = entries[1].response.misbehavior?.evaluations ?? []; + expect(rows.map((row) => row.outcome)).toEqual(["applied"]); + // The same ordinal as the request alone: the rejected one drew none. + expect(rows[0].ordinal).toBe(controlRow?.ordinal); + }); + } + } +}); + +interface PendingRequest { + /** Resolves with the status, or 0 when the request was aborted. */ + done: Promise; + abort: () => void; +} + +function startRequest(url: string, body: Record): PendingRequest { + const payload = JSON.stringify(body); + let req: http.ClientRequest | undefined; + const done = new Promise((resolve) => { + req = http.request( + url, + { + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(payload), + }, + }, + (res) => { + res.resume(); + res.on("end", () => resolve(res.statusCode ?? 0)); + }, + ); + req.on("error", () => resolve(0)); + req.end(payload); + }); + return { done, abort: () => req?.destroy() }; +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Poll the server journal until `ready` holds for its entries, or 5s pass. A + * fixed sleep assumes the request reached the server in time, which a loaded + * machine does not promise. + */ +async function pollJournal( + read: () => Promise | T[], + ready: (entries: T[]) => boolean, +): Promise { + const deadline = Date.now() + 5_000; + let entries = await read(); + while (!ready(entries) && Date.now() < deadline) { + await sleep(10); + entries = await read(); + } + return entries; +} + +interface TimedCell { + name: string; + response: FixtureResponse; + fault: MisbehaviorFault; +} + +// One cell per non-stream Chat branch: fault terminal, text, tool calls, content + tool calls. +const timedCells: TimedCell[] = [ + { name: "refusal terminal", response: { content: text }, fault: { fault: "refusal" } }, + { name: "text", response: { content: text }, fault: { fault: "empty-response" } }, + { name: "tool calls", response: { toolCalls }, fault: { fault: "tool-unknown-name" } }, + { + name: "content + tool calls", + response: { content: text, toolCalls }, + fault: { fault: "tool-unknown-name" }, + }, +]; + +const WAIT_MS = 600; + +describe("a non-stream Chat reply records its fault before the timing wait", () => { + for (const cell of timedCells) { + const fixture = (): Fixture => ({ + match: {}, + response: cell.response, + misbehavior: { faults: [{ ...cell.fault, rate: 1 } as MisbehaviorFault] }, + streamingProfile: { ttft: WAIT_MS, tps: 1_000_000 }, + nonStreamingTiming: true, + }); + + it(`${cell.name}: the journal shows the applied fault while the reply waits`, async () => { + instance = await createServer([fixture()]); + const pending = startRequest(`${instance.url}/v1/chat/completions`, chatBody); + const url = `${instance.url}/__aimock/journal`; + const journal = await pollJournal( + async () => + (await (await fetch(url)).json()) as { + response: { status: number; misbehavior?: { applied: boolean } }; + }[], + (entries) => entries.length > 0, + ); + expect(journal).toHaveLength(1); + expect(journal[0].response.status).toBe(200); + expect(journal[0].response.misbehavior?.applied).toBe(true); + expect(await pending.done).toBe(200); + }); + + it(`${cell.name}: a client that hangs up during the wait is journaled as not served`, async () => { + instance = await createServer([fixture()]); + const pending = startRequest(`${instance.url}/v1/chat/completions`, chatBody); + const journal = instance.journal; + // Hang up only once the server has the request, during its wait. + await pollJournal( + () => journal.getAll(), + (entries) => entries.length > 0, + ); + pending.abort(); + expect(await pending.done).toBe(0); + const entries = await pollJournal( + () => journal.getAll(), + (all) => all[0]?.response.interrupted === true, + ); + expect(entries).toHaveLength(1); + expect(entries[0].response.status).toBe(0); + expect(entries[0].response.interrupted).toBe(true); + expect(entries[0].response.interruptReason).toBe("client aborted"); + // The fault was prepared for this request; the entry still says which. + expect(entries[0].response.misbehavior?.applied).toBe(true); + }); + } +}); diff --git a/src/__tests__/misbehavior-fixture-output-validation.test.ts b/src/__tests__/misbehavior-fixture-output-validation.test.ts new file mode 100644 index 00000000..75587600 --- /dev/null +++ b/src/__tests__/misbehavior-fixture-output-validation.test.ts @@ -0,0 +1,351 @@ +/** + * A fixture's `citations` and `files` are checked for shape and wire support + * before any misbehavior fault rewrites the turn, the same on every wire and + * fault. A malformed value answers its coded 500 with or without a fault; a + * valid value under a fault that alters the reply text is served without the + * citations (they index text that is no longer sent). + * + * Real surface: a real aimock server over HTTP, reading its own journal. + */ +import { afterEach, describe, expect, it } from "vitest"; +import type { ServerInstance } from "../server.js"; +import type { + Fixture, + FixtureResponse, + MisbehaviorFault, + HandlerDefaults, + MockServerOptions, +} from "../types.js"; +import { fixtureCitationsProblem } from "../helpers.js"; +import { + planMisbehavior, + setFixtureMisbehaviorPosition, + supportsMisbehavior, +} from "../misbehavior.js"; +import { Journal } from "../journal.js"; +import { Logger } from "../logger.js"; +import { createServer } from "./helpers/misbehavior-enabled.js"; + +const CITED_URL = "https://example.com/cited"; +const text = "Sunny today."; +const toolCalls = [{ id: "call_1", name: "lookup", arguments: '{"q":"paris"}' }]; + +type WireName = "openai-chat" | "anthropic" | "gemini" | "openai-responses"; + +interface Route { + path: string; + body: Record; +} + +function route(wire: WireName, stream: boolean): Route { + switch (wire) { + case "openai-chat": + return { + path: "/v1/chat/completions", + body: { model: "gpt-4o", stream, messages: [{ role: "user", content: "go" }] }, + }; + case "anthropic": + return { + path: "/v1/messages", + body: { + model: "claude-sonnet-4-5", + max_tokens: 64, + stream, + messages: [{ role: "user", content: "go" }], + }, + }; + case "gemini": + return { + path: `/v1beta/models/gemini-2.0-flash:${stream ? "streamGenerateContent?alt=sse" : "generateContent"}`, + body: { + contents: [{ role: "user", parts: [{ text: "go" }] }], + tools: [{ functionDeclarations: [{ name: "lookup", parameters: { type: "object" } }] }], + }, + }; + case "openai-responses": + return { path: "/v1/responses", body: { model: "gpt-4o", input: "go", stream } }; + } +} + +const WIRES: WireName[] = ["openai-chat", "anthropic", "gemini", "openai-responses"]; +const FAULTS: MisbehaviorFault[] = [ + { fault: "refusal" }, + { fault: "empty-response" }, + { fault: "content-filter" }, + { fault: "reasoning-only", reasoning: "Hm." }, + { fault: "stop-length-mid-tool" }, +]; +/** Faults that replace or empty the reply text (stop-length keeps `content` here). */ +const TEXT_REMOVING = new Set(["refusal", "empty-response", "content-filter", "reasoning-only"]); + +let instance: ServerInstance | undefined; + +afterEach(async () => { + if (instance) { + await new Promise((resolve) => instance!.server.close(() => resolve())); + instance = undefined; + } +}); + +async function serve( + wire: WireName, + stream: boolean, + response: FixtureResponse, + fault: MisbehaviorFault | undefined, + options: MockServerOptions = {}, +): Promise<{ status: number; body: string; fixture: Fixture }> { + const fixture: Fixture = { + match: {}, + response, + ...(fault ? { misbehavior: { faults: [{ ...fault, rate: 1 }] } } : {}), + }; + instance = await createServer([fixture], options); + const { path, body } = route(wire, stream); + const res = await fetch(`${instance.url}${path}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + return { status: res.status, body: await res.text(), fixture }; +} + +/** Malformed fixtures: programmatic fixtures skip load validation. */ +const malformed: { name: string; response: FixtureResponse; code: string }[] = [ + { + name: "malformed citations", + response: { + content: text, + toolCalls, + citations: [{ title: "no url" }], + } as unknown as FixtureResponse, + code: "aimock_invalid_fixture_citation", + }, + { + name: "malformed files", + response: { + content: text, + toolCalls, + files: [{ mimeType: "image/png" }], + } as unknown as FixtureResponse, + code: "aimock_invalid_fixture_file", + }, +]; + +const malformedCells = malformed.flatMap((m) => + FAULTS.flatMap((fault) => + WIRES.flatMap((wire) => + [false, true].map((stream) => ({ ...m, fault, wire, stream, faultName: fault.fault })), + ), + ), +); + +describe("a malformed fixture output is its coded 500 under every fault on every wire", () => { + it.each(malformedCells)( + "$name × $faultName × $wire stream=$stream", + async ({ response, code, fault, wire, stream }) => { + const { status, body, fixture } = await serve(wire, stream, response, fault); + expect(status, body).toBe(500); + expect(body).toContain(code); + const entries = instance!.journal.getAll(); + expect(entries).toHaveLength(1); + expect(entries[0].response.status).toBe(500); + expect(entries[0].response.fixture).toBe(fixture); + expect( + (entries[0].response.misbehavior?.evaluations ?? []).filter( + (row) => row.outcome === "applied", + ), + ).toEqual([]); + }, + ); + + // The same fixture with no fault: the baseline every fault cell must match. + it.each( + malformed.flatMap((m) => + WIRES.flatMap((wire) => [false, true].map((stream) => ({ ...m, wire, stream }))), + ), + )("$name × no fault × $wire stream=$stream", async ({ response, code, wire, stream }) => { + const { status, body } = await serve(wire, stream, response, undefined); + expect(status, body).toBe(500); + expect(body).toContain(code); + }); +}); + +const validCited: FixtureResponse = { + content: text, + toolCalls, + citations: [{ url: CITED_URL, title: "Forecast", startIndex: 0, endIndex: 5 }], +}; + +const validCells = FAULTS.flatMap((fault) => + WIRES.flatMap((wire) => + [false, true] + .filter((stream) => supportsMisbehavior(wire, fault, stream)) + .map((stream) => ({ fault, wire, stream, faultName: fault.fault })), + ), +); + +describe("valid citations under a text-altering fault are served without the citations", () => { + it.each(validCells)("$faultName × $wire stream=$stream", async ({ fault, wire, stream }) => { + const { status, body } = await serve(wire, stream, validCited, fault); + expect(status, body).toBe(200); + expect(body).not.toContain("aimock_invalid_fixture"); + if (TEXT_REMOVING.has(fault.fault)) expect(body).not.toContain(CITED_URL); + const entries = instance!.journal.getAll(); + expect((entries[0].response.misbehavior?.evaluations ?? []).map((r) => r.outcome)).toEqual([ + "applied", + ]); + }); +}); + +describe("citations on a tool-only turn are a 500 under stop-length-mid-tool on every wire", () => { + const toolOnly = { + toolCalls, + citations: [{ url: CITED_URL }], + } as FixtureResponse; + it.each(WIRES.flatMap((wire) => [false, true].map((stream) => ({ wire, stream }))))( + "$wire stream=$stream", + async ({ wire, stream }) => { + const { status, body } = await serve(wire, stream, toolOnly, { + fault: "stop-length-mid-tool", + }); + expect(status, body).toBe(500); + expect(body).toContain("aimock_invalid_fixture_citation"); + }, + ); +}); + +describe("a non-image file under a withholding fault on Responses is its coded 500", () => { + it.each( + FAULTS.filter((f) => TEXT_REMOVING.has(f.fault)).flatMap((fault) => + [false, true].map((stream) => ({ fault, stream, faultName: fault.fault })), + ), + )("$faultName stream=$stream", async ({ fault, stream }) => { + const { status, body } = await serve( + "openai-responses", + stream, + { content: text, files: [{ mimeType: "application/pdf", data: "JVBERi0=" }] }, + fault, + ); + expect(status, body).toBe(500); + expect(body).toContain("aimock_unsupported_generated_file"); + }); +}); + +/** + * A malformed `responsesBlocks` is a fixture error on every wire. The planner + * must not throw its own plain Error for it before the request is journaled: + * it leaves the fixture to the normal path, which answers exactly as it does + * with no fault configured. + */ +describe("a malformed responsesBlocks under a fault on a non-Responses wire", () => { + const response = { + content: text, + toolCalls, + responsesBlocks: [{ type: "text", text: 5 }], + } as unknown as FixtureResponse; + const nonResponses = ["openai-chat", "anthropic", "gemini"] as const; + + it.each(nonResponses)("the planner leaves it to the normal path on %s", (wire) => { + const fixture: Fixture = { match: {}, response }; + setFixtureMisbehaviorPosition(fixture, "code#0"); + const journal = new Journal(); + const defaults: HandlerDefaults = { + latency: 0, + chunkSize: 10, + replaySpeed: 1, + logger: new Logger(), + misbehavior: { + baseline: { faults: [{ fault: "stop-length-mid-tool" }] }, + byTestId: new Map(), + }, + misbehaviorCounters: journal, + }; + expect( + planMisbehavior({ + wire, + fixture, + response, + request: { model: "gpt", messages: [] }, + stream: false, + defaults, + rawHeaders: { "x-test-id": "case" }, + url: undefined, + }), + ).toEqual({ kind: "skipped" }); + }); + + it.each(nonResponses.flatMap((wire) => [false, true].map((stream) => ({ wire, stream }))))( + "$wire stream=$stream answers as with no fault, journaled", + async ({ wire, stream }) => { + const extended: MockServerOptions = { responsesTools: "extended" }; + const baseline = await serve(wire, stream, response, undefined, extended); + await new Promise((resolve) => instance!.server.close(() => resolve())); + instance = undefined; + const faulted = await serve( + wire, + stream, + response, + { fault: "stop-length-mid-tool" }, + extended, + ); + expect(faulted.status, faulted.body).toBe(500); + expect(faulted.status).toBe(baseline.status); + expect(faulted.body).toContain("Invalid fixture block at index 0"); + expect(JSON.parse(faulted.body).error.message).toBe(JSON.parse(baseline.body).error.message); + const entries = instance!.journal.getAll(); + expect(entries).toHaveLength(1); + expect(entries[0].response.status).toBe(500); + }, + ); +}); + +describe("null optional citation fields are absent", () => { + it("fixtureCitationsProblem accepts null optional fields", () => { + expect( + fixtureCitationsProblem( + [ + { url: CITED_URL, title: null, startIndex: null, endIndex: null, type: null }, + { type: "file", fileId: "file-1", filename: "a.pdf", index: null }, + ], + text, + ), + ).toBeUndefined(); + }); + + it.each(WIRES.flatMap((wire) => [false, true].map((stream) => ({ wire, stream }))))( + "$wire stream=$stream serves the citation", + async ({ wire, stream }) => { + const { status, body } = await serve( + wire, + stream, + { + content: text, + citations: [{ url: CITED_URL, title: null, startIndex: null, endIndex: null }], + } as unknown as FixtureResponse, + undefined, + ); + expect(status, body).toBe(200); + expect(body).toContain(CITED_URL); + }, + ); +}); + +describe("a non-array citations on a turn with no text reports its shape", () => { + it("fixtureCitationsProblem names the shape, not the missing text", () => { + expect(fixtureCitationsProblem("oops", "")).toContain("must be an array"); + }); + + it.each(WIRES.flatMap((wire) => [false, true].map((stream) => ({ wire, stream }))))( + "$wire stream=$stream", + async ({ wire, stream }) => { + const { status, body } = await serve( + wire, + stream, + { toolCalls, citations: "oops" } as unknown as FixtureResponse, + undefined, + ); + expect(status, body).toBe(500); + expect(body).toContain("must be an array"); + }, + ); +}); diff --git a/src/__tests__/ws-responses-fault-journal-order.test.ts b/src/__tests__/ws-responses-fault-journal-order.test.ts new file mode 100644 index 00000000..93e1424a --- /dev/null +++ b/src/__tests__/ws-responses-fault-journal-order.test.ts @@ -0,0 +1,169 @@ +/** + * A fixture the Responses WebSocket transport cannot serve (malformed + * `citations`, or a non-image generated file) answers the WebSocket error + * event with the fixture error's aimock code. The journal must say what + * happened, as on HTTP `/v1/responses`: the entry carries the matched fixture + * and status 500, and no misbehavior fault is recorded as applied, in the + * entry or in `aimock_misbehavior_total`. + * + * Real surface: a real aimock server over a real WebSocket, reading its own + * journal and /metrics. + */ +import { afterEach, describe, expect, it } from "vitest"; +import type { ServerInstance } from "../server.js"; +import type { Fixture, FixtureResponse, MisbehaviorFault, TextResponse } from "../types.js"; +import { createServer } from "./helpers/misbehavior-enabled.js"; +import { connectWebSocket } from "./ws-test-client.js"; + +const CITE = "aimock_invalid_fixture_citation"; +const FILES = "aimock_unsupported_generated_file"; +const badCitations = [{ url: "https://example.com", startIndex: 0, endIndex: 999 }]; +const pdf = [{ mimeType: "application/pdf", data: "JVBERi0=" }]; +const text = "Hello world."; +const toolCalls = [{ name: "lookup", arguments: '{"q":"x"}' }]; + +interface Cell { + name: string; + /** A servable fixture response. */ + response: FixtureResponse; + /** What makes it unservable on the Responses API. */ + unservable: Pick; + code: string; + /** What the journal error names. */ + errorNames: string; + fault?: MisbehaviorFault; +} + +const cells: Cell[] = [ + { + name: "text + bad citations, no fault", + response: { content: text }, + unservable: { citations: badCitations }, + code: CITE, + errorNames: "citations", + }, + { + name: "text + non-image file, no fault", + response: { content: text }, + unservable: { files: pdf }, + code: FILES, + errorNames: "application/pdf", + }, + { + name: "tool-unknown-name + bad citations", + response: { content: text, toolCalls }, + unservable: { citations: badCitations }, + code: CITE, + errorNames: "citations", + fault: { fault: "tool-unknown-name" }, + }, + { + name: "tool-unknown-name + non-image file", + response: { content: text, toolCalls }, + unservable: { files: pdf }, + code: FILES, + errorNames: "application/pdf", + fault: { fault: "tool-unknown-name" }, + }, +]; + +let instance: ServerInstance | undefined; + +afterEach(async () => { + if (instance) { + await new Promise((resolve) => instance!.server.close(() => resolve())); + instance = undefined; + } +}); + +function fixtureFor(cell: Cell, response: FixtureResponse): Fixture { + return { + match: {}, + response, + ...(cell.fault ? { misbehavior: { faults: [{ ...cell.fault, rate: 1 }] } } : {}), + }; +} + +/** Send one response.create and collect events through the terminal one. */ +async function exchange(url: string): Promise[]> { + const ws = await connectWebSocket(url, "/v1/responses"); + try { + ws.send(JSON.stringify({ type: "response.create", model: "gpt-4o", input: "go" })); + const events: Record[] = []; + const deadline = Date.now() + 5000; + for (;;) { + const messages = await ws.waitForMessages( + events.length + 1, + Math.max(1, deadline - Date.now()), + ); + const event = JSON.parse(messages.at(-1)!) as Record; + events.push(event); + if (event.type === "error" || event.type === "response.completed") return events; + } + } finally { + ws.close(); + } +} + +function appliedMetricLines(metrics: string): string[] { + return metrics + .split("\n") + .filter( + (line) => line.startsWith("aimock_misbehavior_total{") && line.includes('outcome="applied"'), + ); +} + +describe("Responses WebSocket: an unservable fixture is journaled as a 500 with its fixture", () => { + for (const cell of cells) { + it(cell.name, async () => { + const fixture = fixtureFor(cell, { ...cell.response, ...cell.unservable }); + instance = await createServer([fixture], { metrics: true }); + const events = await exchange(instance.url); + + expect(events).toHaveLength(1); + expect(events[0].type).toBe("error"); + const error = events[0].error as { type?: string; code?: string; message?: string }; + expect(error.type).toBe("server_error"); + expect(error.code).toBe(cell.code); + expect(error.message).toContain(cell.errorNames); + + const entries = instance.journal.getAll(); + expect(entries).toHaveLength(1); + expect(entries[0].method).toBe("WS"); + expect(entries[0].response.status).toBe(500); + expect(entries[0].response.fixture).toBe(fixture); + expect(entries[0].response.error).toContain(cell.errorNames); + const applied = (entries[0].response.misbehavior?.evaluations ?? []).filter( + (row) => row.outcome === "applied", + ); + expect(applied).toEqual([]); + + const metrics = await (await fetch(`${instance.url}/metrics`)).text(); + expect(appliedMetricLines(metrics)).toEqual([]); + }); + } +}); + +describe("Responses WebSocket: the same fixture made servable is still served", () => { + for (const cell of cells) { + it(cell.name, async () => { + const fixture = fixtureFor(cell, cell.response); + instance = await createServer([fixture], { metrics: true }); + const events = await exchange(instance.url); + + expect(events.at(-1)?.type).toBe("response.completed"); + + const entries = instance.journal.getAll(); + expect(entries).toHaveLength(1); + expect(entries[0].response.status).toBe(200); + expect(entries[0].response.fixture).toBe(fixture); + const outcomes = (entries[0].response.misbehavior?.evaluations ?? []).map( + (row) => row.outcome, + ); + expect(outcomes).toEqual(cell.fault ? ["applied"] : []); + + const metrics = await (await fetch(`${instance.url}/metrics`)).text(); + expect(appliedMetricLines(metrics)).toHaveLength(cell.fault ? 1 : 0); + }); + } +}); diff --git a/src/gemini.ts b/src/gemini.ts index 5914f115..73fd35b8 100644 --- a/src/gemini.ts +++ b/src/gemini.ts @@ -1427,6 +1427,16 @@ export async function handleGemini( const replaySpeed = fixture.replaySpeed ?? defaults.replaySpeed; if (misbehavior.kind === "applied") { + // Journal first, as the success branches do, so a fixture error thrown + // while building leaves a 500 entry that keeps the fixture. Record the + // fault only once the reply is built: a thrown error was not served. + const entry = journal.add({ + method: req.method ?? "POST", + path, + headers: flattenHeaders(req.headers), + body: completionReq, + response: { status: 200, fixture }, + }); const prepared = prepareGeminiMisbehavior( misbehavior, completionReq, @@ -1435,13 +1445,6 @@ export async function handleGemini( logger, resolveRequestDecisions(defaults.strict, req.headers).reasoningSuppression, ); - const entry = journal.add({ - method: req.method ?? "POST", - path, - headers: flattenHeaders(req.headers), - body: completionReq, - response: { status: 200, fixture }, - }); recordMisbehaviorOutcome({ entry, summary: prepared.summary, defaults, testId }); if (!streaming) { res.writeHead(200, { "Content-Type": "application/json" }); diff --git a/src/messages.ts b/src/messages.ts index 0b85df92..d8d8219b 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -1909,6 +1909,16 @@ export async function handleMessages( const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); if (misbehavior.kind === "applied") { + // Journal first, as the success branches do, so a fixture error thrown + // below leaves a 500 entry that keeps the fixture. Record the fault only + // once the reply is built: a thrown error is not a fault that was served. + const entry = journal.add({ + method: req.method ?? "POST", + path: req.url ?? "/v1/messages", + headers: flattenHeaders(req.headers), + body: completionReq, + response: { status: 200, fixture }, + }); // Safety net: the planner already skips a fixture with generated `files` // (they have no Anthropic Messages projection), so it is never faulted. // Reject here anyway, before any byte is written, if one gets through. @@ -1921,13 +1931,6 @@ export async function handleMessages( strict: resolveRequestDecisions(defaults.strict, req.headers).reasoningSuppression, citations: true, }); - const entry = journal.add({ - method: req.method ?? "POST", - path: req.url ?? "/v1/messages", - headers: flattenHeaders(req.headers), - body: completionReq, - response: { status: 200, fixture }, - }); recordMisbehaviorOutcome({ entry, summary: prepared.plan.summary, defaults, testId }); if (!prepared.stream) { res.writeHead(200, { "Content-Type": "application/json" }); diff --git a/src/misbehavior.ts b/src/misbehavior.ts index b5369c6f..251f53a8 100644 --- a/src/misbehavior.ts +++ b/src/misbehavior.ts @@ -1,6 +1,8 @@ import { assertCustomToolCalls, assertResponsesToolCalls, + fixtureCitations, + fixtureFiles, isCombinedFixtureResponse, isErrorResponse, isTextResponse, @@ -16,6 +18,8 @@ import { toolArgsForWire, strictToolArgumentsEnabled, keepAuthoredToolArguments, + rejectGeneratedFiles, + UnsupportedGeneratedFileError, } from "./helpers.js"; import { parseChaosNumber } from "./chaos.js"; import type { @@ -1292,7 +1296,10 @@ export function prepareLengthCandidate( }; } -/** Remove all chat output channels while preserving response metadata. */ +/** + * Remove all chat output channels while preserving response metadata. The + * fixture `files` and `citations` keys survive: they are not removed here. + */ function clearChatOutput(response: ContentWithToolCallsResponse): ContentWithToolCallsResponse { const cleared = { ...response, content: "" }; delete cleared.blocks; @@ -1533,15 +1540,21 @@ function misbehaviorRoll(seed: number): number { * function call whose `arguments` are not valid JSON (see * {@link rejectsInvalidToolArguments}). A rejection is either a coded fixture * tool-call error or the plain Error for a malformed text block or an unknown - * block type; all of them mean the wire cannot serve the fixture. + * block type; all of them mean the wire cannot serve the fixture. The ordered + * blocks the planner itself reads ({@link resolveServedBlockOutcome}) are + * checked on every wire, so the planner never throws for them. So are the + * fixture's `citations` and `files` (see {@link assertServableFixtureOutput}), + * on the original turn before any fault rewrites it, so a fixture that will + * be rejected never consumes a fault's budget. */ function servesFixture(wire: WireId, stream: boolean, response: FixtureResponse): boolean { if (typeof response !== "object" || response === null) return true; const blocks = "blocks" in response ? response.blocks : undefined; let functionCalls: Pick[] = []; try { + const outcome = resolveServedBlockOutcome(response); if (wire === "openai-responses") { - if (!resolveServedBlockOutcome(response) && "toolCalls" in response) { + if (!outcome && "toolCalls" in response) { assertCustomToolCalls(response); assertResponsesToolCalls(servedToolCalls(response)); } @@ -1551,6 +1564,7 @@ function servesFixture(wire: WireId, stream: boolean, response: FixtureResponse) functionCalls = resolveFixtureBlocks(blocks).filter((block) => block.type === "toolCall"); } } + assertServableFixtureOutput(wire, response); } catch { return false; } @@ -1562,6 +1576,47 @@ function servesFixture(wire: WireId, stream: boolean, response: FixtureResponse) ); } +/** + * Throw when this wire's normal path would answer a coded 500 for the + * fixture's `citations` or `files`, checked against the original turn before + * any fault rewrites it: a malformed value (`aimock_invalid_fixture_citation` + * / `aimock_invalid_fixture_file`, including citations on a turn with no + * text), or generated files the wire cannot carry + * (`aimock_unsupported_generated_file`: any file on Chat Completions and + * Anthropic Messages, a non-`image/*` file on the OpenAI Responses API). Only + * the wires that read these keys check them; the others ignore them. A valid + * fixture passes, and a fault that alters the reply text then drops its + * citations ({@link citationsForServedText}). + */ +function assertServableFixtureOutput(wire: WireId, response: FixtureResponse): void { + if (!OUTPUT_WIRES.has(wire)) return; + if ( + !( + isCombinedFixtureResponse(response) || + isTextResponse(response) || + isToolCallResponse(response) + ) + ) + return; + // The same checks the handlers run before any byte is written. + if (wire === "openai-chat" || wire === "anthropic") rejectGeneratedFiles(response, wire); + const files = fixtureFiles(response.files); + if ( + wire === "openai-responses" && + !files.every((file) => file.mimeType.toLowerCase().startsWith("image/")) + ) + throw new UnsupportedGeneratedFileError(wire); + fixtureCitations(servedText(response), response.citations); +} + +/** The wires whose normal path reads a fixture's `citations` and `files`. */ +const OUTPUT_WIRES: ReadonlySet = new Set([ + "openai-chat", + "openai-responses", + "anthropic", + "gemini", +]); + /** * Whether the normal path of this wire serves invalid-JSON `arguments` as * `{}` / `"{}"` (the default, as in 1.44.0) instead of the authored text. The @@ -1596,6 +1651,44 @@ function rejectsInvalidToolArguments(wire: WireId, stream: boolean): boolean { } } +/** The reply text a response serves: its ordered text blocks, else `content`. */ +function servedText(response: FixtureResponse): string { + if (isCombinedFixtureResponse(response) || isTextResponse(response)) { + return resolveServedBlockOutcome(response)?.content ?? response.content ?? ""; + } + return ""; +} + +/** + * Fixture `citations` index the reply text. A fault that replaces, truncates + * or empties that text drops them for the turn, so every wire serves no + * citation into text it no longer sends; a fault that serves the original + * text unchanged keeps them. + */ +function citationsForServedText( + original: FixtureResponse, + candidate: MisbehaviorCandidate, +): FixtureResponse { + const response = candidate.response; + if ( + !( + isCombinedFixtureResponse(response) || + isTextResponse(response) || + isToolCallResponse(response) + ) + ) + return response; + if (response.citations === undefined) return response; + const keepsText = + candidate.refusal === undefined && + candidate.reasoning === undefined && + servedText(response) === servedText(original); + if (keepsText) return response; + const withoutCitations = { ...response }; + delete withoutCitations.citations; + return withoutCitations; +} + /** Complete preparation precedes selection; every candidate sees the original response. */ export function planMisbehavior(input: { wire: WireId; @@ -1640,10 +1733,14 @@ export function planMisbehavior(input: { // A fixture this wire cannot serve (a custom tool call off the OpenAI // Responses API, a malformed tool call, invalid JSON arguments that the // normal path replaces or rejects, a malformed text block or an unknown - // block type) is never faulted and never throws here: the request takes the - // normal path exactly as with no misbehavior config, which journals it and - // answers the coded fixture tool-call error (aimock_unsupported_tool_call, - // aimock_invalid_fixture_tool_call or, under strictToolArguments, + // block type, or `citations` / `files` the wire answers a coded 500 for, + // checked on the original turn) is never faulted and never throws here. It + // is skipped before fault selection, so it spends no `times` budget and no + // seed ordinal. The request takes the normal path exactly as with no + // misbehavior config, which journals it and answers the coded fixture error + // (aimock_unsupported_tool_call, aimock_invalid_fixture_tool_call, + // aimock_invalid_fixture_citation, aimock_invalid_fixture_file, + // aimock_unsupported_generated_file or, under strictToolArguments, // aimock_invalid_tool_arguments), the plain block error, or `{}` arguments. if (!servesFixture(wire, input.stream, input.response)) return { kind: "skipped" }; const config = parsed.config; @@ -1771,6 +1868,7 @@ export function planMisbehavior(input: { return { kind: "applied", ...candidate, + response: citationsForServedText(input.response, candidate), summary: { ...summaryFor(row), ...(candidate.target !== undefined ? { target: candidate.target } : {}), diff --git a/src/responses.ts b/src/responses.ts index 08304ab5..a7cb95f3 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -2215,13 +2215,9 @@ export async function handleResponses( const effectiveReplaySpeed = fixture.replaySpeed ?? defaults.replaySpeed; if (misbehavior.kind === "applied") { - const prepared = prepareResponsesMisbehavior(misbehavior); - const output = buildResponsesMisbehavior( - prepared, - completionReq, - chunkSize, - emitEncryptedReasoning, - ); + // Journal first, as the success branches do, so a fixture error thrown + // while building leaves a 500 entry that keeps the fixture. Record the + // fault only once the reply is built: a thrown error was not served. const entry = journal.add({ method: req.method ?? "POST", path: req.url ?? "/v1/responses", @@ -2229,6 +2225,13 @@ export async function handleResponses( body: completionReq, response: { status: 200, fixture }, }); + const prepared = prepareResponsesMisbehavior(misbehavior); + const output = buildResponsesMisbehavior( + prepared, + completionReq, + chunkSize, + emitEncryptedReasoning, + ); recordMisbehaviorOutcome({ entry, summary: prepared.summary, defaults, testId }); if (responsesReq.stream !== true) { res.writeHead(200, { "Content-Type": "application/json" }); diff --git a/src/server.ts b/src/server.ts index 563fd465..592c6b52 100644 --- a/src/server.ts +++ b/src/server.ts @@ -2359,6 +2359,13 @@ async function handleCompletions( evaluation.kind === "applied" ? prepareOpenAIChatMisbehavior(evaluation) : undefined; const response = appliedPlan?.response ?? resolvedResponse; const summary = appliedPlan?.summary ?? evaluation.summary; + // On Chat Completions the success branches journal first (so a coded 500 + // keeps the fixture), then validate and build, and record the outcome only + // once the response is built: a fixture error thrown on the way is not a + // fault that was served. A non-stream reply records it before its timing + // wait (see commitNonStream). Anthropic Messages, Gemini and the Responses + // API do this on their applied-fault path only; their success branches + // record the outcome before validating and building. const recordOutcome = (entry: JournalEntry): void => { recordMisbehaviorOutcome({ entry, summary, defaults, testId }); }; @@ -2450,6 +2457,22 @@ async function handleCompletions( ); }; + // A built non-stream reply is committed to be served: record the outcome + // before the timing wait, so the journal carries it while the reply waits. + // A client that hangs up during the wait got nothing, so the entry records a + // torn socket (status 0, interrupted), as routeError does for one that hangs + // up before its body arrives. False when there is no one left to answer. + const commitNonStream = async (entry: JournalEntry, contentLength: number): Promise => { + recordOutcome(entry); + await awaitNonStreamingTiming(contentLength); + const gone = responseGoneReason(res); + if (gone === null) return true; + entry.response.status = 0; + entry.response.interrupted = true; + entry.response.interruptReason = gone === "client-gone" ? "client aborted" : gone; + return false; + }; + // OpenRouter response shaping (no-op for OpenAI callers). Applied as a // post-pass over the objects the shared OpenAI builders produce so the // OpenAI code path is untouched. All OpenRouter-specific field shapes live in @@ -2550,7 +2573,6 @@ async function handleCompletions( body, response: { status: 200, fixture }, }); - recordOutcome(entry); // Safety net: the planner already skips a fixture with generated `files` // (they have no Chat Completions projection), so it is never faulted. // Reject here anyway, before any byte is written, if one gets through. @@ -2579,7 +2601,7 @@ async function handleCompletions( : appliedPlan.stop === "content_filter" ? "" : (appliedPlan.reasoning ?? ""); - await awaitNonStreamingTiming(nonStreamedContent.length); + if (!(await commitNonStream(entry, nonStreamedContent.length))) return; res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(shapeORCompletion(completion, overrides))); } else { @@ -2605,6 +2627,7 @@ async function handleCompletions( ) : undefined; shapeORChunks(chunks, usageChunk, overrides); + recordOutcome(entry); const interruption = createInterruptionSignal(fixture); const completed = await writeSSEStream(res, chunks, { latency, @@ -2660,7 +2683,6 @@ async function handleCompletions( body, response: { status: 200, fixture }, }); - recordOutcome(journalEntry); // Generated `files` have no Chat Completions projection: reject before any byte is written. rejectGeneratedFiles(response, wire); // A Responses-only custom tool call (customToolCalls / responsesBlocks) is @@ -2674,7 +2696,6 @@ async function handleCompletions( ? blockOutcome.content + blockOutcome.toolCalls.map((tc) => tc.name + tc.arguments).join("") : (response.content ?? "") + toolCalls.map((tc) => tc.name + tc.arguments).join(""); if (!streaming) { - await awaitNonStreamingTiming((effReasoning?.length ?? 0) + completionText.length); const completion = buildContentWithToolCallsCompletion( blockOutcome?.content ?? response.content ?? "", blockOutcome?.toolCalls ?? toolCalls, @@ -2689,6 +2710,8 @@ async function handleCompletions( : overrides, body.messages, ); + const contentLength = (effReasoning?.length ?? 0) + completionText.length; + if (!(await commitNonStream(journalEntry, contentLength))) return; res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(shapeORCompletion(completion, overrides))); } else { @@ -2712,6 +2735,7 @@ async function handleCompletions( ) : undefined; shapeORChunks(chunks, usageChunk, overrides); + recordOutcome(journalEntry); const interruption = createInterruptionSignal(fixture); const completed = await writeSSEStream(res, chunks, { latency, @@ -2760,11 +2784,9 @@ async function handleCompletions( body, response: { status: 200, fixture }, }); - recordOutcome(journalEntry); // Generated `files` have no Chat Completions projection: reject before any byte is written. rejectGeneratedFiles(response, wire); if (body.stream !== true) { - await awaitNonStreamingTiming((effReasoning?.length ?? 0) + response.content.length); const completion = buildTextCompletion( response.content, responseModel, @@ -2772,6 +2794,8 @@ async function handleCompletions( overrides, body.messages, ); + const contentLength = (effReasoning?.length ?? 0) + response.content.length; + if (!(await commitNonStream(journalEntry, contentLength))) return; res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(shapeORCompletion(completion, overrides))); } else { @@ -2792,6 +2816,7 @@ async function handleCompletions( ) : undefined; shapeORChunks(chunks, usageChunk, overrides); + recordOutcome(journalEntry); const interruption = createInterruptionSignal(fixture); const completed = await writeSSEStream(res, chunks, { latency, @@ -2840,13 +2865,11 @@ async function handleCompletions( body, response: { status: 200, fixture }, }); - recordOutcome(journalEntry); // Generated `files` have no Chat Completions projection: reject before any byte is written. rejectGeneratedFiles(response, wire); const toolCalls = requireServedFunctionToolCalls(response, wire); if (body.stream !== true) { const toolCallText = toolCalls.map((tc) => tc.name + tc.arguments).join(""); - await awaitNonStreamingTiming((effReasoning?.length ?? 0) + toolCallText.length); const completion = buildToolCallCompletion( toolCalls, responseModel, @@ -2854,6 +2877,8 @@ async function handleCompletions( overrides, body.messages, ); + const contentLength = (effReasoning?.length ?? 0) + toolCallText.length; + if (!(await commitNonStream(journalEntry, contentLength))) return; res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(shapeORCompletion(completion, overrides))); } else { @@ -2875,6 +2900,7 @@ async function handleCompletions( ) : undefined; shapeORChunks(chunks, usageChunk, overrides); + recordOutcome(journalEntry); const interruption = createInterruptionSignal(fixture); const completed = await writeSSEStream(res, chunks, { latency, diff --git a/src/ws-responses.ts b/src/ws-responses.ts index e5061643..b2615177 100644 --- a/src/ws-responses.ts +++ b/src/ws-responses.ts @@ -25,6 +25,7 @@ import { import { fixtureToolCallErrorCode, isFixtureToolCallError, + journalFixtureToolCallError, isTextResponse, isToolCallResponse, isCombinedFixtureResponse, @@ -289,13 +290,9 @@ async function processMessage( const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); if (misbehavior.kind === "applied") { - const prepared = prepareResponsesMisbehavior(misbehavior); - const output = buildResponsesMisbehavior( - prepared, - completionReq, - chunkSize, - emitEncryptedReasoning, - ); + // Journal first, as HTTP /v1/responses does, so a fixture error thrown + // while building leaves a 500 entry that keeps the fixture. Record the + // fault only once the reply is built: a thrown error was not served. const entry = journal.add({ method: "WS", path: "/v1/responses", @@ -303,6 +300,10 @@ async function processMessage( body: completionReq, response: { status: 200, fixture }, }); + const prepared = prepareResponsesMisbehavior(misbehavior); + const output = journalFixtureToolCallError(entry, () => + buildResponsesMisbehavior(prepared, completionReq, chunkSize, emitEncryptedReasoning), + ); recordMisbehaviorOutcome({ entry, summary: prepared.summary, @@ -436,20 +437,24 @@ async function processMessage( }); recordOutcome(journalEntry); - const events = buildTextStreamEvents( - response.content, - completionReq.model, - chunkSize, - resolveReasoningForModel( - response.reasoning, + // A fixture error (malformed citations, a non-image file) marks the entry + // a 500 carrying the message before the error event goes out. + const events = journalFixtureToolCallError(journalEntry, () => + buildTextStreamEvents( + response.content, completionReq.model, - effectiveStrict, - defaults.logger, + chunkSize, + resolveReasoningForModel( + response.reasoning, + completionReq.model, + effectiveStrict, + defaults.logger, + ), + response.webSearches, + extractOverrides(response), + emitEncryptedReasoning, + synthesizeSummarylessReasoning, ), - response.webSearches, - extractOverrides(response), - emitEncryptedReasoning, - synthesizeSummarylessReasoning, ); const interruption = createInterruptionSignal(fixture); const completed = await sendEvents( From 7bcea7a278093ded69040de4394705029778b752 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 23 Oct 2026 15:15:28 -0700 Subject: [PATCH 5/6] feat: warn at fixture load about citations and files the wires would reject validateFixtures now checks a fixture's files and citations with the same validators the wires use at serve time, and checks citation offsets against the text the wires serve: the ordered text blocks when present, else content. A turn with no text reports any non-empty citations as the wires do. A malformed value stays a warning, not a load error; each warning names the wires that answer a 500 for it and the error code they send. The end-to-end validation tests prove every wire answers the same coded 500 for each malformed value, and that validateFixtures warns with the same message. --- ...fixture-citations-files-validation.test.ts | 937 ++++++++++++++++++ src/fixture-loader.ts | 87 ++ 2 files changed, 1024 insertions(+) create mode 100644 src/__tests__/fixture-citations-files-validation.test.ts diff --git a/src/__tests__/fixture-citations-files-validation.test.ts b/src/__tests__/fixture-citations-files-validation.test.ts new file mode 100644 index 00000000..7fe2990e --- /dev/null +++ b/src/__tests__/fixture-citations-files-validation.test.ts @@ -0,0 +1,937 @@ +/** + * Fixture `citations` and `files` are checked when a request is served, on + * every wire that reads them (Chat Completions, Anthropic Messages, Gemini and + * the OpenAI Responses API), streaming and non-streaming: + * - `files` must be an array of `{ mimeType, data }` string pairs; + * - `citations` must be an array of citation objects. A URL citation has a + * non-empty string `url`, an optional string `title` and optional integer + * offsets that satisfy `0 <= startIndex <= endIndex <= text.length`. A file + * citation (`type: "file"`) has a non-empty string `fileId` and `filename` + * and an optional integer `index` with `0 <= index <= text.length`; it needs + * no `url`. + * + * An invalid value is a coded 500 in the wire's error envelope + * (`aimock_invalid_fixture_file` / `aimock_invalid_fixture_citation`) whose + * message says what is wrong and how to fix the fixture, before any content is + * written. Valid fixtures are served as before. Registration does not throw. + * + * Real surface: a real LLMock over HTTP. + */ +import { afterEach, describe, expect, it } from "vitest"; +import { LLMock } from "../llmock.js"; +import type { Fixture, FixtureResponse } from "../types.js"; +import { validateFixtures } from "../fixture-loader.js"; + +const TEXT = "Hello world."; + +type Shape = "openai" | "messages" | "google"; + +interface Route { + name: string; + path: string; + body: Record; + shape: Shape; +} + +const chatBody = { model: "gpt-4o", messages: [{ role: "user", content: "go" }] }; +const messagesBody = { + model: "claude-sonnet-4-5", + max_tokens: 64, + messages: [{ role: "user", content: "go" }], +}; +const geminiBody = { contents: [{ role: "user", parts: [{ text: "go" }] }] }; +const responsesBody = { model: "gpt-4o", input: "go" }; + +const routes: Route[] = [ + { name: "Chat", path: "/v1/chat/completions", body: chatBody, shape: "openai" }, + { + name: "Chat stream", + path: "/v1/chat/completions", + body: { ...chatBody, stream: true }, + shape: "openai", + }, + { name: "Anthropic", path: "/v1/messages", body: messagesBody, shape: "messages" }, + { + name: "Anthropic stream", + path: "/v1/messages", + body: { ...messagesBody, stream: true }, + shape: "messages", + }, + { + name: "Gemini", + path: "/v1beta/models/gemini-2.0-flash:generateContent", + body: geminiBody, + shape: "google", + }, + { + name: "Gemini stream", + path: "/v1beta/models/gemini-2.0-flash:streamGenerateContent?alt=sse", + body: geminiBody, + shape: "google", + }, + { name: "Responses", path: "/v1/responses", body: responsesBody, shape: "openai" }, + { + name: "Responses stream", + path: "/v1/responses", + body: { ...responsesBody, stream: true }, + shape: "openai", + }, +]; + +interface BadCase { + id: string; + response: Record; + code: "aimock_invalid_fixture_file" | "aimock_invalid_fixture_citation"; + /** Fragments the error message must contain: what is wrong and how to fix it. */ + says: string[]; +} + +const png = { mimeType: "image/png", data: "iVBORw0KGgo=" }; + +const badCases: BadCase[] = [ + { + id: "files is an object", + response: { content: TEXT, files: png }, + code: "aimock_invalid_fixture_file", + says: ['"files" must be an array', "got an object", "wrap a single file in [ ]"], + }, + { + id: "files is a string", + response: { content: TEXT, files: "iVBORw0KGgo=" }, + code: "aimock_invalid_fixture_file", + says: ['"files" must be an array', "got a string"], + }, + { + id: "a file without mimeType", + response: { content: TEXT, files: [{ data: "iVBORw0KGgo=" }] }, + code: "aimock_invalid_fixture_file", + says: ["files[0].mimeType must be a non-empty string", "got undefined"], + }, + { + id: "a file with an empty mimeType", + response: { content: TEXT, files: [{ mimeType: "", data: "iVBORw0KGgo=" }] }, + code: "aimock_invalid_fixture_file", + says: ["files[0].mimeType must be a non-empty string", 'got ""'], + }, + { + id: "a file that is not an object", + response: { content: TEXT, files: ["iVBORw0KGgo="] }, + code: "aimock_invalid_fixture_file", + says: ["files[0] must be a { mimeType, data } object", 'got "iVBORw0KGgo="'], + }, + { + id: "a file that is null", + response: { content: TEXT, files: [null] }, + code: "aimock_invalid_fixture_file", + says: ["files[0] must be a { mimeType, data } object", "got null"], + }, + { + id: "a file that is an array", + response: { content: TEXT, files: [["image/png", "iVBORw0KGgo="]] }, + code: "aimock_invalid_fixture_file", + says: ["files[0] must be a { mimeType, data } object", "got an array"], + }, + { + id: "a file with non-string data", + response: { content: TEXT, files: [{ mimeType: "image/png", data: 42 }] }, + code: "aimock_invalid_fixture_file", + says: ["files[0].data must be a string", "got 42", "set the base64-encoded file bytes"], + }, + { + id: "a file without data", + response: { content: TEXT, files: [{ mimeType: "image/png" }] }, + code: "aimock_invalid_fixture_file", + says: ["files[0].data must be a string", "got undefined"], + }, + { + id: "citations is an object", + response: { content: TEXT, citations: { url: "https://example.com" } }, + code: "aimock_invalid_fixture_citation", + says: ['"citations" must be an array', "got an object"], + }, + { + id: "a citation that is a string", + response: { content: TEXT, citations: ["https://example.com"] }, + code: "aimock_invalid_fixture_citation", + says: [ + "citations[0] must be a { url, title?, startIndex?, endIndex? }", + 'got "https://example.com"', + ], + }, + { + id: "a citation that is null", + response: { content: TEXT, citations: [null] }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0] must be a { url, title?, startIndex?, endIndex? }", "got null"], + }, + { + id: "a citation that is an array", + response: { content: TEXT, citations: [["https://example.com"]] }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0] must be a { url, title?, startIndex?, endIndex? }", "got an array"], + }, + { + id: "a non-string title", + response: { content: TEXT, citations: [{ url: "https://example.com", title: 7 }] }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].title must be a string when set", "got 7"], + }, + { + id: "a non-integer endIndex", + response: { content: TEXT, citations: [{ url: "https://example.com", endIndex: 2.5 }] }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].endIndex must be an integer when set", "got 2.5"], + }, + { + id: "a citation without url", + response: { content: TEXT, citations: [{ title: "Example" }] }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].url must be a non-empty string", "got undefined"], + }, + { + id: "a non-numeric offset", + response: { content: TEXT, citations: [{ url: "https://example.com", startIndex: "0" }] }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].startIndex must be an integer", 'got "0"'], + }, + { + id: "inverted offsets", + response: { + content: TEXT, + citations: [{ url: "https://example.com", startIndex: 5, endIndex: 2 }], + }, + code: "aimock_invalid_fixture_citation", + says: [ + "citations[0] offsets must satisfy 0 <= startIndex <= endIndex <= text length (12)", + "got startIndex 5, endIndex 2", + ], + }, + { + id: "an out-of-range endIndex", + response: { content: TEXT, citations: [{ url: "https://example.com", endIndex: 99 }] }, + code: "aimock_invalid_fixture_citation", + says: ["text length (12)", "got startIndex 0, endIndex 99"], + }, + { + id: "a negative startIndex", + response: { content: TEXT, citations: [{ url: "https://example.com", startIndex: -1 }] }, + code: "aimock_invalid_fixture_citation", + says: ["got startIndex -1, endIndex 12"], + }, + { + id: "a file citation without fileId", + response: { content: TEXT, citations: [{ type: "file", filename: "plan.pdf" }] }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].fileId must be a non-empty string", "got undefined"], + }, + { + id: "a file citation without filename", + response: { content: TEXT, citations: [{ type: "file", fileId: "file-abc123" }] }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].filename must be a non-empty string", "got undefined"], + }, + { + id: "a file citation with a non-integer index", + response: { + content: TEXT, + citations: [{ type: "file", fileId: "file-abc123", filename: "plan.pdf", index: 1.5 }], + }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].index must be an integer"], + }, + { + id: "a file citation with an out-of-range index", + response: { + content: TEXT, + citations: [{ type: "file", fileId: "file-abc123", filename: "plan.pdf", index: 99 }], + }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].index must satisfy 0 <= index <= text length (12)", "got 99"], + }, + { + id: "a citation with an unknown type", + response: { content: TEXT, citations: [{ type: "web", url: "https://example.com" }] }, + code: "aimock_invalid_fixture_citation", + says: ['citations[0].type must be "url" or "file"', 'got "web"'], + }, +]; + +let mock: LLMock | null = null; +afterEach(async () => { + await mock?.stop(); + mock = null; +}); + +async function start(response: Record): Promise { + mock = new LLMock({ port: 0, logLevel: "silent" }); + // Registration must not throw, whatever the fixture holds. + mock.addFixture({ match: {}, response: response as unknown as FixtureResponse } as Fixture); + await mock.start(); + return mock; +} + +async function hit(m: LLMock, path: string, body: Record) { + const res = await fetch(m.url + path, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(5000), + }); + return { status: res.status, text: await res.text() }; +} + +/** The aimock code and message from the wire's error envelope. */ +function codeAndMessage(shape: Shape, text: string): { code: unknown; message: unknown } { + const body = JSON.parse(text); + if (shape === "google") { + return { code: body.error.details?.[0]?.metadata?.code, message: body.error.message }; + } + return { code: body.error.code, message: body.error.message }; +} + +describe.each(badCases)("invalid fixture $id", ({ response, code, says }) => { + it.each(routes)("$name → coded 500 naming the problem and the fix", async (route) => { + const m = await start(response); + const r = await hit(m, route.path, route.body); + expect(r.status, r.text).toBe(500); + expect(r.text).not.toContain(TEXT); + const got = codeAndMessage(route.shape, r.text); + expect(got.code, r.text).toBe(code); + for (const fragment of says) expect(String(got.message)).toContain(fragment); + // The journal keeps the 500 with the matched fixture. + const entry = m.getRequests().at(-1); + expect(entry?.response.status).toBe(500); + expect(entry?.response.fixture).toBe(m.getFixtures()[0]); + }); +}); + +describe("valid citation offsets at the bounds are served", () => { + const cases = [ + { startIndex: 0, endIndex: TEXT.length }, + { startIndex: TEXT.length, endIndex: TEXT.length }, + { startIndex: 3, endIndex: 3 }, + ]; + describe.each(cases)("startIndex $startIndex, endIndex $endIndex", (offsets) => { + it.each(routes)("$name → 200", async (route) => { + const m = await start({ + content: TEXT, + citations: [{ url: "https://example.com", ...offsets }], + }); + const r = await hit(m, route.path, route.body); + expect(r.status, r.text).toBe(200); + expect(r.text).toContain("https://example.com"); + }); + }); +}); + +describe("a file citation needs no url", () => { + const fileCitation = { type: "file", fileId: "file-abc123", filename: "plan.pdf" }; + it.each(routes)("$name → 200", async (route) => { + const m = await start({ content: TEXT, citations: [fileCitation] }); + const r = await hit(m, route.path, route.body); + expect(r.status, r.text).toBe(200); + expect(r.text).not.toContain("aimock_invalid_fixture_citation"); + // Only the Responses API has a file-citation wire shape. + if (route.path === "/v1/responses") expect(r.text).toContain('"file_id":"file-abc123"'); + else expect(r.text).not.toContain("file-abc123"); + }); + + it("validateFixtures gives no citation warning", () => { + const results = validateFixtures([ + { + match: {}, + response: { + content: TEXT, + citations: [{ type: "file", fileId: "file-abc123", filename: "plan.pdf", index: 0 }], + }, + }, + ]); + expect(results.filter((r) => r.message.includes("aimock_invalid_fixture"))).toEqual([]); + }); +}); + +describe("validateFixtures warns about invalid citations and files", () => { + it.each(badCases)( + "$id → warning carrying the serve-time message and code", + ({ response, code, says }) => { + const results = validateFixtures([ + { match: {}, response: response as unknown as FixtureResponse } as Fixture, + ]); + const warning = results.find((r) => r.message.includes(code)); + expect(warning, JSON.stringify(results)).toBeDefined(); + expect(warning?.severity).toBe("warning"); + for (const fragment of says) expect(warning?.message).toContain(fragment); + }, + ); + + it("valid citations and files produce no such warning", () => { + const results = validateFixtures([ + { + match: {}, + response: { + content: TEXT, + files: [png], + citations: [{ url: "https://example.com", startIndex: 0, endIndex: TEXT.length }], + }, + }, + ]); + expect(results.filter((r) => r.message.includes("aimock_invalid_fixture"))).toEqual([]); + }); +}); + +/** A citation case the coded-500 check runs on every route. */ +interface HardeningCase { + id: string; + response: Record; + code: "aimock_invalid_fixture_file" | "aimock_invalid_fixture_citation"; + says: string[]; + /** Fragments the message must not carry (a fix that does not apply). */ + never?: string[]; +} + +// Values JSON cannot carry: a programmatic fixture can still hold them. +const unserializableCases: HardeningCase[] = [ + { + id: "a BigInt url", + response: { content: TEXT, citations: [{ url: 10n }] }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].url must be a non-empty string", "got 10n"], + }, + { + id: "a Symbol url", + response: { content: TEXT, citations: [{ url: Symbol("src") }] }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].url must be a non-empty string", "got Symbol(src)"], + }, + { + id: "a function url", + response: { content: TEXT, citations: [{ url: () => "https://example.com" }] }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].url must be a non-empty string", "got a function"], + }, + { + id: "a BigInt file data", + response: { content: TEXT, files: [{ mimeType: "image/png", data: 1n }] }, + code: "aimock_invalid_fixture_file", + says: ["files[0].data must be a string", "got 1n"], + }, + { + id: "a NaN startIndex", + response: { content: TEXT, citations: [{ url: "https://example.com", startIndex: NaN }] }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].startIndex must be an integer", "got NaN"], + }, + { + id: "an Infinity endIndex", + response: { + content: TEXT, + citations: [{ url: "https://example.com", endIndex: Infinity }], + }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].endIndex must be an integer", "got Infinity"], + }, +]; + +// "😀" is one code point, two UTF-16 units: index 1 falls between them. +const EMOJI_TEXT = "😀 hi"; +const surrogateCases: HardeningCase[] = [ + { + id: "a startIndex inside a surrogate pair", + response: { + content: EMOJI_TEXT, + citations: [{ url: "https://example.com", startIndex: 1, endIndex: 4 }], + }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].startIndex 1 splits a UTF-16 surrogate pair", "fix the offset"], + }, + { + id: "an endIndex inside a surrogate pair", + response: { + content: EMOJI_TEXT, + citations: [{ url: "https://example.com", startIndex: 0, endIndex: 1 }], + }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].endIndex 1 splits a UTF-16 surrogate pair"], + }, + { + id: "a file citation index inside a surrogate pair", + response: { + content: EMOJI_TEXT, + citations: [{ type: "file", fileId: "file-abc123", filename: "plan.pdf", index: 1 }], + }, + code: "aimock_invalid_fixture_citation", + says: ["citations[0].index 1 splits a UTF-16 surrogate pair"], + }, +]; + +const mixedCases: HardeningCase[] = [ + { + id: "a URL citation with file fields", + response: { + content: TEXT, + citations: [{ url: "https://example.com", fileId: "file-abc123", filename: "plan.pdf" }], + }, + code: "aimock_invalid_fixture_citation", + says: [ + "citations[0] mixes URL citation fields (url) with file citation fields (fileId, filename)", + 'set type: "file"', + "remove the stray fields", + ], + }, + { + id: "a file citation with URL fields", + response: { + content: TEXT, + citations: [ + { type: "file", fileId: "file-abc123", filename: "plan.pdf", url: "https://example.com" }, + ], + }, + code: "aimock_invalid_fixture_citation", + says: [ + "citations[0] mixes URL citation fields (url) with file citation fields", + "remove the URL citation fields (url) from this file citation", + ], + never: ['set type: "file"'], + }, + { + id: 'a type "url" citation with file fields', + response: { + content: TEXT, + citations: [ + { type: "url", url: "https://example.com", fileId: "file-abc123", filename: "plan.pdf" }, + ], + }, + code: "aimock_invalid_fixture_citation", + says: [ + "citations[0] mixes URL citation fields (url) with file citation fields (fileId, filename)", + "remove the file citation fields (fileId, filename) from this URL citation", + ], + }, + { + id: "a file-shaped citation with no type", + response: { content: TEXT, citations: [{ fileId: "file-abc123", filename: "plan.pdf" }] }, + code: "aimock_invalid_fixture_citation", + says: [ + "citations[0] has file citation fields (fileId, filename) but no type", + 'set type: "file"', + "remove the stray fields", + ], + }, + { + id: 'a type "url" citation with only file fields', + response: { + content: TEXT, + citations: [{ type: "url", fileId: "file-abc123", filename: "plan.pdf" }], + }, + code: "aimock_invalid_fixture_citation", + says: [ + 'citations[0] has type "url" but file citation fields (fileId, filename)', + "remove the file citation fields (fileId, filename) from this URL citation", + 'or set type: "file"', + ], + }, +]; + +describe.each([...unserializableCases, ...surrogateCases, ...mixedCases])( + "hardened check: $id", + ({ response, code, says, never = [] }) => { + it.each(routes)("$name → coded 500 naming the problem and the fix", async (route) => { + const m = await start(response); + const r = await hit(m, route.path, route.body); + expect(r.status, r.text).toBe(500); + const got = codeAndMessage(route.shape, r.text); + expect(got.code, r.text).toBe(code); + for (const fragment of says) expect(String(got.message)).toContain(fragment); + for (const fragment of never) expect(String(got.message)).not.toContain(fragment); + }); + + it("validateFixtures warns with the same message", () => { + const results = validateFixtures([ + { match: {}, response: response as unknown as FixtureResponse } as Fixture, + ]); + const warning = results.find((r) => r.message.includes(code)); + expect(warning, String(results.map((r) => r.message))).toBeDefined(); + for (const fragment of says) expect(warning?.message).toContain(fragment); + for (const fragment of never) expect(warning?.message).not.toContain(fragment); + }); + }, +); + +describe("surrogate-pair boundaries are valid offsets", () => { + it.each(routes)("$name → 200 for offsets around the pair", async (route) => { + const m = await start({ + content: EMOJI_TEXT, + citations: [ + { url: "https://example.com", startIndex: 0, endIndex: 2 }, + { url: "https://example.org", startIndex: 2, endIndex: EMOJI_TEXT.length }, + ], + }); + const r = await hit(m, route.path, route.body); + expect(r.status, r.text).toBe(200); + // Both citations reach the wire, each covering its side of the pair. + expect(citedSpans(route, r.text)).toEqual([ + ["https://example.com", "😀"], + ["https://example.org", " hi"], + ]); + }); +}); + +/** The JSON payloads of a response body: the body itself, or each SSE data line. */ +function payloads(text: string): T[] { + if (text.trimStart().startsWith("{")) return [JSON.parse(text) as T]; + return text + .split("\n") + .filter((line) => line.startsWith("data: ") && line !== "data: [DONE]") + .map((line) => JSON.parse(line.slice(6)) as T); +} + +interface ClaudeCite { + url: string; + cited_text: string; +} +interface ClaudePayload { + type: string; + content?: { citations?: ClaudeCite[] }[]; + delta?: { type?: string; citation?: ClaudeCite }; +} +interface GeminiPayload { + candidates?: { + groundingMetadata?: { + groundingChunks: { web: { uri: string } }[]; + groundingSupports: { segment: { text: string }; groundingChunkIndices: number[] }[]; + }; + }[]; +} +interface ChatMessage { + content?: string | null; + annotations?: { url_citation: { url: string; start_index: number; end_index: number } }[]; +} +interface ChatPayload { + choices: { message?: ChatMessage; delta?: ChatMessage }[]; +} +interface ResponsesBody { + output: { + content: { + text: string; + annotations: { url: string; start_index: number; end_index: number }[]; + }[]; + }[]; +} +interface ResponsesPayload extends Partial { + object?: string; + type?: string; + response?: ResponsesBody; +} + +/** Each served URL citation as [url, the text it covers], read off the wire. */ +function citedSpans(route: Route, text: string): [string, string][] { + if (route.shape === "messages") { + return payloads(text) + .flatMap((p): ClaudeCite[] => { + if (p.type === "message") return (p.content ?? []).flatMap((b) => b.citations ?? []); + if (p.delta?.type === "citations_delta" && p.delta.citation) return [p.delta.citation]; + return []; + }) + .map((c) => [c.url, c.cited_text]); + } + if (route.shape === "google") { + const meta = payloads(text) + .map((p) => p.candidates?.[0]?.groundingMetadata) + .find((m) => m !== undefined); + if (!meta) throw new Error(`no groundingMetadata in ${text}`); + return meta.groundingSupports.map((s) => [ + meta.groundingChunks[s.groundingChunkIndices[0]].web.uri, + s.segment.text, + ]); + } + if (route.name.startsWith("Chat")) { + const messages = payloads(text).map( + (p) => p.choices[0].message ?? p.choices[0].delta ?? {}, + ); + const content = messages.map((m) => m.content ?? "").join(""); + return messages + .flatMap((m) => m.annotations ?? []) + .map(({ url_citation: c }) => [c.url, content.slice(c.start_index, c.end_index)]); + } + const all = payloads(text); + const body = + all.find((p) => p.object === "response")?.output !== undefined + ? (all.find((p) => p.object === "response") as ResponsesBody) + : all.find((p) => p.type === "response.completed")?.response; + if (!body) throw new Error(`no response body in ${text}`); + const part = body.output[0].content[0]; + return part.annotations.map((a) => [a.url, part.text.slice(a.start_index, a.end_index)]); +} + +/** Mask ids and timestamps, which differ per request. */ +function maskVolatile(body: string): string { + return body + .replace(/"(\w*[iI]d)":"[^"]*"/g, '"$1":""') + .replace(/"(created\w*|completed_at)":\d+/g, '"$1":0'); +} + +describe("null citations and files are served exactly like absent ones", () => { + it.each(routes)("$name → byte-identical to the no-field baseline", async (route) => { + const baseline = await start({ content: TEXT }); + const want = await hit(baseline, route.path, route.body); + await baseline.stop(); + mock = null; + const m = await start({ content: TEXT, citations: null, files: null }); + const got = await hit(m, route.path, route.body); + expect(got.status, got.text).toBe(200); + expect(want.status).toBe(200); + expect(maskVolatile(got.text)).toBe(maskVolatile(want.text)); + }); + + it("validateFixtures gives no citation or file warning", () => { + const results = validateFixtures([ + { + match: {}, + response: { content: TEXT, citations: null, files: null }, + } as unknown as Fixture, + ]); + expect(results.filter((r) => r.message.includes("aimock_invalid_fixture"))).toEqual([]); + }); +}); + +/** + * The load-time check validates citation offsets against the same text the + * serve-time check uses: the joined text of every text block when `blocks` is + * set, else `content`. A turn with no text (tool calls only, or blocks with no + * text block) has nothing to cite: a well-formed non-empty `citations` there + * is the same no-text warning the wires answer as a 500. A malformed value is + * reported by its shape first, with or without text. + */ +describe("validateFixtures checks citation offsets against the served text", () => { + const url = "https://example.com"; + const toolBlock = { type: "toolCall", name: "lookup", arguments: "{}" }; + + function citationWarnings(response: Record) { + return validateFixtures([ + { match: {}, response: response as unknown as FixtureResponse } as Fixture, + ]).filter((r) => r.message.includes("aimock_invalid_fixture_citation")); + } + + it("blocks only, offsets past the joined text → warning", () => { + const warnings = citationWarnings({ + blocks: [ + { type: "text", text: "Hello " }, + { type: "text", text: "world." }, + ], + citations: [{ url, endIndex: 99 }], + }); + expect(warnings).toHaveLength(1); + expect(warnings[0].severity).toBe("warning"); + expect(warnings[0].message).toContain("text length (12)"); + expect(warnings[0].message).toContain("got startIndex 0, endIndex 99"); + }); + + it("blocks only, offsets inside the joined text across a tool block → no warning", () => { + expect( + citationWarnings({ + blocks: [{ type: "text", text: "Hello " }, toolBlock, { type: "text", text: "world." }], + citations: [{ url, startIndex: 3, endIndex: 12 }], + }), + ).toEqual([]); + }); + + it("content and blocks, offsets valid for the blocks text only → no warning", () => { + expect( + citationWarnings({ + content: "Hi", + blocks: [{ type: "text", text: TEXT }], + citations: [{ url, startIndex: 3, endIndex: 12 }], + }), + ).toEqual([]); + }); + + it("content and blocks, offsets valid for the content only → warning", () => { + const warnings = citationWarnings({ + content: "Hello world, and more.", + blocks: [{ type: "text", text: "Hi" }], + citations: [{ url, startIndex: 3, endIndex: 12 }], + }); + expect(warnings).toHaveLength(1); + expect(warnings[0].message).toContain("text length (2)"); + }); + + it("tool-call blocks only, offsets past the empty text → no-text warning", () => { + const warnings = citationWarnings({ + blocks: [toolBlock], + citations: [{ url, startIndex: 0, endIndex: 5 }], + }); + expect(warnings).toHaveLength(1); + expect(warnings[0].message).toContain("this turn has no text"); + }); + + it("tool-call blocks only, no offsets → no-text warning", () => { + const warnings = citationWarnings({ blocks: [toolBlock], citations: [{ url }] }); + expect(warnings).toHaveLength(1); + expect(warnings[0].message).toContain("this turn has no text"); + }); + + it("toolCalls only (no text) → no-text warning for a well-formed value", () => { + const toolCalls = [{ name: "lookup", arguments: "{}" }]; + const warnings = citationWarnings({ toolCalls, citations: [{ url, endIndex: 5 }] }); + expect(warnings).toHaveLength(1); + expect(warnings[0].message).toContain("this turn has no text"); + }); + + it("toolCalls only (no text) → shape warning for a malformed value", () => { + const toolCalls = [{ name: "lookup", arguments: "{}" }]; + const warnings = citationWarnings({ toolCalls, citations: { url } }); + expect(warnings).toHaveLength(1); + expect(warnings[0].message).toContain('"citations" must be an array'); + expect(warnings[0].message).not.toContain("this turn has no text"); + }); +}); + +describe("the served text is the one validateFixtures checks", () => { + it.each(routes)( + "$name: content and blocks, offsets valid for the blocks text → 200", + async (route) => { + const m = await start({ + content: "Hi", + blocks: [{ type: "text", text: TEXT }], + citations: [{ url: "https://example.com", startIndex: 3, endIndex: 12 }], + }); + const r = await hit(m, route.path, route.body); + expect(r.status, r.text).toBe(200); + expect(r.text).toContain("https://example.com"); + }, + ); + + it.each(routes)("$name: blocks only, offsets past the joined text → 500", async (route) => { + const m = await start({ + blocks: [{ type: "text", text: TEXT }], + citations: [{ url: "https://example.com", endIndex: 99 }], + }); + const r = await hit(m, route.path, route.body); + expect(r.status, r.text).toBe(500); + expect(codeAndMessage(route.shape, r.text).code).toBe("aimock_invalid_fixture_citation"); + }); + + it.each(routes)("$name: toolCalls only with a malformed citations value → 500", async (route) => { + const m = await start({ + toolCalls: [{ name: "lookup", arguments: "{}" }], + citations: { url: "https://example.com" }, + }); + const r = await hit(m, route.path, route.body); + expect(r.status, r.text).toBe(500); + const got = codeAndMessage(route.shape, r.text); + expect(got.code).toBe("aimock_invalid_fixture_citation"); + expect(String(got.message)).toContain('"citations" must be an array'); + }); +}); + +/** + * Citations on a turn with no text (tool calls only, `blocks` with no text + * block, or empty `content`) have nothing to point at, so they are a fixture + * error on every wire: a coded 500 + * (`aimock_invalid_fixture_citation`) that says to add text or remove the + * citations, and the same message as a load-time warning. A malformed value + * there names its shape problem instead, the same as on a turn with text. A turn with no + * `citations` key, or an empty array, is served exactly as before. + */ +describe("citations on a turn with no text", () => { + const toolCalls = [{ id: "call_abc123", name: "get_weather", arguments: '{"city":"SF"}' }]; + const says = ["this turn has no text", "add text to the response or remove the citations"]; + const noTextCases: { id: string; response: Record; says?: string[] }[] = [ + { + id: "tool calls with a valid-looking citation", + response: { toolCalls, citations: [{ url: "https://example.com", title: "Example" }] }, + }, + { + id: "tool calls with a malformed citation", + response: { toolCalls, citations: [{ title: "Example" }] }, + says: ["citations[0].url must be a non-empty string"], + }, + { + id: "tool calls with a non-array citations value", + response: { toolCalls, citations: { url: "https://example.com" } }, + says: ['"citations" must be an array'], + }, + { + id: "empty content with a citation", + response: { content: "", citations: [{ url: "https://example.com" }] }, + }, + { + id: "blocks with only a tool call, with a citation", + response: { + blocks: [{ type: "toolCall", ...toolCalls[0] }], + citations: [{ url: "https://example.com" }], + }, + }, + { + id: "blocks with an empty text block and a tool call, with a citation", + response: { + blocks: [ + { type: "text", text: "" }, + { type: "toolCall", ...toolCalls[0] }, + ], + citations: [{ url: "https://example.com" }], + }, + }, + ]; + + describe.each(noTextCases)("$id", ({ response, says: expected = says }) => { + it.each(routes)("$name → coded 500 that names the problem", async (route) => { + const m = await start(response); + const r = await hit(m, route.path, route.body); + expect(r.status, r.text).toBe(500); + const got = codeAndMessage(route.shape, r.text); + expect(got.code, r.text).toBe("aimock_invalid_fixture_citation"); + for (const fragment of expected) expect(String(got.message)).toContain(fragment); + expect(r.text).not.toContain("get_weather"); + const entry = m.getRequests().at(-1); + expect(entry?.response.status).toBe(500); + expect(entry?.response.fixture).toBe(m.getFixtures()[0]); + }); + + it("validateFixtures warns with the serve-time message", async () => { + const m = await start(response); + const r = await hit(m, routes[0].path, routes[0].body); + const served = String(codeAndMessage("openai", r.text).message).replace( + 'aimock: invalid fixture "citations": ', + "", + ); + const results = validateFixtures([ + { match: {}, response: response as unknown as FixtureResponse } as Fixture, + ]); + const warning = results.find((w) => w.message.includes("aimock_invalid_fixture_citation")); + expect(warning, JSON.stringify(results)).toBeDefined(); + expect(warning?.severity).toBe("warning"); + expect(warning?.message).toBe( + `${served} — requests it serves on Chat Completions, Anthropic Messages, Gemini or ` + + `the OpenAI Responses API answer a 500 (aimock_invalid_fixture_citation)`, + ); + }); + }); + + // Random per-response IDs and timestamps differ between two requests. + const normalize = (text: string) => + text.replace(/"(id|item_id|call_id|created|created_at|responseId)":("[^"]*"|\d+)/g, '"$1":_'); + + it.each(routes)( + "$name → an empty citations array on a tool-only turn is unchanged", + async (route) => { + const without = await hit(await start({ toolCalls }), route.path, route.body); + await mock?.stop(); + mock = null; + const withEmpty = await hit( + await start({ toolCalls, citations: [] }), + route.path, + route.body, + ); + expect(without.status, without.text).toBe(200); + expect(withEmpty.status, withEmpty.text).toBe(200); + expect(withEmpty.text).toContain("get_weather"); + expect(normalize(withEmpty.text)).toBe(normalize(without.text)); + }, + ); + + it("validateFixtures gives no citation warning for an empty array on a tool-only turn", () => { + const results = validateFixtures([{ match: {}, response: { toolCalls, citations: [] } }]); + expect(results.filter((r) => r.message.includes("aimock_invalid_fixture"))).toEqual([]); + }); +}); diff --git a/src/fixture-loader.ts b/src/fixture-loader.ts index 93ae590f..2ee7b9e2 100644 --- a/src/fixture-loader.ts +++ b/src/fixture-loader.ts @@ -25,6 +25,9 @@ import { isVideoResponse, isJSONResponse, fixtureListTarget, + fixtureCitationsProblem, + fixtureFilesProblem, + resolveServedBlockOutcome, } from "./helpers.js"; import type { Logger } from "./logger.js"; import { CHAOS_FIELDS, CHAOS_FIELD_NAMES, parseChaosField } from "./chaos.js"; @@ -540,6 +543,84 @@ function validateReasoning( } } +/** The wires that read `files` and `citations`; every other wire ignores them. */ +const CITATIONS_FILES_WIRES = + "Chat Completions, Anthropic Messages, Gemini or the OpenAI Responses API"; + +/** + * Generated `files` and source `citations`. A malformed value is not a load + * error (registration never throws for it, as in the release that added the + * keys); each request the fixture serves on Chat Completions, Anthropic, + * Gemini or the Responses API answers a coded 500 instead, so warn here with + * the same message. Citations are checked against the text they cite when + * served ({@link citedText}). A turn with no text (tool calls only, blocks + * with no text block, or empty content) has nothing to cite, so any + * non-empty `citations` there is reported exactly as the wires report it. + */ +function validateCitationsAndFiles( + response: { + content?: unknown; + files?: unknown; + citations?: unknown; + toolCalls?: unknown; + customToolCalls?: unknown; + blocks?: unknown; + responsesBlocks?: unknown; + }, + fixtureIndex: number, + results: ValidationResult[], +): void { + const filesProblem = fixtureFilesProblem(response.files); + if (filesProblem !== undefined) { + results.push({ + severity: "warning", + fixtureIndex, + message: `${filesProblem} — requests it serves on ${CITATIONS_FILES_WIRES} answer a 500 (aimock_invalid_fixture_file)`, + }); + } + const text = citedText(response); + if (text === undefined) return; + const citationsProblem = fixtureCitationsProblem(response.citations, text); + if (citationsProblem !== undefined) { + results.push({ + severity: "warning", + fixtureIndex, + message: `${citationsProblem} — requests it serves on ${CITATIONS_FILES_WIRES} answer a 500 (aimock_invalid_fixture_citation)`, + }); + } +} + +/** + * The text a fixture's citations index when served: the joined text of every + * text block when `responsesBlocks` or `blocks` is non-empty (through the + * same {@link resolveServedBlockOutcome} the providers use; empty when no + * block is text), else a string `content`, else empty for a tool-only turn. + * Empty text means a turn with nothing to cite. Undefined for a response + * that is neither (validated elsewhere) and for a malformed blocks array, + * which `validateBlocks` reports as an error. + */ +function citedText(response: { + content?: unknown; + blocks?: unknown; + responsesBlocks?: unknown; + toolCalls?: unknown; + customToolCalls?: unknown; +}): string | undefined { + const hasBlocks = + (Array.isArray(response.blocks) && response.blocks.length > 0) || + (Array.isArray(response.responsesBlocks) && response.responsesBlocks.length > 0); + if (hasBlocks) { + try { + return resolveServedBlockOutcome(response)?.content; + } catch { + return undefined; + } + } + if (typeof response.content === "string") return response.content; + if (Array.isArray(response.toolCalls) || Array.isArray(response.customToolCalls)) return ""; + return undefined; +} + function validateWebSearches( response: { webSearches?: unknown }, fixtureIndex: number, @@ -1216,6 +1297,12 @@ export function validateFixtures( ); // OpenAI Responses keys (new): customToolCalls and responsesBlocks. validateResponsesToolKeys(response, i, results); + // Generated `files` and source `citations`. + validateCitationsAndFiles( + response as Parameters[0], + i, + results, + ); // Tool call response checks if (isToolCallResponse(response)) { From 9eaa9f19ec287ed35528ea8a76fd47a9b27726ac Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 23 Oct 2026 15:15:34 -0700 Subject: [PATCH 6/6] docs: document fixture citations and generated files, and update the API report Add the CHANGELOG entry for citations and generated files on every provider that reads them, and regenerate the API report for the new exported types and errors. --- CHANGELOG.md | 7 ++++ etc/aimock-a2a.import.api.md | 27 ++++++++++---- etc/aimock-a2a.require.api.md | 27 ++++++++++---- etc/aimock-agui.import.api.md | 27 ++++++++++---- etc/aimock-agui.require.api.md | 27 ++++++++++---- etc/aimock-index.import.api.md | 60 +++++++++++++++++++++++++++----- etc/aimock-index.require.api.md | 60 +++++++++++++++++++++++++++----- etc/aimock-jest.import.api.md | 31 ++++++++++++----- etc/aimock-jest.require.api.md | 31 ++++++++++++----- etc/aimock-mcp.import.api.md | 27 ++++++++++---- etc/aimock-mcp.require.api.md | 27 ++++++++++---- etc/aimock-vector.import.api.md | 27 ++++++++++---- etc/aimock-vector.require.api.md | 27 ++++++++++---- etc/aimock-vitest.import.api.md | 31 ++++++++++++----- etc/aimock-vitest.require.api.md | 31 ++++++++++++----- 15 files changed, 357 insertions(+), 110 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e31b202..053c5c30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,16 @@ ### Added - Fixture `citations` serve OpenAI Responses `url_citation` annotations; fixture `files` serve Gemini `inlineData` parts (#550 by @abhiaiyer91) +- Fixture `citations` also serve Chat Completions `url_citation` annotations (`message.annotations`, one streamed `delta.annotations` chunk), Anthropic `web_search_result_location` citations on text blocks (`citations_delta` when streaming), and Gemini `groundingMetadata` (`groundingChunks` + `groundingSupports`, byte offsets). A citation with `type: "file"` (`fileId`, `filename`, optional `index`, which defaults to the end of the text) serves an OpenAI Responses `file_citation` annotation on `output_text`, streamed through `response.output_text.annotation.added`; Chat Completions, Anthropic and Gemini skip it. Closes #547. +- Fixture `files` serve OpenAI Responses `image_generation_call` output items (`result` holds the base64 bytes), streamed with the `response.image_generation_call.*` events. Chat Completions and Anthropic Messages reject a fixture with `files` with a 500 carrying `aimock_unsupported_generated_file` (new `UnsupportedGeneratedFileError`). Closes #548. +- Malformed fixture `citations` or `files` (not an array, a missing or mistyped field, a non-integer offset, an offset outside the text, or citations on a turn with no text: tool calls only, `blocks` with no text block, or empty content) answer a 500 before any content is written: `aimock_invalid_fixture_citation` or `aimock_invalid_fixture_file`. A non-`image/*` file on the OpenAI Responses API answers a 500 with `aimock_unsupported_generated_file`. This is the same on every wire that reads these keys, with or without a misbehavior fault: the fixture is checked on its original turn before any fault applies (only an invalid misbehavior config answers its own 400 or 501 first). A valid fixture under a fault that changes the reply text is served without its `citations`. Only Chat Completions, Anthropic Messages, Gemini and OpenAI Responses read these keys; other wires ignore them. +- Fixture load validation warns about malformed `citations` and `files`, with the same problem description the 500 carries (the 500 message adds an `aimock: invalid fixture` prefix; the warning adds which 500 the requests answer). The fixture still loads. +- `InvalidFixtureOutputError` (the error behind `aimock_invalid_fixture_citation` and `aimock_invalid_fixture_file`) is exported from `@copilotkit/aimock`. ### Fixed +- Misbehavior faults on Chat Completions, Anthropic Messages, Gemini and the OpenAI Responses API (HTTP and WebSocket): a fixture the wire rejects (generated `files` on Chat Completions or Anthropic, a non-`image/*` file on Responses, malformed `citations` or `files`) is never faulted. It takes the normal path, is journaled as a 500 that keeps its fixture, and consumes no fault `times` budget or seed ordinal; `aimock_misbehavior_total` does not count it. A non-stream Chat Completions fault is journaled before its timing wait. +- Journal accuracy: a client that hangs up during the timing wait of any non-stream Chat Completions reply (faulted or not) is journaled as interrupted, not as served. - Docker image: the base image is pinned to `node:22.23.4-alpine` by digest. Node 22.23.4 fixes CVE-2026-111497 (Brotli decompression memory bug); the 1.46.0 image shipped Node 22.23.3. ## [1.46.0] - 2026-10-20 diff --git a/etc/aimock-a2a.import.api.md b/etc/aimock-a2a.import.api.md index 4936c553..b2482cfa 100644 --- a/etc/aimock-a2a.import.api.md +++ b/etc/aimock-a2a.import.api.md @@ -361,14 +361,15 @@ type FixtureBlock = { }; // @public -interface FixtureCitation { - // (undocumented) - endIndex?: number; - startIndex?: number; - // (undocumented) - title?: string; +type FixtureCitation = FixtureUrlCitation | FixtureFileCitation; + +// @public +interface FixtureFileCitation { + fileId: string; + filename: string; + index?: number; // (undocumented) - url: string; + type: "file"; } // @public @@ -415,6 +416,18 @@ interface FixtureMatch { // @public (undocumented) type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +// @public +interface FixtureUrlCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + type?: "url"; + // (undocumented) + url: string; +} + // @public (undocumented) interface ImageItem { // (undocumented) diff --git a/etc/aimock-a2a.require.api.md b/etc/aimock-a2a.require.api.md index 4936c553..b2482cfa 100644 --- a/etc/aimock-a2a.require.api.md +++ b/etc/aimock-a2a.require.api.md @@ -361,14 +361,15 @@ type FixtureBlock = { }; // @public -interface FixtureCitation { - // (undocumented) - endIndex?: number; - startIndex?: number; - // (undocumented) - title?: string; +type FixtureCitation = FixtureUrlCitation | FixtureFileCitation; + +// @public +interface FixtureFileCitation { + fileId: string; + filename: string; + index?: number; // (undocumented) - url: string; + type: "file"; } // @public @@ -415,6 +416,18 @@ interface FixtureMatch { // @public (undocumented) type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +// @public +interface FixtureUrlCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + type?: "url"; + // (undocumented) + url: string; +} + // @public (undocumented) interface ImageItem { // (undocumented) diff --git a/etc/aimock-agui.import.api.md b/etc/aimock-agui.import.api.md index 262579c6..2d106358 100644 --- a/etc/aimock-agui.import.api.md +++ b/etc/aimock-agui.import.api.md @@ -990,14 +990,15 @@ type FixtureBlock = { }; // @public -interface FixtureCitation { - // (undocumented) - endIndex?: number; - startIndex?: number; - // (undocumented) - title?: string; +type FixtureCitation = FixtureUrlCitation | FixtureFileCitation; + +// @public +interface FixtureFileCitation { + fileId: string; + filename: string; + index?: number; // (undocumented) - url: string; + type: "file"; } // @public @@ -1044,6 +1045,18 @@ interface FixtureMatch { // @public (undocumented) type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +// @public +interface FixtureUrlCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + type?: "url"; + // (undocumented) + url: string; +} + // @public export function getLastMessageIfToolResult(input: AGUIRunAgentInput): AGUIMessage | null; diff --git a/etc/aimock-agui.require.api.md b/etc/aimock-agui.require.api.md index 262579c6..2d106358 100644 --- a/etc/aimock-agui.require.api.md +++ b/etc/aimock-agui.require.api.md @@ -990,14 +990,15 @@ type FixtureBlock = { }; // @public -interface FixtureCitation { - // (undocumented) - endIndex?: number; - startIndex?: number; - // (undocumented) - title?: string; +type FixtureCitation = FixtureUrlCitation | FixtureFileCitation; + +// @public +interface FixtureFileCitation { + fileId: string; + filename: string; + index?: number; // (undocumented) - url: string; + type: "file"; } // @public @@ -1044,6 +1045,18 @@ interface FixtureMatch { // @public (undocumented) type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +// @public +interface FixtureUrlCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + type?: "url"; + // (undocumented) + url: string; +} + // @public export function getLastMessageIfToolResult(input: AGUIRunAgentInput): AGUIMessage | null; diff --git a/etc/aimock-index.import.api.md b/etc/aimock-index.import.api.md index 8b234513..5fea0202 100644 --- a/etc/aimock-index.import.api.md +++ b/etc/aimock-index.import.api.md @@ -1379,6 +1379,19 @@ export interface ChatCompletion { } & OpenRouterUsageExtras; } +// @public +export interface ChatCompletionAnnotation { + // (undocumented) + type: "url_citation"; + // (undocumented) + url_citation: { + url: string; + title: string; + start_index: number; + end_index: number; + }; +} + // @public (undocumented) export interface ChatCompletionChoice { // (undocumented) @@ -1394,6 +1407,7 @@ export interface ChatCompletionChoice { // @public (undocumented) export interface ChatCompletionMessage { + annotations?: ChatCompletionAnnotation[]; // (undocumented) content: string | null; reasoning?: string | null; @@ -2117,15 +2131,7 @@ export type FixtureBlock = { }; // @public -export interface FixtureCitation { - // (undocumented) - endIndex?: number; - startIndex?: number; - // (undocumented) - title?: string; - // (undocumented) - url: string; -} +export type FixtureCitation = FixtureUrlCitation | FixtureFileCitation; // @public (undocumented) export interface FixtureFile { @@ -2145,6 +2151,15 @@ type FixtureFileBlock = { namespace?: string; }; +// @public +export interface FixtureFileCitation { + fileId: string; + filename: string; + index?: number; + // (undocumented) + type: "file"; +} + // @public (undocumented) export interface FixtureFileContentWithToolCallsResponse extends ResponseOverrides { blocks?: FixtureFileBlock[]; @@ -2376,6 +2391,18 @@ export type FixtureToolCall = (ToolCall & { type?: undefined; }) | CustomToolCall; +// @public +export interface FixtureUrlCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + type?: "url"; + // (undocumented) + url: string; +} + // @public export function flattenHeaders(headers: http$1.IncomingHttpHeaders): Record; @@ -2886,6 +2913,13 @@ export interface InterruptionControl { tick(): void; } +// @public +export class InvalidFixtureOutputError extends Error { + constructor(code: "aimock_invalid_fixture_citation" | "aimock_invalid_fixture_file", problem: string); + // (undocumented) + readonly code: "aimock_invalid_fixture_citation" | "aimock_invalid_fixture_file"; +} + // @public export class InvalidFixtureToolCallError extends Error { constructor(index: number, problem: string, carrier?: "toolCalls" | "customToolCalls" | "blocks"); @@ -4881,6 +4915,7 @@ export interface SSEChunk { // @public (undocumented) export interface SSEDelta { + annotations?: ChatCompletionAnnotation[]; // (undocumented) content?: string | null; // (undocumented) @@ -5050,6 +5085,13 @@ export interface TranscriptionResponse { }; } +// @public +export class UnsupportedGeneratedFileError extends Error { + constructor(wire: string); + // (undocumented) + readonly wire: string; +} + // @public export class UnsupportedToolCallError extends Error { constructor(toolName: string, wire: string); diff --git a/etc/aimock-index.require.api.md b/etc/aimock-index.require.api.md index 8b234513..5fea0202 100644 --- a/etc/aimock-index.require.api.md +++ b/etc/aimock-index.require.api.md @@ -1379,6 +1379,19 @@ export interface ChatCompletion { } & OpenRouterUsageExtras; } +// @public +export interface ChatCompletionAnnotation { + // (undocumented) + type: "url_citation"; + // (undocumented) + url_citation: { + url: string; + title: string; + start_index: number; + end_index: number; + }; +} + // @public (undocumented) export interface ChatCompletionChoice { // (undocumented) @@ -1394,6 +1407,7 @@ export interface ChatCompletionChoice { // @public (undocumented) export interface ChatCompletionMessage { + annotations?: ChatCompletionAnnotation[]; // (undocumented) content: string | null; reasoning?: string | null; @@ -2117,15 +2131,7 @@ export type FixtureBlock = { }; // @public -export interface FixtureCitation { - // (undocumented) - endIndex?: number; - startIndex?: number; - // (undocumented) - title?: string; - // (undocumented) - url: string; -} +export type FixtureCitation = FixtureUrlCitation | FixtureFileCitation; // @public (undocumented) export interface FixtureFile { @@ -2145,6 +2151,15 @@ type FixtureFileBlock = { namespace?: string; }; +// @public +export interface FixtureFileCitation { + fileId: string; + filename: string; + index?: number; + // (undocumented) + type: "file"; +} + // @public (undocumented) export interface FixtureFileContentWithToolCallsResponse extends ResponseOverrides { blocks?: FixtureFileBlock[]; @@ -2376,6 +2391,18 @@ export type FixtureToolCall = (ToolCall & { type?: undefined; }) | CustomToolCall; +// @public +export interface FixtureUrlCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + type?: "url"; + // (undocumented) + url: string; +} + // @public export function flattenHeaders(headers: http$1.IncomingHttpHeaders): Record; @@ -2886,6 +2913,13 @@ export interface InterruptionControl { tick(): void; } +// @public +export class InvalidFixtureOutputError extends Error { + constructor(code: "aimock_invalid_fixture_citation" | "aimock_invalid_fixture_file", problem: string); + // (undocumented) + readonly code: "aimock_invalid_fixture_citation" | "aimock_invalid_fixture_file"; +} + // @public export class InvalidFixtureToolCallError extends Error { constructor(index: number, problem: string, carrier?: "toolCalls" | "customToolCalls" | "blocks"); @@ -4881,6 +4915,7 @@ export interface SSEChunk { // @public (undocumented) export interface SSEDelta { + annotations?: ChatCompletionAnnotation[]; // (undocumented) content?: string | null; // (undocumented) @@ -5050,6 +5085,13 @@ export interface TranscriptionResponse { }; } +// @public +export class UnsupportedGeneratedFileError extends Error { + constructor(wire: string); + // (undocumented) + readonly wire: string; +} + // @public export class UnsupportedToolCallError extends Error { constructor(toolName: string, wire: string); diff --git a/etc/aimock-jest.import.api.md b/etc/aimock-jest.import.api.md index 96f2dee5..2a553853 100644 --- a/etc/aimock-jest.import.api.md +++ b/etc/aimock-jest.import.api.md @@ -297,15 +297,7 @@ type FixtureBlock = { }; // @public -interface FixtureCitation { - // (undocumented) - endIndex?: number; - startIndex?: number; - // (undocumented) - title?: string; - // (undocumented) - url: string; -} +type FixtureCitation = FixtureUrlCitation | FixtureFileCitation; // @public type FixtureFileBlock = { @@ -319,6 +311,15 @@ type FixtureFileBlock = { namespace?: string; }; +// @public +interface FixtureFileCitation { + fileId: string; + filename: string; + index?: number; + // (undocumented) + type: "file"; +} + // @public (undocumented) interface FixtureFileContentWithToolCallsResponse extends ResponseOverrides { blocks?: FixtureFileBlock[]; @@ -488,6 +489,18 @@ type FixtureOpts = Omit; // @public (undocumented) type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +// @public +interface FixtureUrlCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + type?: "url"; + // (undocumented) + url: string; +} + // @public (undocumented) interface ImageItem { // (undocumented) diff --git a/etc/aimock-jest.require.api.md b/etc/aimock-jest.require.api.md index 96f2dee5..2a553853 100644 --- a/etc/aimock-jest.require.api.md +++ b/etc/aimock-jest.require.api.md @@ -297,15 +297,7 @@ type FixtureBlock = { }; // @public -interface FixtureCitation { - // (undocumented) - endIndex?: number; - startIndex?: number; - // (undocumented) - title?: string; - // (undocumented) - url: string; -} +type FixtureCitation = FixtureUrlCitation | FixtureFileCitation; // @public type FixtureFileBlock = { @@ -319,6 +311,15 @@ type FixtureFileBlock = { namespace?: string; }; +// @public +interface FixtureFileCitation { + fileId: string; + filename: string; + index?: number; + // (undocumented) + type: "file"; +} + // @public (undocumented) interface FixtureFileContentWithToolCallsResponse extends ResponseOverrides { blocks?: FixtureFileBlock[]; @@ -488,6 +489,18 @@ type FixtureOpts = Omit; // @public (undocumented) type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +// @public +interface FixtureUrlCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + type?: "url"; + // (undocumented) + url: string; +} + // @public (undocumented) interface ImageItem { // (undocumented) diff --git a/etc/aimock-mcp.import.api.md b/etc/aimock-mcp.import.api.md index ad1b8f9f..d821ab97 100644 --- a/etc/aimock-mcp.import.api.md +++ b/etc/aimock-mcp.import.api.md @@ -229,14 +229,15 @@ type FixtureBlock = { }; // @public -interface FixtureCitation { - // (undocumented) - endIndex?: number; - startIndex?: number; - // (undocumented) - title?: string; +type FixtureCitation = FixtureUrlCitation | FixtureFileCitation; + +// @public +interface FixtureFileCitation { + fileId: string; + filename: string; + index?: number; // (undocumented) - url: string; + type: "file"; } // @public @@ -332,6 +333,18 @@ interface FixtureMatch { // @public (undocumented) type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +// @public +interface FixtureUrlCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + type?: "url"; + // (undocumented) + url: string; +} + // @public (undocumented) interface ImageItem { // (undocumented) diff --git a/etc/aimock-mcp.require.api.md b/etc/aimock-mcp.require.api.md index ad1b8f9f..d821ab97 100644 --- a/etc/aimock-mcp.require.api.md +++ b/etc/aimock-mcp.require.api.md @@ -229,14 +229,15 @@ type FixtureBlock = { }; // @public -interface FixtureCitation { - // (undocumented) - endIndex?: number; - startIndex?: number; - // (undocumented) - title?: string; +type FixtureCitation = FixtureUrlCitation | FixtureFileCitation; + +// @public +interface FixtureFileCitation { + fileId: string; + filename: string; + index?: number; // (undocumented) - url: string; + type: "file"; } // @public @@ -332,6 +333,18 @@ interface FixtureMatch { // @public (undocumented) type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +// @public +interface FixtureUrlCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + type?: "url"; + // (undocumented) + url: string; +} + // @public (undocumented) interface ImageItem { // (undocumented) diff --git a/etc/aimock-vector.import.api.md b/etc/aimock-vector.import.api.md index b6735412..5fd533c2 100644 --- a/etc/aimock-vector.import.api.md +++ b/etc/aimock-vector.import.api.md @@ -223,14 +223,15 @@ type FixtureBlock = { }; // @public -interface FixtureCitation { - // (undocumented) - endIndex?: number; - startIndex?: number; - // (undocumented) - title?: string; +type FixtureCitation = FixtureUrlCitation | FixtureFileCitation; + +// @public +interface FixtureFileCitation { + fileId: string; + filename: string; + index?: number; // (undocumented) - url: string; + type: "file"; } // @public @@ -277,6 +278,18 @@ interface FixtureMatch { // @public (undocumented) type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +// @public +interface FixtureUrlCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + type?: "url"; + // (undocumented) + url: string; +} + // @public (undocumented) interface ImageItem { // (undocumented) diff --git a/etc/aimock-vector.require.api.md b/etc/aimock-vector.require.api.md index b6735412..5fd533c2 100644 --- a/etc/aimock-vector.require.api.md +++ b/etc/aimock-vector.require.api.md @@ -223,14 +223,15 @@ type FixtureBlock = { }; // @public -interface FixtureCitation { - // (undocumented) - endIndex?: number; - startIndex?: number; - // (undocumented) - title?: string; +type FixtureCitation = FixtureUrlCitation | FixtureFileCitation; + +// @public +interface FixtureFileCitation { + fileId: string; + filename: string; + index?: number; // (undocumented) - url: string; + type: "file"; } // @public @@ -277,6 +278,18 @@ interface FixtureMatch { // @public (undocumented) type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +// @public +interface FixtureUrlCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + type?: "url"; + // (undocumented) + url: string; +} + // @public (undocumented) interface ImageItem { // (undocumented) diff --git a/etc/aimock-vitest.import.api.md b/etc/aimock-vitest.import.api.md index 07fdab83..cb667af8 100644 --- a/etc/aimock-vitest.import.api.md +++ b/etc/aimock-vitest.import.api.md @@ -297,15 +297,7 @@ type FixtureBlock = { }; // @public -interface FixtureCitation { - // (undocumented) - endIndex?: number; - startIndex?: number; - // (undocumented) - title?: string; - // (undocumented) - url: string; -} +type FixtureCitation = FixtureUrlCitation | FixtureFileCitation; // @public type FixtureFileBlock = { @@ -319,6 +311,15 @@ type FixtureFileBlock = { namespace?: string; }; +// @public +interface FixtureFileCitation { + fileId: string; + filename: string; + index?: number; + // (undocumented) + type: "file"; +} + // @public (undocumented) interface FixtureFileContentWithToolCallsResponse extends ResponseOverrides { blocks?: FixtureFileBlock[]; @@ -488,6 +489,18 @@ type FixtureOpts = Omit; // @public (undocumented) type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +// @public +interface FixtureUrlCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + type?: "url"; + // (undocumented) + url: string; +} + // @public (undocumented) interface ImageItem { // (undocumented) diff --git a/etc/aimock-vitest.require.api.md b/etc/aimock-vitest.require.api.md index 07fdab83..cb667af8 100644 --- a/etc/aimock-vitest.require.api.md +++ b/etc/aimock-vitest.require.api.md @@ -297,15 +297,7 @@ type FixtureBlock = { }; // @public -interface FixtureCitation { - // (undocumented) - endIndex?: number; - startIndex?: number; - // (undocumented) - title?: string; - // (undocumented) - url: string; -} +type FixtureCitation = FixtureUrlCitation | FixtureFileCitation; // @public type FixtureFileBlock = { @@ -319,6 +311,15 @@ type FixtureFileBlock = { namespace?: string; }; +// @public +interface FixtureFileCitation { + fileId: string; + filename: string; + index?: number; + // (undocumented) + type: "file"; +} + // @public (undocumented) interface FixtureFileContentWithToolCallsResponse extends ResponseOverrides { blocks?: FixtureFileBlock[]; @@ -488,6 +489,18 @@ type FixtureOpts = Omit; // @public (undocumented) type FixtureResponse = TextResponse | ToolCallResponse | ContentWithToolCallsResponse | ErrorResponse | EmbeddingResponse | ImageResponse | AudioResponse | TranscriptionResponse | VideoResponse | RawJSONResponse | LiveFixtureResponse; +// @public +interface FixtureUrlCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + type?: "url"; + // (undocumented) + url: string; +} + // @public (undocumented) interface ImageItem { // (undocumented)