From 220608a060e0653a0a140b90712abf9c92cca6f5 Mon Sep 17 00:00:00 2001 From: Abhi Aiyer Date: Sat, 17 Oct 2026 10:48:48 -0700 Subject: [PATCH 1/6] feat(responses): scriptable url_citation annotations via fixture citations Downstream SDKs map Responses API url_citation annotations to source parts, but fixtures could only produce empty annotations. A citations field on text and content+tool-call responses now fills annotations on output_text parts and emits response.output_text.annotation.added events when streaming. Refs #547 --- src/__tests__/responses.test.ts | 79 +++++++++++++++++++++++++++++++++ src/helpers.ts | 1 + src/responses.ts | 49 +++++++++++++++++--- src/types.ts | 11 +++++ 4 files changed, 134 insertions(+), 6 deletions(-) diff --git a/src/__tests__/responses.test.ts b/src/__tests__/responses.test.ts index 690c3980..9f9b4395 100644 --- a/src/__tests__/responses.test.ts +++ b/src/__tests__/responses.test.ts @@ -2680,3 +2680,82 @@ describe("handleResponses debug logging", () => { expect(debugMessages.some((m) => m.includes("No responses fixture matched"))).toBe(true); }); }); + +describe("POST /v1/responses (citations)", () => { + let instance: ServerInstance | undefined; + afterEach(async () => { + if (instance) { + await new Promise((resolve) => instance!.server.close(() => resolve())); + instance = undefined; + } + }); + + const fixture: Fixture = { + match: { userMessage: "cite-me" }, + response: { + content: "Mastra is a TypeScript framework.", + citations: [ + { url: "https://mastra.ai", title: "Mastra" }, + { url: "https://mastra.ai/docs", startIndex: 0, endIndex: 6 }, + ], + }, + }; + + it("non-streaming output_text carries url_citation annotations", async () => { + instance = await createServer([fixture]); + const res = await post(`${instance.url}/v1/responses`, { + model: "gpt-4", + input: [{ role: "user", content: "cite-me" }], + stream: false, + }); + const body = JSON.parse(res.body); + const msg = body.output.find((o: { type: string }) => o.type === "message"); + expect(msg.content[0].annotations).toEqual([ + { + type: "url_citation", + url: "https://mastra.ai", + title: "Mastra", + start_index: 0, + end_index: 33, + }, + { + type: "url_citation", + url: "https://mastra.ai/docs", + title: "https://mastra.ai/docs", + start_index: 0, + end_index: 6, + }, + ]); + }); + + it("streaming emits annotation.added events and annotated done parts", async () => { + instance = await createServer([fixture]); + const res = await post(`${instance.url}/v1/responses`, { + model: "gpt-4", + input: [{ role: "user", content: "cite-me" }], + stream: true, + }); + const events = parseResponsesSSEEvents(res.body); + const added = events.filter((e) => e.type === "response.output_text.annotation.added"); + expect(added.map((e) => (e.annotation as { url: string }).url)).toEqual([ + "https://mastra.ai", + "https://mastra.ai/docs", + ]); + expect(added.map((e) => e.annotation_index)).toEqual([0, 1]); + const done = events.find((e) => e.type === "response.content_part.done"); + expect((done!.part as { annotations: unknown[] }).annotations).toHaveLength(2); + }); + + it("omitting citations keeps annotations empty", async () => { + instance = await createServer([ + { match: { userMessage: "plain" }, response: { content: "hi" } }, + ]); + const res = await post(`${instance.url}/v1/responses`, { + model: "gpt-4", + input: [{ role: "user", content: "plain" }], + stream: false, + }); + const msg = JSON.parse(res.body).output.find((o: { type: string }) => o.type === "message"); + expect(msg.content[0].annotations).toEqual([]); + }); +}); diff --git a/src/helpers.ts b/src/helpers.ts index aaa826c8..47740b3b 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -1516,6 +1516,7 @@ export function extractOverrides( ): ResponseOverrides { const r = response; return { + ...(r.citations !== undefined && { citations: r.citations }), ...(r.id !== undefined && { id: r.id }), ...(r.created !== undefined && { created: r.created }), ...(r.model !== undefined && { model: r.model }), diff --git a/src/responses.ts b/src/responses.ts index d7f3ebcb..885bd4e9 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -595,6 +595,7 @@ export function buildTextStreamEvents( chunkSize, nextOutputIndex, overrides?.phase, + overrides?.citations, ); events.push(...msgEvents); @@ -1131,13 +1132,25 @@ interface MessageBlockResult { msgItem: object; } +function buildAnnotations(content: string, citations?: ResponseOverrides["citations"]): object[] { + return (citations ?? []).map((c) => ({ + type: "url_citation", + url: c.url, + title: c.title ?? c.url, + start_index: c.startIndex ?? 0, + end_index: c.endIndex ?? content.length, + })); +} + function buildMessageOutputEvents( content: string, chunkSize: number, outputIndex: number, phase?: ResponseOverrides["phase"], + citations?: ResponseOverrides["citations"], ): MessageBlockResult { const msgId = itemId(); + const annotations = buildAnnotations(content, citations); const events: ResponsesSSEEvent[] = []; events.push({ @@ -1177,12 +1190,22 @@ function buildMessageOutputEvents( content_index: 0, text: content, }); + annotations.forEach((annotation, annotationIndex) => { + events.push({ + type: "response.output_text.annotation.added", + item_id: msgId, + output_index: outputIndex, + content_index: 0, + annotation_index: annotationIndex, + annotation, + }); + }); events.push({ type: "response.content_part.done", item_id: msgId, output_index: outputIndex, content_index: 0, - part: { type: "output_text", text: content, annotations: [] }, + part: { type: "output_text", text: content, annotations }, }); const msgItem = { @@ -1191,7 +1214,7 @@ function buildMessageOutputEvents( status: "completed", role: "assistant", ...(phase !== undefined ? { phase } : {}), - content: [{ type: "output_text", text: content, annotations: [] }], + content: [{ type: "output_text", text: content, annotations }], }; events.push({ type: "response.output_item.done", output_index: outputIndex, item: msgItem }); @@ -1337,6 +1360,7 @@ function buildOutputPrefix( emitEncryptedReasoning = false, synthesizeSummarylessReasoning = false, phase?: ResponseOverrides["phase"], + citations?: ResponseOverrides["citations"], ): object[] { const output: object[] = []; @@ -1369,7 +1393,9 @@ function buildOutputPrefix( status: "completed", role: "assistant", ...(phase !== undefined ? { phase } : {}), - content: [{ type: "output_text", text: content, annotations: [] }], + content: [ + { type: "output_text", text: content, annotations: buildAnnotations(content, citations) }, + ], }); return output; @@ -1410,6 +1436,7 @@ export function buildTextResponse( emitEncryptedReasoning, synthesizeSummarylessReasoning, overrides?.phase, + overrides?.citations, ), overrides, ); @@ -1526,6 +1553,7 @@ export function contentWithToolCallsStreamEvents( chunkSize, outputIndex, overrides?.phase, + overrides?.citations, ); events.push(...msgEvents); orderedOutputItems.push(msgItem); @@ -1551,6 +1579,7 @@ export function contentWithToolCallsStreamEvents( chunkSize, nextOutputIndex, overrides?.phase, + overrides?.citations, ); events.push(...msgEvents); orderedOutputItems.push(msgItem); @@ -1602,14 +1631,20 @@ function buildToolCallOutputItem(tc: EmittableToolCall): object { ); } -function buildMessageOutputItem(content: string, phase?: ResponseOverrides["phase"]): object { +function buildMessageOutputItem( + content: string, + phase?: ResponseOverrides["phase"], + citations?: ResponseOverrides["citations"], +): object { return { type: "message", id: itemId(), status: "completed", role: "assistant", ...(phase !== undefined ? { phase } : {}), - content: [{ type: "output_text", text: content, annotations: [] }], + content: [ + { type: "output_text", text: content, annotations: buildAnnotations(content, citations) }, + ], }; } @@ -1654,7 +1689,7 @@ function buildContentWithToolCallsResponse( } for (const block of ordered) { if (block.type === "text") { - output.push(buildMessageOutputItem(block.text, overrides?.phase)); + output.push(buildMessageOutputItem(block.text, overrides?.phase, overrides?.citations)); } else { output.push(buildToolCallOutputItem(block)); } @@ -1673,6 +1708,7 @@ function buildContentWithToolCallsResponse( emitEncryptedReasoning, synthesizeSummarylessReasoning, overrides?.phase, + overrides?.citations, ); for (const tc of toolCalls) { output.push(buildToolCallOutputItem(tc)); @@ -2421,6 +2457,7 @@ export function buildResponsesMisbehavior( chunkSize, outputIndex, overrides.phase, + overrides.citations, ); events.push(...message.events); output.push(message.msgItem); diff --git a/src/types.ts b/src/types.ts index f831ecd2..3651e7db 100644 --- a/src/types.ts +++ b/src/types.ts @@ -270,7 +270,18 @@ export interface FixtureMatch { * API honors five common fields (no role, systemFingerprint) plus optional phase; * Claude (5: no created, systemFingerprint); Gemini (2: only finishReason, usage). */ +/** A source citation attached to a text response. Served on the OpenAI Responses API as `url_citation` annotations. */ +export interface FixtureCitation { + url: string; + title?: string; + /** Character offsets into the text; default to the whole text. */ + startIndex?: number; + endIndex?: number; +} + export interface ResponseOverrides { + /** Source citations for the response text (OpenAI Responses API only). */ + citations?: FixtureCitation[]; id?: string; created?: number; model?: string; From c1511cfa815fc041082740da6741532df929fcc4 Mon Sep 17 00:00:00 2001 From: Abhi Aiyer Date: Sat, 17 Oct 2026 10:49:31 -0700 Subject: [PATCH 2/6] feat(gemini): scriptable model-generated files via fixture files Gemini returns generated images as inlineData parts, which SDKs surface as file outputs. A files field on text responses now emits those parts ahead of the text, both streamed and non-streamed. Refs #548 --- src/__tests__/gemini.test.ts | 38 ++++++++++++++++++++++++++++++++++++ src/gemini.ts | 19 ++++++++++++++++++ src/helpers.ts | 1 + src/types.ts | 9 +++++++++ 4 files changed, 67 insertions(+) diff --git a/src/__tests__/gemini.test.ts b/src/__tests__/gemini.test.ts index e11d9722..6715e4dc 100644 --- a/src/__tests__/gemini.test.ts +++ b/src/__tests__/gemini.test.ts @@ -1255,3 +1255,41 @@ describe("Gemini streaming interruptions", () => { expect(entry!.response.interruptReason).toBe("disconnectAfterMs"); }); }); + +describe("Gemini model-generated files", () => { + const png = "iVBORw0KGgo="; + const fileFixture: Fixture = { + match: { userMessage: "draw" }, + response: { content: "Here is your image.", files: [{ mimeType: "image/png", data: png }] }, + }; + + it("non-streaming emits inlineData parts before the text", async () => { + instance = await createServer([fileFixture]); + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:generateContent`, { + contents: [{ role: "user", parts: [{ text: "draw" }] }], + }); + const parts = JSON.parse(res.body).candidates[0].content.parts; + expect(parts).toEqual([ + { inlineData: { mimeType: "image/png", data: png } }, + { text: "Here is your image." }, + ]); + }); + + it("streaming emits an inlineData chunk before the text chunks", async () => { + instance = await createServer([fileFixture]); + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:streamGenerateContent`, { + contents: [{ role: "user", parts: [{ text: "draw" }] }], + }); + const chunks = parseGeminiSSEChunks(res.body) as { + candidates: { content: { parts: { text?: string; inlineData?: unknown }[] } }[]; + }[]; + expect(chunks[0].candidates[0].content.parts).toEqual([ + { inlineData: { mimeType: "image/png", data: png } }, + ]); + const text = chunks + .slice(1) + .map((c) => c.candidates[0].content.parts[0].text ?? "") + .join(""); + expect(text).toBe("Here is your image."); + }); +}); diff --git a/src/gemini.ts b/src/gemini.ts index 47645248..c4a4999e 100644 --- a/src/gemini.ts +++ b/src/gemini.ts @@ -352,6 +352,22 @@ function buildGeminiTextStreamChunks( } } + if (overrides?.files?.length) { + chunks.push({ + candidates: [ + { + content: { + role: "model", + parts: overrides.files.map((f) => ({ + inlineData: { mimeType: f.mimeType, data: f.data }, + })), + }, + index: 0, + }, + ], + }); + } + // Content chunks for (let i = 0; i < content.length; i += chunkSize) { const slice = content.slice(i, i + chunkSize); @@ -462,6 +478,9 @@ function buildGeminiTextResponse( if (reasoning) { parts.push({ text: reasoning, thought: true }); } + for (const f of overrides?.files ?? []) { + parts.push({ inlineData: { mimeType: f.mimeType, data: f.data } }); + } parts.push({ text: content }); return { diff --git a/src/helpers.ts b/src/helpers.ts index 47740b3b..a1f82a8a 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -1517,6 +1517,7 @@ export function extractOverrides( const r = response; return { ...(r.citations !== undefined && { citations: r.citations }), + ...(r.files !== undefined && { files: r.files }), ...(r.id !== undefined && { id: r.id }), ...(r.created !== undefined && { created: r.created }), ...(r.model !== undefined && { model: r.model }), diff --git a/src/types.ts b/src/types.ts index 3651e7db..e7fc3093 100644 --- a/src/types.ts +++ b/src/types.ts @@ -279,7 +279,16 @@ export interface FixtureCitation { endIndex?: number; } +/** A model-generated file (e.g. an inline image). Served on Gemini as `inlineData` parts. */ +export interface FixtureFile { + mimeType: string; + /** Base64-encoded file bytes. */ + data: string; +} + export interface ResponseOverrides { + /** Model-generated files emitted before the response text (Gemini only). */ + files?: FixtureFile[]; /** Source citations for the response text (OpenAI Responses API only). */ citations?: FixtureCitation[]; id?: string; From 072b4a4cbc6cee847fd8ff8858c04a26f948d70d Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Thu, 22 Oct 2026 10:44:34 -0700 Subject: [PATCH 3/6] fix(types): rename the generated-file fixture type to FixtureGeneratedFile The new FixtureFile interface merged with the existing fixture-document FixtureFile, which broke typecheck and the public type. Rename it, export FixtureCitation and FixtureGeneratedFile, and refresh the API reports. The report change is additive only. --- etc/aimock-a2a.import.api.md | 20 +++++++++++++++++++ etc/aimock-a2a.require.api.md | 20 +++++++++++++++++++ etc/aimock-agui.import.api.md | 20 +++++++++++++++++++ etc/aimock-agui.require.api.md | 20 +++++++++++++++++++ etc/aimock-index.import.api.md | 20 +++++++++++++++++++ etc/aimock-index.require.api.md | 20 +++++++++++++++++++ etc/aimock-jest.import.api.md | 20 +++++++++++++++++++ etc/aimock-jest.require.api.md | 20 +++++++++++++++++++ etc/aimock-mcp.import.api.md | 20 +++++++++++++++++++ etc/aimock-mcp.require.api.md | 20 +++++++++++++++++++ etc/aimock-vector.import.api.md | 20 +++++++++++++++++++ etc/aimock-vector.require.api.md | 20 +++++++++++++++++++ etc/aimock-vitest.import.api.md | 20 +++++++++++++++++++ etc/aimock-vitest.require.api.md | 20 +++++++++++++++++++ src/index.ts | 2 ++ src/types.ts | 34 ++++++++++++++++---------------- 16 files changed, 299 insertions(+), 17 deletions(-) diff --git a/etc/aimock-a2a.import.api.md b/etc/aimock-a2a.import.api.md index 6654eac6..4936c553 100644 --- a/etc/aimock-a2a.import.api.md +++ b/etc/aimock-a2a.import.api.md @@ -360,6 +360,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -822,8 +840,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-a2a.require.api.md b/etc/aimock-a2a.require.api.md index 6654eac6..4936c553 100644 --- a/etc/aimock-a2a.require.api.md +++ b/etc/aimock-a2a.require.api.md @@ -360,6 +360,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -822,8 +840,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-agui.import.api.md b/etc/aimock-agui.import.api.md index 7c06bd55..262579c6 100644 --- a/etc/aimock-agui.import.api.md +++ b/etc/aimock-agui.import.api.md @@ -989,6 +989,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -1460,8 +1478,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-agui.require.api.md b/etc/aimock-agui.require.api.md index 7c06bd55..262579c6 100644 --- a/etc/aimock-agui.require.api.md +++ b/etc/aimock-agui.require.api.md @@ -989,6 +989,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -1460,8 +1478,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-index.import.api.md b/etc/aimock-index.import.api.md index 87f51da1..8b234513 100644 --- a/etc/aimock-index.import.api.md +++ b/etc/aimock-index.import.api.md @@ -2116,6 +2116,17 @@ export type FixtureBlock = { namespace?: string; }; +// @public +export interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + // @public (undocumented) export interface FixtureFile { // (undocumented) @@ -2256,6 +2267,13 @@ export interface FixtureFileToolCallResponse extends ResponseOverrides { webSearches?: string[]; } +// @public +export interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public export class FixtureLoadError extends Error { constructor(init: FixtureLoadErrorInit, options?: ErrorOptions); @@ -4656,8 +4674,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public export interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-index.require.api.md b/etc/aimock-index.require.api.md index 87f51da1..8b234513 100644 --- a/etc/aimock-index.require.api.md +++ b/etc/aimock-index.require.api.md @@ -2116,6 +2116,17 @@ export type FixtureBlock = { namespace?: string; }; +// @public +export interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + // @public (undocumented) export interface FixtureFile { // (undocumented) @@ -2256,6 +2267,13 @@ export interface FixtureFileToolCallResponse extends ResponseOverrides { webSearches?: string[]; } +// @public +export interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public export class FixtureLoadError extends Error { constructor(init: FixtureLoadErrorInit, options?: ErrorOptions); @@ -4656,8 +4674,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public export interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-jest.import.api.md b/etc/aimock-jest.import.api.md index fd278844..96f2dee5 100644 --- a/etc/aimock-jest.import.api.md +++ b/etc/aimock-jest.import.api.md @@ -296,6 +296,17 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + // @public type FixtureFileBlock = { type: "text"; @@ -430,6 +441,13 @@ interface FixtureFileToolCallResponse extends ResponseOverrides { webSearches?: string[]; } +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -1175,8 +1193,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-jest.require.api.md b/etc/aimock-jest.require.api.md index fd278844..96f2dee5 100644 --- a/etc/aimock-jest.require.api.md +++ b/etc/aimock-jest.require.api.md @@ -296,6 +296,17 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + // @public type FixtureFileBlock = { type: "text"; @@ -430,6 +441,13 @@ interface FixtureFileToolCallResponse extends ResponseOverrides { webSearches?: string[]; } +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -1175,8 +1193,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-mcp.import.api.md b/etc/aimock-mcp.import.api.md index 93411782..ad1b8f9f 100644 --- a/etc/aimock-mcp.import.api.md +++ b/etc/aimock-mcp.import.api.md @@ -228,6 +228,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public class FixtureLoadError extends Error { constructor(init: FixtureLoadErrorInit, options?: ErrorOptions); @@ -1384,8 +1402,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-mcp.require.api.md b/etc/aimock-mcp.require.api.md index 93411782..ad1b8f9f 100644 --- a/etc/aimock-mcp.require.api.md +++ b/etc/aimock-mcp.require.api.md @@ -228,6 +228,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public class FixtureLoadError extends Error { constructor(init: FixtureLoadErrorInit, options?: ErrorOptions); @@ -1384,8 +1402,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-vector.import.api.md b/etc/aimock-vector.import.api.md index 32209ff9..b6735412 100644 --- a/etc/aimock-vector.import.api.md +++ b/etc/aimock-vector.import.api.md @@ -222,6 +222,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -699,8 +717,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-vector.require.api.md b/etc/aimock-vector.require.api.md index 32209ff9..b6735412 100644 --- a/etc/aimock-vector.require.api.md +++ b/etc/aimock-vector.require.api.md @@ -222,6 +222,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -699,8 +717,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-vitest.import.api.md b/etc/aimock-vitest.import.api.md index a92ec33b..07fdab83 100644 --- a/etc/aimock-vitest.import.api.md +++ b/etc/aimock-vitest.import.api.md @@ -296,6 +296,17 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + // @public type FixtureFileBlock = { type: "text"; @@ -430,6 +441,13 @@ interface FixtureFileToolCallResponse extends ResponseOverrides { webSearches?: string[]; } +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -1175,8 +1193,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-vitest.require.api.md b/etc/aimock-vitest.require.api.md index a92ec33b..07fdab83 100644 --- a/etc/aimock-vitest.require.api.md +++ b/etc/aimock-vitest.require.api.md @@ -296,6 +296,17 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + // @public type FixtureFileBlock = { type: "text"; @@ -430,6 +441,13 @@ interface FixtureFileToolCallResponse extends ResponseOverrides { webSearches?: string[]; } +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -1175,8 +1193,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/src/index.ts b/src/index.ts index 7b4ed733..dba505a3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -545,6 +545,8 @@ export type { VideoResponse, RawJSONResponse, ResponseOverrides, + FixtureCitation, + FixtureGeneratedFile, ContentWithToolCallsResponse, FixtureFileResponse, FixtureFileToolCall, diff --git a/src/types.ts b/src/types.ts index 322b771d..a1c125ac 100644 --- a/src/types.ts +++ b/src/types.ts @@ -263,6 +263,22 @@ export interface FixtureMatch { // Fixture response types +/** A source citation attached to a text response. Served on the OpenAI Responses API as `url_citation` annotations. */ +export interface FixtureCitation { + url: string; + title?: string; + /** Character offsets into the text; default to the whole text. */ + startIndex?: number; + endIndex?: number; +} + +/** A model-generated file (e.g. an inline image). Served on Gemini as `inlineData` parts. */ +export interface FixtureGeneratedFile { + mimeType: string; + /** Base64-encoded file bytes. */ + data: string; +} + /** * Fields that override auto-generated envelope values in the built response. * Scalar fields (finishReason, role) use OpenAI-canonical values — provider @@ -281,25 +297,9 @@ export interface FixtureMatch { * API honors five common fields (no role, systemFingerprint) plus optional phase; * Claude (5: no created, systemFingerprint); Gemini (2: only finishReason, usage). */ -/** A source citation attached to a text response. Served on the OpenAI Responses API as `url_citation` annotations. */ -export interface FixtureCitation { - url: string; - title?: string; - /** Character offsets into the text; default to the whole text. */ - startIndex?: number; - endIndex?: number; -} - -/** A model-generated file (e.g. an inline image). Served on Gemini as `inlineData` parts. */ -export interface FixtureFile { - mimeType: string; - /** Base64-encoded file bytes. */ - data: string; -} - export interface ResponseOverrides { /** Model-generated files emitted before the response text (Gemini only). */ - files?: FixtureFile[]; + files?: FixtureGeneratedFile[]; /** Source citations for the response text (OpenAI Responses API only). */ citations?: FixtureCitation[]; id?: string; From 69be6c93a0d42803da882265ea1449e02274cefc Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Thu, 22 Oct 2026 10:44:36 -0700 Subject: [PATCH 4/6] fix(gemini): serve fixture files on combined and blocks turns Combined content+toolCalls turns (including toolCalls: [] and blocks) dropped fixture files on both the streaming and non-streaming wires. Emit the same leading inlineData part that text turns emit. Fixtures without files are unchanged. --- src/__tests__/gemini.test.ts | 83 ++++++++++++++++++++++++++++++++++++ src/gemini.ts | 39 +++++++++-------- 2 files changed, 104 insertions(+), 18 deletions(-) diff --git a/src/__tests__/gemini.test.ts b/src/__tests__/gemini.test.ts index 6715e4dc..e577ea90 100644 --- a/src/__tests__/gemini.test.ts +++ b/src/__tests__/gemini.test.ts @@ -1292,4 +1292,87 @@ describe("Gemini model-generated files", () => { .join(""); expect(text).toBe("Here is your image."); }); + + // Combined content+toolCalls turns (legacy and `blocks`) used to drop `files` + // on both wires; they must emit the same leading inlineData part as text turns. + type FilesChunk = { + candidates: { + content: { + parts: { text?: string; inlineData?: unknown; functionCall?: { name: string } }[]; + }; + }[]; + }; + const combinedFileFixtures: Fixture[] = [ + { + match: { userMessage: "draw-empty-tools" }, + response: { + content: "Here is your image.", + toolCalls: [], + files: [{ mimeType: "image/png", data: png }], + }, + }, + { + match: { userMessage: "draw-and-save" }, + response: { + content: "Saving it.", + toolCalls: [{ name: "save", arguments: '{"x":1}' }], + files: [{ mimeType: "image/png", data: png }], + }, + }, + { + match: { userMessage: "draw-blocks" }, + response: { + blocks: [ + { type: "toolCall", name: "save", arguments: '{"x":1}' }, + { type: "text", text: "Saved it." }, + ], + files: [{ mimeType: "image/png", data: png }], + }, + }, + ]; + + it.each([ + ["draw-empty-tools", [{ text: "Here is your image." }]], + ["draw-and-save", [{ text: "Saving it." }, { functionCall: { name: "save", args: { x: 1 } } }]], + ["draw-blocks", [{ functionCall: { name: "save", args: { x: 1 } } }, { text: "Saved it." }]], + ])("non-streaming combined turn %s emits inlineData before its parts", async (msg, rest) => { + instance = await createServer(combinedFileFixtures); + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:generateContent`, { + contents: [{ role: "user", parts: [{ text: msg }] }], + }); + const parts = JSON.parse(res.body).candidates[0].content.parts; + expect(parts).toEqual([{ inlineData: { mimeType: "image/png", data: png } }, ...rest]); + }); + + it.each([ + ["draw-empty-tools", "Here is your image.", []], + ["draw-and-save", "Saving it.", ["save"]], + ["draw-blocks", "Saved it.", ["save"]], + ])("streaming combined turn %s emits an inlineData chunk first", async (msg, text, toolNames) => { + instance = await createServer(combinedFileFixtures); + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:streamGenerateContent`, { + contents: [{ role: "user", parts: [{ text: msg }] }], + }); + const chunks = parseGeminiSSEChunks(res.body) as FilesChunk[]; + expect(chunks[0].candidates[0].content.parts).toEqual([ + { inlineData: { mimeType: "image/png", data: png } }, + ]); + const rest = chunks.slice(1).flatMap((c) => c.candidates[0].content.parts); + expect(rest.some((p) => p.inlineData !== undefined)).toBe(false); + expect(rest.map((p) => p.text ?? "").join("")).toBe(text); + expect(rest.flatMap((p) => (p.functionCall ? [p.functionCall.name] : []))).toEqual(toolNames); + }); + + it("fixtures without files emit no inlineData on combined turns", async () => { + instance = await createServer([ + { + match: { userMessage: "plain-combined" }, + response: { content: "Saving it.", toolCalls: [{ name: "save", arguments: "{}" }] }, + }, + ]); + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:generateContent`, { + contents: [{ role: "user", parts: [{ text: "plain-combined" }] }], + }); + expect(res.body).not.toContain("inlineData"); + }); }); diff --git a/src/gemini.ts b/src/gemini.ts index c7070bc8..11f33a36 100644 --- a/src/gemini.ts +++ b/src/gemini.ts @@ -306,6 +306,22 @@ interface GeminiResponseChunk { }; } +// Fixture `files` as Gemini `inlineData` parts. Every text-bearing builder emits +// these after any thought parts and before the text / functionCall parts. +function geminiFileParts(overrides?: ResponseOverrides): GeminiPart[] { + return (overrides?.files ?? []).map((f) => ({ + inlineData: { mimeType: f.mimeType, data: f.data }, + })); +} + +// Streaming form of {@link geminiFileParts}: one chunk holding every file part, +// or nothing when the fixture has no files. +function geminiFileChunks(overrides?: ResponseOverrides): GeminiResponseChunk[] { + const parts = geminiFileParts(overrides); + if (parts.length === 0) return []; + return [{ candidates: [{ content: { role: "model", parts }, index: 0 }] }]; +} + function buildGeminiTextStreamChunks( content: string, chunkSize: number, @@ -331,21 +347,7 @@ function buildGeminiTextStreamChunks( } } - if (overrides?.files?.length) { - chunks.push({ - candidates: [ - { - content: { - role: "model", - parts: overrides.files.map((f) => ({ - inlineData: { mimeType: f.mimeType, data: f.data }, - })), - }, - index: 0, - }, - ], - }); - } + chunks.push(...geminiFileChunks(overrides)); // Content chunks for (let i = 0; i < content.length; i += chunkSize) { @@ -457,9 +459,7 @@ function buildGeminiTextResponse( if (reasoning) { parts.push({ text: reasoning, thought: true }); } - for (const f of overrides?.files ?? []) { - parts.push({ inlineData: { mimeType: f.mimeType, data: f.data } }); - } + parts.push(...geminiFileParts(overrides)); parts.push({ text: content }); return { @@ -524,6 +524,8 @@ function buildGeminiContentWithToolCallsStreamChunks( } } + chunks.push(...geminiFileChunks(overrides)); + if (blocks && blocks.length > 0) { // NEW path (#274): stream chunks whose parts follow the blocks' ARRAY ORDER, // so a tool-first / interleaved fixture emits its functionCall part before @@ -642,6 +644,7 @@ function buildGeminiContentWithToolCallsResponse( if (reasoning) { parts.push({ text: reasoning, thought: true }); } + parts.push(...geminiFileParts(overrides)); let blockHasTools: boolean | undefined; if (blocks && blocks.length > 0) { From d07d66d91bbd18e222201429dcf6cdf9de989174 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Thu, 22 Oct 2026 10:44:37 -0700 Subject: [PATCH 5/6] docs: add changelog entry for fixture citations and files --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72d7a79c..2a02182e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Fixture `citations` serve OpenAI Responses `url_citation` annotations; fixture `files` serve Gemini `inlineData` parts (#550 by @abhiaiyer91) + ## [1.46.0] - 2026-10-20 ### Added From cf9cb157123f58a9000f62948a5bc124e7cf937f Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Thu, 22 Oct 2026 10:51:16 -0700 Subject: [PATCH 6/6] fix(gemini): serve fixture files on tool-only turns Tool-only turns (no text) dropped fixture files on both wires. Emit the inlineData part after any thought part and before the functionCall parts, the same position as text and combined turns. Fixtures without files are unchanged. --- src/__tests__/gemini.test.ts | 73 ++++++++++++++++++++++++++++++++++++ src/gemini.ts | 7 +++- src/types.ts | 2 +- 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/src/__tests__/gemini.test.ts b/src/__tests__/gemini.test.ts index e577ea90..4ace5ef9 100644 --- a/src/__tests__/gemini.test.ts +++ b/src/__tests__/gemini.test.ts @@ -1363,6 +1363,64 @@ describe("Gemini model-generated files", () => { expect(rest.flatMap((p) => (p.functionCall ? [p.functionCall.name] : []))).toEqual(toolNames); }); + // Tool-only turns (no text) keep their files too: after any thought part, + // before the functionCall parts, matching the text and combined shapes. + const toolOnlyFileFixtures: Fixture[] = [ + { + match: { userMessage: "plain-tool-file" }, + response: { + toolCalls: [{ name: "save", arguments: '{"x":1}' }], + files: [{ mimeType: "image/png", data: png }], + }, + }, + { + match: { userMessage: "thinking-tool-file" }, + response: { + reasoning: "Plan.", + toolCalls: [{ name: "save", arguments: '{"x":1}' }], + files: [{ mimeType: "image/png", data: png }], + }, + }, + ]; + + it.each([ + ["plain-tool-file", []], + ["thinking-tool-file", [{ text: "Plan.", thought: true }]], + ])("non-streaming tool-only turn %s emits inlineData before functionCall", async (msg, lead) => { + instance = await createServer(toolOnlyFileFixtures); + const res = await post(`${instance.url}/v1beta/models/gemini-2.5-flash:generateContent`, { + contents: [{ role: "user", parts: [{ text: msg }] }], + }); + const parts = JSON.parse(res.body).candidates[0].content.parts; + expect(parts).toEqual([ + ...lead, + { inlineData: { mimeType: "image/png", data: png } }, + { functionCall: { name: "save", args: { x: 1 } } }, + ]); + }); + + it.each([ + ["plain-tool-file", []], + ["thinking-tool-file", [[{ text: "Plan.", thought: true }]]], + ])( + "streaming tool-only turn %s emits an inlineData chunk before functionCall", + async (msg, lead) => { + instance = await createServer(toolOnlyFileFixtures); + const res = await post( + `${instance.url}/v1beta/models/gemini-2.5-flash:streamGenerateContent`, + { + contents: [{ role: "user", parts: [{ text: msg }] }], + }, + ); + const chunks = parseGeminiSSEChunks(res.body) as FilesChunk[]; + expect(chunks.map((c) => c.candidates[0].content.parts)).toEqual([ + ...lead, + [{ inlineData: { mimeType: "image/png", data: png } }], + [{ functionCall: { name: "save", args: { x: 1 } } }], + ]); + }, + ); + it("fixtures without files emit no inlineData on combined turns", async () => { instance = await createServer([ { @@ -1375,4 +1433,19 @@ describe("Gemini model-generated files", () => { }); expect(res.body).not.toContain("inlineData"); }); + + it("fixtures without files emit no inlineData on tool-only turns", async () => { + instance = await createServer([ + { + match: { userMessage: "plain-tool" }, + response: { toolCalls: [{ name: "save", arguments: "{}" }] }, + }, + ]); + for (const ep of ["generateContent", "streamGenerateContent"]) { + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:${ep}`, { + contents: [{ role: "user", parts: [{ text: "plain-tool" }] }], + }); + expect(res.body).not.toContain("inlineData"); + } + }); }); diff --git a/src/gemini.ts b/src/gemini.ts index 11f33a36..1e18a293 100644 --- a/src/gemini.ts +++ b/src/gemini.ts @@ -306,8 +306,8 @@ interface GeminiResponseChunk { }; } -// Fixture `files` as Gemini `inlineData` parts. Every text-bearing builder emits -// these after any thought parts and before the text / functionCall parts. +// Fixture `files` as Gemini `inlineData` parts. Every text and tool-call builder +// emits these after any thought parts and before the text / functionCall parts. function geminiFileParts(overrides?: ResponseOverrides): GeminiPart[] { return (overrides?.files ?? []).map((f) => ({ inlineData: { mimeType: f.mimeType, data: f.data }, @@ -431,6 +431,8 @@ function buildGeminiToolCallStreamChunks( } } + chunks.push(...geminiFileChunks(overrides)); + const parts: GeminiPart[] = toolCalls.map((tc) => parseToolCallPart(tc, logger)); // Gemini sends all tool calls in a single response chunk @@ -484,6 +486,7 @@ function buildGeminiToolCallResponse( if (reasoning) { parts.push({ text: reasoning, thought: true }); } + parts.push(...geminiFileParts(overrides)); parts.push(...toolCalls.map((tc) => parseToolCallPart(tc, logger))); return { diff --git a/src/types.ts b/src/types.ts index a1c125ac..85b3b4c0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -298,7 +298,7 @@ export interface FixtureGeneratedFile { * Claude (5: no created, systemFingerprint); Gemini (2: only finishReason, usage). */ export interface ResponseOverrides { - /** Model-generated files emitted before the response text (Gemini only). */ + /** Model-generated files emitted before the response text and tool calls (Gemini only). */ files?: FixtureGeneratedFile[]; /** Source citations for the response text (OpenAI Responses API only). */ citations?: FixtureCitation[];