diff --git a/CHANGELOG.md b/CHANGELOG.md index 72d7a79c..2a02182e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Fixture `citations` serve OpenAI Responses `url_citation` annotations; fixture `files` serve Gemini `inlineData` parts (#550 by @abhiaiyer91) + ## [1.46.0] - 2026-10-20 ### Added diff --git a/etc/aimock-a2a.import.api.md b/etc/aimock-a2a.import.api.md index 6654eac6..4936c553 100644 --- a/etc/aimock-a2a.import.api.md +++ b/etc/aimock-a2a.import.api.md @@ -360,6 +360,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -822,8 +840,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-a2a.require.api.md b/etc/aimock-a2a.require.api.md index 6654eac6..4936c553 100644 --- a/etc/aimock-a2a.require.api.md +++ b/etc/aimock-a2a.require.api.md @@ -360,6 +360,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -822,8 +840,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-agui.import.api.md b/etc/aimock-agui.import.api.md index 7c06bd55..262579c6 100644 --- a/etc/aimock-agui.import.api.md +++ b/etc/aimock-agui.import.api.md @@ -989,6 +989,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -1460,8 +1478,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-agui.require.api.md b/etc/aimock-agui.require.api.md index 7c06bd55..262579c6 100644 --- a/etc/aimock-agui.require.api.md +++ b/etc/aimock-agui.require.api.md @@ -989,6 +989,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -1460,8 +1478,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-index.import.api.md b/etc/aimock-index.import.api.md index 87f51da1..8b234513 100644 --- a/etc/aimock-index.import.api.md +++ b/etc/aimock-index.import.api.md @@ -2116,6 +2116,17 @@ export type FixtureBlock = { namespace?: string; }; +// @public +export interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + // @public (undocumented) export interface FixtureFile { // (undocumented) @@ -2256,6 +2267,13 @@ export interface FixtureFileToolCallResponse extends ResponseOverrides { webSearches?: string[]; } +// @public +export interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public export class FixtureLoadError extends Error { constructor(init: FixtureLoadErrorInit, options?: ErrorOptions); @@ -4656,8 +4674,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public export interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-index.require.api.md b/etc/aimock-index.require.api.md index 87f51da1..8b234513 100644 --- a/etc/aimock-index.require.api.md +++ b/etc/aimock-index.require.api.md @@ -2116,6 +2116,17 @@ export type FixtureBlock = { namespace?: string; }; +// @public +export interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + // @public (undocumented) export interface FixtureFile { // (undocumented) @@ -2256,6 +2267,13 @@ export interface FixtureFileToolCallResponse extends ResponseOverrides { webSearches?: string[]; } +// @public +export interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public export class FixtureLoadError extends Error { constructor(init: FixtureLoadErrorInit, options?: ErrorOptions); @@ -4656,8 +4674,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public export interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-jest.import.api.md b/etc/aimock-jest.import.api.md index fd278844..96f2dee5 100644 --- a/etc/aimock-jest.import.api.md +++ b/etc/aimock-jest.import.api.md @@ -296,6 +296,17 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + // @public type FixtureFileBlock = { type: "text"; @@ -430,6 +441,13 @@ interface FixtureFileToolCallResponse extends ResponseOverrides { webSearches?: string[]; } +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -1175,8 +1193,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-jest.require.api.md b/etc/aimock-jest.require.api.md index fd278844..96f2dee5 100644 --- a/etc/aimock-jest.require.api.md +++ b/etc/aimock-jest.require.api.md @@ -296,6 +296,17 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + // @public type FixtureFileBlock = { type: "text"; @@ -430,6 +441,13 @@ interface FixtureFileToolCallResponse extends ResponseOverrides { webSearches?: string[]; } +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -1175,8 +1193,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-mcp.import.api.md b/etc/aimock-mcp.import.api.md index 93411782..ad1b8f9f 100644 --- a/etc/aimock-mcp.import.api.md +++ b/etc/aimock-mcp.import.api.md @@ -228,6 +228,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public class FixtureLoadError extends Error { constructor(init: FixtureLoadErrorInit, options?: ErrorOptions); @@ -1384,8 +1402,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-mcp.require.api.md b/etc/aimock-mcp.require.api.md index 93411782..ad1b8f9f 100644 --- a/etc/aimock-mcp.require.api.md +++ b/etc/aimock-mcp.require.api.md @@ -228,6 +228,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public class FixtureLoadError extends Error { constructor(init: FixtureLoadErrorInit, options?: ErrorOptions); @@ -1384,8 +1402,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-vector.import.api.md b/etc/aimock-vector.import.api.md index 32209ff9..b6735412 100644 --- a/etc/aimock-vector.import.api.md +++ b/etc/aimock-vector.import.api.md @@ -222,6 +222,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -699,8 +717,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-vector.require.api.md b/etc/aimock-vector.require.api.md index 32209ff9..b6735412 100644 --- a/etc/aimock-vector.require.api.md +++ b/etc/aimock-vector.require.api.md @@ -222,6 +222,24 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -699,8 +717,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-vitest.import.api.md b/etc/aimock-vitest.import.api.md index a92ec33b..07fdab83 100644 --- a/etc/aimock-vitest.import.api.md +++ b/etc/aimock-vitest.import.api.md @@ -296,6 +296,17 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + // @public type FixtureFileBlock = { type: "text"; @@ -430,6 +441,13 @@ interface FixtureFileToolCallResponse extends ResponseOverrides { webSearches?: string[]; } +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -1175,8 +1193,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/etc/aimock-vitest.require.api.md b/etc/aimock-vitest.require.api.md index a92ec33b..07fdab83 100644 --- a/etc/aimock-vitest.require.api.md +++ b/etc/aimock-vitest.require.api.md @@ -296,6 +296,17 @@ type FixtureBlock = { namespace?: string; }; +// @public +interface FixtureCitation { + // (undocumented) + endIndex?: number; + startIndex?: number; + // (undocumented) + title?: string; + // (undocumented) + url: string; +} + // @public type FixtureFileBlock = { type: "text"; @@ -430,6 +441,13 @@ interface FixtureFileToolCallResponse extends ResponseOverrides { webSearches?: string[]; } +// @public +interface FixtureGeneratedFile { + data: string; + // (undocumented) + mimeType: string; +} + // @public (undocumented) interface FixtureMatch { // (undocumented) @@ -1175,8 +1193,10 @@ type ResponseFactory = (req: ChatCompletionRequest) => FixtureResponse | Promise // @public interface ResponseOverrides { + citations?: FixtureCitation[]; // (undocumented) created?: number; + files?: FixtureGeneratedFile[]; // (undocumented) finishReason?: string; // (undocumented) diff --git a/src/__tests__/gemini.test.ts b/src/__tests__/gemini.test.ts index e11d9722..4ace5ef9 100644 --- a/src/__tests__/gemini.test.ts +++ b/src/__tests__/gemini.test.ts @@ -1255,3 +1255,197 @@ describe("Gemini streaming interruptions", () => { expect(entry!.response.interruptReason).toBe("disconnectAfterMs"); }); }); + +describe("Gemini model-generated files", () => { + const png = "iVBORw0KGgo="; + const fileFixture: Fixture = { + match: { userMessage: "draw" }, + response: { content: "Here is your image.", files: [{ mimeType: "image/png", data: png }] }, + }; + + it("non-streaming emits inlineData parts before the text", async () => { + instance = await createServer([fileFixture]); + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:generateContent`, { + contents: [{ role: "user", parts: [{ text: "draw" }] }], + }); + const parts = JSON.parse(res.body).candidates[0].content.parts; + expect(parts).toEqual([ + { inlineData: { mimeType: "image/png", data: png } }, + { text: "Here is your image." }, + ]); + }); + + it("streaming emits an inlineData chunk before the text chunks", async () => { + instance = await createServer([fileFixture]); + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:streamGenerateContent`, { + contents: [{ role: "user", parts: [{ text: "draw" }] }], + }); + const chunks = parseGeminiSSEChunks(res.body) as { + candidates: { content: { parts: { text?: string; inlineData?: unknown }[] } }[]; + }[]; + expect(chunks[0].candidates[0].content.parts).toEqual([ + { inlineData: { mimeType: "image/png", data: png } }, + ]); + const text = chunks + .slice(1) + .map((c) => c.candidates[0].content.parts[0].text ?? "") + .join(""); + expect(text).toBe("Here is your image."); + }); + + // Combined content+toolCalls turns (legacy and `blocks`) used to drop `files` + // on both wires; they must emit the same leading inlineData part as text turns. + type FilesChunk = { + candidates: { + content: { + parts: { text?: string; inlineData?: unknown; functionCall?: { name: string } }[]; + }; + }[]; + }; + const combinedFileFixtures: Fixture[] = [ + { + match: { userMessage: "draw-empty-tools" }, + response: { + content: "Here is your image.", + toolCalls: [], + files: [{ mimeType: "image/png", data: png }], + }, + }, + { + match: { userMessage: "draw-and-save" }, + response: { + content: "Saving it.", + toolCalls: [{ name: "save", arguments: '{"x":1}' }], + files: [{ mimeType: "image/png", data: png }], + }, + }, + { + match: { userMessage: "draw-blocks" }, + response: { + blocks: [ + { type: "toolCall", name: "save", arguments: '{"x":1}' }, + { type: "text", text: "Saved it." }, + ], + files: [{ mimeType: "image/png", data: png }], + }, + }, + ]; + + it.each([ + ["draw-empty-tools", [{ text: "Here is your image." }]], + ["draw-and-save", [{ text: "Saving it." }, { functionCall: { name: "save", args: { x: 1 } } }]], + ["draw-blocks", [{ functionCall: { name: "save", args: { x: 1 } } }, { text: "Saved it." }]], + ])("non-streaming combined turn %s emits inlineData before its parts", async (msg, rest) => { + instance = await createServer(combinedFileFixtures); + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:generateContent`, { + contents: [{ role: "user", parts: [{ text: msg }] }], + }); + const parts = JSON.parse(res.body).candidates[0].content.parts; + expect(parts).toEqual([{ inlineData: { mimeType: "image/png", data: png } }, ...rest]); + }); + + it.each([ + ["draw-empty-tools", "Here is your image.", []], + ["draw-and-save", "Saving it.", ["save"]], + ["draw-blocks", "Saved it.", ["save"]], + ])("streaming combined turn %s emits an inlineData chunk first", async (msg, text, toolNames) => { + instance = await createServer(combinedFileFixtures); + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:streamGenerateContent`, { + contents: [{ role: "user", parts: [{ text: msg }] }], + }); + const chunks = parseGeminiSSEChunks(res.body) as FilesChunk[]; + expect(chunks[0].candidates[0].content.parts).toEqual([ + { inlineData: { mimeType: "image/png", data: png } }, + ]); + const rest = chunks.slice(1).flatMap((c) => c.candidates[0].content.parts); + expect(rest.some((p) => p.inlineData !== undefined)).toBe(false); + expect(rest.map((p) => p.text ?? "").join("")).toBe(text); + expect(rest.flatMap((p) => (p.functionCall ? [p.functionCall.name] : []))).toEqual(toolNames); + }); + + // Tool-only turns (no text) keep their files too: after any thought part, + // before the functionCall parts, matching the text and combined shapes. + const toolOnlyFileFixtures: Fixture[] = [ + { + match: { userMessage: "plain-tool-file" }, + response: { + toolCalls: [{ name: "save", arguments: '{"x":1}' }], + files: [{ mimeType: "image/png", data: png }], + }, + }, + { + match: { userMessage: "thinking-tool-file" }, + response: { + reasoning: "Plan.", + toolCalls: [{ name: "save", arguments: '{"x":1}' }], + files: [{ mimeType: "image/png", data: png }], + }, + }, + ]; + + it.each([ + ["plain-tool-file", []], + ["thinking-tool-file", [{ text: "Plan.", thought: true }]], + ])("non-streaming tool-only turn %s emits inlineData before functionCall", async (msg, lead) => { + instance = await createServer(toolOnlyFileFixtures); + const res = await post(`${instance.url}/v1beta/models/gemini-2.5-flash:generateContent`, { + contents: [{ role: "user", parts: [{ text: msg }] }], + }); + const parts = JSON.parse(res.body).candidates[0].content.parts; + expect(parts).toEqual([ + ...lead, + { inlineData: { mimeType: "image/png", data: png } }, + { functionCall: { name: "save", args: { x: 1 } } }, + ]); + }); + + it.each([ + ["plain-tool-file", []], + ["thinking-tool-file", [[{ text: "Plan.", thought: true }]]], + ])( + "streaming tool-only turn %s emits an inlineData chunk before functionCall", + async (msg, lead) => { + instance = await createServer(toolOnlyFileFixtures); + const res = await post( + `${instance.url}/v1beta/models/gemini-2.5-flash:streamGenerateContent`, + { + contents: [{ role: "user", parts: [{ text: msg }] }], + }, + ); + const chunks = parseGeminiSSEChunks(res.body) as FilesChunk[]; + expect(chunks.map((c) => c.candidates[0].content.parts)).toEqual([ + ...lead, + [{ inlineData: { mimeType: "image/png", data: png } }], + [{ functionCall: { name: "save", args: { x: 1 } } }], + ]); + }, + ); + + it("fixtures without files emit no inlineData on combined turns", async () => { + instance = await createServer([ + { + match: { userMessage: "plain-combined" }, + response: { content: "Saving it.", toolCalls: [{ name: "save", arguments: "{}" }] }, + }, + ]); + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:generateContent`, { + contents: [{ role: "user", parts: [{ text: "plain-combined" }] }], + }); + expect(res.body).not.toContain("inlineData"); + }); + + it("fixtures without files emit no inlineData on tool-only turns", async () => { + instance = await createServer([ + { + match: { userMessage: "plain-tool" }, + response: { toolCalls: [{ name: "save", arguments: "{}" }] }, + }, + ]); + for (const ep of ["generateContent", "streamGenerateContent"]) { + const res = await post(`${instance.url}/v1beta/models/gemini-2.0-flash:${ep}`, { + contents: [{ role: "user", parts: [{ text: "plain-tool" }] }], + }); + expect(res.body).not.toContain("inlineData"); + } + }); +}); diff --git a/src/__tests__/responses.test.ts b/src/__tests__/responses.test.ts index 690c3980..9f9b4395 100644 --- a/src/__tests__/responses.test.ts +++ b/src/__tests__/responses.test.ts @@ -2680,3 +2680,82 @@ describe("handleResponses debug logging", () => { expect(debugMessages.some((m) => m.includes("No responses fixture matched"))).toBe(true); }); }); + +describe("POST /v1/responses (citations)", () => { + let instance: ServerInstance | undefined; + afterEach(async () => { + if (instance) { + await new Promise((resolve) => instance!.server.close(() => resolve())); + instance = undefined; + } + }); + + const fixture: Fixture = { + match: { userMessage: "cite-me" }, + response: { + content: "Mastra is a TypeScript framework.", + citations: [ + { url: "https://mastra.ai", title: "Mastra" }, + { url: "https://mastra.ai/docs", startIndex: 0, endIndex: 6 }, + ], + }, + }; + + it("non-streaming output_text carries url_citation annotations", async () => { + instance = await createServer([fixture]); + const res = await post(`${instance.url}/v1/responses`, { + model: "gpt-4", + input: [{ role: "user", content: "cite-me" }], + stream: false, + }); + const body = JSON.parse(res.body); + const msg = body.output.find((o: { type: string }) => o.type === "message"); + expect(msg.content[0].annotations).toEqual([ + { + type: "url_citation", + url: "https://mastra.ai", + title: "Mastra", + start_index: 0, + end_index: 33, + }, + { + type: "url_citation", + url: "https://mastra.ai/docs", + title: "https://mastra.ai/docs", + start_index: 0, + end_index: 6, + }, + ]); + }); + + it("streaming emits annotation.added events and annotated done parts", async () => { + instance = await createServer([fixture]); + const res = await post(`${instance.url}/v1/responses`, { + model: "gpt-4", + input: [{ role: "user", content: "cite-me" }], + stream: true, + }); + const events = parseResponsesSSEEvents(res.body); + const added = events.filter((e) => e.type === "response.output_text.annotation.added"); + expect(added.map((e) => (e.annotation as { url: string }).url)).toEqual([ + "https://mastra.ai", + "https://mastra.ai/docs", + ]); + expect(added.map((e) => e.annotation_index)).toEqual([0, 1]); + const done = events.find((e) => e.type === "response.content_part.done"); + expect((done!.part as { annotations: unknown[] }).annotations).toHaveLength(2); + }); + + it("omitting citations keeps annotations empty", async () => { + instance = await createServer([ + { match: { userMessage: "plain" }, response: { content: "hi" } }, + ]); + const res = await post(`${instance.url}/v1/responses`, { + model: "gpt-4", + input: [{ role: "user", content: "plain" }], + stream: false, + }); + const msg = JSON.parse(res.body).output.find((o: { type: string }) => o.type === "message"); + expect(msg.content[0].annotations).toEqual([]); + }); +}); diff --git a/src/gemini.ts b/src/gemini.ts index 18c466f8..1e18a293 100644 --- a/src/gemini.ts +++ b/src/gemini.ts @@ -306,6 +306,22 @@ interface GeminiResponseChunk { }; } +// Fixture `files` as Gemini `inlineData` parts. Every text and tool-call builder +// emits these after any thought parts and before the text / functionCall parts. +function geminiFileParts(overrides?: ResponseOverrides): GeminiPart[] { + return (overrides?.files ?? []).map((f) => ({ + inlineData: { mimeType: f.mimeType, data: f.data }, + })); +} + +// Streaming form of {@link geminiFileParts}: one chunk holding every file part, +// or nothing when the fixture has no files. +function geminiFileChunks(overrides?: ResponseOverrides): GeminiResponseChunk[] { + const parts = geminiFileParts(overrides); + if (parts.length === 0) return []; + return [{ candidates: [{ content: { role: "model", parts }, index: 0 }] }]; +} + function buildGeminiTextStreamChunks( content: string, chunkSize: number, @@ -331,6 +347,8 @@ function buildGeminiTextStreamChunks( } } + chunks.push(...geminiFileChunks(overrides)); + // Content chunks for (let i = 0; i < content.length; i += chunkSize) { const slice = content.slice(i, i + chunkSize); @@ -413,6 +431,8 @@ function buildGeminiToolCallStreamChunks( } } + chunks.push(...geminiFileChunks(overrides)); + const parts: GeminiPart[] = toolCalls.map((tc) => parseToolCallPart(tc, logger)); // Gemini sends all tool calls in a single response chunk @@ -441,6 +461,7 @@ function buildGeminiTextResponse( if (reasoning) { parts.push({ text: reasoning, thought: true }); } + parts.push(...geminiFileParts(overrides)); parts.push({ text: content }); return { @@ -465,6 +486,7 @@ function buildGeminiToolCallResponse( if (reasoning) { parts.push({ text: reasoning, thought: true }); } + parts.push(...geminiFileParts(overrides)); parts.push(...toolCalls.map((tc) => parseToolCallPart(tc, logger))); return { @@ -505,6 +527,8 @@ function buildGeminiContentWithToolCallsStreamChunks( } } + chunks.push(...geminiFileChunks(overrides)); + if (blocks && blocks.length > 0) { // NEW path (#274): stream chunks whose parts follow the blocks' ARRAY ORDER, // so a tool-first / interleaved fixture emits its functionCall part before @@ -623,6 +647,7 @@ function buildGeminiContentWithToolCallsResponse( if (reasoning) { parts.push({ text: reasoning, thought: true }); } + parts.push(...geminiFileParts(overrides)); let blockHasTools: boolean | undefined; if (blocks && blocks.length > 0) { diff --git a/src/helpers.ts b/src/helpers.ts index 54c05354..abe95146 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -1508,6 +1508,8 @@ export function extractOverrides( ): ResponseOverrides { const r = response; return { + ...(r.citations !== undefined && { citations: r.citations }), + ...(r.files !== undefined && { files: r.files }), ...(r.id !== undefined && { id: r.id }), ...(r.created !== undefined && { created: r.created }), ...(r.model !== undefined && { model: r.model }), diff --git a/src/index.ts b/src/index.ts index 7b4ed733..dba505a3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -545,6 +545,8 @@ export type { VideoResponse, RawJSONResponse, ResponseOverrides, + FixtureCitation, + FixtureGeneratedFile, ContentWithToolCallsResponse, FixtureFileResponse, FixtureFileToolCall, diff --git a/src/responses.ts b/src/responses.ts index 8e0d2790..c3c2e727 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -578,6 +578,7 @@ export function buildTextStreamEvents( chunkSize, nextOutputIndex, overrides?.phase, + overrides?.citations, ); events.push(...msgEvents); @@ -1114,13 +1115,25 @@ interface MessageBlockResult { msgItem: object; } +function buildAnnotations(content: string, citations?: ResponseOverrides["citations"]): object[] { + return (citations ?? []).map((c) => ({ + type: "url_citation", + url: c.url, + title: c.title ?? c.url, + start_index: c.startIndex ?? 0, + end_index: c.endIndex ?? content.length, + })); +} + function buildMessageOutputEvents( content: string, chunkSize: number, outputIndex: number, phase?: ResponseOverrides["phase"], + citations?: ResponseOverrides["citations"], ): MessageBlockResult { const msgId = itemId(); + const annotations = buildAnnotations(content, citations); const events: ResponsesSSEEvent[] = []; events.push({ @@ -1160,12 +1173,22 @@ function buildMessageOutputEvents( content_index: 0, text: content, }); + annotations.forEach((annotation, annotationIndex) => { + events.push({ + type: "response.output_text.annotation.added", + item_id: msgId, + output_index: outputIndex, + content_index: 0, + annotation_index: annotationIndex, + annotation, + }); + }); events.push({ type: "response.content_part.done", item_id: msgId, output_index: outputIndex, content_index: 0, - part: { type: "output_text", text: content, annotations: [] }, + part: { type: "output_text", text: content, annotations }, }); const msgItem = { @@ -1174,7 +1197,7 @@ function buildMessageOutputEvents( status: "completed", role: "assistant", ...(phase !== undefined ? { phase } : {}), - content: [{ type: "output_text", text: content, annotations: [] }], + content: [{ type: "output_text", text: content, annotations }], }; events.push({ type: "response.output_item.done", output_index: outputIndex, item: msgItem }); @@ -1320,6 +1343,7 @@ function buildOutputPrefix( emitEncryptedReasoning = false, synthesizeSummarylessReasoning = false, phase?: ResponseOverrides["phase"], + citations?: ResponseOverrides["citations"], ): object[] { const output: object[] = []; @@ -1352,7 +1376,9 @@ function buildOutputPrefix( status: "completed", role: "assistant", ...(phase !== undefined ? { phase } : {}), - content: [{ type: "output_text", text: content, annotations: [] }], + content: [ + { type: "output_text", text: content, annotations: buildAnnotations(content, citations) }, + ], }); return output; @@ -1393,6 +1419,7 @@ export function buildTextResponse( emitEncryptedReasoning, synthesizeSummarylessReasoning, overrides?.phase, + overrides?.citations, ), overrides, ); @@ -1509,6 +1536,7 @@ export function contentWithToolCallsStreamEvents( chunkSize, outputIndex, overrides?.phase, + overrides?.citations, ); events.push(...msgEvents); orderedOutputItems.push(msgItem); @@ -1534,6 +1562,7 @@ export function contentWithToolCallsStreamEvents( chunkSize, nextOutputIndex, overrides?.phase, + overrides?.citations, ); events.push(...msgEvents); orderedOutputItems.push(msgItem); @@ -1585,14 +1614,20 @@ function buildToolCallOutputItem(tc: EmittableToolCall): object { ); } -function buildMessageOutputItem(content: string, phase?: ResponseOverrides["phase"]): object { +function buildMessageOutputItem( + content: string, + phase?: ResponseOverrides["phase"], + citations?: ResponseOverrides["citations"], +): object { return { type: "message", id: itemId(), status: "completed", role: "assistant", ...(phase !== undefined ? { phase } : {}), - content: [{ type: "output_text", text: content, annotations: [] }], + content: [ + { type: "output_text", text: content, annotations: buildAnnotations(content, citations) }, + ], }; } @@ -1637,7 +1672,7 @@ function buildContentWithToolCallsResponse( } for (const block of ordered) { if (block.type === "text") { - output.push(buildMessageOutputItem(block.text, overrides?.phase)); + output.push(buildMessageOutputItem(block.text, overrides?.phase, overrides?.citations)); } else { output.push(buildToolCallOutputItem(block)); } @@ -1656,6 +1691,7 @@ function buildContentWithToolCallsResponse( emitEncryptedReasoning, synthesizeSummarylessReasoning, overrides?.phase, + overrides?.citations, ); for (const tc of toolCalls) { output.push(buildToolCallOutputItem(tc)); @@ -2407,6 +2443,7 @@ export function buildResponsesMisbehavior( chunkSize, outputIndex, overrides.phase, + overrides.citations, ); events.push(...message.events); output.push(message.msgItem); diff --git a/src/types.ts b/src/types.ts index ae3900a5..85b3b4c0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -263,6 +263,22 @@ export interface FixtureMatch { // Fixture response types +/** A source citation attached to a text response. Served on the OpenAI Responses API as `url_citation` annotations. */ +export interface FixtureCitation { + url: string; + title?: string; + /** Character offsets into the text; default to the whole text. */ + startIndex?: number; + endIndex?: number; +} + +/** A model-generated file (e.g. an inline image). Served on Gemini as `inlineData` parts. */ +export interface FixtureGeneratedFile { + mimeType: string; + /** Base64-encoded file bytes. */ + data: string; +} + /** * Fields that override auto-generated envelope values in the built response. * Scalar fields (finishReason, role) use OpenAI-canonical values — provider @@ -282,6 +298,10 @@ export interface FixtureMatch { * Claude (5: no created, systemFingerprint); Gemini (2: only finishReason, usage). */ export interface ResponseOverrides { + /** Model-generated files emitted before the response text and tool calls (Gemini only). */ + files?: FixtureGeneratedFile[]; + /** Source citations for the response text (OpenAI Responses API only). */ + citations?: FixtureCitation[]; id?: string; created?: number; model?: string;