From 3b564897a31b35f149af18a445513f56d8db7d02 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 29 Oct 2026 08:11:18 +0000 Subject: [PATCH 1/2] fix(agent): flatten content tool outputs to text for chatd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit chatd stores a client tool result as opaque JSON and gives it to the model as text. A `content` output (from a tool's `toModelOutput`) reached the model as a JSON array with the file bytes as base64 characters. Keep the text parts, replace each file part with an "[ omitted]" note, and submit one string. Closes #404 Signed-off-by: Thomas Kosiewski --- _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-opus-5-5` • Thinking: `high`_ Change-Id: Ic94476e1bfc5933b2a18442dc3a3dcf49a6b1c07 --- packages/agent/README.md | 6 ++++ packages/agent/src/model/prompt.ts | 29 +++++++++++++++- packages/agent/test/unit/prompt.test.ts | 45 +++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/packages/agent/README.md b/packages/agent/README.md index d99fc470..308e1854 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -124,6 +124,12 @@ SDK tool loop: - Coder's own server‑side tools (file editing, shell, MCP, …) still run on the server. They appear in the transcript as `providerExecuted` tool calls/results: you observe them, you don't execute them. +- Coder Agents gives a client tool result to the model as text. If a tool's + `toModelOutput` returns `content` parts, this package keeps the text parts + and replaces each file or image part with a note such as + `[image/png chart.png omitted]`. The model does not see the image. To show + the model a file, put it in a user message as a `file` part (see + [Files](#files)).
Since v0.2.1: server tools stream as dynamic-tool parts diff --git a/packages/agent/src/model/prompt.ts b/packages/agent/src/model/prompt.ts index a153ff6d..f628ecdb 100644 --- a/packages/agent/src/model/prompt.ts +++ b/packages/agent/src/model/prompt.ts @@ -85,6 +85,33 @@ export async function userContentToInputParts( return parts.filter((p): p is ChatInputPart => p !== null); } +type ContentOutputPart = Extract< + LanguageModelV4ToolResultOutput, + { type: "content" } +>["value"][number]; + +/** + * Flattens a `content` tool output (e.g. from a tool's `toModelOutput`) to one + * text string. chatd stores a client tool result as opaque JSON and gives it to + * the model as text, so a file part would reach the model as base64 characters, + * never as an image. Text parts pass through. A file part becomes a short + * placeholder, in the format of effect's `Prompt.toolModelOutputToText`. + */ +function contentOutputToText(parts: readonly ContentOutputPart[]): string { + return parts + .map((part) => { + switch (part.type) { + case "text": + return part.text; + case "file": + return `[${part.mediaType}${part.filename === undefined ? "" : ` ${part.filename}`} omitted]`; + default: + return `[${part.type} content omitted]`; + } + }) + .join("\n"); +} + function toolResultOutputToChatd(output: LanguageModelV4ToolResultOutput): { value: unknown; isError: boolean; @@ -101,7 +128,7 @@ function toolResultOutputToChatd(output: LanguageModelV4ToolResultOutput): { case "execution-denied": return { value: { execution_denied: true, reason: output.reason }, isError: true }; case "content": - return { value: output.value, isError: false }; + return { value: contentOutputToText(output.value), isError: false }; default: return { value: output, isError: false }; } diff --git a/packages/agent/test/unit/prompt.test.ts b/packages/agent/test/unit/prompt.test.ts index dbed5d9c..3d0040dc 100644 --- a/packages/agent/test/unit/prompt.test.ts +++ b/packages/agent/test/unit/prompt.test.ts @@ -69,6 +69,51 @@ describe("classifyTurnAction", () => { throw new Error("expected resume"); } }); + + it("flattens a content output to text and replaces file parts with placeholders", () => { + // chatd hands a client tool result to the model as text, so file bytes + // must not reach the wire as base64 (or as a JSON-serialized Uint8Array). + const prompt: LanguageModelV4Prompt = [ + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "c", + toolName: "screenshot", + output: { + type: "content", + value: [ + { type: "text", text: "Swatch label: red" }, + { + type: "file", + data: { type: "data", data: "iVBORw0KGgo=" }, + mediaType: "image/png", + filename: "swatch.png", + }, + { + type: "file", + data: { type: "data", data: new Uint8Array([1, 2, 3]) }, + mediaType: "application/pdf", + }, + { type: "custom", providerOptions: { x: { y: 1 } } }, + ], + }, + }, + ], + }, + ]; + const action = classifyTurnAction(prompt); + if (action.kind !== "resume") throw new Error("expected resume"); + expect(action.toolResults).toEqual([ + { + tool_call_id: "c", + output: + "Swatch label: red\n[image/png swatch.png omitted]\n[application/pdf omitted]\n[custom content omitted]", + is_error: false, + }, + ]); + }); }); describe("extractSystemPrompt", () => { From ac8b57774c8e0d43ae1ab97056a9ee87aa576d45 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 29 Oct 2026 08:20:59 +0000 Subject: [PATCH 2/2] fix(agent): keep inline text files and file URLs in content outputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An inline text file (`{ type: "text" }` data) now passes through as text, and a URL file keeps its URL in the note. Both reached the model before this branch as part of the raw JSON array. _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-opus-5-5` • Thinking: `high`_ Change-Id: I034d3bf0abec212700176931e1516e31edba5c8f Signed-off-by: Thomas Kosiewski --- packages/agent/README.md | 5 +++-- packages/agent/src/model/prompt.ts | 13 +++++++++---- packages/agent/test/unit/prompt.test.ts | 25 ++++++++++++++++++++++--- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/packages/agent/README.md b/packages/agent/README.md index 308e1854..b2764705 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -126,8 +126,9 @@ SDK tool loop: calls/results: you observe them, you don't execute them. - Coder Agents gives a client tool result to the model as text. If a tool's `toModelOutput` returns `content` parts, this package keeps the text parts - and replaces each file or image part with a note such as - `[image/png chart.png omitted]`. The model does not see the image. To show + and inline text files. It replaces each other file or image part with a note + such as `[image/png chart.png omitted]` (a URL file keeps its URL in the + note). The model does not see the image. To show the model a file, put it in a user message as a `file` part (see [Files](#files)). diff --git a/packages/agent/src/model/prompt.ts b/packages/agent/src/model/prompt.ts index f628ecdb..c5f9a5d1 100644 --- a/packages/agent/src/model/prompt.ts +++ b/packages/agent/src/model/prompt.ts @@ -94,8 +94,9 @@ type ContentOutputPart = Extract< * Flattens a `content` tool output (e.g. from a tool's `toModelOutput`) to one * text string. chatd stores a client tool result as opaque JSON and gives it to * the model as text, so a file part would reach the model as base64 characters, - * never as an image. Text parts pass through. A file part becomes a short - * placeholder, in the format of effect's `Prompt.toolModelOutputToText`. + * never as an image. Text parts and inline text files pass through. A URL file + * becomes a note with its URL. Any other file becomes a short placeholder, in + * the format of effect's `Prompt.toolModelOutputToText`. */ function contentOutputToText(parts: readonly ContentOutputPart[]): string { return parts @@ -103,8 +104,12 @@ function contentOutputToText(parts: readonly ContentOutputPart[]): string { switch (part.type) { case "text": return part.text; - case "file": - return `[${part.mediaType}${part.filename === undefined ? "" : ` ${part.filename}`} omitted]`; + case "file": { + if (part.data.type === "text") return part.data.text; + const label = `${part.mediaType}${part.filename === undefined ? "" : ` ${part.filename}`}`; + if (part.data.type === "url") return `[${label} at ${String(part.data.url)}]`; + return `[${label} omitted]`; + } default: return `[${part.type} content omitted]`; } diff --git a/packages/agent/test/unit/prompt.test.ts b/packages/agent/test/unit/prompt.test.ts index 3d0040dc..fa8f2176 100644 --- a/packages/agent/test/unit/prompt.test.ts +++ b/packages/agent/test/unit/prompt.test.ts @@ -70,7 +70,7 @@ describe("classifyTurnAction", () => { } }); - it("flattens a content output to text and replaces file parts with placeholders", () => { + it("flattens a content output to text and replaces binary file parts with notes", () => { // chatd hands a client tool result to the model as text, so file bytes // must not reach the wire as base64 (or as a JSON-serialized Uint8Array). const prompt: LanguageModelV4Prompt = [ @@ -96,6 +96,18 @@ describe("classifyTurnAction", () => { data: { type: "data", data: new Uint8Array([1, 2, 3]) }, mediaType: "application/pdf", }, + { + type: "file", + data: { type: "reference", reference: { openai: "file-1" } }, + mediaType: "image/jpeg", + }, + { type: "file", data: { type: "text", text: "a,b\n1,2" }, mediaType: "text/csv" }, + { + type: "file", + data: { type: "url", url: new URL("https://example.com/chart.png") }, + mediaType: "image/png", + filename: "chart.png", + }, { type: "custom", providerOptions: { x: { y: 1 } } }, ], }, @@ -108,8 +120,15 @@ describe("classifyTurnAction", () => { expect(action.toolResults).toEqual([ { tool_call_id: "c", - output: - "Swatch label: red\n[image/png swatch.png omitted]\n[application/pdf omitted]\n[custom content omitted]", + output: [ + "Swatch label: red", + "[image/png swatch.png omitted]", + "[application/pdf omitted]", + "[image/jpeg omitted]", + "a,b\n1,2", + "[image/png chart.png at https://example.com/chart.png]", + "[custom content omitted]", + ].join("\n"), is_error: false, }, ]);