fix(agent): flatten content tool outputs to text for chatd

packages/agent/README.md

@@ -124,6 +124,13 @@ SDK tool loop:
 - Coder's own server‑side tools (file editing, shell, MCP, …) still run on the
   server. They appear in the transcript as `providerExecuted` tool
   calls/results: you observe them, you don't execute them.
+- Coder Agents gives a client tool result to the model as text. If a tool's
+  `toModelOutput` returns `content` parts, this package keeps the text parts
+  and inline text files. It replaces each other file or image part with a note
+  such as `[image/png chart.png omitted]` (a URL file keeps its URL in the
+  note). The model does not see the image. To show
+  the model a file, put it in a user message as a `file` part (see
+  [Files](#files)).
 
 <details>
 <summary>Since v0.2.1: server tools stream as <code>dynamic-tool</code> parts</summary>

packages/agent/src/model/prompt.ts

@@ -85,6 +85,38 @@ export async function userContentToInputParts(
   return parts.filter((p): p is ChatInputPart => p !== null);
 }
 
+type ContentOutputPart = Extract<
+  LanguageModelV4ToolResultOutput,
+  { type: "content" }
+>["value"][number];
+
+/**
+ * Flattens a `content` tool output (e.g. from a tool's `toModelOutput`) to one
+ * text string. chatd stores a client tool result as opaque JSON and gives it to
+ * the model as text, so a file part would reach the model as base64 characters,
+ * never as an image. Text parts and inline text files pass through. A URL file
+ * becomes a note with its URL. Any other file becomes a short placeholder, in
+ * the format of effect's `Prompt.toolModelOutputToText`.
+ */
+function contentOutputToText(parts: readonly ContentOutputPart[]): string {
+  return parts
+    .map((part) => {
+      switch (part.type) {
+        case "text":
+          return part.text;
+        case "file": {
+          if (part.data.type === "text") return part.data.text;
+          const label = `${part.mediaType}${part.filename === undefined ? "" : ` ${part.filename}`}`;
+          if (part.data.type === "url") return `[${label} at ${String(part.data.url)}]`;
+          return `[${label} omitted]`;
+        }
+        default:
+          return `[${part.type} content omitted]`;
+      }
+    })
+    .join("\n");
+}
+
 function toolResultOutputToChatd(output: LanguageModelV4ToolResultOutput): {
   value: unknown;
   isError: boolean;
@@ -101,7 +133,7 @@ function toolResultOutputToChatd(output: LanguageModelV4ToolResultOutput): {
     case "execution-denied":
       return { value: { execution_denied: true, reason: output.reason }, isError: true };
     case "content":
-      return { value: output.value, isError: false };
+      return { value: contentOutputToText(output.value), isError: false };
     default:
       return { value: output, isError: false };
   }

packages/agent/test/unit/prompt.test.ts

@@ -69,6 +69,70 @@ describe("classifyTurnAction", () => {
       throw new Error("expected resume");
     }
   });
+
+  it("flattens a content output to text and replaces binary file parts with notes", () => {
+    // chatd hands a client tool result to the model as text, so file bytes
+    // must not reach the wire as base64 (or as a JSON-serialized Uint8Array).
+    const prompt: LanguageModelV4Prompt = [
+      {
+        role: "tool",
+        content: [
+          {
+            type: "tool-result",
+            toolCallId: "c",
+            toolName: "screenshot",
+            output: {
+              type: "content",
+              value: [
+                { type: "text", text: "Swatch label: red" },
+                {
+                  type: "file",
+                  data: { type: "data", data: "iVBORw0KGgo=" },
+                  mediaType: "image/png",
+                  filename: "swatch.png",
+                },
+                {
+                  type: "file",
+                  data: { type: "data", data: new Uint8Array([1, 2, 3]) },
+                  mediaType: "application/pdf",
+                },
+                {
+                  type: "file",
+                  data: { type: "reference", reference: { openai: "file-1" } },
+                  mediaType: "image/jpeg",
+                },
+                { type: "file", data: { type: "text", text: "a,b\n1,2" }, mediaType: "text/csv" },
+                {
+                  type: "file",
+                  data: { type: "url", url: new URL("https://example.com/chart.png") },
+                  mediaType: "image/png",
+                  filename: "chart.png",
+                },
+                { type: "custom", providerOptions: { x: { y: 1 } } },
+              ],
+            },
+          },
+        ],
+      },
+    ];
+    const action = classifyTurnAction(prompt);
+    if (action.kind !== "resume") throw new Error("expected resume");
+    expect(action.toolResults).toEqual([
+      {
+        tool_call_id: "c",
+        output: [
+          "Swatch label: red",
+          "[image/png swatch.png omitted]",
+          "[application/pdf omitted]",
+          "[image/jpeg omitted]",
+          "a,b\n1,2",
+          "[image/png chart.png at https://example.com/chart.png]",
+          "[custom content omitted]",
+        ].join("\n"),
+        is_error: false,
+      },
+    ]);
+  });
 });
 
 describe("extractSystemPrompt", () => {