feat(chat): translate inline audio to Chat Completions

docs-site/src/content/docs/guides/pi.md

@@ -147,12 +147,14 @@ An explicit reasoning effort of `none` survives Chat conversion. Output limits a
 controls are preserved for generic API-key Responses targets; the canonical ChatGPT target
 still applies its own restrictions. This does not make all providers' controls equivalent.
 
-**Audio and most file attachments need a native input wire that supports them.** A document
-that carries its own base64 bytes in a user message is the exception: it survives translation
-and reaches the Anthropic, OpenAI Chat and Google wires as a native document, file part and
-inline data part. Everything else still returns an explicit error rather than succeeding
-without the attachment — audio, a file-ID or remote reference the proxy cannot dereference, an
-attachment in a tool output or a system message, and a document routed to a wire with no byte
+Inline WAV/MP3 audio in a user message or tool result survives translation to OpenAI Chat as an
+`input_audio` content part; tool-result audio is promoted into a follow-up user message after
+the paired tool results because Chat tool messages are text-only. A document that carries its
+own base64 bytes in a user message is also carried: it reaches the Anthropic, OpenAI Chat and
+Google wires as a native document, file part and inline data part. Everything else still
+returns an explicit error rather than succeeding without the attachment — remote or
+unsupported audio, a file-ID or remote reference the proxy cannot dereference, an attachment
+in a system message, a document in a tool result, and a document routed to a wire with no byte
 carrier. File-ID-only images have the same restriction because translated adapters cannot
 resolve those IDs. Convert the attachment to text first, or use a native wire and model that
 support it. Native Chat and raw Responses (including Azure) retain their existing behavior;

scripts/test-layout/layout.json

@@ -118,6 +118,7 @@
     "restart-replacement.test.ts": "server", "deepseek-artifact-tool-schema.test.ts": "providers",
     "client-config-export-output-limit.test.ts": "config", "qoder-client.test.ts": "clients", "responses-skills-snapshot.test.ts": "responses", "responses-code-mode-mcp-direct.test.ts": "responses", "local-read-response-proof.test.ts": "server",
     "claude-desktop-system-proxy.test.ts": "claude-integration", "codex-config-drift-heal.test.ts": "codex-integration",
+    "openai-chat-audio.test.ts": "adapters/openai",
     "openai-chat-serialized-tool-call-scaling.test.ts": "adapters/openai",
     "openai-chat-tool-call-id-remint.test.ts": "adapters/openai",
     "coding-agent-json-lines-scaling.test.ts": "providers",

src/adapters/cursor/protobuf-request.ts

@@ -869,6 +869,9 @@ function decodeResultParts(message: OcxToolResultMessage): DecodedResultPart[] |
     // Without this the document falls through to decodeInlineImage(part.imageUrl) below and is
     // treated as an image it is not.
     if (part.type === "document") return { kind: "text", text: part.text };
+    // The translated-adapter guard rejects audio before this wire; retain its marker if this
+    // decoder is ever called directly.
+    if (part.type === "audio") return { kind: "text", text: part.text };
     const decoded = decodeInlineImage(part.imageUrl);
     return decoded ? { kind: "image", ...decoded } : { kind: "undecodable" };
   });

src/adapters/image.ts

@@ -18,9 +18,11 @@ export function parseDataUrl(url: string): { mediaType: string; base64: string }
  */
 export function contentPartsToText(content: string | OcxContentPart[]): string {
   if (typeof content === "string") return content;
-  // A document carries its own marker, so this wire states the attachment instead of
-  // mislabelling it as a video.
+  // Documents and audio carry their own markers, so this wire states the attachment instead
+  // of mislabelling it as a video.
   const text = content.map(p =>
-    p.type === "text" || p.type === "document" ? p.text : p.type === "image" ? "[image]" : "[video]").join("");
+    p.type === "text" || p.type === "document" || p.type === "audio"
+      ? p.text
+      : p.type === "image" ? "[image]" : "[video]").join("");
   return text || "[image]";
 }

src/adapters/input-media-guard.ts

@@ -10,15 +10,18 @@ import type { OcxParsedRequest } from "../types";
  * out because it uses the original body rather than the lossy normalized content.
  *
  * Two of the three checks are wire-scoped, and that is the point. A constraint the normalized
- * request CAN carry — a caller restriction on a tool, an attached document's bytes — still has
+ * request CAN carry — a caller restriction on a tool, an attached document's bytes, or inline
+ * audio in the translated Chat content model — still has
  * to reach a wire that can express it. Leaving that to each adapter means an adapter that never
  * learned about the carrier rebuilds without it and answers normally, so the allowlist in
  * `declaration-carrier.ts` is default-deny and this is the one place every registered adapter
  * passes through.
  */
 export function withInputMediaGuard<T extends ProviderAdapter>(adapter: T, wire: AdapterWire): T {
   const refusal = (parsed: OcxParsedRequest): string | undefined => {
-    const kind = untranslatedResponsesInputMedia(parsed._rawBody);
+    // Chat Completions has a native input_audio user-content part. The adapter promotes
+    // tool-result audio into a follow-up user message, so these blocks no longer lose bytes.
+    const kind = untranslatedResponsesInputMedia(parsed._rawBody, { allowAudio: wire === "openai-chat" });
     return kind ? untranslatedInputMediaMessage(kind) : unrepresentableDeclaration(parsed, wire);
   };
   const build = adapter.buildRequest.bind(adapter);

src/adapters/ollama-native.ts

@@ -290,6 +290,7 @@ function contentToNative(
       text += part.text;
       continue;
     }
+    if (part.type === "audio") throw new Error(`ollama-native cannot send audio content in ${label}`);
     // Ollama's native /api/chat message shape carries `images: string[]` and has no video
     // counterpart, so a video part is refused rather than silently dropped or mis-sent as an image.
     if (part.type === "video") throw new Error(`ollama-native cannot send video content in ${label}`);

src/adapters/openai-chat/messages.ts

@@ -27,8 +27,8 @@ const VIDEO_UNSUPPORTED_MARKER = "[video omitted: the translated Chat route has
 /**
  * Chat-completions image_url parts for images carried inside a tool result (issue #888). role:"tool"
  * content is text-only on every chat provider, so these ride in a follow-up user message instead of
- * being flattened to the "[image]" marker the model can't actually see. Data URLs and remote https
- * URLs are both valid in image_url.url, unlike Gemini inline_data which needs base64.
+ * being flattened to the "[image]" marker the model can't actually see. The same follow-up carries
+ * inline audio as input_audio, which is also user-content-only on this wire.
  */
 export function toolResultTextForWire(content: string | OcxContentPart[], annotateEmpty = false): string {
   // An empty content array is a present-but-empty result; `contentPartsToText` would
@@ -63,6 +63,14 @@ export function toolResultImageChatParts(content: string | OcxContentPart[]): un
   return parts;
 }
 
+/** Chat input_audio parts for audio carried inside a tool result. */
+export function toolResultAudioChatParts(content: string | OcxContentPart[]): unknown[] {
+  if (typeof content === "string") return [];
+  return content
+    .filter((p): p is Extract<OcxContentPart, { type: "audio" }> => p.type === "audio")
+    .map(p => ({ type: "input_audio", input_audio: { data: p.data, format: p.format } }));
+}
+
 /**
  * Whether this wire serializes a replayed thinking block for the given model.
  *
@@ -98,6 +106,7 @@ export function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProv
   let pendingToolCalls: PendingToolCall[] = [];
   let deferredBarrierMessages: unknown[] = [];
   let pendingToolResultImageParts: unknown[] = [];
+  let pendingToolResultAudioParts: unknown[] = [];
   let mintedIdSeq = 0;
   const seenWireCallIds = new Set<string>();
 
@@ -128,6 +137,18 @@ export function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProv
     pendingToolResultImageParts = [];
   };
 
+  const flushToolResultAudio = (): void => {
+    if (pendingToolResultAudioParts.length === 0) return;
+    out.push({
+      role: "user",
+      content: [
+        { type: "text", text: "[ocx] audio output from the preceding tool result(s):" },
+        ...pendingToolResultAudioParts,
+      ],
+    });
+    pendingToolResultAudioParts = [];
+  };
+
   const flushPendingToolCalls = (): void => {
     if (pendingToolCalls.length === 0) return;
     for (const call of pendingToolCalls) {
@@ -139,6 +160,7 @@ export function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProv
     }
     pendingToolCalls = [];
     flushToolResultImages();
+    flushToolResultAudio();
     releaseDeferredBarriers();
   };
 
@@ -185,9 +207,10 @@ export function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProv
       case "developer": {
         const parts = typeof msg.content === "string" ? undefined : msg.content as OcxContentPart[];
         const hasImages = parts?.some(p => p.type === "image") ?? false;
+        const hasAudio = parts?.some(p => p.type === "audio") ?? false;
         // A document has a structured counterpart on this wire, so it needs the parts array for
         // the same reason an image does: flattening it to a string would drop the bytes.
-        const hasStructured = hasImages || (parts?.some(p => p.type === "document") ?? false);
+        const hasStructured = hasImages || hasAudio || (parts?.some(p => p.type === "document") ?? false);
         let chatMsg: Record<string, unknown>;
         if (msg.role === "developer" && !hasStructured) {
           const text = typeof msg.content === "string"
@@ -219,6 +242,9 @@ export function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProv
             if (p.type === "image") {
               return { type: "image_url", image_url: { url: p.imageUrl, ...(p.detail ? { detail: p.detail } : {}) } };
             }
+            if (p.type === "audio") {
+              return { type: "input_audio", input_audio: { data: p.data, format: p.format } };
+            }
             // Chat Completions carries an attached document as a file part with inline bytes,
             // the direct counterpart of the Anthropic document block the caller sent.
             if (p.type === "document") {
@@ -240,7 +266,7 @@ export function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProv
           // this wire. One carrying only a document has no such precedent, and demoting it would
           // undo the role this adapter just finished preserving.
           chatMsg = {
-            role: msg.role === "developer" && !hasImages ? developerWireRole : "user",
+            role: msg.role === "developer" && !hasImages && !hasAudio ? developerWireRole : "user",
             content: chatParts,
           };
         }
@@ -330,9 +356,11 @@ export function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProv
             content: toolResultTextForWire(msg.content, provider.annotateEmptyToolOutputs === true),
           });
           pendingToolResultImageParts.push(...toolResultImageChatParts(msg.content));
+          pendingToolResultAudioParts.push(...toolResultAudioChatParts(msg.content));
           pendingToolCalls.splice(matchIdx, 1);
           if (pendingToolCalls.length === 0) {
             flushToolResultImages();
+            flushToolResultAudio();
             releaseDeferredBarriers();
           }
         } else {
@@ -380,7 +408,9 @@ export function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProv
             content: toolResultTextForWire(msg.content, provider.annotateEmptyToolOutputs === true),
           });
           pendingToolResultImageParts.push(...toolResultImageChatParts(msg.content));
+          pendingToolResultAudioParts.push(...toolResultAudioChatParts(msg.content));
           flushToolResultImages();
+          flushToolResultAudio();
         }
         break;
       }

src/responses/audio.ts

@@ -0,0 +1,46 @@
+import type { OcxAudioContent } from "../types";
+
+export type ChatAudioFormat = OcxAudioContent["format"];
+
+const MIME_FORMATS: Readonly<Record<string, ChatAudioFormat>> = {
+  "audio/mp3": "mp3",
+  "audio/mpeg": "mp3",
+  "audio/wav": "wav",
+  "audio/wave": "wav",
+  "audio/x-wav": "wav",
+};
+
+function normalizedFormat(value: unknown): ChatAudioFormat | undefined {
+  if (typeof value !== "string") return undefined;
+  const format = value.trim().toLowerCase();
+  return format === "mp3" || format === "wav" ? format : undefined;
+}
+
+function validBase64(value: string): boolean {
+  return value.length > 0
+    && value.length % 4 === 0
+    && /^[A-Za-z0-9+/]+={0,2}$/.test(value);
+}
+
+/**
+ * Parse the inline audio shape Codex puts in a Responses content block. Chat Completions
+ * needs bare base64 plus `wav`/`mp3`, while Codex supplies an RFC 2397 data URL. A raw
+ * base64 value with an explicit supported format is accepted as well; remote URLs are not.
+ */
+export function audioContentFromRaw(audioUrl: unknown, declaredFormat?: unknown): OcxAudioContent | undefined {
+  if (typeof audioUrl !== "string" || audioUrl.length === 0) return undefined;
+  const dataUrl = /^data:([^;,]+);base64,([\s\S]*)$/.exec(audioUrl);
+  let data: string;
+  let format: ChatAudioFormat | undefined;
+  if (dataUrl) {
+    const declared = normalizedFormat(declaredFormat);
+    const mime = dataUrl[1].trim().toLowerCase();
+    format = MIME_FORMATS[mime] ?? (mime === "application/octet-stream" ? declared : undefined);
+    data = dataUrl[2].replace(/\s+/g, "");
+  } else {
+    format = normalizedFormat(declaredFormat);
+    data = audioUrl.replace(/\s+/g, "");
+  }
+  if (!format || !validBase64(data)) return undefined;
+  return { type: "audio", data, format, text: `[audio: ${format}]` };
+}

src/responses/input-media.ts

@@ -1,4 +1,5 @@
 import { isInlineDocumentDataUrl } from "./inline-document";
+import { audioContentFromRaw } from "./audio";
 
 /** Input kinds for which the normalized request has no lossless content carrier. */
 export type UntranslatedInputMedia = "audio" | "file";
@@ -10,23 +11,35 @@ function isRecord(value: unknown): value is RecordValue {
 }
 
 /**
- * Whether the position being scanned has a converter that builds a document part.
+ * Whether the position being scanned has a converter that builds a lossless media part.
  *
- * Only user content does. A tool output, a system or assistant message, and a Chat `developer`
- * message are all flattened to text by their converters, so exempting an attachment there would
- * turn today's explicit refusal into the silent drop this scanner exists to prevent.
+ * User content carries documents and inline audio; developer content carries documents only.
+ * Tool output carries inline audio through the Chat adapter's follow-up user message. System and
+ * assistant content have no lossless translated media carrier, so exempting an attachment there
+ * would turn today's explicit refusal into the silent drop this scanner exists to prevent.
  */
-type CarrierPosition = "user-content" | "flattened";
+type CarrierPosition = "user-content" | "developer-content" | "tool-output" | "flattened";
 
-function mediaKind(value: unknown, position: CarrierPosition): UntranslatedInputMedia | undefined {
+function mediaKind(value: unknown, position: CarrierPosition, allowAudio: boolean): UntranslatedInputMedia | undefined {
   if (!isRecord(value)) return undefined;
-  if (value.type === "input_audio" || value.type === "audio") return "audio";
+  if (value.type === "input_audio" || value.type === "audio") {
+    // A Chat content part has a nested input_audio object, while Codex's Responses
+    // block uses audio_url. Only the latter is carried by the translated Chat adapter; the
+    // Chat-shaped `audio` spelling has no parser path and must never pass the guard.
+    return value.type === "input_audio"
+      && allowAudio
+      && (position === "user-content" || position === "tool-output")
+      && audioContentFromRaw(value.audio_url, value.format)
+      ? undefined
+      : "audio";
+  }
   if (value.type === "input_file" || value.type === "file" || value.type === "document") {
     // A document that carries its own bytes has a lossless carrier in user content, so refusing
     // it there would reject the very request #5212 exists to preserve. A reference with no
     // payload has none anywhere: translated adapters cannot dereference a file_id or a remote
     // source.
-    return position === "user-content" && carriesInlineDocumentBytes(value) ? undefined : "file";
+    return (position === "user-content" || position === "developer-content")
+      && carriesInlineDocumentBytes(value) ? undefined : "file";
   }
   // A file-id-only image is not pixels: translated adapters cannot dereference it.
   if (value.type === "input_image" && typeof value.file_id === "string" && value.file_id.length > 0
@@ -46,10 +59,10 @@ function carriesInlineDocumentBytes(value: RecordValue): boolean {
     && value.source.data.length > 0;
 }
 
-function contentMedia(content: unknown, position: CarrierPosition): UntranslatedInputMedia | undefined {
+function contentMedia(content: unknown, position: CarrierPosition, allowAudio: boolean): UntranslatedInputMedia | undefined {
   if (!Array.isArray(content)) return undefined;
   for (const part of content) {
-    const kind = mediaKind(part, position);
+    const kind = mediaKind(part, position, allowAudio);
     if (kind) return kind;
   }
   return undefined;
@@ -60,20 +73,29 @@ function contentMedia(content: unknown, position: CarrierPosition): Untranslated
  * arguments, schema properties, or arbitrary nested objects. No payload is copied,
  * decoded, fetched or included in the returned value.
  */
-export function untranslatedResponsesInputMedia(body: unknown): UntranslatedInputMedia | undefined {
+export function untranslatedResponsesInputMedia(
+  body: unknown,
+  options: { allowAudio?: boolean } = {},
+): UntranslatedInputMedia | undefined {
   if (!isRecord(body) || !Array.isArray(body.input)) return undefined;
   for (const item of body.input) {
     if (!isRecord(item)) continue;
-    const direct = mediaKind(item, "flattened");
+    const direct = mediaKind(item, "flattened", options.allowAudio === true);
     if (direct) return direct;
     if (item.type === "function_call_output" || item.type === "custom_tool_call_output") {
-      const kind = contentMedia(item.output, "flattened");
+      const kind = contentMedia(item.output, "tool-output", options.allowAudio === true);
       if (kind) return kind;
     } else if (item.type === "message" || item.type === undefined) {
       // `inputContentParts` runs for user and developer messages only; a system message is
       // flattened to text by the parser.
       const role = item.role;
-      const kind = contentMedia(item.content, role === "user" || role === "developer" ? "user-content" : "flattened");
+      const kind = contentMedia(
+        item.content,
+        role === "user" ? "user-content"
+          : role === "developer" ? "developer-content"
+          : "flattened",
+        options.allowAudio === true,
+      );
       if (kind) return kind;
     }
   }
@@ -87,7 +109,7 @@ export function untranslatedChatInputMedia(body: unknown): UntranslatedInputMedi
     if (!isRecord(message)) continue;
     // Only the `user` branch of the Chat projection builds content blocks. `system`,
     // `developer`, `assistant` and `tool` all reduce their content to a string.
-    const kind = contentMedia(message.content, message.role === "user" ? "user-content" : "flattened");
+    const kind = contentMedia(message.content, message.role === "user" ? "user-content" : "flattened", false);
     if (kind) return kind;
   }
   return undefined;