diff --git a/providers/internal/prompttest/cases.go b/providers/internal/prompttest/cases.go index 8ca5e9a09..b8c9eb85b 100644 --- a/providers/internal/prompttest/cases.go +++ b/providers/internal/prompttest/cases.go @@ -96,6 +96,16 @@ func Cases() []Case { }}, }, }, + { + Name: "user_text_file", + Model: "gpt-4o", + Prompt: fantasy.Prompt{ + {Role: fantasy.MessageRoleUser, Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "Summarize the notes."}, + fantasy.FilePart{Filename: "notes.txt", MediaType: "text/plain", Data: []byte("hello")}, + }}, + }, + }, { Name: "user_unsupported_media", Model: "gpt-4o", @@ -307,6 +317,43 @@ func Cases() []Case { }}, }, }, + { + // A PDF result converts with the user file rules: a file part + // that keeps the tool's filename, in the deferred user message. + Name: "tool_result_media_pdf", + Model: "gpt-4o", + Prompt: fantasy.Prompt{ + {Role: fantasy.MessageRoleAssistant, Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ToolCallID: "c1", ToolName: "read", Input: "{}"}, + }}, + {Role: fantasy.MessageRoleTool, Content: []fantasy.MessagePart{ + fantasy.ToolResultPart{ToolCallID: "c1", Output: fantasy.ToolResultOutputContentMedia{ + Data: b64([]byte("%PDF-1.4")), + MediaType: "application/pdf", + Filename: "report.pdf", + }}, + }}, + }, + }, + { + // Text media is the exception to the user file rules, which would + // send it as a bare base64 file part that providers can reject. + // It gets the unsupported placeholder and warning instead. + Name: "tool_result_media_text", + Model: "gpt-4o", + Prompt: fantasy.Prompt{ + {Role: fantasy.MessageRoleAssistant, Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ToolCallID: "c1", ToolName: "read", Input: "{}"}, + }}, + {Role: fantasy.MessageRoleTool, Content: []fantasy.MessagePart{ + fantasy.ToolResultPart{ToolCallID: "c1", Output: fantasy.ToolResultOutputContentMedia{ + Data: b64([]byte("hello")), + MediaType: "text/plain", + Filename: "notes.txt", + }}, + }}, + }, + }, { // An unsupported media type still has to answer the tool_call, so // the text message goes out alone with a warning naming the type. @@ -324,6 +371,25 @@ func Cases() []Case { }}, }, }, + { + // Accompanying text is kept after the notice, which takes the + // media's place, so the model does not assume it saw the media + // the text describes. + Name: "tool_result_media_unsupported_with_text", + Model: "gpt-4o", + Prompt: fantasy.Prompt{ + {Role: fantasy.MessageRoleAssistant, Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ToolCallID: "c1", ToolName: "record", Input: "{}"}, + }}, + {Role: fantasy.MessageRoleTool, Content: []fantasy.MessagePart{ + fantasy.ToolResultPart{ToolCallID: "c1", Output: fantasy.ToolResultOutputContentMedia{ + Data: b64([]byte{1, 2, 3}), + MediaType: "video/mp4", + Text: "Recording of the session.", + }}, + }}, + }, + }, { // The ordering case from the media fix: a parallel batch must keep // its tool messages contiguous. diff --git a/providers/internal/prompttest/testdata/tool_result_media_pdf.json b/providers/internal/prompttest/testdata/tool_result_media_pdf.json new file mode 100644 index 000000000..1ece9bfdb --- /dev/null +++ b/providers/internal/prompttest/testdata/tool_result_media_pdf.json @@ -0,0 +1,37 @@ +{ + "openai,openaicompat,openrouter,vercel": { + "messages": [ + { + "tool_calls": [ + { + "id": "c1", + "function": { + "arguments": "{}", + "name": "read" + }, + "type": "function" + } + ], + "role": "assistant" + }, + { + "content": "The tool returned application/pdf content; see the following user message.", + "tool_call_id": "c1", + "role": "tool" + }, + { + "content": [ + { + "file": { + "file_data": "data:application/pdf;base64,JVBERi0xLjQ=", + "filename": "report.pdf" + }, + "type": "file" + } + ], + "role": "user" + } + ], + "warnings": [] + } +} diff --git a/providers/internal/prompttest/testdata/tool_result_media_text.json b/providers/internal/prompttest/testdata/tool_result_media_text.json new file mode 100644 index 000000000..68ad4439e --- /dev/null +++ b/providers/internal/prompttest/testdata/tool_result_media_text.json @@ -0,0 +1,27 @@ +{ + "openai,openaicompat,openrouter,vercel": { + "messages": [ + { + "tool_calls": [ + { + "id": "c1", + "function": { + "arguments": "{}", + "name": "read" + }, + "type": "function" + } + ], + "role": "assistant" + }, + { + "content": "The tool returned text/plain content, which cannot be displayed.", + "tool_call_id": "c1", + "role": "tool" + } + ], + "warnings": [ + "other: tool result media type text/plain not supported, sending text placeholder only" + ] + } +} diff --git a/providers/internal/prompttest/testdata/tool_result_media_unsupported_with_text.json b/providers/internal/prompttest/testdata/tool_result_media_unsupported_with_text.json new file mode 100644 index 000000000..4845ea2ed --- /dev/null +++ b/providers/internal/prompttest/testdata/tool_result_media_unsupported_with_text.json @@ -0,0 +1,27 @@ +{ + "openai,openaicompat,openrouter,vercel": { + "messages": [ + { + "tool_calls": [ + { + "id": "c1", + "function": { + "arguments": "{}", + "name": "record" + }, + "type": "function" + } + ], + "role": "assistant" + }, + { + "content": "The tool returned video/mp4 content, which cannot be displayed.\n\nRecording of the session.", + "tool_call_id": "c1", + "role": "tool" + } + ], + "warnings": [ + "other: tool result media type video/mp4 not supported, sending text placeholder only" + ] + } +} diff --git a/providers/internal/prompttest/testdata/user_text_file.json b/providers/internal/prompttest/testdata/user_text_file.json new file mode 100644 index 000000000..a12182e6d --- /dev/null +++ b/providers/internal/prompttest/testdata/user_text_file.json @@ -0,0 +1,18 @@ +{ + "openai,openaicompat,openrouter,vercel": { + "messages": [ + { + "content": [ + { + "text": "Summarize the notes.", + "type": "text" + } + ], + "role": "user" + } + ], + "warnings": [ + "other: file part media type text/plain not supported" + ] + } +} diff --git a/providers/openai/language_model_hooks.go b/providers/openai/language_model_hooks.go index 234a83b1a..c892534e1 100644 --- a/providers/openai/language_model_hooks.go +++ b/providers/openai/language_model_hooks.go @@ -1,12 +1,12 @@ package openai import ( - "encoding/base64" "fmt" "net/http" "strings" "charm.land/fantasy" + "charm.land/fantasy/providers/internal/filepart" "github.com/openai/openai-go/v3" "github.com/openai/openai-go/v3/packages/param" "github.com/openai/openai-go/v3/shared" @@ -412,90 +412,12 @@ func DefaultToPrompt(prompt fantasy.Prompt, _, _ string) ([]openai.ChatCompletio continue } - switch { - case strings.HasPrefix(filePart.MediaType, "text/"): - base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data) - documentBlock := openai.ChatCompletionContentPartFileFileParam{ - FileData: param.NewOpt(base64Encoded), - } - content = append(content, openai.FileContentPart(documentBlock)) - - case strings.HasPrefix(filePart.MediaType, "image/"): - // Handle image files - base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data) - data := "data:" + filePart.MediaType + ";base64," + base64Encoded - imageURL := openai.ChatCompletionContentPartImageImageURLParam{URL: data} - - // Check for provider-specific options like image detail - if providerOptions, ok := filePart.ProviderOptions[Name]; ok { - if detail, ok := providerOptions.(*ProviderFileOptions); ok { - imageURL.Detail = detail.ImageDetail - } - } - - imageBlock := openai.ChatCompletionContentPartImageParam{ImageURL: imageURL} - content = append(content, openai.ChatCompletionContentPartUnionParam{OfImageURL: &imageBlock}) - - case filePart.MediaType == "audio/wav": - // Handle WAV audio files - base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data) - audioBlock := openai.ChatCompletionContentPartInputAudioParam{ - InputAudio: openai.ChatCompletionContentPartInputAudioInputAudioParam{ - Data: base64Encoded, - Format: "wav", - }, - } - content = append(content, openai.ChatCompletionContentPartUnionParam{OfInputAudio: &audioBlock}) - - case filePart.MediaType == "audio/mpeg" || filePart.MediaType == "audio/mp3": - // Handle MP3 audio files - base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data) - audioBlock := openai.ChatCompletionContentPartInputAudioParam{ - InputAudio: openai.ChatCompletionContentPartInputAudioInputAudioParam{ - Data: base64Encoded, - Format: "mp3", - }, - } - content = append(content, openai.ChatCompletionContentPartUnionParam{OfInputAudio: &audioBlock}) - - case filePart.MediaType == "application/pdf": - // Handle PDF files - dataStr := string(filePart.Data) - - // Check if data looks like a file ID (starts with "file-") - if strings.HasPrefix(dataStr, "file-") { - fileBlock := openai.ChatCompletionContentPartFileParam{ - File: openai.ChatCompletionContentPartFileFileParam{ - FileID: param.NewOpt(dataStr), - }, - } - content = append(content, openai.ChatCompletionContentPartUnionParam{OfFile: &fileBlock}) - } else { - // Handle as base64 data - base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data) - data := "data:application/pdf;base64," + base64Encoded - - filename := filePart.Filename - if filename == "" { - // Generate default filename based on content index - filename = fmt.Sprintf("part-%d.pdf", len(content)) - } - - fileBlock := openai.ChatCompletionContentPartFileParam{ - File: openai.ChatCompletionContentPartFileFileParam{ - Filename: param.NewOpt(filename), - FileData: param.NewOpt(data), - }, - } - content = append(content, openai.ChatCompletionContentPartUnionParam{OfFile: &fileBlock}) - } - - default: - warnings = append(warnings, fantasy.CallWarning{ - Type: fantasy.CallWarningTypeOther, - Message: fmt.Sprintf("file part media type %s not supported", filePart.MediaType), - }) + part, warning := FilePartToChatContentPart(filePart, len(content)) + if warning != nil { + warnings = append(warnings, *warning) + continue } + content = append(content, part) } } if !HasVisibleUserContent(content) { @@ -579,6 +501,90 @@ func DefaultToPrompt(prompt fantasy.Prompt, _, _ string) ([]openai.ChatCompletio return media.Close(messages), warnings } +// FilePartToChatContentPart converts a file part to a Chat Completions user +// content part. A PDF without a filename is named after index, its position in +// the message. Unsupported media types return a warning instead of a part. +// Text files are unsupported, as in the Responses converter, so callers must +// send their content as a text part. OpenAI-compatible providers share this +// converter. +func FilePartToChatContentPart(filePart fantasy.FilePart, index int) (openai.ChatCompletionContentPartUnionParam, *fantasy.CallWarning) { + // PDF data that looks like a file ID refers to an uploaded file. + if filePart.MediaType == "application/pdf" && strings.HasPrefix(string(filePart.Data), "file-") { + fileBlock := openai.ChatCompletionContentPartFileParam{ + File: openai.ChatCompletionContentPartFileFileParam{ + FileID: param.NewOpt(string(filePart.Data)), + }, + } + return openai.ChatCompletionContentPartUnionParam{OfFile: &fileBlock}, nil + } + return chatFileContentPart(filepart.FromFilePart(filePart), index) +} + +// chatFileContentPart converts a user file part or tool-result media to a Chat +// Completions user content part, as FilePartToChatContentPart describes. +func chatFileContentPart(file filepart.File, index int) (openai.ChatCompletionContentPartUnionParam, *fantasy.CallWarning) { + switch { + case strings.HasPrefix(file.MediaType, "image/"): + // Handle image files + data := "data:" + file.MediaType + ";base64," + file.Base64() + imageURL := openai.ChatCompletionContentPartImageImageURLParam{URL: data} + + // Check for provider-specific options like image detail + if providerOptions, ok := file.ProviderOptions[Name]; ok { + if detail, ok := providerOptions.(*ProviderFileOptions); ok { + imageURL.Detail = detail.ImageDetail + } + } + + imageBlock := openai.ChatCompletionContentPartImageParam{ImageURL: imageURL} + return openai.ChatCompletionContentPartUnionParam{OfImageURL: &imageBlock}, nil + + case file.MediaType == "audio/wav": + // Handle WAV audio files + audioBlock := openai.ChatCompletionContentPartInputAudioParam{ + InputAudio: openai.ChatCompletionContentPartInputAudioInputAudioParam{ + Data: file.Base64(), + Format: "wav", + }, + } + return openai.ChatCompletionContentPartUnionParam{OfInputAudio: &audioBlock}, nil + + case file.MediaType == "audio/mpeg" || file.MediaType == "audio/mp3": + // Handle MP3 audio files + audioBlock := openai.ChatCompletionContentPartInputAudioParam{ + InputAudio: openai.ChatCompletionContentPartInputAudioInputAudioParam{ + Data: file.Base64(), + Format: "mp3", + }, + } + return openai.ChatCompletionContentPartUnionParam{OfInputAudio: &audioBlock}, nil + + case file.MediaType == "application/pdf": + // Handle PDF files + data := "data:application/pdf;base64," + file.Base64() + + filename := file.Filename + if filename == "" { + // Generate default filename based on content index + filename = fmt.Sprintf("part-%d.pdf", index) + } + + fileBlock := openai.ChatCompletionContentPartFileParam{ + File: openai.ChatCompletionContentPartFileFileParam{ + Filename: param.NewOpt(filename), + FileData: param.NewOpt(data), + }, + } + return openai.ChatCompletionContentPartUnionParam{OfFile: &fileBlock}, nil + + default: + return openai.ChatCompletionContentPartUnionParam{}, &fantasy.CallWarning{ + Type: fantasy.CallWarningTypeOther, + Message: fmt.Sprintf("file part media type %s not supported", file.MediaType), + } + } +} + // HasVisibleUserContent reports whether a user message carries anything the // model can actually see. A message that converts to no visible parts must be // dropped rather than sent: the API rejects a content-less user message. diff --git a/providers/openai/responses_language_model.go b/providers/openai/responses_language_model.go index bcb432ebd..93db8cd9c 100644 --- a/providers/openai/responses_language_model.go +++ b/providers/openai/responses_language_model.go @@ -2,7 +2,6 @@ package openai import ( "context" - "encoding/base64" "encoding/json" "errors" "fmt" @@ -15,6 +14,7 @@ import ( "charm.land/fantasy" "charm.land/fantasy/object" + "charm.land/fantasy/providers/internal/filepart" "charm.land/fantasy/schema" "github.com/google/uuid" "github.com/openai/openai-go/v3" @@ -562,35 +562,12 @@ func toResponsesPromptWithValidation(prompt fantasy.Prompt, systemMessageMode st continue } - if strings.HasPrefix(filePart.MediaType, "image/") { - base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data) - imageURL := fmt.Sprintf("data:%s;base64,%s", filePart.MediaType, base64Encoded) - contentParts = append(contentParts, responses.ResponseInputContentUnionParam{ - OfInputImage: &responses.ResponseInputImageParam{ - Type: "input_image", - ImageURL: param.NewOpt(imageURL), - }, - }) - } else if filePart.MediaType == "application/pdf" { - base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data) - fileData := fmt.Sprintf("data:application/pdf;base64,%s", base64Encoded) - filename := filePart.Filename - if filename == "" { - filename = fmt.Sprintf("part-%d.pdf", i) - } - contentParts = append(contentParts, responses.ResponseInputContentUnionParam{ - OfInputFile: &responses.ResponseInputFileParam{ - Type: "input_file", - Filename: param.NewOpt(filename), - FileData: param.NewOpt(fileData), - }, - }) - } else { - warnings = append(warnings, fantasy.CallWarning{ - Type: fantasy.CallWarningTypeOther, - Message: fmt.Sprintf("file part media type %s not supported", filePart.MediaType), - }) + part, warning := filePartToResponsesContent(filepart.FromFilePart(filePart), i) + if warning != nil { + warnings = append(warnings, *warning) + continue } + contentParts = append(contentParts, part) } } @@ -776,7 +753,6 @@ func toResponsesPromptWithValidation(prompt fantasy.Prompt, systemMessageMode st } var outputStr string - var followupParts responses.ResponseInputMessageContentListParam switch toolResultPart.Output.GetType() { case fantasy.ToolResultContentTypeText: output, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentText](toolResultPart.Output) @@ -807,29 +783,12 @@ func toResponsesPromptWithValidation(prompt fantasy.Prompt, systemMessageMode st }) continue } - // The Responses API function_call_output only accepts a - // string. Emit a text placeholder (preserving any - // accompanying text) so the tool_call/tool_result pairing - // stays valid, then attach the media as a synthetic user - // input_image so vision-capable models still receive it. - outputStr = output.Text - if outputStr == "" { - outputStr = fmt.Sprintf("The tool returned %s content; see the following user message.", output.MediaType) - } - if strings.HasPrefix(output.MediaType, "image/") { - imageURL := fmt.Sprintf("data:%s;base64,%s", output.MediaType, output.Data) - followupParts = append(followupParts, responses.ResponseInputContentUnionParam{ - OfInputImage: &responses.ResponseInputImageParam{ - Type: "input_image", - ImageURL: param.NewOpt(imageURL), - }, - }) - } else { - warnings = append(warnings, fantasy.CallWarning{ - Type: fantasy.CallWarningTypeOther, - Message: fmt.Sprintf("tool result media type %s not supported, sending text placeholder only", output.MediaType), - }) + mediaOutput, warning := responsesToolResultMediaOutput(output, toolResultPart.ToolCallID) + if warning != nil { + warnings = append(warnings, *warning) } + input = append(input, mediaOutput) + continue default: warnings = append(warnings, fantasy.CallWarning{ Type: fantasy.CallWarningTypeOther, @@ -839,9 +798,6 @@ func toResponsesPromptWithValidation(prompt fantasy.Prompt, systemMessageMode st } input = append(input, responses.ResponseInputItemParamOfFunctionCallOutput(toolResultPart.ToolCallID, outputStr)) - if len(followupParts) > 0 { - input = append(input, responses.ResponseInputItemParamOfMessage(followupParts, responses.EasyInputMessageRoleUser)) - } } } } @@ -853,6 +809,110 @@ func toResponsesPromptWithValidation(prompt fantasy.Prompt, systemMessageMode st return input, warnings, nil } +// filePartToResponsesContent converts a user file part or tool-result media to +// a Responses input content part. A PDF without a filename is named after +// index, its position in the message. Unsupported media types return a warning +// instead of a part. +func filePartToResponsesContent(file filepart.File, index int) (responses.ResponseInputContentUnionParam, *fantasy.CallWarning) { + switch { + case strings.HasPrefix(file.MediaType, "image/"): + imageURL := fmt.Sprintf("data:%s;base64,%s", file.MediaType, file.Base64()) + return responses.ResponseInputContentUnionParam{ + OfInputImage: &responses.ResponseInputImageParam{ + Type: "input_image", + ImageURL: param.NewOpt(imageURL), + }, + }, nil + case file.MediaType == "application/pdf": + fileData := fmt.Sprintf("data:application/pdf;base64,%s", file.Base64()) + filename := file.Filename + if filename == "" { + filename = fmt.Sprintf("part-%d.pdf", index) + } + return responses.ResponseInputContentUnionParam{ + OfInputFile: &responses.ResponseInputFileParam{ + Type: "input_file", + Filename: param.NewOpt(filename), + FileData: param.NewOpt(fileData), + }, + }, nil + default: + return responses.ResponseInputContentUnionParam{}, &fantasy.CallWarning{ + Type: fantasy.CallWarningTypeOther, + Message: fmt.Sprintf("file part media type %s not supported", file.MediaType), + } + } +} + +// toFunctionCallOutputItem copies an input_image or input_file content part +// into the function_call_output item type, which has the same fields under +// different type names. +func toFunctionCallOutputItem(part responses.ResponseInputContentUnionParam) responses.ResponseFunctionCallOutputItemUnionParam { + switch { + case part.OfInputImage != nil: + return responses.ResponseFunctionCallOutputItemUnionParam{ + OfInputImage: &responses.ResponseInputImageContentParam{ + FileID: part.OfInputImage.FileID, + ImageURL: part.OfInputImage.ImageURL, + Detail: responses.ResponseInputImageContentDetail(part.OfInputImage.Detail), + }, + } + case part.OfInputFile != nil: + return responses.ResponseFunctionCallOutputItemUnionParam{ + OfInputFile: &responses.ResponseInputFileContentParam{ + FileData: part.OfInputFile.FileData, + FileID: part.OfInputFile.FileID, + FileURL: part.OfInputFile.FileURL, + Filename: part.OfInputFile.Filename, + Detail: responses.ResponseInputFileContentDetail(part.OfInputFile.Detail), + }, + } + default: + return responses.ResponseFunctionCallOutputItemUnionParam{} + } +} + +// responsesToolResultMediaOutput builds the function_call_output for a media +// tool result. The output holds the media, converted with the same rules as a +// user file, then the accompanying text, if any, in the same order as the +// Anthropic tool_result content. Media that cannot be converted +// becomes a text output that says so, plus a warning. +func responsesToolResultMediaOutput(output fantasy.ToolResultOutputContentMedia, callID string) (responses.ResponseInputItemUnionParam, *fantasy.CallWarning) { + part, warning := filePartToResponsesContent(filepart.FromToolResult(output), 0) + if warning != nil { + return responses.ResponseInputItemParamOfFunctionCallOutput(callID, undisplayableToolMediaText(output)), unsupportedToolMediaWarning(output) + } + items := responses.ResponseFunctionCallOutputItemListParam{toFunctionCallOutputItem(part)} + if output.Text != "" { + items = append(items, responses.ResponseFunctionCallOutputItemUnionParam{ + OfInputText: &responses.ResponseInputTextContentParam{Text: output.Text}, + }) + } + return responses.ResponseInputItemParamOfFunctionCallOutput(callID, items), nil +} + +// undisplayableToolMediaText is the tool output text for media that is not +// sent to the model. The notice takes the place of the media, so it comes +// before the accompanying text. +func undisplayableToolMediaText(output fantasy.ToolResultOutputContentMedia) string { + mediaType := output.MediaType + if mediaType == "" { + mediaType = "file" + } + notice := fmt.Sprintf("The tool returned %s content, which cannot be displayed.", mediaType) + if output.Text == "" { + return notice + } + return notice + "\n\n" + output.Text +} + +func unsupportedToolMediaWarning(output fantasy.ToolResultOutputContentMedia) *fantasy.CallWarning { + return &fantasy.CallWarning{ + Type: fantasy.CallWarningTypeOther, + Message: fmt.Sprintf("tool result media type %s not supported, sending text placeholder only", output.MediaType), + } +} + func responsesReasoningInputItem(meta *ResponsesReasoningMetadata) (responses.ResponseInputItemUnionParam, bool) { if !meta.Finalized || meta.EncryptedContent == nil || *meta.EncryptedContent == "" { return responses.ResponseInputItemUnionParam{}, false diff --git a/providers/openai/tool_messages.go b/providers/openai/tool_messages.go index 9d3abf178..d191a8fc2 100644 --- a/providers/openai/tool_messages.go +++ b/providers/openai/tool_messages.go @@ -2,11 +2,11 @@ package openai import ( "fmt" - "strings" "github.com/openai/openai-go/v3" "charm.land/fantasy" + "charm.land/fantasy/providers/internal/filepart" ) // ToolCacheTypeFunc returns the prompt-cache hint to attach to the tool result @@ -104,20 +104,20 @@ func tagToolCacheControl(msg *openai.ChatCompletionToolMessageParam, cacheType s // ToolResultMediaMessages maps a tool-result media output to the chat // completions messages that convey it: a text tool message that keeps the -// tool_call/tool_result pairing valid, plus the user messages holding the image -// or audio itself, which a tool message cannot carry. +// tool_call/tool_result pairing valid, plus the user messages holding the media +// itself, which a tool message cannot carry. // // The two are returned separately because the tool message must stay inside the // run of tool messages while the media has to land after it. Unsupported media -// types return only the tool message, plus a warning. +// types return only the tool message, which says so, plus a warning. func ToolResultMediaMessages(output fantasy.ToolResultOutputContentMedia, toolCallID string) (openai.ChatCompletionMessageParamUnion, []openai.ChatCompletionMessageParamUnion, []fantasy.CallWarning) { mediaPart, warning, emit := toolResultMediaUserPart(output) placeholder := output.Text if placeholder == "" && emit { placeholder = fmt.Sprintf("The tool returned %s content; see the following user message.", output.MediaType) - } else if placeholder == "" { - placeholder = fmt.Sprintf("The tool returned %s content, which cannot be displayed.", output.MediaType) + } else if !emit { + placeholder = undisplayableToolMediaText(output) } toolMessage := openai.ToolMessage(placeholder, toolCallID) @@ -133,33 +133,14 @@ func ToolResultMediaMessages(output fantasy.ToolResultOutputContentMedia, toolCa } // toolResultMediaUserPart maps a tool-result media output to a user content -// part, reporting whether the caller should emit it. +// part with the same rules as a user file, reporting whether the caller should +// emit it. func toolResultMediaUserPart(output fantasy.ToolResultOutputContentMedia) (openai.ChatCompletionContentPartUnionParam, *fantasy.CallWarning, bool) { - switch { - case strings.HasPrefix(output.MediaType, "image/"): - data := "data:" + output.MediaType + ";base64," + output.Data - imageBlock := openai.ChatCompletionContentPartImageParam{ - ImageURL: openai.ChatCompletionContentPartImageImageURLParam{URL: data}, - } - return openai.ChatCompletionContentPartUnionParam{OfImageURL: &imageBlock}, nil, true - case output.MediaType == "audio/wav", output.MediaType == "audio/mpeg", output.MediaType == "audio/mp3": - format := "wav" - if output.MediaType != "audio/wav" { - format = "mp3" - } - audioBlock := openai.ChatCompletionContentPartInputAudioParam{ - InputAudio: openai.ChatCompletionContentPartInputAudioInputAudioParam{ - Data: output.Data, - Format: format, - }, - } - return openai.ChatCompletionContentPartUnionParam{OfInputAudio: &audioBlock}, nil, true - default: - return openai.ChatCompletionContentPartUnionParam{}, &fantasy.CallWarning{ - Type: fantasy.CallWarningTypeOther, - Message: fmt.Sprintf("tool result media type %s not supported, sending text placeholder only", output.MediaType), - }, false + part, warning := chatFileContentPart(filepart.FromToolResult(output), 0) + if warning != nil { + return openai.ChatCompletionContentPartUnionParam{}, unsupportedToolMediaWarning(output), false } + return part, nil, true } // ToolRunBuffer holds synthetic user messages carrying tool-result media back diff --git a/providers/openai/tool_result_media_test.go b/providers/openai/tool_result_media_test.go index aa86cbd67..cef117034 100644 --- a/providers/openai/tool_result_media_test.go +++ b/providers/openai/tool_result_media_test.go @@ -2,6 +2,7 @@ package openai import ( "encoding/base64" + "encoding/json" "testing" "charm.land/fantasy" @@ -331,22 +332,63 @@ func TestToResponsesPrompt_MediaToolResult_ImagePNG(t *testing.T) { input, warnings, err := toResponsesPrompt(prompt, "system", false) require.NoError(t, err) require.Empty(t, warnings) - // Assistant function call + function_call_output + synthetic user image - // message. - require.Len(t, input, 3) + // Assistant function call + function_call_output that holds the image. + require.Len(t, input, 2) + requireJSONEq(t, `{ + "type": "function_call_output", + "call_id": "img-resp-1", + "output": [{"type": "input_image", "image_url": "data:image/png;base64,`+imageData+`"}] + }`, input[1]) +} - funcOut := input[1].OfFunctionCallOutput - require.NotNil(t, funcOut) - require.Equal(t, "img-resp-1", funcOut.CallID) - require.Contains(t, funcOut.Output.OfString.Value, "image/png") +// Media stays inside its function_call_output, so a parallel batch has no +// user message between its outputs. +func TestToResponsesPrompt_MediaToolResult_PDFInParallelBatch(t *testing.T) { + t.Parallel() - userMsg := input[2].OfMessage - require.NotNil(t, userMsg) - parts := userMsg.Content.OfInputItemContentList - require.Len(t, parts, 1) - imagePart := parts[0].OfInputImage - require.NotNil(t, imagePart) - require.Equal(t, "data:image/png;base64,"+imageData, imagePart.ImageURL.Value) + pdfData := base64.StdEncoding.EncodeToString([]byte("%PDF-1.4 fake")) + prompt := fantasy.Prompt{ + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ToolCallID: "pdf-resp-1", ToolName: "read", Input: "{}"}, + fantasy.ToolCallPart{ToolCallID: "txt-resp-1", ToolName: "read", Input: "{}"}, + }, + }, + { + Role: fantasy.MessageRoleTool, + Content: []fantasy.MessagePart{ + fantasy.ToolResultPart{ + ToolCallID: "pdf-resp-1", + Output: fantasy.ToolResultOutputContentMedia{ + Data: pdfData, + MediaType: "application/pdf", + Text: "Quarterly report.", + Filename: "report.pdf", + }, + }, + fantasy.ToolResultPart{ + ToolCallID: "txt-resp-1", + Output: fantasy.ToolResultOutputContentText{Text: "ok"}, + }, + }, + }, + } + + input, warnings, err := toResponsesPrompt(prompt, "system", false) + require.NoError(t, err) + require.Empty(t, warnings) + // Two function calls + two function_call_outputs, and no user message. + require.Len(t, input, 4) + requireJSONEq(t, `{ + "type": "function_call_output", + "call_id": "pdf-resp-1", + "output": [ + {"type": "input_file", "filename": "report.pdf", "file_data": "data:application/pdf;base64,`+pdfData+`"}, + {"type": "input_text", "text": "Quarterly report."} + ] + }`, input[2]) + requireJSONEq(t, `{"type": "function_call_output", "call_id": "txt-resp-1", "output": "ok"}`, input[3]) } func TestToResponsesPrompt_MediaToolResult_UnsupportedMediaType(t *testing.T) { @@ -376,11 +418,20 @@ func TestToResponsesPrompt_MediaToolResult_UnsupportedMediaType(t *testing.T) { input, warnings, err := toResponsesPrompt(prompt, "system", false) require.NoError(t, err) - // Assistant function call + function_call_output, but no synthetic user - // image message. + // Assistant function call + text function_call_output. require.Len(t, input, 2) - require.NotNil(t, input[1].OfFunctionCallOutput) - require.Equal(t, "vid-resp-1", input[1].OfFunctionCallOutput.CallID) + requireJSONEq(t, `{ + "type": "function_call_output", + "call_id": "vid-resp-1", + "output": "The tool returned video/mp4 content, which cannot be displayed." + }`, input[1]) require.Len(t, warnings, 1) require.Contains(t, warnings[0].Message, "video/mp4") } + +func requireJSONEq(t *testing.T, want string, got any) { + t.Helper() + data, err := json.Marshal(got) + require.NoError(t, err) + require.JSONEq(t, want, string(data)) +} diff --git a/providers/openaicompat/language_model_hooks.go b/providers/openaicompat/language_model_hooks.go index 740292d34..83b3aee46 100644 --- a/providers/openaicompat/language_model_hooks.go +++ b/providers/openaicompat/language_model_hooks.go @@ -1,7 +1,6 @@ package openaicompat import ( - "encoding/base64" "encoding/json" "fmt" "strings" @@ -284,90 +283,12 @@ func ToPromptFunc(prompt fantasy.Prompt, _, _ string) ([]openaisdk.ChatCompletio continue } - switch { - case strings.HasPrefix(filePart.MediaType, "text/"): - base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data) - documentBlock := openaisdk.ChatCompletionContentPartFileFileParam{ - FileData: param.NewOpt(base64Encoded), - } - content = append(content, openaisdk.FileContentPart(documentBlock)) - - case strings.HasPrefix(filePart.MediaType, "image/"): - // Handle image files - base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data) - data := "data:" + filePart.MediaType + ";base64," + base64Encoded - imageURL := openaisdk.ChatCompletionContentPartImageImageURLParam{URL: data} - - // Check for provider-specific options like image detail - if providerOptions, ok := filePart.ProviderOptions[openai.Name]; ok { - if detail, ok := providerOptions.(*openai.ProviderFileOptions); ok { - imageURL.Detail = detail.ImageDetail - } - } - - imageBlock := openaisdk.ChatCompletionContentPartImageParam{ImageURL: imageURL} - content = append(content, openaisdk.ChatCompletionContentPartUnionParam{OfImageURL: &imageBlock}) - - case filePart.MediaType == "audio/wav": - // Handle WAV audio files - base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data) - audioBlock := openaisdk.ChatCompletionContentPartInputAudioParam{ - InputAudio: openaisdk.ChatCompletionContentPartInputAudioInputAudioParam{ - Data: base64Encoded, - Format: "wav", - }, - } - content = append(content, openaisdk.ChatCompletionContentPartUnionParam{OfInputAudio: &audioBlock}) - - case filePart.MediaType == "audio/mpeg" || filePart.MediaType == "audio/mp3": - // Handle MP3 audio files - base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data) - audioBlock := openaisdk.ChatCompletionContentPartInputAudioParam{ - InputAudio: openaisdk.ChatCompletionContentPartInputAudioInputAudioParam{ - Data: base64Encoded, - Format: "mp3", - }, - } - content = append(content, openaisdk.ChatCompletionContentPartUnionParam{OfInputAudio: &audioBlock}) - - case filePart.MediaType == "application/pdf": - // Handle PDF files - dataStr := string(filePart.Data) - - // Check if data looks like a file ID (starts with "file-") - if strings.HasPrefix(dataStr, "file-") { - fileBlock := openaisdk.ChatCompletionContentPartFileParam{ - File: openaisdk.ChatCompletionContentPartFileFileParam{ - FileID: param.NewOpt(dataStr), - }, - } - content = append(content, openaisdk.ChatCompletionContentPartUnionParam{OfFile: &fileBlock}) - } else { - // Handle as base64 data - base64Encoded := base64.StdEncoding.EncodeToString(filePart.Data) - data := "data:application/pdf;base64," + base64Encoded - - filename := filePart.Filename - if filename == "" { - // Generate default filename based on content index - filename = fmt.Sprintf("part-%d.pdf", len(content)) - } - - fileBlock := openaisdk.ChatCompletionContentPartFileParam{ - File: openaisdk.ChatCompletionContentPartFileFileParam{ - Filename: param.NewOpt(filename), - FileData: param.NewOpt(data), - }, - } - content = append(content, openaisdk.ChatCompletionContentPartUnionParam{OfFile: &fileBlock}) - } - - default: - warnings = append(warnings, fantasy.CallWarning{ - Type: fantasy.CallWarningTypeOther, - Message: fmt.Sprintf("file part media type %s not supported", filePart.MediaType), - }) + part, warning := openai.FilePartToChatContentPart(filePart, len(content)) + if warning != nil { + warnings = append(warnings, *warning) + continue } + content = append(content, part) } } if !openai.HasVisibleUserContent(content) {