Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions provider/openaiprovider/responses.go
Original file line number Diff line number Diff line change
Expand Up @@ -1656,7 +1656,7 @@ func responsesProcessStreamingUpdate(update responses.ResponseStreamEventUnion,
// The in-progress event already emitted the ImageGenerationToolCallContent
// for this item, so emit only the finished result here to avoid a
// duplicate tool call.
u.Contents = []message.Content{imageGenerationResult(item.ID, item.Result, "png", item)}
u.Contents = []message.Content{imageGenerationResult(item.ID, item.Result, cmp.Or(item.OutputFormat, "png"), item)}
}
case responses.ResponseReasoningItem:
// Carry the completed reasoning item's encrypted content so it can be
Expand Down Expand Up @@ -1820,7 +1820,7 @@ func mcpToolCallErrorMessage(err responses.McpToolCallErrorUnion) string {
func imageGenerationContents(item responses.ResponseOutputItemImageGenerationCall) message.Contents {
return message.Contents{
&message.ImageGenerationToolCallContent{CallID: item.ID},
imageGenerationResult(item.ID, item.Result, "png", item),
imageGenerationResult(item.ID, item.Result, cmp.Or(item.OutputFormat, "png"), item),
}
}

Expand Down
73 changes: 73 additions & 0 deletions provider/openaiprovider/responses_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5119,6 +5119,38 @@ func TestResponsesUserMessageWithVariousContentTypes_ConvertsCorrectly(t *testin
}
}

// The generated image's media type must follow the call's output_format rather
// than always being image/png, matching the Python client (and the streaming
// partial path).
func TestResponsesImageGenerationCall_UsesOutputFormat(t *testing.T) {
const imageBase64 = "iVBORw0KGgo="
const input = `{"model":"gpt-4o-mini","input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"draw"}]}]}`
const output = `{
"id":"resp_fmt","object":"response","created_at":1741892091,"status":"completed","model":"gpt-4o-mini",
"output":[{"type":"image_generation_call","id":"ig_fmt","status":"completed","output_format":"webp","result":"` + imageBase64 + `"}]
}`

server := newTestResponsesServer(t, input, output)
defer server.Close()

resp, err := newTestResponsesClient(server, "gpt-4o-mini").RunText(t.Context(), "draw").Collect()
if err != nil {
t.Fatalf("error = %v", err)
}
var image *message.DataContent
for content := range resp.Contents() {
if r, ok := content.(*message.ImageGenerationToolResultContent); ok && len(r.Outputs) == 1 {
image, _ = r.Outputs[0].(*message.DataContent)
}
}
if image == nil {
t.Fatal("no image data content surfaced")
}
if image.MediaType != "image/webp" {
t.Errorf("MediaType = %q, want %q", image.MediaType, "image/webp")
}
}

func TestResponsesNonStreamingImageGenerationCall_MapsToToolContents(t *testing.T) {
const imageBase64 = "iVBORw0KGgo="
const input = `
Expand Down Expand Up @@ -5252,6 +5284,47 @@ data: {"type":"response.completed","sequence_number":4,"response":{"id":"resp_00
}
}

// With no partial_image event, the finished image is emitted from
// output_item.done and its media type must follow the call's output_format.
func TestResponsesStreamingImageGenerationCall_NoPartial_UsesOutputFormat(t *testing.T) {
const imageBase64 = "iVBORw0KGgo="
const input = `{"model":"gpt-4o-mini","input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"draw"}]}],"stream":true}`
const output = `event: response.created
data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_002","object":"response","created_at":1741892091,"status":"in_progress","model":"gpt-4o-mini","output":[]}}

event: response.image_generation_call.in_progress
data: {"type":"response.image_generation_call.in_progress","sequence_number":1,"output_index":0,"item_id":"ig_np"}

event: response.output_item.done
data: {"type":"response.output_item.done","sequence_number":2,"output_index":0,"item":{"type":"image_generation_call","id":"ig_np","status":"completed","output_format":"webp","result":"` + imageBase64 + `"}}

event: response.completed
data: {"type":"response.completed","sequence_number":3,"response":{"id":"resp_002","object":"response","created_at":1741892091,"status":"completed","model":"gpt-4o-mini","output":[]}}

`

server := newTestResponsesServerStreaming(t, input, output)
defer server.Close()

var image *message.DataContent
for update, err := range newTestResponsesClient(server, "gpt-4o-mini").RunText(t.Context(), "draw", agent.Stream(true)) {
if err != nil {
t.Fatalf("error = %v", err)
}
for _, content := range update.Contents {
if r, ok := content.(*message.ImageGenerationToolResultContent); ok && len(r.Outputs) == 1 {
image, _ = r.Outputs[0].(*message.DataContent)
}
}
}
if image == nil {
t.Fatal("no image data content surfaced from output_item.done")
}
if image.MediaType != "image/webp" {
t.Errorf("MediaType = %q, want image/webp", image.MediaType)
}
}

func TestResponsesNonStreamingWebSearchCall_MapsToToolContents(t *testing.T) {
const input = `
{
Expand Down
Loading