feat(api): support audio fixture playback in Live sessions

.castiron.stats.yml

@@ -1,8 +1,8 @@
 schema_version: 1
-generation_id: 4b4aafa4-81c7-4a2b-a856-845ab3ce3fd3
-openapi_spec_hash: e004507264375728a48a36cfa571544e
-openapi_transformed_spec_hash: bb7d73817be9bf23b859a5dfd4e15b11
-config_hash: 47b922b06d16fe0e7595c5eacbbb5495
-codegen_sha: 5228c89d4e6bd4c1e5eca339d0116d6b40ecbdf1
-codegen_hash: f18269d01828f09deac5c4ac6561a04f03fd7dbc6281f596cdb1ab91074e8560
-public_codegen_sha: 81d6121e965a7884690bcfb26b6f5859a1ed106b
+generation_id: fb116796-2ea5-4163-b68f-b61bf0010915
+openapi_spec_hash: e81fa0fd2133715dabef9f00a42557a9
+openapi_transformed_spec_hash: bea7a30d23deed3fc8f1c6413a38d354
+config_hash: 77036ac910eb540010c265da4ea569fa
+codegen_sha: fb779116bbfc76b8edaca001ad8b4b91caaab006
+codegen_hash: ba4243ffb54f4298c746fa063ec51f6945b7057f9ca92e2743803b4ee9a624da
+public_codegen_sha: 1a345e61a29b52a1329b349e89df7e076c9b3514

api_reference/openapi.transformed.yml

@@ -50637,6 +50637,7 @@ components:
       - $ref: '#/components/schemas/LiveInputAudioAppendEvent'
       - $ref: '#/components/schemas/LiveInputAudioMuteParam'
       - $ref: '#/components/schemas/LiveInputAudioUnmuteParam'
+      - $ref: '#/components/schemas/LiveOutputAudioFixturePlayParam'
       - $ref: '#/components/schemas/LiveInstructionsAppendParam'
       - $ref: '#/components/schemas/LiveThinkingAppendParam'
       - $ref: '#/components/schemas/LiveCommentaryAppendParam'
@@ -50694,6 +50695,7 @@ components:
       - $ref: '#/components/schemas/LiveInputAudioAppendEvent'
       - $ref: '#/components/schemas/LiveInputAudioMuteParam'
       - $ref: '#/components/schemas/LiveInputAudioUnmuteParam'
+      - $ref: '#/components/schemas/LiveOutputAudioFixturePlayParam'
       - $ref: '#/components/schemas/LiveInstructionsAppendParam'
       - $ref: '#/components/schemas/LiveThinkingAppendParam'
       - $ref: '#/components/schemas/LiveCommentaryAppendParam'
@@ -50881,13 +50883,14 @@ components:
       - session
       - transport
     LiveSidebandClientEvent:
-      description: Client events accepted by an attached Live sideband WebSocket. The session is already started; send audio over the primary connection.
+      description: Client events accepted by an attached Live sideband WebSocket. The session has already started. Send input audio over the primary connection.
       discriminator:
         propertyName: type
       anyOf:
       - $ref: '#/components/schemas/LiveSessionUpdateParam'
       - $ref: '#/components/schemas/LiveInputAudioMuteParam'
       - $ref: '#/components/schemas/LiveInputAudioUnmuteParam'
+      - $ref: '#/components/schemas/LiveOutputAudioFixturePlayParam'
       - $ref: '#/components/schemas/LiveInstructionsAppendParam'
       - $ref: '#/components/schemas/LiveThinkingAppendParam'
       - $ref: '#/components/schemas/LiveCommentaryAppendParam'
@@ -50920,6 +50923,8 @@ components:
       - $ref: '#/components/schemas/LiveSessionUpdated'
       - $ref: '#/components/schemas/LiveInputAudioMuted'
       - $ref: '#/components/schemas/LiveInputAudioUnmuted'
+      - $ref: '#/components/schemas/LiveOutputAudioFixturePlaying'
+      - $ref: '#/components/schemas/LiveOutputAudioFixturePlayed'
       - $ref: '#/components/schemas/LiveInstructionsAppended'
       - $ref: '#/components/schemas/LiveThinkingAppended'
       - $ref: '#/components/schemas/LiveCommentaryAppended'
@@ -92107,6 +92112,39 @@ components:
         example:
           type: session.input_audio.unmute
           event_id: evt_unmute_001
+    LiveOutputAudioFixturePlayParam:
+      description: Play one complete audio fixture through the session output. Send this event over the primary WebSocket or a server sideband. Audio must be raw mono PCM16 little-endian at 24 kHz, regardless of the session audio format, and at most 30 seconds. The entire payload is validated before model speech is interrupted and input is muted. Only one fixture can be active. fixture.playing acknowledges acceptance and fixture.played reports server mixer drain. These notifications are not exact transport-media boundaries. Input arriving during playback is dropped and not replayed.
+      example:
+        type: session.output_audio.fixture.play
+        event_id: evt_play_001
+        audio: AAAAAA==
+      type: object
+      properties:
+        event_id:
+          anyOf:
+          - description: Optional client identifier for correlating this command with a server event's client_event_id or error.client_event_id.
+            type: string
+            maxLength: 512
+          - type: 'null'
+        type:
+          default: session.output_audio.fixture.play
+          x-stainless-const: true
+          type: string
+          enum:
+          - session.output_audio.fixture.play
+        audio:
+          description: Base64-encoded raw mono PCM16 audio, little-endian, at 24 kHz. Do not include a WAV or other container header.
+          type: string
+          minLength: 1
+          maxLength: 1920000
+      required:
+      - type
+      - audio
+      x-oaiMeta:
+        example:
+          type: session.output_audio.fixture.play
+          event_id: evt_play_001
+          audio: AAAAAA==
     LiveInstructionsAppendParam:
       description: Append instructions to the Live conversation while it is running, optionally associating them with an existing client delegation.
       example:
@@ -92624,6 +92662,62 @@ components:
           type: session.input_audio.unmuted
           event_id: evt_unmuted_001
           client_event_id: evt_unmute_001
+    LiveOutputAudioFixturePlaying:
+      description: The existing playback path has accepted the validated audio fixture. This acknowledgment does not identify when the listener receives the first sample. The optional client_event_id echoes the play command.
+      example:
+        type: session.output_audio.fixture.playing
+        event_id: evt_playing_001
+        client_event_id: evt_play_001
+      type: object
+      properties:
+        event_id:
+          description: The unique ID of the Live server event.
+          type: string
+        client_event_id:
+          description: The event_id of the client command associated with this server event, when supplied.
+          type: string
+        type:
+          default: session.output_audio.fixture.playing
+          x-stainless-const: true
+          type: string
+          enum:
+          - session.output_audio.fixture.playing
+      required:
+      - event_id
+      - type
+      x-oaiMeta:
+        example:
+          type: session.output_audio.fixture.playing
+          event_id: evt_playing_001
+          client_event_id: evt_play_001
+    LiveOutputAudioFixturePlayed:
+      description: The complete audio fixture has drained from the server mixer and normal inference has resumed. Transport buffers may still contain audio; this event does not prove listener-side completion. The optional client_event_id echoes the play command. This event is not emitted for canceled or failed playback.
+      example:
+        type: session.output_audio.fixture.played
+        event_id: evt_played_001
+        client_event_id: evt_play_001
+      type: object
+      properties:
+        event_id:
+          description: The unique ID of the Live server event.
+          type: string
+        client_event_id:
+          description: The event_id of the client command associated with this server event, when supplied.
+          type: string
+        type:
+          default: session.output_audio.fixture.played
+          x-stainless-const: true
+          type: string
+          enum:
+          - session.output_audio.fixture.played
+      required:
+      - event_id
+      - type
+      x-oaiMeta:
+        example:
+          type: session.output_audio.fixture.played
+          event_id: evt_played_001
+          client_event_id: evt_play_001
     LiveInstructionsAppended:
       description: Returned when a session.instructions.append command is accepted into the Live session timeline. Acknowledges the appended instructions without guaranteeing that the model has acted on them.
       example:
@@ -93487,6 +93581,8 @@ components:
       - $ref: '#/components/schemas/LiveSessionUpdated'
       - $ref: '#/components/schemas/LiveInputAudioMuted'
       - $ref: '#/components/schemas/LiveInputAudioUnmuted'
+      - $ref: '#/components/schemas/LiveOutputAudioFixturePlaying'
+      - $ref: '#/components/schemas/LiveOutputAudioFixturePlayed'
       - $ref: '#/components/schemas/LiveInstructionsAppended'
       - $ref: '#/components/schemas/LiveThinkingAppended'
       - $ref: '#/components/schemas/LiveCommentaryAppended'
@@ -107960,6 +108056,9 @@ x-oaiMeta:
     - type: object
       key: LiveInputAudioUnmuteParam
       path: <auto>
+    - type: object
+      key: LiveOutputAudioFixturePlayParam
+      path: <auto>
     - type: object
       key: LiveInstructionsAppendParam
       path: <auto>
@@ -108004,6 +108103,12 @@ x-oaiMeta:
     - type: object
       key: LiveInputAudioUnmuted
       path: <auto>
+    - type: object
+      key: LiveOutputAudioFixturePlaying
+      path: <auto>
+    - type: object
+      key: LiveOutputAudioFixturePlayed
+      path: <auto>
     - type: object
       key: LiveInstructionsAppended
       path: <auto>

openai-java-client-okhttp/src/test/kotlin/com/openai/client/okhttp/LargePayloadContractTest.kt

@@ -1,11 +1,19 @@
 package com.openai.client.okhttp
 
+import com.openai.core.ClientOptions
+import com.openai.core.jsonMapper
 import com.openai.helpers.ChatCompletionAccumulator
+import com.openai.helpers.LiveSidebandConnection
 import com.openai.helpers.ResponseAccumulator
 import com.openai.models.chat.completions.ChatCompletionCreateParams
+import com.openai.models.live.OutputAudioFixturePlayEvent
 import com.openai.models.responses.Response
 import com.openai.models.responses.ResponseCreateParams
+import java.util.Base64
+import java.util.concurrent.CompletableFuture
 import java.util.concurrent.TimeUnit
+import okhttp3.WebSocket
+import okhttp3.WebSocketListener
 import okhttp3.mockwebserver.MockResponse
 import okhttp3.mockwebserver.MockWebServer
 import org.assertj.core.api.Assertions.assertThat
@@ -23,6 +31,40 @@ internal class LargePayloadContractTest {
     // Generate data in memory and keep these few cases sequential.
     private val payloadSize = 16 * 1024 * 1024 + 1
 
+    @Test
+    fun liveSidebandSendsMaximumAudioFixture() {
+        // The maximum is 30 seconds of 24 kHz mono PCM16: 1,440,000 raw bytes.
+        val audio = Base64.getEncoder().encodeToString(ByteArray(30 * 24_000 * 2))
+        val received = CompletableFuture<String>()
+        MockWebServer().use { server ->
+            server.enqueue(
+                MockResponse()
+                    .withWebSocketUpgrade(
+                        object : WebSocketListener() {
+                            override fun onMessage(webSocket: WebSocket, text: String) {
+                                received.complete(text)
+                            }
+                        }
+                    )
+            )
+            OkHttpClient.builder().build().use { http ->
+                val options =
+                    ClientOptions.builder()
+                        .httpClient(http)
+                        .apiKey("fake-key")
+                        .baseUrl(server.url("/v1").toString())
+                        .build()
+                LiveSidebandConnection.connect(options, "fixture-session").use { sideband ->
+                    sideband.send(OutputAudioFixturePlayEvent.builder().audio(audio).build())
+                    val event = jsonMapper().readTree(received.get(10, TimeUnit.SECONDS))
+                    assertThat(event.path("type").asText())
+                        .isEqualTo("session.output_audio.fixture.play")
+                    assertCompleteText(event.path("audio").asText(), audio)
+                }
+            }
+        }
+    }
+
     @Test
     fun blockingResponsesJsonPreservesLargeOutputText() {
         val text = "x".repeat(payloadSize)

openai-java-client-okhttp/src/test/kotlin/com/openai/client/okhttp/LiveSidebandConnectionTest.kt

@@ -14,6 +14,7 @@ import com.openai.helpers.LiveSidebandConnection
 import com.openai.helpers.LiveWebSocketOptions
 import com.openai.models.live.InputAudioMuteEvent
 import com.openai.models.live.InputAudioUnmuteEvent
+import com.openai.models.live.OutputAudioFixturePlayEvent
 import com.openai.models.live.SessionCloseEvent
 import java.io.IOException
 import java.net.URI
@@ -180,6 +181,63 @@ class LiveSidebandConnectionTest {
         }
     }
 
+    @Test
+    fun outputAudioFixturePlaybackLifecycleOverSideband() {
+        MockWebServer().use { server ->
+            val peer = Peer()
+            server.enqueue(MockResponse().withWebSocketUpgrade(peer))
+            OkHttpClient.builder().build().use { http ->
+                LiveSidebandConnection.connect(options(server, http), "signaling").use { sideband ->
+                    val fixture =
+                        OutputAudioFixturePlayEvent.builder()
+                            .audio("AAABAA==")
+                            .eventId("fixture-1")
+                            .build()
+                    assertThatThrownBy {
+                            sideband.send(
+                                fixture
+                                    .toBuilder()
+                                    .type(JsonValue.from("session.input_audio.append"))
+                                    .build()
+                            )
+                        }
+                        .isInstanceOf(IllegalArgumentException::class.java)
+                    sideband.send(fixture)
+                    val play = mapper.readTree(peer.messages.poll(8, TimeUnit.SECONDS))
+                    assertThat(play)
+                        .isEqualTo(
+                            mapper.readTree(
+                                """{"type":"session.output_audio.fixture.play","audio":"AAABAA==","event_id":"fixture-1"}"""
+                            )
+                        )
+                    val clientEventId = play.path("event_id").asText()
+                    peer.socket.get(8, TimeUnit.SECONDS).apply {
+                        send(
+                            """{"type":"session.output_audio.fixture.playing","event_id":"playing-1","client_event_id":"$clientEventId"}"""
+                        )
+                        send(
+                            """{"type":"session.output_audio.fixture.played","event_id":"played-1","client_event_id":"$clientEventId"}"""
+                        )
+                    }
+                    val playing =
+                        sideband
+                            .receiveAsync()
+                            .get(8, TimeUnit.SECONDS)
+                            .asSessionOutputAudioFixturePlaying()
+                    assertThat(playing.eventId()).isEqualTo("playing-1")
+                    assertThat(playing.clientEventId()).contains(clientEventId)
+                    val played =
+                        sideband
+                            .receiveAsync()
+                            .get(8, TimeUnit.SECONDS)
+                            .asSessionOutputAudioFixturePlayed()
+                    assertThat(played.eventId()).isEqualTo("played-1")
+                    assertThat(played.clientEventId()).contains(clientEventId)
+                }
+            }
+        }
+    }
+
     @Test
     fun explicitSessionCloseDrainsErrorsAndFutureEventsWhileObserverCloseOnlyDetaches() {
         MockWebServer().use { server ->

openai-java-core/src/main/kotlin/com/openai/helpers/LiveSidebandConnection.kt

@@ -9,6 +9,7 @@ import com.openai.models.live.CommentaryAppendEvent
 import com.openai.models.live.InputAudioMuteEvent
 import com.openai.models.live.InputAudioUnmuteEvent
 import com.openai.models.live.InstructionsAppendEvent
+import com.openai.models.live.OutputAudioFixturePlayEvent
 import com.openai.models.live.ResponseCreateEvent
 import com.openai.models.live.ResponseItemCreateEvent
 import com.openai.models.live.ServerEvent
@@ -22,7 +23,8 @@ import java.util.concurrent.CompletableFuture
  * credentials. Configure the intended bearer in ClientOptions or per-connection headers; matching a
  * project alone does not guarantee access. A completed connection is transport-open, not a fresh
  * session.started: the server can replay recent events followed by live delivery. No session.start,
- * reconnect, or audio append is sent by this helper. Send audio over the primary transport.
+ * reconnect, or input audio append is sent by this helper. Send input audio over the primary
+ * transport.
  *
  * Receive typed Live server events (including reflected audio and signaling events) in wire order.
  * Closing the observer releases its transport without ending the underlying session or its client.
@@ -93,6 +95,8 @@ private constructor(
 
     fun send(event: InputAudioUnmuteEvent) = sendCommand(event)
 
+    fun send(event: OutputAudioFixturePlayEvent) = sendCommand(event)
+
     fun send(event: InstructionsAppendEvent) = sendCommand(event)
 
     fun send(event: ThinkingAppendEvent) = sendCommand(event)