diff --git a/.changeset/audio-compression.md b/.changeset/audio-compression.md new file mode 100644 index 000000000..7a8f834b9 --- /dev/null +++ b/.changeset/audio-compression.md @@ -0,0 +1,6 @@ +--- +"@braintrust/audio": minor +"braintrust": minor +--- + +feat(audio): Compressed voice recordings with `@braintrust/audio` diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index 6156a2e46..c3092fe02 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -417,6 +417,34 @@ jobs: working-directory: integrations/temporal-js run: pnpm run test + audio: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + node-version: [20, 22, 24, 26] + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: ${{ matrix.node-version == 20 && 'true' || '' }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: ${{ matrix.node-version }} + - name: Setup pnpm + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Build braintrust + working-directory: js + run: pnpm run build + - name: Build @braintrust/audio + working-directory: integrations/audio + run: pnpm run build + - name: Run @braintrust/audio tests + working-directory: integrations/audio + run: pnpm run test + templates-nunjucks-js: runs-on: ubuntu-latest timeout-minutes: 15 @@ -530,6 +558,7 @@ jobs: - js-smoke-discover - js-smoke-test - temporal-js + - audio - templates-nunjucks-js - otel-js - js-zod-versions @@ -563,6 +592,7 @@ jobs: check_result "js-smoke-discover" "${{ needs.js-smoke-discover.result }}" check_result "js-smoke-test" "${{ needs.js-smoke-test.result }}" check_result "temporal-js" "${{ needs.temporal-js.result }}" + check_result "audio" "${{ needs.audio.result }}" check_result "templates-nunjucks-js" "${{ needs.templates-nunjucks-js.result }}" check_result "otel-js" "${{ needs.otel-js.result }}" check_result "js-zod-versions" "${{ needs.js-zod-versions.result }}" diff --git a/e2e/package.json b/e2e/package.json index cb9ad5d74..3fba053c1 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -9,6 +9,7 @@ "test:e2e:cassettes": "node ./scripts/run-e2e-tests.mjs --check-cassettes" }, "devDependencies": { + "@braintrust/audio": "workspace:^", "@braintrust/deepseek-harness": "workspace:^", "@braintrust/langchain-js": "workspace:^", "@braintrust/otel": "workspace:^", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.json index 32898bbbd..8501eca8d 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.json @@ -201,8 +201,8 @@ "input": { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -218,10 +218,9 @@ "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.txt index 590ae2d7a..1ca50928d 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.txt @@ -3,8 +3,8 @@ span_tree: input: { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -20,10 +20,9 @@ span_tree: "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.json index b44d84fcd..aaff0cc25 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.json @@ -574,8 +574,8 @@ "input": { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -591,10 +591,9 @@ "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.txt index f61c67976..a39aa2114 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.txt @@ -3,8 +3,8 @@ span_tree: input: { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -20,10 +20,9 @@ span_tree: "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.json index 32898bbbd..8501eca8d 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.json @@ -201,8 +201,8 @@ "input": { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -218,10 +218,9 @@ "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.txt index 590ae2d7a..1ca50928d 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.txt @@ -3,8 +3,8 @@ span_tree: input: { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -20,10 +20,9 @@ span_tree: "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.json index 5c1a8042f..031773773 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.json @@ -556,8 +556,8 @@ "input": { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -573,10 +573,9 @@ "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.txt index e1a2539ea..fb0d22143 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.txt @@ -3,8 +3,8 @@ span_tree: input: { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -20,10 +20,9 @@ span_tree: "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.json index b9f24e612..58a6ab1e1 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.json @@ -561,8 +561,8 @@ "input": { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -578,10 +578,9 @@ "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.txt index b562da2b5..784c165a2 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.txt @@ -3,8 +3,8 @@ span_tree: input: { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -20,10 +20,9 @@ span_tree: "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.json index fcc8d5756..344eb69e1 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.json @@ -868,8 +868,8 @@ "input": { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -885,10 +885,9 @@ "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.txt index 24646b5f5..2164a0052 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.txt @@ -3,8 +3,8 @@ span_tree: input: { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -20,10 +20,9 @@ span_tree: "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.json index 70fd4ad6a..864b8dfa7 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.json @@ -763,8 +763,8 @@ "input": { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -780,10 +780,9 @@ "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "session_input", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.txt index 665d4d773..3f0188cc3 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.txt @@ -3,8 +3,8 @@ span_tree: input: { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -20,10 +20,9 @@ span_tree: "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "session_input", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.json index 7f8aa7a85..b97ec2534 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.json @@ -579,8 +579,8 @@ "input": { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -596,10 +596,9 @@ "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.txt index af616a14a..306e9f1a4 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.txt @@ -3,8 +3,8 @@ span_tree: input: { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -20,10 +20,9 @@ span_tree: "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.json index 879f6e19b..5b2c64fd7 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.json @@ -904,8 +904,8 @@ "input": { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -921,10 +921,9 @@ "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.txt index 9a72bdcae..0d2094dc4 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.txt @@ -3,8 +3,8 @@ span_tree: input: { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -20,10 +20,9 @@ span_tree: "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "local_audio_output", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.json index 90b4e03af..b3c0459a7 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.json @@ -724,8 +724,8 @@ "input": { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -741,10 +741,9 @@ "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "session_input", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.txt index ada7d0c0b..715d5baed 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.txt @@ -3,8 +3,8 @@ span_tree: input: { "audio": { "call-0000": { - "content_type": "audio/pcm", - "filename": "call-0000.pcm", + "content_type": "audio/ogg", + "filename": "call-0000.ogg", "key": "", "type": "braintrust_attachment" } @@ -20,10 +20,9 @@ span_tree: "channel_count": 2, "duration_ms": 0, "id": "", - "mime_type": "audio/pcm", + "mime_type": "audio/ogg", "recording_group_id": "call", - "sample_format": "s16le", - "sample_rate_hz": 24000, + "sample_rate_hz": 48000, "sources": [ { "boundary": "session_input", diff --git a/e2e/scenarios/livekit-instrumentation/configure.mjs b/e2e/scenarios/livekit-instrumentation/configure.mjs index a1d711778..5cb438662 100644 --- a/e2e/scenarios/livekit-instrumentation/configure.mjs +++ b/e2e/scenarios/livekit-instrumentation/configure.mjs @@ -3,7 +3,14 @@ configureInstrumentation({ ...(process.env.LIVEKIT_SEGMENTS === "1" ? { streamingAttachmentBufferBytes: 19200 } // 200 ms of 24 kHz stereo s16le. : {}), - integrations: { livekit: process.env.LIVEKIT_INTEGRATION !== "off" }, + integrations: { + livekit: + process.env.LIVEKIT_INTEGRATION === "off" + ? false + : process.env.LIVEKIT_AUDIO + ? { audio: JSON.parse(process.env.LIVEKIT_AUDIO) } + : true, + }, spanCustomizers: [ { onSpanExport(data) { diff --git a/e2e/scenarios/livekit-instrumentation/scenario.mjs b/e2e/scenarios/livekit-instrumentation/scenario.mjs index 9681db5b0..1d2e4bd39 100644 --- a/e2e/scenarios/livekit-instrumentation/scenario.mjs +++ b/e2e/scenarios/livekit-instrumentation/scenario.mjs @@ -42,6 +42,37 @@ if (process.env.LIVEKIT_SEGMENTS === "1") { return response; }); } +// Attachment bytes go to object storage through the global fetch, as a PUT +// to a signed URL. +if (process.env.LIVEKIT_PROBE_UPLOADS === "1") { + const upload = globalThis.fetch; + globalThis.fetch = async (input, init) => { + if (init?.method === "PUT" && typeof init.body !== "string") + console.log(JSON.stringify({ uploadedAudio: await probe(init.body) })); + return upload(input, init); + }; +} +/** Each uploaded file's format, and for Ogg its decoded duration. */ +async function probe(body) { + const file = new Uint8Array(await new Response(body).arrayBuffer()); + return new TextDecoder().decode(file.subarray(0, 4)) === "OggS" + ? { format: "ogg", decodedMs: oggDurationMs(file) } + : { format: "raw" }; +} +/** Final granule minus the OpusHead pre-skip, at 48 kHz (RFC 7845). */ +function oggDurationMs(file) { + const view = new DataView(file.buffer, file.byteOffset, file.byteLength); + const preSkip = view.getUint16(28 + 10, true); + let granule = 0; + for (let at = 0; at < file.length; ) { + granule = Number(view.getBigUint64(at + 6, true)); + const count = file[at + 26]; + at += file + .subarray(at + 27, at + 27 + count) + .reduce((sum, n) => sum + n, 27 + count); + } + return (granule - preSkip) / 48; +} const logger = initLogger({ projectName: scopedName("e2e-livekit") }); const providerMode = process.env.LIVEKIT_PROVIDER === "1"; class Output extends voice.AudioOutput { diff --git a/e2e/scenarios/livekit-instrumentation/scenario.test.ts b/e2e/scenarios/livekit-instrumentation/scenario.test.ts index 57e6afd6c..8e22e3d4b 100644 --- a/e2e/scenarios/livekit-instrumentation/scenario.test.ts +++ b/e2e/scenarios/livekit-instrumentation/scenario.test.ts @@ -85,6 +85,16 @@ type RunOptions = { cassette?: string; recording?: boolean; }; +/** What the scenario reported about each uploaded audio file, in order. */ +function uploadedAudio( + stdout: string, +): { format: string; decodedMs?: number }[] { + return stdout + .split("\n") + .filter((line) => line.startsWith('{"uploadedAudio":')) + .map((line) => JSON.parse(line).uploadedAudio); +} + function scenarioOptions( variant: string, { @@ -184,6 +194,91 @@ for (const variant of variants) { 120000, ); + // With @braintrust/audio installed, recordings upload as Ogg Opus. + for (const [name, audio] of [ + ["by default", undefined], + [ + "with configured options", + { codec: { name: "opus", options: { bitrate: 16000 } } }, + ], + ] as const) + test.sequential( + `${variant} compressed recording ${name}`, + async () => { + await withScenarioHarness(async (harness) => { + const result = await harness.runNodeScenarioDir( + scenarioOptions(variant, { + env: { + LIVEKIT_PROBE_UPLOADS: "1", + ...(audio ? { LIVEKIT_AUDIO: JSON.stringify(audio) } : {}), + }, + }), + ); + const root = findLatestSpan( + assertAudioTrace(harness.events()), + "session", + )!; + const [recording] = root.metadata?.["audio.recordings"] as { + duration_ms: number; + }[]; + expect(recording).toMatchObject({ + id: "call-0000", + state: "ready", + mime_type: "audio/ogg", + sample_rate_hz: 48000, + channel_count: 2, + }); + expect(recording).not.toHaveProperty("sample_format"); + // The uploaded file decodes to exactly the recorded duration. + const [upload] = uploadedAudio(result.stdout); + expect(upload.format).toBe("ogg"); + expect(upload.decodedMs).toBeCloseTo(recording.duration_ms, 1); + expect( + (root.row.input as { audio: Record }).audio[ + "call-0000" + ], + ).toMatchObject({ + filename: "call-0000.ogg", + content_type: "audio/ogg", + }); + expect(result.stderr).not.toContain("uncompressed PCM"); + }); + }, + 120000, + ); + + // Turning compression off records raw PCM, without a warning. + test.sequential( + `${variant} raw PCM recording with codec pcm`, + async () => { + await withScenarioHarness(async (harness) => { + const result = await harness.runNodeScenarioDir( + scenarioOptions(variant, { + env: { + LIVEKIT_PROBE_UPLOADS: "1", + LIVEKIT_AUDIO: JSON.stringify({ codec: { name: "pcm" } }), + }, + }), + ); + const root = findLatestSpan( + assertAudioTrace(harness.events()), + "session", + )!; + expect(root.metadata?.["audio.recordings"]).toMatchObject([ + { + id: "call-0000", + state: "ready", + mime_type: "audio/pcm", + sample_format: "s16le", + }, + ]); + expect(uploadedAudio(result.stdout)).toEqual([{ format: "raw" }]); + expect(result.stderr).not.toContain("uncompressed PCM"); + }); + }, + 120000, + ); + test.sequential( `${variant} recording requires attachment capture`, async () => { diff --git a/integrations/audio/README.md b/integrations/audio/README.md new file mode 100644 index 000000000..6d0ecc674 --- /dev/null +++ b/integrations/audio/README.md @@ -0,0 +1,54 @@ +# @braintrust/audio + +Provides audio plugins for audio instrumentation recordings, such as compression and containers. + +## Installation + +```sh +npm install @braintrust/audio +``` + +Requires Node 20+ and `braintrust` 3.41+. + +When recording is activated with `BRAINTRUST_CAPTURE_USER_AUDIO_ATTACHMENTS` or `BRAINTRUST_CAPTURE_AGENT_AUDIO_ATTACHMENTS`, the SDK will automatically use this package if available. + +## Configuration + +Integrations that support audio recording may offer the following configuration options: + +```ts +import { configureInstrumentation } from "braintrust/instrumentation"; + +configureInstrumentation({ + integrations: { + livekit: { + audio: { + codec: { name: "opus", options: { bitrate: 32_000 } }, + container: { name: "ogg" }, + }, + }, + }, +}); +``` + +| Setting | Default | Meaning | +| --------------------------- | --------- | ------------------------------------------------------------ | +| `codec.name` | `"opus"` | `"opus"`, or `"pcm"` to record uncompressed audio. | +| `codec.options.bitrate` | `64000` | Target bitrate in whole bits per second, from 500 to 512000. | +| `codec.options.complexity` | `5` | Encoder effort, from 0 (fastest) to 10. | +| `codec.options.application` | `"audio"` | Encoder tuning: `"audio"`, `"voip"`, or `"lowdelay"`. | +| `container.name` | `"ogg"` | The file format. | + +If the package is missing or a setting is unsupported, recordings upload as raw PCM and a warning explains why. + +## Bundled apps + +Braintrust loads this package from `node_modules` at runtime. If you deploy a bundle without `node_modules`, import the package in your app so your bundler includes it: + +```ts +import "@braintrust/audio"; +``` + +## Performance + +Encoding runs on the main thread in 20 ms frames and yields to the event loop after each second of audio. diff --git a/integrations/audio/package.json b/integrations/audio/package.json new file mode 100644 index 000000000..aac8489a2 --- /dev/null +++ b/integrations/audio/package.json @@ -0,0 +1,58 @@ +{ + "name": "@braintrust/audio", + "version": "0.0.1", + "description": "Audio codecs for Braintrust voice recordings", + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "README.md" + ], + "exports": { + "./package.json": "./package.json", + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.mjs", + "module": "./dist/index.mjs", + "require": "./dist/index.js" + } + }, + "scripts": { + "build": "tsup", + "check:typings": "tsc --noEmit", + "watch": "tsup --watch", + "clean": "rm -r dist/*", + "test": "vitest run" + }, + "author": "Braintrust Data Inc.", + "license": "MIT", + "dependencies": { + "@audio/encode-opus": "1.3.1" + }, + "devDependencies": { + "@types/node": "^20.10.5", + "braintrust": "workspace:^", + "ogg-opus-decoder": "1.7.5", + "tsup": "^8.5.1", + "typescript": "^5.3.3", + "vitest": "4.1.11" + }, + "peerDependencies": { + "braintrust": ">=3.41.0" + }, + "engines": { + "node": ">=20" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/braintrustdata/braintrust-sdk-javascript.git", + "directory": "integrations/audio" + }, + "homepage": "https://www.braintrust.dev/docs", + "publishConfig": { + "access": "public", + "registry": "https://registry.npmjs.org/", + "provenance": true + } +} diff --git a/integrations/audio/src/index.ts b/integrations/audio/src/index.ts new file mode 100644 index 000000000..decee229d --- /dev/null +++ b/integrations/audio/src/index.ts @@ -0,0 +1,9 @@ +import { createStack } from "./stack"; + +// Lets the SDK build compressed recordings without depending on this package. +// The SDK loads this package itself; the first copy loaded wins, so duplicate +// installs agree on one implementation. +(globalThis as Record)[Symbol.for("braintrust.audio")] ??= { + version: 1, + createStack, +}; diff --git a/integrations/audio/src/ogg.test.ts b/integrations/audio/src/ogg.test.ts new file mode 100644 index 000000000..6247fae5c --- /dev/null +++ b/integrations/audio/src/ogg.test.ts @@ -0,0 +1,175 @@ +import { expect, test } from "vitest"; +import oggOpus from "@audio/encode-opus"; +import { OggOpusDecoder } from "ogg-opus-decoder"; +import type { AudioSegmentMeta } from "./types"; +import { ogg } from "./ogg"; +import { opus } from "./opus"; +import type { EncodedPackets } from "./packets"; + +const meta = (frames: number): AudioSegmentMeta => ({ + index: 1, + startOffsetMs: 3000, + durationMs: frames / 24, + sampleRate: 24000, + channels: [ + { sources: ["session_input"] }, + { sources: ["local_audio_output"] }, + ], + mimeType: "audio/pcm", + encoding: "pcm", + attributes: { + "pcm.sample_format": "s16le", + "pcm.channel_layout": "interleaved", + "acme.call_id": "c-1", + }, +}); + +/** Speech-like tones at 24 kHz, a different pitch per channel. */ +function tones(frames: number) { + const pcm = new Int16Array(frames * 2); + for (let i = 0; i < frames; i++) + for (let c = 0; c < 2; c++) + pcm[i * 2 + c] = Math.round( + 9000 * Math.sin((2 * Math.PI * (220 + 110 * c) * i) / 24000), + ); + return pcm; +} + +/** Runs the opus() → ogg() stack and returns the file it produces. */ +async function encodeOgg(pcm: Int16Array, input: AudioSegmentMeta) { + return opus()( + new Uint8Array(pcm.buffer), + input, + async (packets, packetMeta) => + ogg()(packets, packetMeta, async (segment, meta) => ({ segment, meta })), + ); +} + +async function decode(file: Uint8Array) { + const decoder = new OggOpusDecoder(); + await decoder.ready; + try { + return await decoder.decodeFile(file); + } finally { + decoder.free(); + } +} + +/** Pages with their header fields, verifying each page's CRC. */ +function pages(file: Uint8Array) { + const view = new DataView(file.buffer, file.byteOffset, file.byteLength); + const result: { flags: number; granule: number; crcValid: boolean }[] = []; + for (let at = 0; at < file.length; ) { + expect(new TextDecoder().decode(file.subarray(at, at + 4))).toBe("OggS"); + const count = file[at + 26]; + const size = file + .subarray(at + 27, at + 27 + count) + .reduce((sum, n) => sum + n, 27 + count); + const page = file.slice(at, at + size); + const crc = view.getUint32(at + 22, true); + page.fill(0, 22, 26); + result.push({ + flags: file[at + 5], + granule: Number(view.getBigUint64(at + 6, true)), + crcValid: crc === crc32(page), + }); + at += size; + } + return result; +} + +function crc32(bytes: Uint8Array) { + let crc = 0; + for (const byte of bytes) { + crc ^= byte << 24; + for (let j = 0; j < 8; j++) + crc = crc & 0x80000000 ? (crc << 1) ^ 0x04c11db7 : crc << 1; + crc >>>= 0; + } + return crc; +} + +test.each([2400, 24001, 72123, 5 * 24000])( + "%i frames decode to the reference encoder's audio, exactly as long as the input", + async (frames) => { + const pcm = tones(frames); + const { segment: file } = await encodeOgg(pcm, meta(frames)); + const decoded = await decode(file); + expect(decoded.errors).toEqual([]); + expect(decoded.channelData).toHaveLength(2); + + const encoder = await oggOpus({ + sampleRate: 24000, + channels: 2, + bitrate: 64, + complexity: 5, + application: "audio", + }); + let reference: Awaited>; + try { + const planar = [0, 1].map((c) => + Float32Array.from({ length: frames }, (_, i) => pcm[i * 2 + c] / 32768), + ); + const parts = [encoder.encode(planar), encoder.flush()]; + reference = await decode(Uint8Array.from([...parts[0], ...parts[1]])); + } finally { + encoder.free(); + } + expect(decoded.samplesDecoded).toBe(reference.samplesDecoded); + for (const c of [0, 1]) + expect(decoded.channelData[c]).toEqual(reference.channelData[c]); + + // The final granule trims padding: 48 kHz input length plus the pre-skip. + const preSkip = new DataView(file.buffer).getUint16(28 + 10, true); + const layout = pages(file); + expect(layout.at(-1)!.granule - preSkip).toBe(frames * 2); + }, +); + +test("pages follow RFC 7845: headers alone, then audio, with valid CRCs", async () => { + const frames = 3 * 24000; + const { segment: file } = await encodeOgg(tones(frames), meta(frames)); + const layout = pages(file); + expect(layout.every((page) => page.crcValid)).toBe(true); + expect(layout[0]).toMatchObject({ flags: 0x02, granule: 0 }); // OpusHead + expect(layout[1]).toMatchObject({ flags: 0, granule: 0 }); // OpusTags + expect(layout.slice(2, -1).every((page) => page.flags === 0)).toBe(true); + expect(layout.at(-1)!.flags).toBe(0x04); + // Granules increase page by page, each page holding at most a second of audio. + const granules = layout.slice(2).map((page) => page.granule); + for (const [i, granule] of granules.entries()) + expect(granule - (granules[i - 1] ?? 0)).toBeLessThanOrEqual(48000); + expect(new TextDecoder().decode(file.subarray(28, 36))).toBe("OpusHead"); +}); + +test("metadata becomes an Ogg file on the same timeline and channels", async () => { + const input = meta(2400); + const { meta: output } = await encodeOgg(tones(2400), input); + // The container keeps the codec and its attributes. + expect(output).toEqual({ + ...input, + sampleRate: 48000, + mimeType: "audio/ogg", + encoding: "opus", + attributes: { + "acme.call_id": "c-1", + "opus.bitrate_bps": 64000, + "opus.complexity": 5, + "opus.application": "audio", + }, + }); +}); + +test.each([ + ["raw bytes", new Uint8Array(4)], + [ + "another codec", + { codec: "vorbis", headers: [], packets: [], delay: 0, padding: 0 }, + ], +])("rejects %s", (_name, segment) => { + expect(() => + ogg()(segment as EncodedPackets, meta(1), () => { + throw new Error("next must not run"); + }), + ).toThrow(/ogg requires Opus packets/); +}); diff --git a/integrations/audio/src/ogg.ts b/integrations/audio/src/ogg.ts new file mode 100644 index 000000000..9c74c5c33 --- /dev/null +++ b/integrations/audio/src/ogg.ts @@ -0,0 +1,123 @@ +import type { AudioMiddleware } from "./types"; +import type { EncodedPackets } from "./packets"; + +/** A page closes past 4 KB or one second of 20 ms packets, as libogg pages Opus. */ +const PAGE_BYTES = 4096; +const PAGE_PACKETS = 50; +const MAX_SEGMENTS = 255; +const BOS = 0x02; +const EOS = 0x04; + +/** + * Wraps encoded packets in a self-contained Ogg file (RFC 7845 for Opus). + * Place it after an encoder such as `opus()`. + */ +export function ogg(): AudioMiddleware { + return function ogg(segment, meta, next) { + if (segment?.codec !== "opus" || !segment.headers?.[0]) + throw new Error("ogg requires Opus packets, such as from opus()"); + const serial = Math.floor(Math.random() * 2 ** 32); + const pages: Uint8Array[] = []; + const page = (packets: Uint8Array[], granule: number, flags = 0) => + pages.push(oggPage(packets, granule, flags, serial, pages.length)); + page([segment.headers[0]], 0, BOS); + page([opusTags()], 0); + + let queue: Uint8Array[] = []; + let bytes = 0; + let segments = 0; + let granule = 0; + for (const [i, packet] of segment.packets.entries()) { + const lacing = Math.floor(packet.data.length / 255) + 1; + if ( + queue.length && + (queue.length >= PAGE_PACKETS || + bytes + packet.data.length > PAGE_BYTES || + segments + lacing > MAX_SEGMENTS) + ) { + page(queue, granule); + queue = []; + bytes = segments = 0; + } + queue.push(packet.data); + bytes += packet.data.length; + segments += lacing; + granule += packet.samples; + // The last page's granule trims the padding (RFC 7845 §4.4). + if (i === segment.packets.length - 1) + page(queue, granule - segment.padding, EOS); + } + return next(concat(pages), { ...meta, mimeType: "audio/ogg" }); + }; +} + +/** Opus comment header with no tags (RFC 7845 §5.2). */ +function opusTags() { + const vendor = new TextEncoder().encode("@braintrust/audio"); + const tags = new Uint8Array(8 + 4 + vendor.length + 4); + const view = new DataView(tags.buffer); + tags.set(new TextEncoder().encode("OpusTags")); + view.setUint32(8, vendor.length, true); + tags.set(vendor, 12); + view.setUint32(12 + vendor.length, 0, true); // user comment count + return tags; +} + +/** One page of whole packets (RFC 3533). */ +function oggPage( + packets: Uint8Array[], + granule: number, + flags: number, + serial: number, + sequence: number, +) { + const lacing: number[] = []; + for (const packet of packets) { + for (let n = packet.length; n >= 255; n -= 255) lacing.push(255); + lacing.push(packet.length % 255); + } + const header = 27 + lacing.length; + const page = new Uint8Array( + header + packets.reduce((sum, packet) => sum + packet.length, 0), + ); + const view = new DataView(page.buffer); + page.set([0x4f, 0x67, 0x67, 0x53]); // "OggS" + page[5] = flags; + view.setBigUint64(6, BigInt(granule), true); + view.setUint32(14, serial, true); + view.setUint32(18, sequence, true); + page[26] = lacing.length; + page.set(lacing, 27); + let at = header; + for (const packet of packets) { + page.set(packet, at); + at += packet.length; + } + view.setUint32(22, crc32(page), true); + return page; +} + +const CRC_TABLE = Uint32Array.from({ length: 256 }, (_, i) => { + let r = i << 24; + for (let j = 0; j < 8; j++) + r = r & 0x80000000 ? (r << 1) ^ 0x04c11db7 : r << 1; + return r >>> 0; +}); + +/** Ogg's CRC-32: polynomial 0x04c11db7, no reflection, zero initial value. */ +function crc32(bytes: Uint8Array) { + let crc = 0; + for (const byte of bytes) + crc = ((crc << 8) ^ CRC_TABLE[((crc >>> 24) ^ byte) & 0xff]) >>> 0; + return crc; +} + +function concat(parts: Uint8Array[]) { + const out = new Uint8Array(parts.reduce((sum, part) => sum + part.length, 0)); + let at = 0; + for (const part of parts) { + out.set(part, at); + at += part.length; + } + return out; +} diff --git a/integrations/audio/src/opus.test.ts b/integrations/audio/src/opus.test.ts new file mode 100644 index 000000000..586791dbe --- /dev/null +++ b/integrations/audio/src/opus.test.ts @@ -0,0 +1,254 @@ +import { expect, test } from "vitest"; +import oggOpus from "@audio/encode-opus"; +import type { AudioSegmentMeta } from "./types"; +import { opus, type OpusOptions } from "./opus"; +import type { EncodedPackets } from "./packets"; + +const meta = (sampleRate: number, channelCount: number): AudioSegmentMeta => ({ + index: 2, + startOffsetMs: 4000, + durationMs: 1000, + sampleRate, + channels: Array.from({ length: channelCount }, (_, channel) => ({ + sources: channel === 0 ? ["session_input"] : [], + })), + mimeType: "audio/pcm", + encoding: "pcm", + attributes: { + "pcm.sample_format": "s16le", + "pcm.channel_layout": "interleaved", + "acme.call_id": "c-1", + }, +}); + +/** Speech-like test tones, a different pitch per channel, as interleaved s16le. */ +function tones(frames: number, sampleRate: number, channels: number) { + const pcm = new Int16Array(frames * channels); + for (let i = 0; i < frames; i++) + for (let c = 0; c < channels; c++) + pcm[i * channels + c] = Math.round( + 9000 * Math.sin((2 * Math.PI * (220 + 110 * c) * i) / sampleRate), + ); + return pcm; +} +const bytes = (pcm: Int16Array) => new Uint8Array(pcm.buffer); + +/** Runs opus() and captures what it passes down the stack. */ +async function encode( + pcm: Int16Array, + segmentMeta: AudioSegmentMeta, + options?: OpusOptions, +) { + let down: { segment: EncodedPackets; meta: AudioSegmentMeta } | undefined; + await opus(options)(bytes(pcm), segmentMeta, async (segment, meta) => { + down = { segment, meta }; + return { segment: new Uint8Array(), meta }; + }); + return down!; +} + +/** Packets and the final granule position from an Ogg Opus file (RFC 3533). */ +function oggPackets(file: Uint8Array) { + const view = new DataView(file.buffer, file.byteOffset, file.byteLength); + const packets: Uint8Array[] = []; + let partial: number[] = []; + let granule = 0; + for (let at = 0; at < file.length; ) { + granule = Number(view.getBigUint64(at + 6, true)); + const count = file[at + 26]; + let data = at + 27 + count; + for (let i = 0; i < count; i++) { + const size = file[at + 27 + i]; + partial.push(...file.subarray(data, data + size)); + data += size; + if (size < 255) { + packets.push(Uint8Array.from(partial)); + partial = []; + } + } + at = data; + } + return { head: packets[0], audio: packets.slice(2), granule }; +} + +/** The library's own Ogg encoder, with opus() defaults, as the reference. */ +async function reference( + pcm: Int16Array, + sampleRate: number, + channels: number, +) { + const planar = Array.from({ length: channels }, (_, c) => + Float32Array.from( + { length: pcm.length / channels }, + (_, i) => pcm[i * channels + c] / 32768, + ), + ); + const encoder = await oggOpus({ + sampleRate, + channels, + bitrate: 64, + complexity: 5, + application: "audio", + }); + try { + const parts = [encoder.encode(planar), encoder.flush()]; + const file = new Uint8Array(parts[0].length + parts[1].length); + file.set(parts[0]); + file.set(parts[1], parts[0].length); + return oggPackets(file); + } finally { + encoder.free(); + } +} + +test.each([ + { frames: 1, sampleRate: 24000, channels: 2 }, + { frames: 3, sampleRate: 24000, channels: 2 }, + { frames: 24001, sampleRate: 24000, channels: 2 }, + { frames: 72123, sampleRate: 24000, channels: 2 }, + { frames: 24000, sampleRate: 24000, channels: 1 }, + { frames: 48017, sampleRate: 48000, channels: 2 }, +])( + "packets match the reference encoder for $frames frames of $channels-channel $sampleRate Hz", + async ({ frames, sampleRate, channels }) => { + const pcm = tones(frames, sampleRate, channels); + const { segment } = await encode(pcm, meta(sampleRate, channels)); + const expected = await reference(pcm, sampleRate, channels); + expect(segment.codec).toBe("opus"); + expect(segment.headers).toEqual([expected.head]); + expect(segment.packets.map((p) => p.data)).toEqual(expected.audio); + // Decoded length after trimming the delay and padding is the input's, at 48 kHz. + const decoded = segment.packets.reduce((sum, p) => sum + p.samples, 0); + expect(decoded - segment.delay - segment.padding).toBe( + (frames * 48000) / sampleRate, + ); + expect(decoded - segment.padding).toBe(expected.granule); + }, +); + +test("PCM at an odd byte offset encodes like aligned PCM", async () => { + const pcm = tones(4800, 24000, 2); + const shifted = new Uint8Array(pcm.byteLength + 1); + shifted.set(bytes(pcm), 1); + let down: EncodedPackets | undefined; + await opus()(shifted.subarray(1), meta(24000, 2), async (segment, m) => { + down = segment; + return { segment: new Uint8Array(), meta: m }; + }); + const aligned = await encode(pcm, meta(24000, 2)); + expect(down!.packets).toEqual(aligned.segment.packets); +}); + +test("metadata describes Opus at 48 kHz on the same timeline and channels", async () => { + const input = meta(24000, 2); + const { meta: output } = await encode(tones(2400, 24000, 2), input); + // The encoder's settings, defaults included, replace the PCM details. + expect(output).toEqual({ + ...input, + sampleRate: 48000, + mimeType: "audio/opus", + encoding: "opus", + attributes: { + "acme.call_id": "c-1", + "opus.bitrate_bps": 64000, + "opus.complexity": 5, + "opus.application": "audio", + }, + }); +}); + +test("attributes report configured settings", async () => { + const { meta: output } = await encode(tones(480, 24000, 1), meta(24000, 1), { + bitrate: 32_500, + complexity: 9, + application: "voip", + }); + expect(output.attributes).toMatchObject({ + "opus.bitrate_bps": 32500, + "opus.complexity": 9, + "opus.application": "voip", + }); +}); + +test.each([ + [{ bitrate: 0 }, "bitrate"], + [{ bitrate: Number.NaN }, "bitrate"], + [{ bitrate: 499 }, "bitrate"], + [{ bitrate: 512_001 }, "bitrate"], + [{ bitrate: 32_000.5 }, "bitrate"], + [{ bitrate: "32000" }, "bitrate"], + [{ complexity: 2.5 }, "complexity"], + [{ complexity: 11 }, "complexity"], + [{ complexity: -1 }, "complexity"], + [{ application: "music" }, "application"], +])("rejects option %j", (options, option) => { + expect(() => opus(options as OpusOptions)).toThrow(`Opus ${option} must`); +}); + +test.each([ + [{ bitrate: 500 }], + [{ bitrate: 512_000 }], + [{ complexity: 0 }], + [{ complexity: 10 }], + [{ bitrate: undefined }], +])("accepts option %j", (options) => { + expect(() => opus(options)).not.toThrow(); +}); + +test("options change the encoding", async () => { + const pcm = tones(48000, 24000, 2); + const size = async (options: OpusOptions) => + (await encode(pcm, meta(24000, 2), options)).segment.packets.reduce( + (sum, p) => sum + p.data.length, + 0, + ); + expect(await size({ bitrate: 16_000 })).toBeLessThan(await size({})); +}); + +test.each([ + ["non-PCM input", { ...meta(24000, 2), mimeType: "audio/ogg" }], + [ + "another sample format", + { ...meta(24000, 2), attributes: { "pcm.sample_format": "f32le" } }, + ], + ["more than two channels", meta(24000, 3)], + ["an unsupported sample rate", meta(16000, 2)], +])("rejects %s", async (_name, input) => { + await expect( + opus()(bytes(tones(160, 16000, 3)), input, () => { + throw new Error("next must not run"); + }), + ).rejects.toThrow(/opus/); +}); + +test("long segments yield to the event loop while encoding", async () => { + await encode(tones(240, 24000, 2), meta(24000, 2)); // Load the codec first. + let ticks = 0; + let timer: NodeJS.Immediate; + const tick = () => { + ticks++; + timer = setImmediate(tick); + }; + timer = setImmediate(tick); + try { + await encode(tones(5 * 24000, 24000, 2), meta(24000, 2)); + } finally { + clearImmediate(timer); + } + // Five seconds of audio yields after each of its first four seconds. + expect(ticks).toBeGreaterThanOrEqual(4); +}); + +test("concurrent segments encode independently", async () => { + const a = tones(36000, 24000, 2); + const b = tones(30000, 24000, 2).map((v) => -v); + const sequential = [ + await encode(a, meta(24000, 2)), + await encode(b, meta(24000, 2)), + ]; + const concurrent = await Promise.all([ + encode(a, meta(24000, 2)), + encode(b, meta(24000, 2)), + ]); + expect(concurrent).toEqual(sequential); +}); diff --git a/integrations/audio/src/opus.ts b/integrations/audio/src/opus.ts new file mode 100644 index 000000000..64199b214 --- /dev/null +++ b/integrations/audio/src/opus.ts @@ -0,0 +1,228 @@ +import type { AudioMiddleware } from "./types"; +import type { EncodedPacket, EncodedPackets } from "./packets"; + +export interface OpusOptions { + /** Target bitrate in bits per second, from 500 to 512000. Defaults to 64000. */ + bitrate?: number; + /** Encoder effort from 0 (fastest) to 10. Defaults to 5. */ + complexity?: number; + /** Encoder tuning. Defaults to `"audio"`; `"voip"` favors speech. */ + application?: "audio" | "voip" | "lowdelay"; +} + +const OPUS_RATE = 48000; +const APPLICATIONS = ["audio", "voip", "lowdelay"] as const; +/** Frames (20 ms each) encoded between yields to the event loop: 1 s of audio. */ +const FRAMES_PER_YIELD = 50; + +type Codec = typeof import("@audio/encode-opus/core"); +let codec: Promise | undefined; + +/** + * Encodes raw PCM segments into Opus packets for a container such as `ogg()`. + * Accepts 1 or 2 channels of interleaved s16le PCM at 24 or 48 kHz. Encoding + * runs in 20 ms frames and yields to the event loop after each second of audio. + * Throws if an option is invalid. + */ +export function opus( + options: OpusOptions = {}, +): AudioMiddleware { + const settings = validate({ + bitrate: options.bitrate ?? 64_000, + complexity: options.complexity ?? 5, + application: options.application ?? "audio", + }); + return async function opus(segment, meta, next) { + if ( + meta.mimeType !== "audio/pcm" || + meta.encoding !== "pcm" || + meta.attributes?.["pcm.sample_format"] !== "s16le" || + meta.attributes?.["pcm.channel_layout"] !== "interleaved" + ) + throw new Error("opus requires interleaved s16le PCM input"); + const channels = meta.channels.length; + if (channels !== 1 && channels !== 2) + throw new Error("opus supports 1 or 2 channels"); + const { createOpusEncoder, toOpusRate, FRAME } = await (codec ??= + import("@audio/encode-opus/core")); + const encoder = await createOpusEncoder({ + channels, + ...settings, + // The codec takes kbps. + bitrate: settings.bitrate / 1000, + }); + const packets: EncodedPacket[] = []; + let delay: number; + let padding: number; + try { + delay = encoder.lookahead; + const frame = new Float32Array(FRAME * channels); + let filled = 0; + let total = 0; + const encode = () => { + frame.fill(0, filled); + packets.push({ data: encoder.encode(frame), samples: FRAME }); + filled = 0; + }; + // Samples are 16-bit, so an odd byte offset needs an aligned copy. + const bytes = segment.byteOffset % 2 ? segment.slice() : segment; + const pcm = new Int16Array( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength / 2, + ); + for (const block of planarBlocks( + chunks(pcm, meta.sampleRate, channels), + meta.sampleRate, + channels, + toOpusRate, + )) { + total += block[0].length; + for (let i = 0; i < block[0].length; i++) { + for (let c = 0; c < channels; c++) frame[filled++] = block[c][i]; + if (filled === frame.length) { + encode(); + if (packets.length % FRAMES_PER_YIELD === 0) + await yieldToEventLoop(); + } + } + } + // Push the encoder delay out with silence, then trim it when decoding. + while (packets.length * FRAME < total + delay) encode(); + padding = packets.length * FRAME - total - delay; + } finally { + encoder.free(); + } + return next( + { + codec: "opus", + headers: [opusHead(channels, delay, meta.sampleRate)], + packets, + delay, + padding, + }, + { + ...meta, + sampleRate: OPUS_RATE, + mimeType: "audio/opus", + encoding: "opus", + attributes: { + // PCM details no longer describe the file; other attributes do. + ...Object.fromEntries( + Object.entries(meta.attributes ?? {}).filter( + ([key]) => !key.startsWith("pcm."), + ), + ), + "opus.bitrate_bps": settings.bitrate, + "opus.complexity": settings.complexity, + "opus.application": settings.application, + }, + }, + ); + }; +} + +function validate(settings: Required): Required { + const { bitrate, complexity, application } = settings; + if (!Number.isInteger(bitrate) || bitrate < 500 || bitrate > 512_000) + throw new Error( + "Opus bitrate must be a whole number of bits per second from 500 to 512000", + ); + if (!Number.isInteger(complexity) || complexity < 0 || complexity > 10) + throw new Error("Opus complexity must be an integer from 0 to 10"); + if (!APPLICATIONS.includes(application)) + throw new Error( + `Opus application must be one of ${APPLICATIONS.map((a) => `"${a}"`).join(", ")}`, + ); + return settings; +} + +/** Loads and compiles the codec ahead of the first segment. */ +export function warmOpus(): Promise { + return (codec ??= import("@audio/encode-opus/core")) + .then(({ createOpusEncoder }) => createOpusEncoder()) + .then((encoder) => encoder.free()); +} + +const yieldToEventLoop = () => + new Promise((resolve) => setImmediate(resolve)); + +/** 100 ms blocks of interleaved PCM. */ +function* chunks(pcm: Int16Array, sampleRate: number, channels: number) { + const size = (sampleRate / 10) * channels; + for (let at = 0; at < pcm.length; at += size) + yield pcm.subarray(at, at + size); +} + +/** + * Resamples interleaved PCM blocks to planar 48 kHz, preserving the codec's + * Lanczos filter across blocks with three samples of history and lookahead. + * Only the start and end of the segment clamp the filter. + */ +function* planarBlocks( + chunks: Iterable, + sampleRate: number, + channels: number, + toOpusRate: Codec["toOpusRate"], +) { + if (sampleRate !== 24000 && sampleRate !== OPUS_RATE) + throw new Error("opus supports 24 or 48 kHz input"); + const tail = Array.from({ length: channels }, () => new Float32Array(6)); + let retained = 3; + let initialized = false; + let input: Float32Array[] = []; + let output: Float32Array[] = []; + function convert(length: number, end: number) { + const interleaved = toOpusRate( + input.map((c) => c.subarray(0, length)), + sampleRate, + ); + const ratio = OPUS_RATE / sampleRate; + const count = (end - 3) * ratio; + if (!output.length || output[0].length < count) + output = Array.from({ length: channels }, () => new Float32Array(count)); + for (let i = 0; i < count; i++) + for (let c = 0; c < channels; c++) + output[c][i] = interleaved[(i + 3 * ratio) * channels + c]; + return output.map((c) => c.subarray(0, count)); + } + for (const pcm of chunks) { + if (!pcm.length) continue; + const length = retained + pcm.length / channels; + if (!input.length || input[0].length < length) + input = Array.from({ length: channels }, () => new Float32Array(length)); + if (!initialized) { + for (let c = 0; c < channels; c++) tail[c].fill(pcm[c] / 32768, 0, 3); + initialized = true; + } + for (let c = 0; c < channels; c++) { + input[c].set(tail[c].subarray(0, retained)); + for (let i = 0; i < pcm.length / channels; i++) + input[c][retained + i] = pcm[i * channels + c] / 32768; + } + const end = Math.max(3, length - 3); + if (end > 3) yield convert(length, end); + retained = length - end + 3; + for (let c = 0; c < channels; c++) + tail[c].set(input[c].subarray(end - 3, length)); + } + if (initialized) { + for (let c = 0; c < channels; c++) + input[c].set(tail[c].subarray(0, retained)); + yield convert(retained, retained); + } +} + +/** Opus identification header (RFC 7845 §5.1), channel mapping family 0. */ +function opusHead(channels: number, preSkip: number, inputSampleRate: number) { + const head = new Uint8Array(19); + const view = new DataView(head.buffer); + head.set(new TextEncoder().encode("OpusHead")); + head[8] = 1; // version + head[9] = channels; + view.setUint16(10, preSkip, true); + view.setUint32(12, inputSampleRate, true); + view.setInt16(16, 0, true); // output gain + head[18] = 0; // mapping family + return head; +} diff --git a/integrations/audio/src/packets.ts b/integrations/audio/src/packets.ts new file mode 100644 index 000000000..c6e0e21d8 --- /dev/null +++ b/integrations/audio/src/packets.ts @@ -0,0 +1,21 @@ +/** + * Encoded audio passed from an encoder to a container, such as `opus()` to + * `ogg()`. Sample counts use the codec clock (`meta.sampleRate`). + */ +export interface EncodedPackets { + /** Codec name, such as `"opus"`. */ + readonly codec: string; + /** Codec setup data a container stores ahead of the packets, such as OpusHead. */ + readonly headers: readonly Uint8Array[]; + readonly packets: readonly EncodedPacket[]; + /** Leading decoded samples to discard: the encoder delay (Ogg pre-skip). */ + readonly delay: number; + /** Trailing decoded samples to discard after the last packet. */ + readonly padding: number; +} + +export interface EncodedPacket { + readonly data: Uint8Array; + /** Decoded samples per channel in this packet. */ + readonly samples: number; +} diff --git a/integrations/audio/src/stack.test.ts b/integrations/audio/src/stack.test.ts new file mode 100644 index 000000000..8d7ab6496 --- /dev/null +++ b/integrations/audio/src/stack.test.ts @@ -0,0 +1,98 @@ +import { afterEach, expect, test, vi } from "vitest"; +import { createStack } from "./stack"; +import type { AudioMiddleware, AudioSegmentMeta } from "./types"; + +const meta: AudioSegmentMeta = { + index: 0, + startOffsetMs: 0, + durationMs: 100, + sampleRate: 24000, + channels: [ + { sources: ["session_input"] }, + { sources: ["local_audio_output"] }, + ], + mimeType: "audio/pcm", + encoding: "pcm", + attributes: { + "pcm.sample_format": "s16le", + "pcm.channel_layout": "interleaved", + }, +}; + +/** Runs one segment through the stack, as the SDK does. */ +async function run(stack: AudioMiddleware[], segment: Uint8Array) { + const next = (layer: number) => async (s: unknown, m: AudioSegmentMeta) => + layer === stack.length + ? { segment: s as Uint8Array, meta: m } + : stack[layer](s, m, next(layer + 1)); + return next(0)(segment, meta); +} + +afterEach(() => { + delete (globalThis as Record)[ + Symbol.for("braintrust.audio") + ]; + vi.resetModules(); +}); + +test("opus in ogg encodes a PCM segment into an Ogg Opus file", async () => { + const stack = createStack({ + codec: { name: "opus", options: { bitrate: 32_000 } }, + container: { name: "ogg" }, + }); + expect(stack.map((middleware) => middleware.name)).toEqual(["opus", "ogg"]); + const result = await run(stack, new Uint8Array(2400 * 2 * 2)); + expect(new TextDecoder().decode(result.segment.subarray(0, 4))).toBe("OggS"); + expect(result.meta).toMatchObject({ + mimeType: "audio/ogg", + encoding: "opus", + }); +}); + +test("each recording gets its own middleware", () => { + const config = { codec: { name: "opus" }, container: { name: "ogg" } }; + const [first, second] = [createStack(config), createStack(config)]; + expect(first[0]).not.toBe(second[0]); + expect(first[1]).not.toBe(second[1]); +}); + +test.each([ + [ + { codec: { name: "mp3" }, container: { name: "ogg" } }, + 'Unknown audio codec "mp3" (supported: "opus")', + ], + [ + { codec: { name: "opus" }, container: { name: "webm" } }, + 'Unknown audio container "webm" (supported: "ogg")', + ], + [{ codec: { name: "opus" } }, 'Missing audio container (supported: "ogg")'], + [ + { codec: { name: "toString" }, container: { name: "ogg" } }, + 'Unknown audio codec "toString"', + ], +])("rejects %j", (config, message) => { + expect(() => createStack(config)).toThrow(message); +}); + +test("invalid codec options are rejected by the codec", () => { + expect(() => + createStack({ + codec: { name: "opus", options: { bitrate: 0 } }, + container: { name: "ogg" }, + }), + ).toThrow("Opus bitrate must"); +}); + +test("importing the package registers it for the SDK; the first copy wins", async () => { + const key = Symbol.for("braintrust.audio"); + const registry = globalThis as Record; + await import("./index"); + expect(registry[key]).toEqual({ + version: 1, + createStack: expect.any(Function), + }); + const first = registry[key]; + vi.resetModules(); + await import("./index"); + expect(registry[key]).toBe(first); +}); diff --git a/integrations/audio/src/stack.ts b/integrations/audio/src/stack.ts new file mode 100644 index 000000000..e236d4124 --- /dev/null +++ b/integrations/audio/src/stack.ts @@ -0,0 +1,54 @@ +import { ogg } from "./ogg"; +import { opus, warmOpus, type OpusOptions } from "./opus"; +import type { AudioMiddleware } from "./types"; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Each layer has its own segment types. +type AnyAudioMiddleware = AudioMiddleware; + +/** What the SDK asks for, after applying the integration's defaults. */ +interface AudioConfig { + codec: { name: string; options?: Record }; + container?: { name: string; options?: Record }; +} + +const CODECS: Record< + string, + (options: Record) => AnyAudioMiddleware +> = { + // Options are validated by the codec. + opus: (options) => opus(options as OpusOptions), +}; +const CONTAINERS: Record AnyAudioMiddleware> = { + ogg: () => ogg(), +}; + +let warmed = false; + +/** + * Builds the middleware for one recording: the codec, then its container. + * Throws with an explanation when the config names something unsupported. + */ +export function createStack(config: AudioConfig): AnyAudioMiddleware[] { + const codec = lookup(CODECS, "codec", config.codec?.name); + const container = lookup(CONTAINERS, "container", config.container?.name); + const stack = [codec(config.codec.options ?? {}), container()]; + // Compile the codec now so the first segment does not. + if (!warmed) { + warmed = true; + warmOpus().catch(() => {}); + } + return stack; +} + +function lookup(table: Record, kind: string, name: unknown): T { + if (typeof name === "string" && Object.hasOwn(table, name)) + return table[name]; + const supported = Object.keys(table) + .map((key) => `"${key}"`) + .join(", "); + throw new Error( + name === undefined + ? `Missing audio ${kind} (supported: ${supported})` + : `Unknown audio ${kind} ${JSON.stringify(name)} (supported: ${supported})`, + ); +} diff --git a/integrations/audio/src/types.ts b/integrations/audio/src/types.ts new file mode 100644 index 000000000..9faf309f8 --- /dev/null +++ b/integrations/audio/src/types.ts @@ -0,0 +1,28 @@ +/** + * The SDK's internal audio middleware contract, which this package implements. + * Keep in sync with `js/src/audio-middleware.ts`; the registry `version` gates + * compatibility at runtime. + */ +export interface AudioSegmentMeta { + readonly index: number; + readonly startOffsetMs: number; + readonly durationMs: number; + readonly sampleRate: number; + /** Every output channel in order, with the sources heard in it. */ + readonly channels: readonly { readonly sources: readonly string[] }[]; + /** Media or container type of the segment bytes, such as `audio/pcm`. */ + readonly mimeType: string; + /** Audio codec, such as `pcm` or `opus`. */ + readonly encoding: string; + readonly attributes?: Readonly>; + readonly state?: Readonly>; +} + +type AudioResult = { segment: Uint8Array; meta: AudioSegmentMeta }; + +/** Transforms a segment, Rack/Koa style; `next` runs the rest of the stack. */ +export type AudioMiddleware = ( + segment: In, + meta: AudioSegmentMeta, + next: (segment: Down, meta: AudioSegmentMeta) => Promise, +) => AudioResult | Promise; diff --git a/integrations/audio/tsconfig.json b/integrations/audio/tsconfig.json new file mode 100644 index 000000000..ea6ad1861 --- /dev/null +++ b/integrations/audio/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "declaration": true, + "lib": ["es2022"], + "module": "commonjs", + "target": "es2022", + "moduleResolution": "node", + "strict": true, + "noUnusedLocals": true, + "esModuleInterop": true, + "skipLibCheck": true + }, + "include": ["."], + "exclude": ["node_modules/**", "**/dist/**"] +} diff --git a/integrations/audio/tsup.config.ts b/integrations/audio/tsup.config.ts new file mode 100644 index 000000000..e49826d4f --- /dev/null +++ b/integrations/audio/tsup.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from "tsup"; + +export default defineConfig([ + { + entry: ["src/index.ts"], + format: ["cjs", "esm"], + outDir: "dist", + dts: true, + clean: true, + // The codec is ESM-only; load it with import() from both builds. + external: ["@audio/encode-opus"], + esbuildOptions(options) { + options.supported = { ...options.supported, "dynamic-import": true }; + }, + }, +]); diff --git a/integrations/audio/turbo.json b/integrations/audio/turbo.json new file mode 100644 index 000000000..69a3e1241 --- /dev/null +++ b/integrations/audio/turbo.json @@ -0,0 +1,12 @@ +{ + "extends": ["//"], + "tasks": { + "build": { + "dependsOn": ["^build"], + "outputs": ["dist/**"] + }, + "test": { + "dependsOn": ["build"] + } + } +} diff --git a/integrations/audio/vitest.config.ts b/integrations/audio/vitest.config.ts new file mode 100644 index 000000000..85dceba75 --- /dev/null +++ b/integrations/audio/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + reporters: ["default"], + include: ["src/**/*.test.ts"], + }, +}); diff --git a/js/src/audio-compression.test.ts b/js/src/audio-compression.test.ts new file mode 100644 index 000000000..26450683b --- /dev/null +++ b/js/src/audio-compression.test.ts @@ -0,0 +1,143 @@ +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { + _exportsForTestingOnly, + audioEncodingStack, + type AudioEncodingDefaults, +} from "./audio-compression"; +import type { AudioMiddleware } from "./audio-middleware"; +import iso from "./isomorph"; +import { configureNode } from "./node/config"; + +configureNode(); + +const REGISTRY = Symbol.for("braintrust.audio"); +const registry = globalThis as Record; +const DEFAULTS: AudioEncodingDefaults = { + codec: { name: "opus" }, + container: { name: "ogg" }, +}; +const PCM = "audio is recorded as uncompressed PCM, which can be very large."; +const tag: AudioMiddleware = (segment, meta, next) => next(segment, meta); + +let warnings: string[]; +let load: ReturnType unknown>>; +const original = iso.loadOptionalModule; + +/** Simulates installing @braintrust/audio: loading it registers createStack. */ +function install(createStack: (config: unknown) => unknown = () => [tag]) { + load.mockImplementation(() => { + registry[REGISTRY] = { version: 1, createStack }; + return {}; + }); + return createStack; +} + +beforeEach(() => { + warnings = []; + vi.spyOn(console, "warn").mockImplementation((...args: unknown[]) => { + warnings.push(args.join(" ")); + }); + load = vi.fn(() => undefined); + iso.loadOptionalModule = load; +}); +afterEach(() => { + delete registry[REGISTRY]; + iso.loadOptionalModule = original; + _exportsForTestingOnly.reset(); + vi.restoreAllMocks(); +}); + +test("installed, recordings compress with the integration's defaults", () => { + const createStack = vi.fn(() => [tag]); + install(createStack); + expect(audioEncodingStack(undefined, DEFAULTS)).toEqual([tag]); + expect(load).toHaveBeenCalledWith("@braintrust/audio"); + expect(createStack).toHaveBeenCalledWith(DEFAULTS); + expect(warnings).toEqual([]); +}); + +test("configured settings win over the defaults, field by field", () => { + const createStack = vi.fn(() => [tag]); + install(createStack); + const codec = { name: "opus", options: { bitrate: 32_000 } } as const; + audioEncodingStack({ codec }, DEFAULTS); + expect(createStack).toHaveBeenCalledWith({ + codec, + container: DEFAULTS.container, + }); +}); + +test("the package loads once; each recording gets a new stack", () => { + const createStack = vi.fn(() => [tag]); + install(createStack); + audioEncodingStack(undefined, DEFAULTS); + audioEncodingStack(undefined, DEFAULTS); + expect(load).toHaveBeenCalledTimes(1); + expect(createStack).toHaveBeenCalledTimes(2); +}); + +test("an app that imports the package itself needs no loading", () => { + registry[REGISTRY] = { version: 1, createStack: () => [tag] }; + expect(audioEncodingStack(undefined, DEFAULTS)).toEqual([tag]); + expect(load).not.toHaveBeenCalled(); +}); + +test("codec pcm records raw PCM without loading or warning", () => { + install(); + expect(audioEncodingStack({ codec: { name: "pcm" } }, DEFAULTS)).toEqual([]); + expect(load).not.toHaveBeenCalled(); + expect(warnings).toEqual([]); +}); + +test("not installed: raw PCM, warned once per process", () => { + expect(audioEncodingStack(undefined, DEFAULTS)).toEqual([]); + expect(audioEncodingStack(undefined, DEFAULTS)).toEqual([]); + expect(load).toHaveBeenCalledTimes(1); + expect(warnings).toEqual([ + expect.stringContaining( + `@braintrust/audio is not installed: ${PCM} Install @braintrust/audio to compress recordings.`, + ), + ]); +}); + +test("fails to load: raw PCM with the reason", () => { + load.mockImplementation(() => { + throw new Error("Cannot find module '@audio/encode-opus'"); + }); + expect(audioEncodingStack(undefined, DEFAULTS)).toEqual([]); + expect(warnings).toEqual([ + expect.stringContaining( + `@braintrust/audio failed to load (Cannot find module '@audio/encode-opus'): ${PCM} Reinstall @braintrust/audio to compress recordings.`, + ), + ]); +}); + +test.each([ + ["a newer contract", { version: 2, createStack: () => [tag] }], + ["no factory", { version: 1 }], +])("incompatible package (%s): raw PCM", (_name, registered) => { + load.mockImplementation(() => { + registry[REGISTRY] = registered; + return {}; + }); + expect(audioEncodingStack(undefined, DEFAULTS)).toEqual([]); + expect(warnings).toEqual([ + expect.stringContaining( + `@braintrust/audio is incompatible with this version of braintrust: ${PCM} Install matching versions of braintrust and @braintrust/audio to compress recordings.`, + ), + ]); +}); + +test("settings the package rejects: raw PCM with its explanation, warned once", () => { + install(() => { + throw new Error('Unknown audio codec "mp3" (supported: "opus")'); + }); + const configured = { codec: { name: "mp3" } } as never; + expect(audioEncodingStack(configured, DEFAULTS)).toEqual([]); + expect(audioEncodingStack(configured, DEFAULTS)).toEqual([]); + expect(warnings).toEqual([ + expect.stringContaining( + `Unknown audio codec "mp3" (supported: "opus"): ${PCM}`, + ), + ]); +}); diff --git a/js/src/audio-compression.ts b/js/src/audio-compression.ts new file mode 100644 index 000000000..c4003554b --- /dev/null +++ b/js/src/audio-compression.ts @@ -0,0 +1,98 @@ +import type { AudioMiddlewareStack } from "./audio-middleware"; +import { debugLogger } from "./debug-logger"; +import type { AudioEncodingConfig } from "./instrumentation/config"; +import iso from "./isomorph"; +import { isObject } from "../util"; + +const AUDIO_PACKAGE = "@braintrust/audio"; +/** Set by @braintrust/audio when it loads. */ +const REGISTRY = Symbol.for("braintrust.audio"); +const PCM = "audio is recorded as uncompressed PCM, which can be very large."; + +type AudioCodecs = { + version: 1; + createStack(config: Required): AudioMiddlewareStack; +}; + +/** Each integration's encoding when it is not configured. */ +export type AudioEncodingDefaults = Required; + +/** Loaded once per process; null when unavailable. */ +let codecs: AudioCodecs | null | undefined; +const warned = new Set(); + +/** + * Returns the middleware that encodes one recording, or an empty stack for raw + * PCM. Configured settings win, then the integration's defaults apply. Never + * throws: problems are warned about once per process and the recording stays + * raw PCM. + */ +export function audioEncodingStack( + configured: AudioEncodingConfig | undefined, + defaults: AudioEncodingDefaults, +): AudioMiddlewareStack { + const codec = configured?.codec ?? defaults.codec; + if (codec.name === "pcm") return []; + if (codecs === undefined) codecs = loadCodecs() ?? null; + if (!codecs) return []; + try { + return codecs.createStack({ + codec, + container: configured?.container ?? defaults.container, + }); + } catch (error) { + return warnOnce(`${message(error)}: ${PCM}`); + } +} + +function loadCodecs(): AudioCodecs | undefined { + if (Reflect.get(globalThis, REGISTRY) === undefined) { + let loaded: unknown; + try { + loaded = iso.loadOptionalModule?.(AUDIO_PACKAGE); + } catch (error) { + warnOnce( + `${AUDIO_PACKAGE} failed to load (${message(error)}): ${PCM} Reinstall ${AUDIO_PACKAGE} to compress recordings.`, + ); + return; + } + if (loaded === undefined) { + warnOnce( + `${AUDIO_PACKAGE} is not installed: ${PCM} Install ${AUDIO_PACKAGE} to compress recordings.`, + ); + return; + } + } + const registered: unknown = Reflect.get(globalThis, REGISTRY); + if (isAudioCodecs(registered)) return registered; + warnOnce( + `${AUDIO_PACKAGE} is incompatible with this version of braintrust: ${PCM} Install matching versions of braintrust and ${AUDIO_PACKAGE} to compress recordings.`, + ); +} + +function isAudioCodecs(value: unknown): value is AudioCodecs { + return ( + isObject(value) && + value.version === 1 && + typeof value.createStack === "function" + ); +} + +function message(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function warnOnce(text: string): [] { + if (!warned.has(text)) { + warned.add(text); + debugLogger.notice.warn(text); + } + return []; +} + +export const _exportsForTestingOnly = { + reset() { + codecs = undefined; + warned.clear(); + }, +}; diff --git a/js/src/audio-recorder.ts b/js/src/audio-recorder.ts index 8bca00922..e40bc2439 100644 --- a/js/src/audio-recorder.ts +++ b/js/src/audio-recorder.ts @@ -28,6 +28,9 @@ const PCM_ATTRIBUTES = { "pcm.channel_layout": "interleaved", }; +/** Encoding failures are reported once per process. */ +let warnedEncodingFailure = false; + /** * Assembles PCM on a channel timeline, cuts bounded segments behind a confirmed * offset, and passes them through middleware to `onFlush`, one at a time. @@ -39,7 +42,6 @@ export class AudioRecorder { private index = 0; private job?: Promise; private finishing?: Promise; - private warned = false; private stopReason?: string; private isClosed = false; @@ -203,12 +205,12 @@ export class AudioRecorder { meta, ); } catch (error) { - if (!this.warned) + if (!warnedEncodingFailure) { + warnedEncodingFailure = true; debugLogger.notice.warn( - "Audio middleware failed; the recording omits this segment and stops.", - error, + `Audio encoding failed (${error instanceof Error ? error.message : String(error)}): the rest of this recording is not uploaded.`, ); - this.warned = true; + } return this.fail("recording_transform_failed", error, meta); } try { diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts index 188712d80..b223e34e6 100644 --- a/js/src/instrumentation/braintrust-plugin.ts +++ b/js/src/instrumentation/braintrust-plugin.ts @@ -216,6 +216,9 @@ export class BraintrustPlugin extends BasePlugin { if (integrations.livekit !== false) { this.liveKitPlugin = new LiveKitPlugin( this.config.streamingAttachmentBufferBytes, + typeof integrations.livekit === "object" + ? integrations.livekit?.audio + : undefined, ); this.liveKitPlugin.enable(); } diff --git a/js/src/instrumentation/config.ts b/js/src/instrumentation/config.ts index 1cf7f09a8..ad544b7a9 100644 --- a/js/src/instrumentation/config.ts +++ b/js/src/instrumentation/config.ts @@ -63,7 +63,49 @@ export interface InstrumentationIntegrationsConfig { typesafe?: boolean; workersAI?: boolean; elevenlabs?: boolean; - livekit?: boolean; + /** Set to false to disable, or pass options; options keep LiveKit enabled. */ + livekit?: boolean | LiveKitIntegrationConfig; +} + +export interface LiveKitIntegrationConfig { + /** + * How recorded call audio is encoded. Recording itself still requires + * BRAINTRUST_CAPTURE_USER_AUDIO_ATTACHMENTS or + * BRAINTRUST_CAPTURE_AGENT_AUDIO_ATTACHMENTS. + * + * Defaults to Opus in Ogg when `@braintrust/audio` is installed, and raw + * PCM otherwise. Set `codec: { name: "pcm" }` to always record raw PCM. + * + * @example + * ```ts + * configureInstrumentation({ + * integrations: { + * livekit: { + * audio: { codec: { name: "opus", options: { bitrate: 32_000 } } }, + * }, + * }, + * }); + * ``` + */ + audio?: AudioEncodingConfig; +} + +export interface AudioEncodingConfig { + codec?: + | { name: "pcm" } + | { + name: "opus"; + options?: { + /** Target bitrate in whole bits per second, from 500 to 512000. Defaults to 64000. */ + bitrate?: number; + /** Encoder effort from 0 (fastest) to 10. Defaults to 5. */ + complexity?: number; + /** Encoder tuning. Defaults to `"audio"`; `"voip"` favors speech. */ + application?: "audio" | "voip" | "lowdelay"; + }; + }; + /** The file format for encoded audio. Defaults to `ogg` for Opus. */ + container?: { name: "ogg" }; } export const DEFAULT_STREAMING_ATTACHMENT_BUFFER_BYTES = 4 * 1024 * 1024; @@ -74,6 +116,8 @@ export interface InstrumentationConfig { * Defaults to 4 MiB. Must be a positive safe integer. * Instrumentations flush at valid media boundaries and on recording completion. * Counts binary bytes after decoding and format conversion, excluding metadata. + * Audio recordings count raw PCM before compression, so compressed uploads + * are smaller than this size. * Whole-file responses remain intact and may exceed this threshold. * This is not a total memory limit and does not enable attachment capture. * Configure before initializing the SDK or enabling instrumentation. diff --git a/js/src/instrumentation/plugins/livekit-plugin.ts b/js/src/instrumentation/plugins/livekit-plugin.ts index 63772704c..b0369b93e 100644 --- a/js/src/instrumentation/plugins/livekit-plugin.ts +++ b/js/src/instrumentation/plugins/livekit-plugin.ts @@ -31,9 +31,16 @@ import { isObject } from "../../../util"; import { zodToJsonSchema } from "../../zod/utils"; import { debugLogger } from "../../debug-logger"; import { isEnvFlagEnabled } from "../../wrappers/attachment-utils"; -import { DEFAULT_STREAMING_ATTACHMENT_BUFFER_BYTES } from "../config"; +import { + DEFAULT_STREAMING_ATTACHMENT_BUFFER_BYTES, + type AudioEncodingConfig, +} from "../config"; import { pcmBytesToMs, msToSamples, samplesToMs } from "../../pcm-buffer"; import { AudioRecorder } from "../../audio-recorder"; +import { + audioEncodingStack, + type AudioEncodingDefaults, +} from "../../audio-compression"; import type { AudioMiddlewareStack, AudioSegmentMeta, @@ -52,6 +59,11 @@ const FINAL_TRANSCRIPT = 2; // SpeechEventType.FINAL_TRANSCRIPT // than the shared BRAINTRUST_CAPTURE_ATTACHMENTS, which also enables other media. const CAPTURE_USER_AUDIO = "BRAINTRUST_CAPTURE_USER_AUDIO_ATTACHMENTS"; const CAPTURE_AGENT_AUDIO = "BRAINTRUST_CAPTURE_AGENT_AUDIO_ATTACHMENTS"; +/** Speech compresses well as Opus, and Ogg is its standard file format. */ +const AUDIO_DEFAULTS: AudioEncodingDefaults = { + codec: { name: "opus" }, + container: { name: "ogg" }, +}; // Native spans with a Braintrust name. Other session spans export as `livekit.`. const SPANS = new Map([ @@ -78,6 +90,7 @@ export class LiveKitPlugin extends BasePlugin { private tracing?: LiveKitTracing; constructor( private streamingAttachmentBufferBytes = DEFAULT_STREAMING_ATTACHMENT_BUFFER_BYTES, + private audio?: AudioEncodingConfig, ) { super(); } @@ -85,6 +98,7 @@ export class LiveKitPlugin extends BasePlugin { protected onEnable(): void { const tracing = (this.tracing = new LiveKitTracing( this.streamingAttachmentBufferBytes, + () => audioEncodingStack(this.audio, AUDIO_DEFAULTS), )); const observeActive = (fn: (span: LiveKitSpan) => T, options: TraceOptions) => diff --git a/js/src/instrumentation/registry.test.ts b/js/src/instrumentation/registry.test.ts index 27324857f..7ac402482 100644 --- a/js/src/instrumentation/registry.test.ts +++ b/js/src/instrumentation/registry.test.ts @@ -202,6 +202,26 @@ describe("Plugin Registry", () => { }, ); + it("LiveKit options keep it enabled and pass audio settings across bundles", () => { + const bootstrap = new (registry.constructor as any)(); + const sdk = new (registry.constructor as any)(); + const audio = { codec: { name: "pcm" } }; + const received: unknown[] = []; + const enable = vi + .spyOn(LiveKitPlugin.prototype, "enable") + .mockImplementation(function (this: LiveKitPlugin) { + received.push(Reflect.get(this, "audio")); + }); + try { + bootstrap.configure({ integrations: { livekit: { audio } } }); + sdk.enable(); + expect(received).toEqual([audio]); + } finally { + sdk.disable(); + enable.mockRestore(); + } + }); + it("should allow configuration before enable", () => { const testRegistry = new (registry.constructor as any)(); const warnSpy = vi @@ -297,6 +317,26 @@ describe("Environment Variable Configuration", () => { testRegistry.disable(); }); + it("BRAINTRUST_DISABLE_INSTRUMENTATION disables LiveKit configured with options", async () => { + const iso = (await import("../isomorph")).default; + iso.getEnv = (name: string) => + name === "BRAINTRUST_DISABLE_INSTRUMENTATION" + ? "livekit" + : originalGetEnv(name); + const enable = vi.spyOn(LiveKitPlugin.prototype, "enable"); + const testRegistry = new (registry.constructor as any)(); + try { + testRegistry.configure({ + integrations: { livekit: { audio: { codec: { name: "pcm" } } } }, + }); + testRegistry.enable(); + expect(enable).not.toHaveBeenCalled(); + } finally { + testRegistry.disable(); + enable.mockRestore(); + } + }); + it("should parse BRAINTRUST_DISABLE_INSTRUMENTATION with multiple SDKs", async () => { const iso = (await import("../isomorph")).default; iso.getEnv = (name: string) => { diff --git a/js/src/isomorph.ts b/js/src/isomorph.ts index 0b6cdcde8..b4eb89ff4 100644 --- a/js/src/isomorph.ts +++ b/js/src/isomorph.ts @@ -48,6 +48,10 @@ interface Common { // eslint-disable-next-line @typescript-eslint/no-explicit-any processOn: (event: string, handler: (code: any) => void) => void; + // Loads an installed optional package, or returns undefined if it is not + // installed. Throws if the package is installed but fails to load. + loadOptionalModule?: (name: string) => unknown; + // hash a string. not guaranteed to be crypto safe. hash?: (data: string) => string; diff --git a/js/src/node/apply-auto-instrumentation.ts b/js/src/node/apply-auto-instrumentation.ts index d8bef2c27..f6af91ca2 100644 --- a/js/src/node/apply-auto-instrumentation.ts +++ b/js/src/node/apply-auto-instrumentation.ts @@ -1,5 +1,4 @@ import * as nodeModule from "node:module"; -import { pathToFileURL } from "node:url"; import { getDefaultAutoInstrumentationConfigs } from "../auto-instrumentations/configs/all"; import { patchCommonJS } from "../auto-instrumentations/loader/cjs-patch"; import { @@ -8,6 +7,7 @@ import { syncTransformedSources, } from "../auto-instrumentations/loader/module-hooks.mjs"; import { debugLogger } from "../debug-logger"; +import { getCurrentModuleUrl } from "./module-url"; import { GLOBAL_INSTRUMENTATION_HOOKS_PROTOCOL_VERSION } from "../global-instrumentation-hooks"; interface ApplyAutoInstrumentationState { @@ -95,18 +95,3 @@ export function supportsSyncHooks(): boolean { if (major === 22) return minor > 22 || (minor === 22 && patch >= 3); return false; } - -function getCurrentModuleUrl(): string { - if (typeof __filename !== "undefined") { - return pathToFileURL(__filename).href; - } - - const stack = new Error().stack ?? ""; - const match = - stack.match(/\((file:\/\/[^)]+)\)/) ?? stack.match(/\s(file:\/\/\S+)/); - if (match) { - return match[1].replace(/:\d+:\d+$/, ""); - } - - return pathToFileURL(process.argv[1] ?? process.cwd()).href; -} diff --git a/js/src/node/config.test.ts b/js/src/node/config.test.ts index 97ed25839..157605e72 100644 --- a/js/src/node/config.test.ts +++ b/js/src/node/config.test.ts @@ -148,3 +148,44 @@ describe("configureNode .env.braintrust API key lookup", () => { await expect(iso.getBraintrustApiKey()).resolves.toBe("file-key"); }); }); + +describe("configureNode optional module loading", () => { + let originalCwd: string; + let app: string; + + async function installPackage(name: string, source: string) { + const dir = path.join(app, "node_modules", name); + await mkdir(dir, { recursive: true }); + await writeFile( + path.join(dir, "package.json"), + JSON.stringify({ name, main: "index.js" }), + ); + await writeFile(path.join(dir, "index.js"), source); + } + + beforeEach(async () => { + configureNode(); + originalCwd = process.cwd(); + app = await mkdtemp(path.join(tmpdir(), "bt-optional-")); + process.chdir(app); + }); + + afterEach(async () => { + process.chdir(originalCwd); + await rm(app, { force: true, recursive: true }); + }); + + test("loads a package the app installed", async () => { + await installPackage("@bt-test/installed", "module.exports = 'loaded';"); + expect(iso.loadOptionalModule!("@bt-test/installed")).toBe("loaded"); + }); + + test("returns undefined for a package that is not installed", () => { + expect(iso.loadOptionalModule!("@bt-test/missing")).toBeUndefined(); + }); + + test("throws when an installed package fails to load", async () => { + await installPackage("@bt-test/broken", "throw new Error('broken');"); + expect(() => iso.loadOptionalModule!("@bt-test/broken")).toThrow("broken"); + }); +}); diff --git a/js/src/node/config.ts b/js/src/node/config.ts index 329989c3c..8f340c5ed 100644 --- a/js/src/node/config.ts +++ b/js/src/node/config.ts @@ -1,4 +1,6 @@ import { AsyncLocalStorage } from "node:async_hooks"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; import * as path from "node:path"; import * as fs from "node:fs/promises"; import * as os from "node:os"; @@ -19,6 +21,7 @@ import { } from "../instrumentation/config"; import { installMastraExporterFactory } from "../auto-instrumentations/loader/mastra-observability-patch"; import { BraintrustObservabilityExporter } from "../wrappers/mastra"; +import { getCurrentModuleUrl } from "./module-url"; const BRAINTRUST_ENV_SEARCH_PARENT_LIMIT = 64; @@ -149,6 +152,20 @@ export function configureNode() { iso.openFile = fs.open; iso.gzip = promisify(zlib.gzip); iso.gunzip = promisify(zlib.gunzip); + iso.loadOptionalModule = (name) => { + const url = getCurrentModuleUrl(); + const load = createRequire(url); + let resolved: string; + try { + // Look beside the SDK, then in the app, for linked or global SDK installs. + resolved = load.resolve(name, { + paths: [path.dirname(fileURLToPath(url)), process.cwd()], + }); + } catch { + return undefined; + } + return load(resolved); + }; iso.hash = (data) => crypto.createHash("sha256").update(data).digest("hex"); iso.hmacSha256 = (key, data) => crypto.createHmac("sha256", key).update(data).digest("hex"); diff --git a/js/src/node/module-url.ts b/js/src/node/module-url.ts new file mode 100644 index 000000000..c77ad4634 --- /dev/null +++ b/js/src/node/module-url.ts @@ -0,0 +1,17 @@ +import { pathToFileURL } from "node:url"; + +/** The URL of the built SDK file running this code, in both CJS and ESM builds. */ +export function getCurrentModuleUrl(): string { + if (typeof __filename !== "undefined") { + return pathToFileURL(__filename).href; + } + + const stack = new Error().stack ?? ""; + const match = + stack.match(/\((file:\/\/[^)]+)\)/) ?? stack.match(/\s(file:\/\/\S+)/); + if (match) { + return match[1].replace(/:\d+:\d+$/, ""); + } + + return pathToFileURL(process.argv[1] ?? process.cwd()).href; +} diff --git a/js/src/wrappers/livekit.ts b/js/src/wrappers/livekit.ts index 9c56b977c..4104fbc4c 100644 --- a/js/src/wrappers/livekit.ts +++ b/js/src/wrappers/livekit.ts @@ -75,9 +75,10 @@ function wrapSession(session: AgentSession): AgentSession { * Trace a LiveKit Agents (1.9.x) module with Braintrust. * * Captures cascade voice sessions: user and agent turns, speech, STT, LLM - * responses, TTS and function tools. Call audio is recorded as PCM attachments - * per direction: set BRAINTRUST_CAPTURE_USER_AUDIO_ATTACHMENTS for caller audio - * and BRAINTRUST_CAPTURE_AGENT_AUDIO_ATTACHMENTS for agent audio. Wrap the module before constructing + * responses, TTS and function tools. Call audio is recorded as attachments per + * direction: set BRAINTRUST_CAPTURE_USER_AUDIO_ATTACHMENTS for caller audio and + * BRAINTRUST_CAPTURE_AGENT_AUDIO_ATTACHMENTS for agent audio. Recordings are + * Ogg Opus when @braintrust/audio is installed, and raw PCM otherwise. Wrap the module before constructing * sessions; only sessions created through the wrapped module are traced. * * @example diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 99ba8fdea..8140e03fb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -92,6 +92,9 @@ importers: e2e: devDependencies: + '@braintrust/audio': + specifier: workspace:^ + version: link:../integrations/audio '@braintrust/deepseek-harness': specifier: workspace:^ version: link:../integrations/deepseek-harness @@ -132,6 +135,31 @@ importers: specifier: 4.3.6 version: 4.3.6 + integrations/audio: + dependencies: + '@audio/encode-opus': + specifier: 1.3.1 + version: 1.3.1 + devDependencies: + '@types/node': + specifier: ^20.10.5 + version: 20.19.40 + braintrust: + specifier: workspace:^ + version: link:../../js + ogg-opus-decoder: + specifier: 1.7.5 + version: 1.7.5 + tsup: + specifier: ^8.5.1 + version: 8.5.1(@swc/core@1.15.8)(jiti@2.6.1)(postcss@8.5.26)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.9.0) + typescript: + specifier: ^5.3.3 + version: 5.9.3 + vitest: + specifier: 4.1.11 + version: 4.1.11(@opentelemetry/api@1.9.0)(@types/node@20.19.40)(msw@2.13.6(@types/node@20.19.40)(typescript@5.9.3))(vite@8.1.5(@types/node@20.19.40)(esbuild@0.28.1)(jiti@2.6.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.9.0)) + integrations/browser-js: dependencies: als-browser: @@ -632,6 +660,10 @@ packages: zod: optional: true + '@audio/encode-opus@1.3.1': + resolution: {integrity: sha512-FFNQgim/doIajnOyDidaIx541HI51VoMKWQk0NXFf2XP4bLgduR7kSmwoPXzF5TPCt7axC+Wu6IuTOGyuVWQPg==} + engines: {node: '>=18'} + '@babel/runtime@7.28.4': resolution: {integrity: sha512-Q/N6JNWvIvPnLDvjlE1OUBLPQHH6l3CltCEsHIujp45zQUSSh8K+gHnaEX45yAT1nyngnINhvWtzN+Nb9D8RAQ==} engines: {node: '>=6.9.0'} @@ -1035,6 +1067,9 @@ packages: cpu: [x64] os: [win32] + '@eshaz/web-worker@1.2.2': + resolution: {integrity: sha512-WxXiHFmD9u/owrzempiDlBB1ZYqiLnm9s6aPc8AlFQalq2tKmqdmMr9GXOupDgzXtqnBipj8Un0gkIm7Sjf8mw==} + '@eslint-community/eslint-utils@4.9.1': resolution: {integrity: sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -2148,6 +2183,12 @@ packages: '@vitest/utils@4.1.11': resolution: {integrity: sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==} + '@wasm-audio-decoders/common@9.0.7': + resolution: {integrity: sha512-WRaUuWSKV7pkttBygml/a6dIEpatq2nnZGFIoPTc5yPLkxL6Wk4YaslPM98OPQvWacvNZ+Py9xROGDtrFBDzag==} + + '@wasm-audio-decoders/opus-ml@0.0.3': + resolution: {integrity: sha512-SEeTmk0eb3q5etJA/NHCRWW6BivTx9tqvNTmG0kg2ynuATEQRUZnwQE6Ibvl1BBvqIGHqI57+qyJMWmJBwb5XQ==} + '@webassemblyjs/ast@1.14.1': resolution: {integrity: sha512-nuBEDgQfm1ccRp/8bCQrx1frohyufl4JlbMMZ4P1wpeOfDhF6FQkxZJ1b/e+PLwr6X1Nhw6OLme5usuBWYBvuQ==} @@ -2460,6 +2501,9 @@ packages: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} engines: {node: '>=12'} + codec-parser@2.5.0: + resolution: {integrity: sha512-Ru9t80fV8B0ZiixQl8xhMTLru+dzuis/KQld32/x5T/+3LwZb0/YvQdSKytX9JqCnRdiupvAvyYJINKrXieziQ==} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -3509,6 +3553,9 @@ packages: resolution: {integrity: sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==} engines: {node: '>=12.20.0'} + ogg-opus-decoder@1.7.5: + resolution: {integrity: sha512-08WfRbuBBew1H63PqQ3zj2B/J0Eh9ISfNOTi+Dk6vuXmkBGSx8BiPJGOJ470EYeKezTa7/GhOAwHe3XlXwOZMw==} + on-finished@2.4.1: resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} engines: {node: '>= 0.8'} @@ -3560,6 +3607,9 @@ packages: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} + opus-decoder@0.7.12: + resolution: {integrity: sha512-6zpHFDi/MhYvMwyYiuJPqJ2lcRkfm7wHblSvT3lUtx/kLvHkunoe/rhTn1dLKxqVGLeQj3gQi7CnR0N2p3RuOg==} + outdent@0.5.0: resolution: {integrity: sha512-/jHxFIzoMXdqPzTaCpFzAAWhpkSjZPF4Vsn6jAfNpmbH/ymsmd7Qc6VE9BGn0L6YMj6uwpQLxCECpus4ukKS9Q==} @@ -3904,6 +3954,9 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + simple-yenc@1.0.4: + resolution: {integrity: sha512-5gvxpSd79e9a3V4QDYUqnqxeD4HGlhCakVpb6gMnDD7lexJggSBJRBO5h52y/iJrdXRilX9UCuDaIJhSWm5OWw==} + sisteransi@1.0.5: resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} @@ -4561,6 +4614,8 @@ snapshots: optionalDependencies: zod: 3.25.76 + '@audio/encode-opus@1.3.1': {} + '@babel/runtime@7.28.4': {} '@braintrust/bt-darwin-arm64@0.12.0': @@ -4954,6 +5009,8 @@ snapshots: '@esbuild/win32-x64@0.28.1': optional: true + '@eshaz/web-worker@1.2.2': {} + '@eslint-community/eslint-utils@4.9.1(eslint@9.39.4(jiti@2.6.1))': dependencies: eslint: 9.39.4(jiti@2.6.1) @@ -6129,6 +6186,15 @@ snapshots: convert-source-map: 2.0.0 tinyrainbow: 3.1.1 + '@wasm-audio-decoders/common@9.0.7': + dependencies: + '@eshaz/web-worker': 1.2.2 + simple-yenc: 1.0.4 + + '@wasm-audio-decoders/opus-ml@0.0.3': + dependencies: + '@wasm-audio-decoders/common': 9.0.7 + '@webassemblyjs/ast@1.14.1': dependencies: '@webassemblyjs/helper-numbers': 1.13.2 @@ -6442,6 +6508,8 @@ snapshots: strip-ansi: 6.0.1 wrap-ansi: 7.0.0 + codec-parser@2.5.0: {} + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -7543,6 +7611,13 @@ snapshots: obug@2.2.1: {} + ogg-opus-decoder@1.7.5: + dependencies: + '@wasm-audio-decoders/common': 9.0.7 + '@wasm-audio-decoders/opus-ml': 0.0.3 + codec-parser: 2.5.0 + opus-decoder: 0.7.12 + on-finished@2.4.1: dependencies: ee-first: 1.1.1 @@ -7580,6 +7655,10 @@ snapshots: type-check: 0.4.0 word-wrap: 1.2.5 + opus-decoder@0.7.12: + dependencies: + '@wasm-audio-decoders/common': 9.0.7 + outdent@0.5.0: {} outvariant@1.4.3: {} @@ -7971,6 +8050,8 @@ snapshots: signal-exit@4.1.0: {} + simple-yenc@1.0.4: {} + sisteransi@1.0.5: {} slash@3.0.0: {} diff --git a/scripts/release/_shared.mjs b/scripts/release/_shared.mjs index 2af80e54c..0bb25507a 100644 --- a/scripts/release/_shared.mjs +++ b/scripts/release/_shared.mjs @@ -14,6 +14,7 @@ export const DOCS_HOMEPAGE = "https://www.braintrust.dev/docs"; export const PUBLISHABLE_PACKAGES = [ { dir: "js", name: "braintrust" }, + { dir: "integrations/audio", name: "@braintrust/audio" }, { dir: "integrations/browser-js", name: "@braintrust/browser" }, { dir: "integrations/deepseek-harness",