From 50b73b025a1a9ade03bb380ffa7fe43b2a22acd8 Mon Sep 17 00:00:00 2001 From: Kev Choi Date: Wed, 30 Sep 2026 13:11:50 -0400 Subject: [PATCH 1/3] feat: add speech clarity and turn-taking voice scorers Co-Authored-By: Claude Opus 5.5 --- fixtures/tone.ogg | Bin 0 -> 1813 bytes js/llm.test.ts | 60 +++++++++++++++++ js/llm.ts | 115 ++++++++++++++++++++++++++++++++- js/manifest.ts | 14 ++++ js/templates.ts | 5 ++ package.json | 2 + pnpm-lock.yaml | 93 +++++++++++++++++++++++++++ py/autoevals/llm.py | 110 +++++++++++++++++++++++++++++++- py/autoevals/test_llm.py | 54 +++++++++++++++- pyproject.toml | 1 + templates/speech_clarity.yaml | 12 ++++ templates/turn_taking.yaml | 12 ++++ tsconfig.json | 1 + uv.lock | 117 ++++++++++++++++++++++++++-------- 14 files changed, 565 insertions(+), 31 deletions(-) create mode 100644 fixtures/tone.ogg create mode 100644 templates/speech_clarity.yaml create mode 100644 templates/turn_taking.yaml diff --git a/fixtures/tone.ogg b/fixtures/tone.ogg new file mode 100644 index 0000000000000000000000000000000000000000..bf9025029e2c2a2f228755f0a9a612cdd602378c GIT binary patch literal 1813 zcmeZIPY-5bVt|4>B31Gr=9gt}&oc`97nBxzq$Z{?GFmV;>;^0G2PuaGka9*KuX1K_ zEzAG^|D`}WLK4%9xq*T?nMwITu7aVS30%V{V>ARtUG|!+3a;}% zZ}ynh?%B-wuTv~$fB7+)VwUbx9X=awut@cpt>`-?dFj+GzVnt(K0jU@cKOz`=I^;b zHb?fBGj5EpuW7QEUz6#zlD*w%?wY_dRm#~!!7 zqFdKp#QWAN>|5)xR6H`vRdfDh?jTM!72|w1t}Q;k0nK5xlQg#M_$(cB)O26tqND3i z^W;?A)Dc{!y(n|tkG1s-D^`^jU;H|^{v|ulev{eZD#o#WefyjgoeaIrXIe(EPyW(y zt=ZQ?u_}GOQ=XM^`1=*Ig3f7epRE!$nlN=atYu*R)UH-!c}cTi5l5(#Aj37y)4I$N zv!7J|EC~OlQ)c8~#eT!~BcJ$b=cg;q-?A~g|9fa zvaX2f4>RQ+$1o@?KfZFO;%8^cbqi#47lNI*@pAj>P<0lu<~5=21^fQ2{;(=J(5$Z1 zZ_=m54$N=(pKor=+j8O0GBfe~Cx<5Gw4eX`qOaaN>+4lDt*8%=eD-8sy>?TyiQ}~I zl^bd$H774DnH3}6wa71I(wWB#3{va<{&zQ>YtGP+)X!DRaH>MpScqG5#+it_mc`PX zzk_3Dd+y|@-ZJ;Xt4lJGY8N^#e3CCdy-=Uc?Y5%RyuL&a*V=VI)~vbfbH4uZ?N6sI z1pT)-*Etza-6`TYpY2v#yrtVE6Uo1tnfHozPyS_{oGbFr%8f79*mkGJ9{rx#<(@hh zdbTWnb|511?CRgztm|&rPdBx6x7WKhf8(D!nT&_auHIc^n6gE4f5EAPUQ-o1g3Z|P z9p5@dkn#1i!-r+{UTy8%dN1*J;{wM(?$?+9C(U=y^XN_gl(%PF#Towp34ebIPtsyl gU-zMIo5!kGd*`^FHhZ)|dY|tOqvB~zo0ne#0Ih}U`2YX_ literal 0 HcmV?d00001 diff --git a/js/llm.test.ts b/js/llm.test.ts index 5863eecc..7ff317de 100644 --- a/js/llm.test.ts +++ b/js/llm.test.ts @@ -1,3 +1,4 @@ +import { readFileSync } from "node:fs"; import { bypass, http, HttpResponse } from "msw"; import { setupServer } from "msw/node"; import { OpenAI } from "openai"; @@ -8,6 +9,7 @@ import { buildClassificationTools, LLMClassifierFromTemplate, OpenAIClassifier, + SpeechClarity, templateUsesThreadVariables, } from "../js/llm"; import { @@ -662,3 +664,61 @@ Issue Description: {{page_content}} init({ client }); }); }); + +describe("SpeechClarity", () => { + test("sends OGG audio as MP3 to gemini-3.8-flash through chat completions", async () => { + let body: any; + server.use( + http.post( + "https://api.openai.com/v1/chat/completions", + async ({ request }) => { + body = await request.json(); + return HttpResponse.json({ + choices: [ + { + message: { + role: "assistant", + tool_calls: [ + { + id: "call_test", + type: "function", + function: { + name: "select_choice", + arguments: '{"reasons":"Clear.","choice":"A"}', + }, + }, + ], + }, + }, + ], + }); + }, + ), + ); + + const score = await SpeechClarity({ + input: { + audio: { + data: readFileSync("fixtures/tone.ogg"), + content_type: "audio/ogg", + }, + }, + output: undefined, + openAiApiKey: "test-api-key", + }); + + expect(score.score).toBe(1); + expect(body.model).toBe("gemini-3.8-flash"); + expect(body.messages[0].content[0].type).toBe("text"); + expect(body.messages[0].content[1].input_audio.format).toBe("mp3"); + }); + + test("skips when there is no audio", async () => { + const score = await SpeechClarity({ + input: { text: "hello" }, + output: undefined, + openAiApiKey: "test-api-key", + }); + expect(score.score).toBeNull(); + }); +}); diff --git a/js/llm.ts b/js/llm.ts index 1b7f318e..50b60dcb 100644 --- a/js/llm.ts +++ b/js/llm.ts @@ -7,6 +7,7 @@ import { } from "./oai"; import { ModelGradedSpec, templates } from "./templates"; import { + ChatCompletionContentPartInputAudio, ChatCompletionMessage, ChatCompletionMessageParam, ChatCompletionTool, @@ -62,6 +63,81 @@ function filterSystemMessagesFromThread(thread: unknown[]): unknown[] { }); } +function getPath(args: unknown, path: string): unknown { + let value = args; + for (const key of path.split(".")) { + value = + value && typeof value === "object" && !Array.isArray(value) + ? Reflect.get(value, key) + : undefined; + } + return value; +} + +async function oggToMp3(ogg: Uint8Array): Promise { + const { OggOpusDecoder } = await import("ogg-opus-decoder"); + const { Mp3Encoder } = await import("@breezystack/lamejs"); + const decoder = new OggOpusDecoder(); + await decoder.ready; + try { + const { channelData, samplesDecoded, sampleRate, errors } = + await decoder.decodeFile(ogg); + if (errors.length > 0 || samplesDecoded === 0) { + throw new Error("Could not decode OGG audio"); + } + const [left, right] = channelData + .slice(0, 2) + .map((samples) => + Int16Array.from(samples, (s) => Math.max(-1, Math.min(1, s)) * 0x7fff), + ); + const encoder = new Mp3Encoder(right ? 2 : 1, sampleRate, 128); + const chunks = [encoder.encodeBuffer(left, right), encoder.flush()]; + return Buffer.concat(chunks.map((c) => Buffer.from(c))); + } finally { + decoder.free(); + } +} + +export type Audio = { data: Uint8Array; content_type: string }; + +const AUDIO_FORMATS: Record = { + "audio/wav": "wav", + "audio/x-wav": "wav", + "audio/mpeg": "mp3", + "audio/mp3": "mp3", + "audio/ogg": "ogg", +}; + +async function audioPart( + audio: unknown, +): Promise { + const data = Reflect.get(Object(audio), "data"); + if (!(data instanceof Uint8Array)) { + throw new TypeError( + "Audio must be an object with `data` bytes and a `content_type`", + ); + } + const contentType = String(Reflect.get(Object(audio), "content_type") ?? "") + .split(";")[0] + .trim() + .toLowerCase(); + const format = AUDIO_FORMATS[contentType]; + if (!format) { + throw new Error( + `Audio must be WAV, MP3, or OGG, got content type "${contentType}"`, + ); + } + const [bytes, sentFormat] = + format === "ogg" ? [await oggToMp3(data), "mp3" as const] : [data, format]; + return { + type: "input_audio", + input_audio: { + data: Buffer.from(bytes).toString("base64"), + format: sentFormat, + }, + }; +} + const NO_COT_SUFFIX = "Answer the question by calling `select_choice` with a single choice from {{__choices}}."; @@ -142,6 +218,7 @@ export type OpenAIClassifierArgs = { messages: ChatCompletionMessageParam[]; choiceScores: Record; classificationTools: ChatCompletionTool[]; + inputAudio?: ChatCompletionContentPartInputAudio; cache?: ChatCache; } & LLMArgs & RenderArgs; @@ -174,6 +251,7 @@ export async function OpenAIClassifier( reasoningEnabled, reasoningBudget, useResponsesApi, + inputAudio, cache, ...remainingRenderArgs } = remaining; @@ -212,6 +290,13 @@ export async function OpenAIClassifier( }; const messages = renderMessages(messagesArg, renderArgs); + if (inputAudio) { + const last = messages[messages.length - 1]; + messages[messages.length - 1] = { + role: "user", + content: [{ type: "text", text: String(last.content) }, inputAudio], + }; + } const resp = await cachedChatCompletion( { @@ -306,6 +391,7 @@ export function LLMClassifierFromTemplate({ reasoningEnabled, reasoningBudget, useResponsesApi, + audio, }: { name: string; promptTemplate: string; @@ -318,11 +404,17 @@ export function LLMClassifierFromTemplate({ reasoningEnabled?: boolean; reasoningBudget?: number; useResponsesApi?: boolean; + audio?: string; }): Scorer> { const choiceStrings = Object.keys(choiceScores); const ret = async ( runtimeArgs: ScorerArgs>, ) => { + const audioValue = audio ? getPath(runtimeArgs, audio) : undefined; + if (audio && audioValue == null) { + return { name, score: null }; + } + const useCoT = runtimeArgs.useCoT ?? useCoTArg ?? true; // Use runtime model > template model > configured default model const model = runtimeArgs.model ?? modelArg ?? getDefaultModel(); @@ -374,6 +466,7 @@ export function LLMClassifierFromTemplate({ // Since the logic is a bit funky for computing this, include // it at the end to prevent overrides useCoT, + inputAudio: audio ? await audioPart(audioValue) : undefined, }; return await OpenAIClassifier(classifierArgs); @@ -398,6 +491,7 @@ export function LLMClassifierFromSpec( useCoT: spec.use_cot, temperature: spec.temperature, maxTokens: spec.max_tokens, + audio: spec.audio, }); } @@ -409,10 +503,10 @@ export function LLMClassifierFromSpecFile( return LLMClassifierFromSpec(name, doc); } -function buildLLMClassifier( +function buildLLMClassifier( name: string, templateName: keyof typeof templates, -): ScorerWithPartial> { +): ScorerWithPartial> { if (!(templateName in templates)) { throw new Error(`Model template ${name} not found`); } @@ -492,3 +586,20 @@ export const Translation = buildLLMClassifier<{ language: string; input: string; }>("Translation", "translation"); + +/** + * Test how clearly an agent speaks, from a recording of the whole conversation (`input.audio`). + */ +export const SpeechClarity = buildLLMClassifier< + { input: { audio?: Audio; [key: string]: unknown } }, + unknown +>("SpeechClarity", "speech_clarity"); + +/** + * Test whether an agent takes turns without talking over or cutting off the caller, + * from a recording of the whole conversation (`input.audio`). + */ +export const TurnTaking = buildLLMClassifier< + { input: { audio?: Audio; [key: string]: unknown } }, + unknown +>("TurnTaking", "turn_taking"); diff --git a/js/manifest.ts b/js/manifest.ts index c21d3703..c6faf632 100644 --- a/js/manifest.ts +++ b/js/manifest.ts @@ -6,9 +6,11 @@ import { Humor, Possible, Security, + SpeechClarity, Sql, Summary, Translation, + TurnTaking, } from "./llm"; import { NumericDiff } from "./number"; import { EmbeddingSimilarity, Levenshtein } from "./string"; @@ -82,6 +84,12 @@ export const Evaluators: { description: "Test whether an output is malicious.", template: templates.security, }, + { + method: SpeechClarity, + description: + "Test how clearly an agent's speech can be understood, from a recording of the whole conversation (`input.audio`).", + template: templates.speech_clarity, + }, { method: Sql, description: @@ -101,6 +109,12 @@ export const Evaluators: { template: templates.translation, requiresExtraParams: true, }, + { + method: TurnTaking, + description: + "Test whether an agent takes turns well, without talking over or cutting off the caller, from a recording of the whole conversation (`input.audio`).", + template: templates.turn_taking, + }, ], }, { diff --git a/js/templates.ts b/js/templates.ts index 69f4637c..e1867094 100644 --- a/js/templates.ts +++ b/js/templates.ts @@ -7,9 +7,11 @@ import factuality from "../templates/factuality.yaml"; import humor from "../templates/humor.yaml"; import possible from "../templates/possible.yaml"; import security from "../templates/security.yaml"; +import speech_clarity from "../templates/speech_clarity.yaml"; import sql from "../templates/sql.yaml"; import summary from "../templates/summary.yaml"; import translation from "../templates/translation.yaml"; +import turn_taking from "../templates/turn_taking.yaml"; export const modelGradedSpecSchema = z.object({ prompt: z.string(), @@ -18,6 +20,7 @@ export const modelGradedSpecSchema = z.object({ use_cot: z.boolean().optional(), temperature: z.number().optional(), max_tokens: z.number().optional(), + audio: z.string().optional(), }); export type ModelGradedSpec = z.infer; @@ -29,9 +32,11 @@ const templateStrings = { humor, possible, security, + speech_clarity, sql, summary, translation, + turn_taking, } as const; // eslint-disable-next-line @typescript-eslint/consistent-type-assertions diff --git a/package.json b/package.json index ce662eae..6e088ebd 100644 --- a/package.json +++ b/package.json @@ -50,12 +50,14 @@ "zod": "^3.25.0 || ^4.0.0" }, "dependencies": { + "@breezystack/lamejs": "1.2.7", "ajv": "^8.17.1", "compute-cosine-similarity": "^1.1.0", "js-levenshtein": "^1.1.6", "js-yaml": "^4.1.0", "linear-sum-assignment": "^1.0.7", "mustache": "^4.2.0", + "ogg-opus-decoder": "1.7.5", "openai": "^6.7.0", "zod-to-json-schema": "3.25.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a42b8cd8..d652d580 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,6 +7,9 @@ settings: importers: .: dependencies: + "@breezystack/lamejs": + specifier: 1.2.7 + version: 1.2.7 ajv: specifier: ^8.17.1 version: 8.17.1 @@ -25,6 +28,9 @@ importers: mustache: specifier: ^4.2.0 version: 4.2.0 + ogg-opus-decoder: + specifier: 1.7.5 + version: 1.7.5 openai: specifier: ^6.7.0 version: 6.15.0(zod@3.25.67) @@ -102,6 +108,12 @@ packages: } engines: { node: ">=18" } + "@breezystack/lamejs@1.2.7": + resolution: + { + integrity: sha512-6wc7ck65ctA75Hq7FYHTtTvGnYs6msgdxiSUICQ+A01nVOWg6rqouZB8IdyteRlfpYYiFovkf67dIeOgWIUzTA==, + } + "@bundled-es-modules/cookie@2.0.1": resolution: { @@ -1000,6 +1012,12 @@ packages: cpu: [x64] os: [win32] + "@eshaz/web-worker@1.2.2": + resolution: + { + integrity: sha512-WxXiHFmD9u/owrzempiDlBB1ZYqiLnm9s6aPc8AlFQalq2tKmqdmMr9GXOupDgzXtqnBipj8Un0gkIm7Sjf8mw==, + } + "@gar/promisify@1.1.3": resolution: { @@ -1240,6 +1258,7 @@ packages: } cpu: [arm] os: [linux] + libc: [glibc] "@rollup/rollup-linux-arm-musleabihf@4.46.4": resolution: @@ -1248,6 +1267,7 @@ packages: } cpu: [arm] os: [linux] + libc: [musl] "@rollup/rollup-linux-arm64-gnu@4.46.4": resolution: @@ -1256,6 +1276,7 @@ packages: } cpu: [arm64] os: [linux] + libc: [glibc] "@rollup/rollup-linux-arm64-musl@4.46.4": resolution: @@ -1264,6 +1285,7 @@ packages: } cpu: [arm64] os: [linux] + libc: [musl] "@rollup/rollup-linux-loongarch64-gnu@4.46.4": resolution: @@ -1272,6 +1294,7 @@ packages: } cpu: [loong64] os: [linux] + libc: [glibc] "@rollup/rollup-linux-ppc64-gnu@4.46.4": resolution: @@ -1280,6 +1303,7 @@ packages: } cpu: [ppc64] os: [linux] + libc: [glibc] "@rollup/rollup-linux-riscv64-gnu@4.46.4": resolution: @@ -1288,6 +1312,7 @@ packages: } cpu: [riscv64] os: [linux] + libc: [glibc] "@rollup/rollup-linux-riscv64-musl@4.46.4": resolution: @@ -1296,6 +1321,7 @@ packages: } cpu: [riscv64] os: [linux] + libc: [musl] "@rollup/rollup-linux-s390x-gnu@4.46.4": resolution: @@ -1304,6 +1330,7 @@ packages: } cpu: [s390x] os: [linux] + libc: [glibc] "@rollup/rollup-linux-x64-gnu@4.46.4": resolution: @@ -1312,6 +1339,7 @@ packages: } cpu: [x64] os: [linux] + libc: [glibc] "@rollup/rollup-linux-x64-musl@4.46.4": resolution: @@ -1320,6 +1348,7 @@ packages: } cpu: [x64] os: [linux] + libc: [musl] "@rollup/rollup-win32-arm64-msvc@4.46.4": resolution: @@ -1474,6 +1503,18 @@ packages: integrity: sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==, } + "@wasm-audio-decoders/common@9.0.7": + resolution: + { + integrity: sha512-WRaUuWSKV7pkttBygml/a6dIEpatq2nnZGFIoPTc5yPLkxL6Wk4YaslPM98OPQvWacvNZ+Py9xROGDtrFBDzag==, + } + + "@wasm-audio-decoders/opus-ml@0.0.3": + resolution: + { + integrity: sha512-SEeTmk0eb3q5etJA/NHCRWW6BivTx9tqvNTmG0kg2ynuATEQRUZnwQE6Ibvl1BBvqIGHqI57+qyJMWmJBwb5XQ==, + } + a-sync-waterfall@1.0.1: resolution: { @@ -1841,6 +1882,12 @@ packages: } engines: { node: ">=12" } + codec-parser@2.5.0: + resolution: + { + integrity: sha512-Ru9t80fV8B0ZiixQl8xhMTLru+dzuis/KQld32/x5T/+3LwZb0/YvQdSKytX9JqCnRdiupvAvyYJINKrXieziQ==, + } + color-convert@2.0.1: resolution: { @@ -3066,6 +3113,12 @@ packages: } engines: { node: ">= 0.4" } + ogg-opus-decoder@1.7.5: + resolution: + { + integrity: sha512-08WfRbuBBew1H63PqQ3zj2B/J0Eh9ISfNOTi+Dk6vuXmkBGSx8BiPJGOJ470EYeKezTa7/GhOAwHe3XlXwOZMw==, + } + on-finished@2.4.1: resolution: { @@ -3094,6 +3147,12 @@ packages: zod: optional: true + opus-decoder@0.7.12: + resolution: + { + integrity: sha512-6zpHFDi/MhYvMwyYiuJPqJ2lcRkfm7wHblSvT3lUtx/kLvHkunoe/rhTn1dLKxqVGLeQj3gQi7CnR0N2p3RuOg==, + } + outvariant@1.4.3: resolution: { @@ -3494,6 +3553,12 @@ packages: integrity: sha512-ivHoFS9Yi9GY49ogc6/YAi3Fl9ROnF4VyubNylgCkA+RVqLaKWnDSzXOVzya8csELIaWaYNutsEuAhZrtOjozA==, } + simple-yenc@1.0.4: + resolution: + { + integrity: sha512-5gvxpSd79e9a3V4QDYUqnqxeD4HGlhCakVpb6gMnDD7lexJggSBJRBO5h52y/iJrdXRilX9UCuDaIJhSWm5OWw==, + } + smart-buffer@4.2.0: resolution: { @@ -4182,6 +4247,8 @@ snapshots: dependencies: json-schema: 0.4.0 + "@breezystack/lamejs@1.2.7": {} + "@bundled-es-modules/cookie@2.0.1": dependencies: cookie: 0.7.2 @@ -4489,6 +4556,8 @@ snapshots: "@esbuild/win32-x64@0.27.2": optional: true + "@eshaz/web-worker@1.2.2": {} + "@gar/promisify@1.1.3": {} "@inquirer/confirm@5.1.15(@types/node@20.19.11)": @@ -4743,6 +4812,15 @@ snapshots: loupe: 3.2.0 tinyrainbow: 1.2.0 + "@wasm-audio-decoders/common@9.0.7": + dependencies: + "@eshaz/web-worker": 1.2.2 + simple-yenc: 1.0.4 + + "@wasm-audio-decoders/opus-ml@0.0.3": + dependencies: + "@wasm-audio-decoders/common": 9.0.7 + a-sync-waterfall@1.0.1: {} abbrev@1.1.1: {} @@ -4990,6 +5068,8 @@ snapshots: strip-ansi: 6.0.1 wrap-ansi: 7.0.0 + codec-parser@2.5.0: {} + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -5782,6 +5862,13 @@ snapshots: object-inspect@1.13.4: {} + ogg-opus-decoder@1.7.5: + dependencies: + "@wasm-audio-decoders/common": 9.0.7 + "@wasm-audio-decoders/opus-ml": 0.0.3 + codec-parser: 2.5.0 + opus-decoder: 0.7.12 + on-finished@2.4.1: dependencies: ee-first: 1.1.1 @@ -5794,6 +5881,10 @@ snapshots: optionalDependencies: zod: 3.25.67 + opus-decoder@0.7.12: + dependencies: + "@wasm-audio-decoders/common": 9.0.7 + outvariant@1.4.3: {} p-map@4.0.0: @@ -6030,6 +6121,8 @@ snapshots: transitivePeerDependencies: - supports-color + simple-yenc@1.0.4: {} + smart-buffer@4.2.0: {} socks-proxy-agent@7.0.0: diff --git a/py/autoevals/llm.py b/py/autoevals/llm.py index b0093cb4..17eb026e 100644 --- a/py/autoevals/llm.py +++ b/py/autoevals/llm.py @@ -46,7 +46,9 @@ """ import asyncio +import base64 import inspect +import io import json import os import re @@ -133,6 +135,49 @@ def build_classification_tools(useCoT, choice_strings): ] +def get_path(args, path): + value = args + for key in path.split("."): + value = value.get(key) if isinstance(value, dict) else None + return value + + +def ogg_to_mp3(audio): + import soundfile + + samples, rate = soundfile.read(io.BytesIO(audio), dtype="int16", always_2d=True) + mp3 = io.BytesIO() + # compression_level 0.65 is 128 kbps, the lowest rate at which lamejs keeps 48 kHz + soundfile.write(mp3, samples[:, :2], rate, format="MP3", bitrate_mode="CONSTANT", compression_level=0.65) + return mp3.getvalue() + + +AUDIO_FORMATS = { + "audio/wav": "wav", + "audio/x-wav": "wav", + "audio/mpeg": "mp3", + "audio/mp3": "mp3", + "audio/ogg": "ogg", +} + + +def audio_part(audio): + if not isinstance(audio, dict) or not isinstance(audio.get("data"), (bytes, bytearray)): + raise TypeError("Audio must be a dict with `data` bytes and a `content_type`") + content_type = str(audio.get("content_type", "")).split(";")[0].strip().lower() + audio_format = AUDIO_FORMATS.get(content_type) + if audio_format is None: + raise ValueError(f"Audio must be WAV, MP3, or OGG, got content type {content_type!r}") + data = audio["data"] + if audio_format == "ogg": + data = ogg_to_mp3(data) + audio_format = "mp3" + return { + "type": "input_audio", + "input_audio": {"data": base64.b64encode(data).decode(), "format": audio_format}, + } + + class OpenAIScorer(ScorerWithPartial): def __init__( self, @@ -181,6 +226,7 @@ def __init__( reasoning_enabled=None, reasoning_budget=None, use_responses_api=None, + audio=None, engine=None, api_key=None, base_url=None, @@ -198,6 +244,7 @@ def __init__( self.model = model self.engine = engine self.messages = messages + self.audio = audio if max_tokens is not None: self.extra_args["max_tokens"] = max(max_tokens, 5) @@ -234,13 +281,21 @@ def _build_args(self, output, expected, **kwargs): def _render_messages(self, **kwargs): kwargs.update(self.render_args) - return [ + messages = [ { **m, "content": chevron.render(m["content"].strip(), kwargs, warn=True), } for m in self.messages ] + if self.audio: + text = messages[-1]["content"] + messages[-1]["content"] = [{"type": "text", "text": text}, audio_part(get_path(kwargs, self.audio))] + return messages + + def _missing_audio(self, output, expected, **kwargs): + args = {"output": output, "expected": expected, **kwargs, **self.render_args} + return self.audio and get_path(args, self.audio) is None def _request_args(self, output, expected, **kwargs): ret = { @@ -284,11 +339,15 @@ def _postprocess_response(self, resp): raise ValueError("Empty response from OpenAI") async def _run_eval_async(self, output, expected, **kwargs): + if self._missing_audio(output, expected, **kwargs): + return Score(name=self.name, score=None) return self._postprocess_response( await arun_cached_request(**(await self._request_args_async(output, expected, **kwargs))) ) def _run_eval_sync(self, output, expected, **kwargs): + if self._missing_audio(output, expected, **kwargs): + return Score(name=self.name, score=None) return self._postprocess_response(run_cached_request(**self._request_args(output, expected, **kwargs))) @@ -301,6 +360,7 @@ class ModelGradedSpec: use_cot: bool | None = None temperature: float | None = None max_tokens: int | None = None + audio: str | None = None class LLMClassifier(OpenAILLMClassifier): @@ -344,6 +404,7 @@ class LLMClassifier(OpenAILLMClassifier): reasoning_effort: Controls reasoning depth for o-series models (e.g., "low", "medium", "high"). reasoning_enabled: Enable extended thinking for supported models (e.g., Claude). Defaults to None. reasoning_budget: Token allocation for model's internal reasoning. Defaults to None. + audio: Path in the arguments (e.g. `input.audio`) to a dict with `data` bytes and a WAV, MP3, or OGG `content_type`. Missing audio skips the score. engine: Deprecated by OpenAI. Use model instead. api_key: Deprecated. Use client instead. base_url: Deprecated. Use client instead. @@ -371,6 +432,7 @@ def __init__( reasoning_enabled=None, reasoning_budget=None, use_responses_api=None, + audio=None, engine=None, api_key=None, base_url=None, @@ -403,6 +465,7 @@ def __init__( reasoning_enabled=reasoning_enabled, reasoning_budget=reasoning_budget, use_responses_api=use_responses_api, + audio=audio, engine=engine, api_key=api_key, base_url=base_url, @@ -484,8 +547,11 @@ def from_spec(cls, name: str, spec: ModelGradedSpec, client: Client | None = Non spec_kwargs["temperature"] = spec.temperature if spec.max_tokens is not None: spec_kwargs["max_tokens"] = spec.max_tokens + if spec.audio is not None: + spec_kwargs["audio"] = spec.audio # kwargs can override spec values - return cls(name, spec.prompt, spec.choice_scores, client=client, **spec_kwargs, **kwargs) + spec_kwargs.update(kwargs) + return cls(name, spec.prompt, spec.choice_scores, client=client, **spec_kwargs) @classmethod def from_spec_file(cls, name: str, path: str, client: Client | None = None, **kwargs): @@ -851,3 +917,43 @@ class Translation(SpecFileClassifier): """ pass + + +class SpeechClarity(SpecFileClassifier): + """Rate how clearly an agent speaks in a recording of the whole conversation. + + Example: + ```python + from pathlib import Path + from autoevals import SpeechClarity + + audio = {"data": Path("call.ogg").read_bytes(), "content_type": "audio/ogg"} + result = SpeechClarity().eval(input={"audio": audio}, output=None) + print(result.score) # 1 if clear, 0.5 if flawed but easy to follow, 0 if words are lost + ``` + + Args: + input: Dict whose `audio` is the recording, as `{"data": bytes, "content_type": "audio/ogg"}` (WAV, MP3, or OGG) + """ + + pass + + +class TurnTaking(SpecFileClassifier): + """Rate whether an agent takes turns without talking over or cutting off the caller. + + Example: + ```python + from pathlib import Path + from autoevals import TurnTaking + + audio = {"data": Path("call.ogg").read_bytes(), "content_type": "audio/ogg"} + result = TurnTaking().eval(input={"audio": audio}, output=None) + print(result.score) # 1 if smooth, 0.5 if brief overlaps, 0 if caller words are lost + ``` + + Args: + input: Dict whose `audio` is the recording, as `{"data": bytes, "content_type": "audio/ogg"}` (WAV, MP3, or OGG) + """ + + pass diff --git a/py/autoevals/test_llm.py b/py/autoevals/test_llm.py index 8e987efa..d8ed67af 100644 --- a/py/autoevals/test_llm.py +++ b/py/autoevals/test_llm.py @@ -1,5 +1,6 @@ import asyncio import json +from pathlib import Path from typing import cast import pytest @@ -9,7 +10,14 @@ from pydantic import BaseModel from autoevals import init -from autoevals.llm import Battle, Factuality, LLMClassifier, OpenAILLMClassifier, build_classification_tools +from autoevals.llm import ( + Battle, + Factuality, + LLMClassifier, + OpenAILLMClassifier, + SpeechClarity, + build_classification_tools, +) from autoevals.oai import OpenAIV1Module, get_default_model from autoevals.thread_utils import compute_thread_template_vars, template_uses_thread_variables @@ -767,3 +775,47 @@ async def get_thread(self): classifier.eval(output="x", expected="y", trace=trace) assert trace.calls == 0 + + +@respx.mock +def test_speech_clarity_sends_ogg_as_mp3_to_chat_completions(): + route = respx.route(method="POST", path__regex=r".*/chat/completions$").respond( + json={ + "choices": [ + { + "message": { + "role": "assistant", + "tool_calls": [ + { + "id": "call_test", + "type": "function", + "function": { + "name": "select_choice", + "arguments": '{"reasons":"Clear.","choice":"A"}', + }, + } + ], + }, + } + ], + } + ) + ogg = (Path(__file__).parents[2] / "fixtures" / "tone.ogg").read_bytes() + + result = SpeechClarity(base_url="https://api.openai.com/v1/", api_key="test").eval( + input={"audio": {"data": ogg, "content_type": "audio/ogg"}}, output=None + ) + + assert result.score == 1 + body = json.loads(route.calls.last.request.content) + assert body["model"] == "gemini-3.8-flash" + assert body["messages"][0]["content"][0]["type"] == "text" + assert body["messages"][0]["content"][1]["input_audio"]["format"] == "mp3" + + +def test_speech_clarity_accepts_model_override(): + assert SpeechClarity(model="gpt-4o-audio-preview").model == "gpt-4o-audio-preview" + + +def test_speech_clarity_skips_without_audio(): + assert SpeechClarity().eval(input={"text": "hello"}, output=None).score is None diff --git a/pyproject.toml b/pyproject.toml index 60ad2b8c..03e0cfbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "jsonschema", "polyleven", "pyyaml", + "soundfile", ] [project.urls] diff --git a/templates/speech_clarity.yaml b/templates/speech_clarity.yaml new file mode 100644 index 00000000..9f9e23fd --- /dev/null +++ b/templates/speech_clarity.yaml @@ -0,0 +1,12 @@ +prompt: |- + Listen to the attached recording of a conversation between an agent and a caller. Judge only the agent's speech. + How easily can a listener on a phone call understand every word on first hearing? Consider garbled or cut-off words, distortion, dropouts, echo, and background noise. Do not fill in unclear words from context, and ignore accent unless it makes words hard to understand. + (A) Words are clear and understandable without effort. + (B) Some flaws are audible, but the message remains easy to understand. + (C) Important words are unclear or lost. +choice_scores: + "A": 1.0 + "B": 0.5 + "C": 0.0 +model: gemini-3.8-flash +audio: input.audio diff --git a/templates/turn_taking.yaml b/templates/turn_taking.yaml new file mode 100644 index 00000000..3471e9f6 --- /dev/null +++ b/templates/turn_taking.yaml @@ -0,0 +1,12 @@ +prompt: |- + Listen to the attached recording of a conversation between an agent and a caller. Judge only how the agent handles turn-taking. + Consider whether the agent talks over the caller, cuts the caller off before they finish, or keeps talking when the caller interrupts. Brief backchannels such as "mm-hm" are fine unless they cover the caller's words. Ignore how long the agent takes to respond. + (A) The agent takes turns smoothly: it lets the caller finish and stops promptly when interrupted. + (B) There are brief overlaps or a slow yield, but no caller words are lost. + (C) The agent talks over the caller, ignores an interruption, or cuts the caller off, so caller words are lost. +choice_scores: + "A": 1.0 + "B": 0.5 + "C": 0.0 +model: gemini-3.8-flash +audio: input.audio diff --git a/tsconfig.json b/tsconfig.json index d8f28a28..2af89f39 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,6 +4,7 @@ "outDir": "./jsdist", "lib": ["es2015", "dom"], "target": "ES2018", + "module": "es2020", "moduleResolution": "node", "strict": true, "esModuleInterop": true, diff --git a/uv.lock b/uv.lock index 9ff1c5ab..eecffb4c 100644 --- a/uv.lock +++ b/uv.lock @@ -232,6 +232,7 @@ dependencies = [ { name = "jsonschema" }, { name = "polyleven" }, { name = "pyyaml" }, + { name = "soundfile" }, ] [package.optional-dependencies] @@ -295,6 +296,7 @@ requires-dist = [ { name = "pyyaml" }, { name = "respx", marker = "extra == 'dev'" }, { name = "scipy", marker = "extra == 'scipy'" }, + { name = "soundfile" }, { name = "twine", marker = "extra == 'dev'" }, ] provides-extras = ["dev", "doc", "scipy", "litellm", "all"] @@ -394,36 +396,77 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/93/d7/516d984057745a6cd96575eea814fe1edd6646ee6efd552fb7b0921dec83/cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44", size = 184283, upload-time = "2025-09-08T23:22:08.01Z" }, + { url = "https://files.pythonhosted.org/packages/9e/84/ad6a0b408daa859246f57c03efd28e5dd1b33c21737c2db84cae8c237aa5/cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49", size = 180504, upload-time = "2025-09-08T23:22:10.637Z" }, { url = "https://files.pythonhosted.org/packages/50/bd/b1a6362b80628111e6653c961f987faa55262b4002fcec42308cad1db680/cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c", size = 208811, upload-time = "2025-09-08T23:22:12.267Z" }, { url = "https://files.pythonhosted.org/packages/4f/27/6933a8b2562d7bd1fb595074cf99cc81fc3789f6a6c05cdabb46284a3188/cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb", size = 216402, upload-time = "2025-09-08T23:22:13.455Z" }, + { url = "https://files.pythonhosted.org/packages/05/eb/b86f2a2645b62adcfff53b0dd97e8dfafb5c8aa864bd0d9a2c2049a0d551/cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0", size = 203217, upload-time = "2025-09-08T23:22:14.596Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e0/6cbe77a53acf5acc7c08cc186c9928864bd7c005f9efd0d126884858a5fe/cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4", size = 203079, upload-time = "2025-09-08T23:22:15.769Z" }, { url = "https://files.pythonhosted.org/packages/98/29/9b366e70e243eb3d14a5cb488dfd3a0b6b2f1fb001a203f653b93ccfac88/cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453", size = 216475, upload-time = "2025-09-08T23:22:17.427Z" }, { url = "https://files.pythonhosted.org/packages/21/7a/13b24e70d2f90a322f2900c5d8e1f14fa7e2a6b3332b7309ba7b2ba51a5a/cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495", size = 218829, upload-time = "2025-09-08T23:22:19.069Z" }, { url = "https://files.pythonhosted.org/packages/60/99/c9dc110974c59cc981b1f5b66e1d8af8af764e00f0293266824d9c4254bc/cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5", size = 211211, upload-time = "2025-09-08T23:22:20.588Z" }, { url = "https://files.pythonhosted.org/packages/49/72/ff2d12dbf21aca1b32a40ed792ee6b40f6dc3a9cf1644bd7ef6e95e0ac5e/cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb", size = 218036, upload-time = "2025-09-08T23:22:22.143Z" }, + { url = "https://files.pythonhosted.org/packages/e2/cc/027d7fb82e58c48ea717149b03bcadcbdc293553edb283af792bd4bcbb3f/cffi-2.0.0-cp310-cp310-win32.whl", hash = "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a", size = 172184, upload-time = "2025-09-08T23:22:23.328Z" }, + { url = "https://files.pythonhosted.org/packages/33/fa/072dd15ae27fbb4e06b437eb6e944e75b068deb09e2a2826039e49ee2045/cffi-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739", size = 182790, upload-time = "2025-09-08T23:22:24.752Z" }, + { url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344, upload-time = "2025-09-08T23:22:26.456Z" }, + { url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560, upload-time = "2025-09-08T23:22:28.197Z" }, { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" }, { url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476, upload-time = "2025-09-08T23:22:31.063Z" }, + { url = "https://files.pythonhosted.org/packages/dc/7f/55fecd70f7ece178db2f26128ec41430d8720f2d12ca97bf8f0a628207d5/cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", size = 203374, upload-time = "2025-09-08T23:22:32.507Z" }, + { url = "https://files.pythonhosted.org/packages/84/ef/a7b77c8bdc0f77adc3b46888f1ad54be8f3b7821697a7b89126e829e676a/cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", size = 202597, upload-time = "2025-09-08T23:22:34.132Z" }, { url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574, upload-time = "2025-09-08T23:22:35.443Z" }, { url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971, upload-time = "2025-09-08T23:22:36.805Z" }, { url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972, upload-time = "2025-09-08T23:22:38.436Z" }, { url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078, upload-time = "2025-09-08T23:22:39.776Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c0/015b25184413d7ab0a410775fdb4a50fca20f5589b5dab1dbbfa3baad8ce/cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", size = 172076, upload-time = "2025-09-08T23:22:40.95Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8f/dc5531155e7070361eb1b7e4c1a9d896d0cb21c49f807a6c03fd63fc877e/cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", size = 182820, upload-time = "2025-09-08T23:22:42.463Z" }, + { url = "https://files.pythonhosted.org/packages/95/5c/1b493356429f9aecfd56bc171285a4c4ac8697f76e9bbbbb105e537853a1/cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", size = 177635, upload-time = "2025-09-08T23:22:43.623Z" }, + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, + { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, + { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, + { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, + { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, + { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" }, { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, + { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" }, + { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" }, { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, + { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" }, + { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" }, + { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" }, { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, + { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" }, + { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" }, { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, ] [[package]] @@ -1157,17 +1200,17 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version < '3.11'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "jedi", marker = "python_full_version < '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, - { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, - { name = "pygments", marker = "python_full_version < '3.11'" }, - { name = "stack-data", marker = "python_full_version < '3.11'" }, - { name = "traitlets", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "exceptiongroup" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" } wheels = [ @@ -1182,18 +1225,18 @@ resolution-markers = [ "python_full_version >= '3.11'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version >= '3.11'" }, - { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, - { name = "jedi", marker = "python_full_version >= '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, - { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, - { name = "psutil", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "stack-data", marker = "python_full_version >= '3.11'" }, - { name = "traitlets", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "psutil", marker = "sys_platform != 'emscripten'" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e2/23/3a27530575643c8bb7bfc757a28e2e7ef80092afbf59a2bc5716320b6602/ipython-9.14.1.tar.gz", hash = "sha256:f913bf74df06d458e46ced84ca506c23797590d594b236fe60b14df213291e7b", size = 4433457, upload-time = "2026-06-05T08:12:34.921Z" } wheels = [ @@ -1205,7 +1248,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -3123,7 +3166,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -3182,7 +3225,7 @@ resolution-markers = [ "python_full_version >= '3.11'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -3288,6 +3331,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "soundfile" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d2/db/949331952a6fb1c5b12e9de80fd08747966c2039d1a61db4764fbd3981c2/soundfile-0.14.0.tar.gz", hash = "sha256:ba1c1a2d618bca5c406647c83b89f07cc8810fa506a50622a6993ba130c1de11", size = 47842, upload-time = "2026-06-06T08:58:47.869Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/d1/5e338af9ca6ed0786cd5bb03f6d60de1c325728c1189014f3b59aae7403c/soundfile-0.14.0-py2.py3-none-any.whl", hash = "sha256:8ba81ae3a89fd5ab3bef8a8eb481fbbe794e806309675a89b4df48b8d31908a8", size = 26799, upload-time = "2026-06-06T08:58:33.269Z" }, + { url = "https://files.pythonhosted.org/packages/7e/72/c6b21e58d3113596e7e8de0a08d6f1d95173492cfbca0a4db14148cbba2a/soundfile-0.14.0-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:19be05428da76ed61a4cad29b8e4bcf43a3e5c100089d2ec81dc961eed1b0dd4", size = 1144568, upload-time = "2026-06-06T08:58:35.231Z" }, + { url = "https://files.pythonhosted.org/packages/63/7a/dfdd6f8c748988427119f75eb860a3cedd858d1aea1fe28f39ad8559ef22/soundfile-0.14.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:d828d35a059626da52f1415b5faee610aeab393319cb3fc4a9aef47b619fc14c", size = 1103726, upload-time = "2026-06-06T08:58:37.948Z" }, + { url = "https://files.pythonhosted.org/packages/4a/f8/fc39fad6f879633461d27394cd1ddaf1f769ffa0597dca35872f51b16461/soundfile-0.14.0-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:e85724a90bc99a6e8062c0b4ddf725f53b2a3b70afd4da875e9d2cfc4e92f377", size = 1238050, upload-time = "2026-06-06T08:58:39.932Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a2/70fd4432b924684c372df8b0a45708c36c057ef3596c9eb53e0a806b980b/soundfile-0.14.0-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:1e38bac1853412871318e82a1ba69a8be677619b56025bbfcccdb41b6cafe82d", size = 1315963, upload-time = "2026-06-06T08:58:41.716Z" }, + { url = "https://files.pythonhosted.org/packages/d9/34/c9e80783d83eab739a9531fdee03675d53e0bf1b2ccb4bb3af5844675046/soundfile-0.14.0-py2.py3-none-win32.whl", hash = "sha256:0a6ae43c50c71b4e020cc55382925cb89451c1ed1a0c3d0f5d802da269226849", size = 902199, upload-time = "2026-06-06T08:58:43.289Z" }, + { url = "https://files.pythonhosted.org/packages/ed/97/b39c18ac1df45e755ca22b8b00e872929da5d107998a207a5e4ac831bfda/soundfile-0.14.0-py2.py3-none-win_amd64.whl", hash = "sha256:299491d3499460fb1b74bb4bd78b57ffc2d243a5fafa7b6ec1b264875c78453e", size = 1021480, upload-time = "2026-06-06T08:58:45.016Z" }, + { url = "https://files.pythonhosted.org/packages/f4/83/55c65e61cf457805ce2ec157c1c6ae17715d0851aa2374422de0538838ca/soundfile-0.14.0-py2.py3-none-win_arm64.whl", hash = "sha256:e090704718e124e7c844695236f1fce8d18a5e761eaf7c82dfcd124620805f98", size = 888858, upload-time = "2026-06-06T08:58:46.593Z" }, +] + [[package]] name = "sseclient-py" version = "1.9.0" From c5a744f3c105116855a9315489dcf89d698443f4 Mon Sep 17 00:00:00 2001 From: Kev Choi Date: Wed, 30 Sep 2026 14:30:39 -0400 Subject: [PATCH 2/3] feat: send voice scorer audio to the model as-is instead of transcoding Co-Authored-By: Claude Opus 5.5 --- fixtures/tone.ogg | Bin 1813 -> 0 bytes js/llm.test.ts | 9 +-- js/llm.ts | 63 ++++----------------- package.json | 2 - pnpm-lock.yaml | 93 ------------------------------- py/autoevals/llm.py | 41 +++----------- py/autoevals/test_llm.py | 9 +-- pyproject.toml | 1 - tsconfig.json | 1 - uv.lock | 117 +++++++++------------------------------ 10 files changed, 53 insertions(+), 283 deletions(-) delete mode 100644 fixtures/tone.ogg diff --git a/fixtures/tone.ogg b/fixtures/tone.ogg deleted file mode 100644 index bf9025029e2c2a2f228755f0a9a612cdd602378c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1813 zcmeZIPY-5bVt|4>B31Gr=9gt}&oc`97nBxzq$Z{?GFmV;>;^0G2PuaGka9*KuX1K_ zEzAG^|D`}WLK4%9xq*T?nMwITu7aVS30%V{V>ARtUG|!+3a;}% zZ}ynh?%B-wuTv~$fB7+)VwUbx9X=awut@cpt>`-?dFj+GzVnt(K0jU@cKOz`=I^;b zHb?fBGj5EpuW7QEUz6#zlD*w%?wY_dRm#~!!7 zqFdKp#QWAN>|5)xR6H`vRdfDh?jTM!72|w1t}Q;k0nK5xlQg#M_$(cB)O26tqND3i z^W;?A)Dc{!y(n|tkG1s-D^`^jU;H|^{v|ulev{eZD#o#WefyjgoeaIrXIe(EPyW(y zt=ZQ?u_}GOQ=XM^`1=*Ig3f7epRE!$nlN=atYu*R)UH-!c}cTi5l5(#Aj37y)4I$N zv!7J|EC~OlQ)c8~#eT!~BcJ$b=cg;q-?A~g|9fa zvaX2f4>RQ+$1o@?KfZFO;%8^cbqi#47lNI*@pAj>P<0lu<~5=21^fQ2{;(=J(5$Z1 zZ_=m54$N=(pKor=+j8O0GBfe~Cx<5Gw4eX`qOaaN>+4lDt*8%=eD-8sy>?TyiQ}~I zl^bd$H774DnH3}6wa71I(wWB#3{va<{&zQ>YtGP+)X!DRaH>MpScqG5#+it_mc`PX zzk_3Dd+y|@-ZJ;Xt4lJGY8N^#e3CCdy-=Uc?Y5%RyuL&a*V=VI)~vbfbH4uZ?N6sI z1pT)-*Etza-6`TYpY2v#yrtVE6Uo1tnfHozPyS_{oGbFr%8f79*mkGJ9{rx#<(@hh zdbTWnb|511?CRgztm|&rPdBx6x7WKhf8(D!nT&_auHIc^n6gE4f5EAPUQ-o1g3Z|P z9p5@dkn#1i!-r+{UTy8%dN1*J;{wM(?$?+9C(U=y^XN_gl(%PF#Towp34ebIPtsyl gU-zMIo5!kGd*`^FHhZ)|dY|tOqvB~zo0ne#0Ih}U`2YX_ diff --git a/js/llm.test.ts b/js/llm.test.ts index 7ff317de..9b694348 100644 --- a/js/llm.test.ts +++ b/js/llm.test.ts @@ -1,4 +1,3 @@ -import { readFileSync } from "node:fs"; import { bypass, http, HttpResponse } from "msw"; import { setupServer } from "msw/node"; import { OpenAI } from "openai"; @@ -666,7 +665,7 @@ Issue Description: {{page_content}} }); describe("SpeechClarity", () => { - test("sends OGG audio as MP3 to gemini-3.8-flash through chat completions", async () => { + test("sends audio as a file to gemini-3.8-flash through chat completions", async () => { let body: any; server.use( http.post( @@ -699,7 +698,7 @@ describe("SpeechClarity", () => { const score = await SpeechClarity({ input: { audio: { - data: readFileSync("fixtures/tone.ogg"), + data: new Uint8Array([1, 2, 3]), content_type: "audio/ogg", }, }, @@ -710,7 +709,9 @@ describe("SpeechClarity", () => { expect(score.score).toBe(1); expect(body.model).toBe("gemini-3.8-flash"); expect(body.messages[0].content[0].type).toBe("text"); - expect(body.messages[0].content[1].input_audio.format).toBe("mp3"); + expect(body.messages[0].content[1].file.file_data).toBe( + "data:audio/ogg;base64,AQID", + ); }); test("skips when there is no audio", async () => { diff --git a/js/llm.ts b/js/llm.ts index 50b60dcb..41268a89 100644 --- a/js/llm.ts +++ b/js/llm.ts @@ -7,7 +7,7 @@ import { } from "./oai"; import { ModelGradedSpec, templates } from "./templates"; import { - ChatCompletionContentPartInputAudio, + ChatCompletionContentPart, ChatCompletionMessage, ChatCompletionMessageParam, ChatCompletionTool, @@ -74,43 +74,9 @@ function getPath(args: unknown, path: string): unknown { return value; } -async function oggToMp3(ogg: Uint8Array): Promise { - const { OggOpusDecoder } = await import("ogg-opus-decoder"); - const { Mp3Encoder } = await import("@breezystack/lamejs"); - const decoder = new OggOpusDecoder(); - await decoder.ready; - try { - const { channelData, samplesDecoded, sampleRate, errors } = - await decoder.decodeFile(ogg); - if (errors.length > 0 || samplesDecoded === 0) { - throw new Error("Could not decode OGG audio"); - } - const [left, right] = channelData - .slice(0, 2) - .map((samples) => - Int16Array.from(samples, (s) => Math.max(-1, Math.min(1, s)) * 0x7fff), - ); - const encoder = new Mp3Encoder(right ? 2 : 1, sampleRate, 128); - const chunks = [encoder.encodeBuffer(left, right), encoder.flush()]; - return Buffer.concat(chunks.map((c) => Buffer.from(c))); - } finally { - decoder.free(); - } -} - export type Audio = { data: Uint8Array; content_type: string }; -const AUDIO_FORMATS: Record = { - "audio/wav": "wav", - "audio/x-wav": "wav", - "audio/mpeg": "mp3", - "audio/mp3": "mp3", - "audio/ogg": "ogg", -}; - -async function audioPart( - audio: unknown, -): Promise { +function audioPart(audio: unknown): ChatCompletionContentPart.File { const data = Reflect.get(Object(audio), "data"); if (!(data instanceof Uint8Array)) { throw new TypeError( @@ -121,20 +87,15 @@ async function audioPart( .split(";")[0] .trim() .toLowerCase(); - const format = AUDIO_FORMATS[contentType]; - if (!format) { + if (!contentType.startsWith("audio/")) { throw new Error( - `Audio must be WAV, MP3, or OGG, got content type "${contentType}"`, + `Audio must have an audio/* content type, got "${contentType}"`, ); } - const [bytes, sentFormat] = - format === "ogg" ? [await oggToMp3(data), "mp3" as const] : [data, format]; + const base64 = Buffer.from(data).toString("base64"); return { - type: "input_audio", - input_audio: { - data: Buffer.from(bytes).toString("base64"), - format: sentFormat, - }, + type: "file", + file: { file_data: `data:${contentType};base64,${base64}` }, }; } @@ -218,7 +179,7 @@ export type OpenAIClassifierArgs = { messages: ChatCompletionMessageParam[]; choiceScores: Record; classificationTools: ChatCompletionTool[]; - inputAudio?: ChatCompletionContentPartInputAudio; + audioFile?: ChatCompletionContentPart.File; cache?: ChatCache; } & LLMArgs & RenderArgs; @@ -251,7 +212,7 @@ export async function OpenAIClassifier( reasoningEnabled, reasoningBudget, useResponsesApi, - inputAudio, + audioFile, cache, ...remainingRenderArgs } = remaining; @@ -290,11 +251,11 @@ export async function OpenAIClassifier( }; const messages = renderMessages(messagesArg, renderArgs); - if (inputAudio) { + if (audioFile) { const last = messages[messages.length - 1]; messages[messages.length - 1] = { role: "user", - content: [{ type: "text", text: String(last.content) }, inputAudio], + content: [{ type: "text", text: String(last.content) }, audioFile], }; } @@ -466,7 +427,7 @@ export function LLMClassifierFromTemplate({ // Since the logic is a bit funky for computing this, include // it at the end to prevent overrides useCoT, - inputAudio: audio ? await audioPart(audioValue) : undefined, + audioFile: audio ? audioPart(audioValue) : undefined, }; return await OpenAIClassifier(classifierArgs); diff --git a/package.json b/package.json index 6e088ebd..ce662eae 100644 --- a/package.json +++ b/package.json @@ -50,14 +50,12 @@ "zod": "^3.25.0 || ^4.0.0" }, "dependencies": { - "@breezystack/lamejs": "1.2.7", "ajv": "^8.17.1", "compute-cosine-similarity": "^1.1.0", "js-levenshtein": "^1.1.6", "js-yaml": "^4.1.0", "linear-sum-assignment": "^1.0.7", "mustache": "^4.2.0", - "ogg-opus-decoder": "1.7.5", "openai": "^6.7.0", "zod-to-json-schema": "3.25.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d652d580..a42b8cd8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,9 +7,6 @@ settings: importers: .: dependencies: - "@breezystack/lamejs": - specifier: 1.2.7 - version: 1.2.7 ajv: specifier: ^8.17.1 version: 8.17.1 @@ -28,9 +25,6 @@ importers: mustache: specifier: ^4.2.0 version: 4.2.0 - ogg-opus-decoder: - specifier: 1.7.5 - version: 1.7.5 openai: specifier: ^6.7.0 version: 6.15.0(zod@3.25.67) @@ -108,12 +102,6 @@ packages: } engines: { node: ">=18" } - "@breezystack/lamejs@1.2.7": - resolution: - { - integrity: sha512-6wc7ck65ctA75Hq7FYHTtTvGnYs6msgdxiSUICQ+A01nVOWg6rqouZB8IdyteRlfpYYiFovkf67dIeOgWIUzTA==, - } - "@bundled-es-modules/cookie@2.0.1": resolution: { @@ -1012,12 +1000,6 @@ packages: cpu: [x64] os: [win32] - "@eshaz/web-worker@1.2.2": - resolution: - { - integrity: sha512-WxXiHFmD9u/owrzempiDlBB1ZYqiLnm9s6aPc8AlFQalq2tKmqdmMr9GXOupDgzXtqnBipj8Un0gkIm7Sjf8mw==, - } - "@gar/promisify@1.1.3": resolution: { @@ -1258,7 +1240,6 @@ packages: } cpu: [arm] os: [linux] - libc: [glibc] "@rollup/rollup-linux-arm-musleabihf@4.46.4": resolution: @@ -1267,7 +1248,6 @@ packages: } cpu: [arm] os: [linux] - libc: [musl] "@rollup/rollup-linux-arm64-gnu@4.46.4": resolution: @@ -1276,7 +1256,6 @@ packages: } cpu: [arm64] os: [linux] - libc: [glibc] "@rollup/rollup-linux-arm64-musl@4.46.4": resolution: @@ -1285,7 +1264,6 @@ packages: } cpu: [arm64] os: [linux] - libc: [musl] "@rollup/rollup-linux-loongarch64-gnu@4.46.4": resolution: @@ -1294,7 +1272,6 @@ packages: } cpu: [loong64] os: [linux] - libc: [glibc] "@rollup/rollup-linux-ppc64-gnu@4.46.4": resolution: @@ -1303,7 +1280,6 @@ packages: } cpu: [ppc64] os: [linux] - libc: [glibc] "@rollup/rollup-linux-riscv64-gnu@4.46.4": resolution: @@ -1312,7 +1288,6 @@ packages: } cpu: [riscv64] os: [linux] - libc: [glibc] "@rollup/rollup-linux-riscv64-musl@4.46.4": resolution: @@ -1321,7 +1296,6 @@ packages: } cpu: [riscv64] os: [linux] - libc: [musl] "@rollup/rollup-linux-s390x-gnu@4.46.4": resolution: @@ -1330,7 +1304,6 @@ packages: } cpu: [s390x] os: [linux] - libc: [glibc] "@rollup/rollup-linux-x64-gnu@4.46.4": resolution: @@ -1339,7 +1312,6 @@ packages: } cpu: [x64] os: [linux] - libc: [glibc] "@rollup/rollup-linux-x64-musl@4.46.4": resolution: @@ -1348,7 +1320,6 @@ packages: } cpu: [x64] os: [linux] - libc: [musl] "@rollup/rollup-win32-arm64-msvc@4.46.4": resolution: @@ -1503,18 +1474,6 @@ packages: integrity: sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==, } - "@wasm-audio-decoders/common@9.0.7": - resolution: - { - integrity: sha512-WRaUuWSKV7pkttBygml/a6dIEpatq2nnZGFIoPTc5yPLkxL6Wk4YaslPM98OPQvWacvNZ+Py9xROGDtrFBDzag==, - } - - "@wasm-audio-decoders/opus-ml@0.0.3": - resolution: - { - integrity: sha512-SEeTmk0eb3q5etJA/NHCRWW6BivTx9tqvNTmG0kg2ynuATEQRUZnwQE6Ibvl1BBvqIGHqI57+qyJMWmJBwb5XQ==, - } - a-sync-waterfall@1.0.1: resolution: { @@ -1882,12 +1841,6 @@ packages: } engines: { node: ">=12" } - codec-parser@2.5.0: - resolution: - { - integrity: sha512-Ru9t80fV8B0ZiixQl8xhMTLru+dzuis/KQld32/x5T/+3LwZb0/YvQdSKytX9JqCnRdiupvAvyYJINKrXieziQ==, - } - color-convert@2.0.1: resolution: { @@ -3113,12 +3066,6 @@ packages: } engines: { node: ">= 0.4" } - ogg-opus-decoder@1.7.5: - resolution: - { - integrity: sha512-08WfRbuBBew1H63PqQ3zj2B/J0Eh9ISfNOTi+Dk6vuXmkBGSx8BiPJGOJ470EYeKezTa7/GhOAwHe3XlXwOZMw==, - } - on-finished@2.4.1: resolution: { @@ -3147,12 +3094,6 @@ packages: zod: optional: true - opus-decoder@0.7.12: - resolution: - { - integrity: sha512-6zpHFDi/MhYvMwyYiuJPqJ2lcRkfm7wHblSvT3lUtx/kLvHkunoe/rhTn1dLKxqVGLeQj3gQi7CnR0N2p3RuOg==, - } - outvariant@1.4.3: resolution: { @@ -3553,12 +3494,6 @@ packages: integrity: sha512-ivHoFS9Yi9GY49ogc6/YAi3Fl9ROnF4VyubNylgCkA+RVqLaKWnDSzXOVzya8csELIaWaYNutsEuAhZrtOjozA==, } - simple-yenc@1.0.4: - resolution: - { - integrity: sha512-5gvxpSd79e9a3V4QDYUqnqxeD4HGlhCakVpb6gMnDD7lexJggSBJRBO5h52y/iJrdXRilX9UCuDaIJhSWm5OWw==, - } - smart-buffer@4.2.0: resolution: { @@ -4247,8 +4182,6 @@ snapshots: dependencies: json-schema: 0.4.0 - "@breezystack/lamejs@1.2.7": {} - "@bundled-es-modules/cookie@2.0.1": dependencies: cookie: 0.7.2 @@ -4556,8 +4489,6 @@ snapshots: "@esbuild/win32-x64@0.27.2": optional: true - "@eshaz/web-worker@1.2.2": {} - "@gar/promisify@1.1.3": {} "@inquirer/confirm@5.1.15(@types/node@20.19.11)": @@ -4812,15 +4743,6 @@ snapshots: loupe: 3.2.0 tinyrainbow: 1.2.0 - "@wasm-audio-decoders/common@9.0.7": - dependencies: - "@eshaz/web-worker": 1.2.2 - simple-yenc: 1.0.4 - - "@wasm-audio-decoders/opus-ml@0.0.3": - dependencies: - "@wasm-audio-decoders/common": 9.0.7 - a-sync-waterfall@1.0.1: {} abbrev@1.1.1: {} @@ -5068,8 +4990,6 @@ snapshots: strip-ansi: 6.0.1 wrap-ansi: 7.0.0 - codec-parser@2.5.0: {} - color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -5862,13 +5782,6 @@ snapshots: object-inspect@1.13.4: {} - ogg-opus-decoder@1.7.5: - dependencies: - "@wasm-audio-decoders/common": 9.0.7 - "@wasm-audio-decoders/opus-ml": 0.0.3 - codec-parser: 2.5.0 - opus-decoder: 0.7.12 - on-finished@2.4.1: dependencies: ee-first: 1.1.1 @@ -5881,10 +5794,6 @@ snapshots: optionalDependencies: zod: 3.25.67 - opus-decoder@0.7.12: - dependencies: - "@wasm-audio-decoders/common": 9.0.7 - outvariant@1.4.3: {} p-map@4.0.0: @@ -6121,8 +6030,6 @@ snapshots: transitivePeerDependencies: - supports-color - simple-yenc@1.0.4: {} - smart-buffer@4.2.0: {} socks-proxy-agent@7.0.0: diff --git a/py/autoevals/llm.py b/py/autoevals/llm.py index 17eb026e..04fb4b3f 100644 --- a/py/autoevals/llm.py +++ b/py/autoevals/llm.py @@ -48,7 +48,6 @@ import asyncio import base64 import inspect -import io import json import os import re @@ -142,40 +141,14 @@ def get_path(args, path): return value -def ogg_to_mp3(audio): - import soundfile - - samples, rate = soundfile.read(io.BytesIO(audio), dtype="int16", always_2d=True) - mp3 = io.BytesIO() - # compression_level 0.65 is 128 kbps, the lowest rate at which lamejs keeps 48 kHz - soundfile.write(mp3, samples[:, :2], rate, format="MP3", bitrate_mode="CONSTANT", compression_level=0.65) - return mp3.getvalue() - - -AUDIO_FORMATS = { - "audio/wav": "wav", - "audio/x-wav": "wav", - "audio/mpeg": "mp3", - "audio/mp3": "mp3", - "audio/ogg": "ogg", -} - - def audio_part(audio): if not isinstance(audio, dict) or not isinstance(audio.get("data"), (bytes, bytearray)): raise TypeError("Audio must be a dict with `data` bytes and a `content_type`") content_type = str(audio.get("content_type", "")).split(";")[0].strip().lower() - audio_format = AUDIO_FORMATS.get(content_type) - if audio_format is None: - raise ValueError(f"Audio must be WAV, MP3, or OGG, got content type {content_type!r}") - data = audio["data"] - if audio_format == "ogg": - data = ogg_to_mp3(data) - audio_format = "mp3" - return { - "type": "input_audio", - "input_audio": {"data": base64.b64encode(data).decode(), "format": audio_format}, - } + if not content_type.startswith("audio/"): + raise ValueError(f"Audio must have an audio/* content type, got {content_type!r}") + data = base64.b64encode(audio["data"]).decode() + return {"type": "file", "file": {"file_data": f"data:{content_type};base64,{data}"}} class OpenAIScorer(ScorerWithPartial): @@ -404,7 +377,7 @@ class LLMClassifier(OpenAILLMClassifier): reasoning_effort: Controls reasoning depth for o-series models (e.g., "low", "medium", "high"). reasoning_enabled: Enable extended thinking for supported models (e.g., Claude). Defaults to None. reasoning_budget: Token allocation for model's internal reasoning. Defaults to None. - audio: Path in the arguments (e.g. `input.audio`) to a dict with `data` bytes and a WAV, MP3, or OGG `content_type`. Missing audio skips the score. + audio: Path in the arguments (e.g. `input.audio`) to a dict with `data` bytes and an audio `content_type`, such as `audio/ogg`. Missing audio skips the score. engine: Deprecated by OpenAI. Use model instead. api_key: Deprecated. Use client instead. base_url: Deprecated. Use client instead. @@ -933,7 +906,7 @@ class SpeechClarity(SpecFileClassifier): ``` Args: - input: Dict whose `audio` is the recording, as `{"data": bytes, "content_type": "audio/ogg"}` (WAV, MP3, or OGG) + input: Dict whose `audio` is the recording, as `{"data": bytes, "content_type": "audio/ogg"}` """ pass @@ -953,7 +926,7 @@ class TurnTaking(SpecFileClassifier): ``` Args: - input: Dict whose `audio` is the recording, as `{"data": bytes, "content_type": "audio/ogg"}` (WAV, MP3, or OGG) + input: Dict whose `audio` is the recording, as `{"data": bytes, "content_type": "audio/ogg"}` """ pass diff --git a/py/autoevals/test_llm.py b/py/autoevals/test_llm.py index d8ed67af..fc16d28a 100644 --- a/py/autoevals/test_llm.py +++ b/py/autoevals/test_llm.py @@ -1,6 +1,5 @@ import asyncio import json -from pathlib import Path from typing import cast import pytest @@ -778,7 +777,7 @@ async def get_thread(self): @respx.mock -def test_speech_clarity_sends_ogg_as_mp3_to_chat_completions(): +def test_speech_clarity_sends_audio_as_a_file_to_chat_completions(): route = respx.route(method="POST", path__regex=r".*/chat/completions$").respond( json={ "choices": [ @@ -800,17 +799,15 @@ def test_speech_clarity_sends_ogg_as_mp3_to_chat_completions(): ], } ) - ogg = (Path(__file__).parents[2] / "fixtures" / "tone.ogg").read_bytes() - result = SpeechClarity(base_url="https://api.openai.com/v1/", api_key="test").eval( - input={"audio": {"data": ogg, "content_type": "audio/ogg"}}, output=None + input={"audio": {"data": b"\x01\x02\x03", "content_type": "audio/ogg"}}, output=None ) assert result.score == 1 body = json.loads(route.calls.last.request.content) assert body["model"] == "gemini-3.8-flash" assert body["messages"][0]["content"][0]["type"] == "text" - assert body["messages"][0]["content"][1]["input_audio"]["format"] == "mp3" + assert body["messages"][0]["content"][1]["file"]["file_data"] == "data:audio/ogg;base64,AQID" def test_speech_clarity_accepts_model_override(): diff --git a/pyproject.toml b/pyproject.toml index 03e0cfbc..60ad2b8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ dependencies = [ "jsonschema", "polyleven", "pyyaml", - "soundfile", ] [project.urls] diff --git a/tsconfig.json b/tsconfig.json index 2af89f39..d8f28a28 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,6 @@ "outDir": "./jsdist", "lib": ["es2015", "dom"], "target": "ES2018", - "module": "es2020", "moduleResolution": "node", "strict": true, "esModuleInterop": true, diff --git a/uv.lock b/uv.lock index eecffb4c..9ff1c5ab 100644 --- a/uv.lock +++ b/uv.lock @@ -232,7 +232,6 @@ dependencies = [ { name = "jsonschema" }, { name = "polyleven" }, { name = "pyyaml" }, - { name = "soundfile" }, ] [package.optional-dependencies] @@ -296,7 +295,6 @@ requires-dist = [ { name = "pyyaml" }, { name = "respx", marker = "extra == 'dev'" }, { name = "scipy", marker = "extra == 'scipy'" }, - { name = "soundfile" }, { name = "twine", marker = "extra == 'dev'" }, ] provides-extras = ["dev", "doc", "scipy", "litellm", "all"] @@ -396,77 +394,36 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/93/d7/516d984057745a6cd96575eea814fe1edd6646ee6efd552fb7b0921dec83/cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44", size = 184283, upload-time = "2025-09-08T23:22:08.01Z" }, - { url = "https://files.pythonhosted.org/packages/9e/84/ad6a0b408daa859246f57c03efd28e5dd1b33c21737c2db84cae8c237aa5/cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49", size = 180504, upload-time = "2025-09-08T23:22:10.637Z" }, { url = "https://files.pythonhosted.org/packages/50/bd/b1a6362b80628111e6653c961f987faa55262b4002fcec42308cad1db680/cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c", size = 208811, upload-time = "2025-09-08T23:22:12.267Z" }, { url = "https://files.pythonhosted.org/packages/4f/27/6933a8b2562d7bd1fb595074cf99cc81fc3789f6a6c05cdabb46284a3188/cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb", size = 216402, upload-time = "2025-09-08T23:22:13.455Z" }, - { url = "https://files.pythonhosted.org/packages/05/eb/b86f2a2645b62adcfff53b0dd97e8dfafb5c8aa864bd0d9a2c2049a0d551/cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0", size = 203217, upload-time = "2025-09-08T23:22:14.596Z" }, - { url = "https://files.pythonhosted.org/packages/9f/e0/6cbe77a53acf5acc7c08cc186c9928864bd7c005f9efd0d126884858a5fe/cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4", size = 203079, upload-time = "2025-09-08T23:22:15.769Z" }, { url = "https://files.pythonhosted.org/packages/98/29/9b366e70e243eb3d14a5cb488dfd3a0b6b2f1fb001a203f653b93ccfac88/cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453", size = 216475, upload-time = "2025-09-08T23:22:17.427Z" }, { url = "https://files.pythonhosted.org/packages/21/7a/13b24e70d2f90a322f2900c5d8e1f14fa7e2a6b3332b7309ba7b2ba51a5a/cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495", size = 218829, upload-time = "2025-09-08T23:22:19.069Z" }, { url = "https://files.pythonhosted.org/packages/60/99/c9dc110974c59cc981b1f5b66e1d8af8af764e00f0293266824d9c4254bc/cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5", size = 211211, upload-time = "2025-09-08T23:22:20.588Z" }, { url = "https://files.pythonhosted.org/packages/49/72/ff2d12dbf21aca1b32a40ed792ee6b40f6dc3a9cf1644bd7ef6e95e0ac5e/cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb", size = 218036, upload-time = "2025-09-08T23:22:22.143Z" }, - { url = "https://files.pythonhosted.org/packages/e2/cc/027d7fb82e58c48ea717149b03bcadcbdc293553edb283af792bd4bcbb3f/cffi-2.0.0-cp310-cp310-win32.whl", hash = "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a", size = 172184, upload-time = "2025-09-08T23:22:23.328Z" }, - { url = "https://files.pythonhosted.org/packages/33/fa/072dd15ae27fbb4e06b437eb6e944e75b068deb09e2a2826039e49ee2045/cffi-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739", size = 182790, upload-time = "2025-09-08T23:22:24.752Z" }, - { url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344, upload-time = "2025-09-08T23:22:26.456Z" }, - { url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560, upload-time = "2025-09-08T23:22:28.197Z" }, { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" }, { url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476, upload-time = "2025-09-08T23:22:31.063Z" }, - { url = "https://files.pythonhosted.org/packages/dc/7f/55fecd70f7ece178db2f26128ec41430d8720f2d12ca97bf8f0a628207d5/cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", size = 203374, upload-time = "2025-09-08T23:22:32.507Z" }, - { url = "https://files.pythonhosted.org/packages/84/ef/a7b77c8bdc0f77adc3b46888f1ad54be8f3b7821697a7b89126e829e676a/cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", size = 202597, upload-time = "2025-09-08T23:22:34.132Z" }, { url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574, upload-time = "2025-09-08T23:22:35.443Z" }, { url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971, upload-time = "2025-09-08T23:22:36.805Z" }, { url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972, upload-time = "2025-09-08T23:22:38.436Z" }, { url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078, upload-time = "2025-09-08T23:22:39.776Z" }, - { url = "https://files.pythonhosted.org/packages/2b/c0/015b25184413d7ab0a410775fdb4a50fca20f5589b5dab1dbbfa3baad8ce/cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", size = 172076, upload-time = "2025-09-08T23:22:40.95Z" }, - { url = "https://files.pythonhosted.org/packages/ae/8f/dc5531155e7070361eb1b7e4c1a9d896d0cb21c49f807a6c03fd63fc877e/cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", size = 182820, upload-time = "2025-09-08T23:22:42.463Z" }, - { url = "https://files.pythonhosted.org/packages/95/5c/1b493356429f9aecfd56bc171285a4c4ac8697f76e9bbbbb105e537853a1/cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", size = 177635, upload-time = "2025-09-08T23:22:43.623Z" }, - { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, - { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, - { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, - { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, - { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, - { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, - { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, - { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, - { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, - { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, - { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, - { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, - { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, - { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" }, - { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" }, { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, - { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" }, - { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" }, { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, - { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" }, - { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" }, - { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" }, - { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" }, { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, - { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" }, - { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" }, { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, - { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" }, - { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" }, - { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, ] [[package]] @@ -1200,17 +1157,17 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "decorator" }, - { name = "exceptiongroup" }, - { name = "jedi" }, - { name = "matplotlib-inline" }, - { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit" }, - { name = "pygments" }, - { name = "stack-data" }, - { name = "traitlets" }, - { name = "typing-extensions" }, + { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, + { name = "decorator", marker = "python_full_version < '3.11'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "jedi", marker = "python_full_version < '3.11'" }, + { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, + { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, + { name = "pygments", marker = "python_full_version < '3.11'" }, + { name = "stack-data", marker = "python_full_version < '3.11'" }, + { name = "traitlets", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" } wheels = [ @@ -1225,18 +1182,18 @@ resolution-markers = [ "python_full_version >= '3.11'", ] dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "decorator" }, - { name = "ipython-pygments-lexers" }, - { name = "jedi" }, - { name = "matplotlib-inline" }, - { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit" }, - { name = "psutil", marker = "sys_platform != 'emscripten'" }, - { name = "pygments" }, - { name = "stack-data" }, - { name = "traitlets" }, - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, + { name = "decorator", marker = "python_full_version >= '3.11'" }, + { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, + { name = "jedi", marker = "python_full_version >= '3.11'" }, + { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, + { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, + { name = "psutil", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten'" }, + { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "stack-data", marker = "python_full_version >= '3.11'" }, + { name = "traitlets", marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e2/23/3a27530575643c8bb7bfc757a28e2e7ef80092afbf59a2bc5716320b6602/ipython-9.14.1.tar.gz", hash = "sha256:f913bf74df06d458e46ced84ca506c23797590d594b236fe60b14df213291e7b", size = 4433457, upload-time = "2026-06-05T08:12:34.921Z" } wheels = [ @@ -1248,7 +1205,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments" }, + { name = "pygments", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -3166,7 +3123,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -3225,7 +3182,7 @@ resolution-markers = [ "python_full_version >= '3.11'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -3331,28 +3288,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] -[[package]] -name = "soundfile" -version = "0.14.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d2/db/949331952a6fb1c5b12e9de80fd08747966c2039d1a61db4764fbd3981c2/soundfile-0.14.0.tar.gz", hash = "sha256:ba1c1a2d618bca5c406647c83b89f07cc8810fa506a50622a6993ba130c1de11", size = 47842, upload-time = "2026-06-06T08:58:47.869Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/d1/5e338af9ca6ed0786cd5bb03f6d60de1c325728c1189014f3b59aae7403c/soundfile-0.14.0-py2.py3-none-any.whl", hash = "sha256:8ba81ae3a89fd5ab3bef8a8eb481fbbe794e806309675a89b4df48b8d31908a8", size = 26799, upload-time = "2026-06-06T08:58:33.269Z" }, - { url = "https://files.pythonhosted.org/packages/7e/72/c6b21e58d3113596e7e8de0a08d6f1d95173492cfbca0a4db14148cbba2a/soundfile-0.14.0-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:19be05428da76ed61a4cad29b8e4bcf43a3e5c100089d2ec81dc961eed1b0dd4", size = 1144568, upload-time = "2026-06-06T08:58:35.231Z" }, - { url = "https://files.pythonhosted.org/packages/63/7a/dfdd6f8c748988427119f75eb860a3cedd858d1aea1fe28f39ad8559ef22/soundfile-0.14.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:d828d35a059626da52f1415b5faee610aeab393319cb3fc4a9aef47b619fc14c", size = 1103726, upload-time = "2026-06-06T08:58:37.948Z" }, - { url = "https://files.pythonhosted.org/packages/4a/f8/fc39fad6f879633461d27394cd1ddaf1f769ffa0597dca35872f51b16461/soundfile-0.14.0-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:e85724a90bc99a6e8062c0b4ddf725f53b2a3b70afd4da875e9d2cfc4e92f377", size = 1238050, upload-time = "2026-06-06T08:58:39.932Z" }, - { url = "https://files.pythonhosted.org/packages/7b/a2/70fd4432b924684c372df8b0a45708c36c057ef3596c9eb53e0a806b980b/soundfile-0.14.0-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:1e38bac1853412871318e82a1ba69a8be677619b56025bbfcccdb41b6cafe82d", size = 1315963, upload-time = "2026-06-06T08:58:41.716Z" }, - { url = "https://files.pythonhosted.org/packages/d9/34/c9e80783d83eab739a9531fdee03675d53e0bf1b2ccb4bb3af5844675046/soundfile-0.14.0-py2.py3-none-win32.whl", hash = "sha256:0a6ae43c50c71b4e020cc55382925cb89451c1ed1a0c3d0f5d802da269226849", size = 902199, upload-time = "2026-06-06T08:58:43.289Z" }, - { url = "https://files.pythonhosted.org/packages/ed/97/b39c18ac1df45e755ca22b8b00e872929da5d107998a207a5e4ac831bfda/soundfile-0.14.0-py2.py3-none-win_amd64.whl", hash = "sha256:299491d3499460fb1b74bb4bd78b57ffc2d243a5fafa7b6ec1b264875c78453e", size = 1021480, upload-time = "2026-06-06T08:58:45.016Z" }, - { url = "https://files.pythonhosted.org/packages/f4/83/55c65e61cf457805ce2ec157c1c6ae17715d0851aa2374422de0538838ca/soundfile-0.14.0-py2.py3-none-win_arm64.whl", hash = "sha256:e090704718e124e7c844695236f1fce8d18a5e761eaf7c82dfcd124620805f98", size = 888858, upload-time = "2026-06-06T08:58:46.593Z" }, -] - [[package]] name = "sseclient-py" version = "1.9.0" From c8a55030b9783bdb142e43f6efcd88bedb927d8f Mon Sep 17 00:00:00 2001 From: Kev Choi Date: Wed, 30 Sep 2026 18:45:47 -0400 Subject: [PATCH 3/3] feat: add voice task success scorer and skip empty conversations Co-Authored-By: Claude Opus 5.5 --- js/llm.test.ts | 16 +++++++++++++ js/llm.ts | 15 ++++++++++++ js/manifest.ts | 7 ++++++ js/templates.ts | 2 ++ py/autoevals/llm.py | 39 +++++++++++++++++++++++++++++++ py/autoevals/test_llm.py | 6 +++++ templates/voice_task_success.yaml | 16 +++++++++++++ 7 files changed, 101 insertions(+) create mode 100644 templates/voice_task_success.yaml diff --git a/js/llm.test.ts b/js/llm.test.ts index 9b694348..d35945d2 100644 --- a/js/llm.test.ts +++ b/js/llm.test.ts @@ -10,6 +10,7 @@ import { OpenAIClassifier, SpeechClarity, templateUsesThreadVariables, + VoiceTaskSuccess, } from "../js/llm"; import { openaiClassifierShouldEvaluateArithmeticExpressions, @@ -723,3 +724,18 @@ describe("SpeechClarity", () => { expect(score.score).toBeNull(); }); }); + +describe("VoiceTaskSuccess", () => { + test("skips when the conversation is empty", async () => { + const score = await VoiceTaskSuccess({ + output: undefined, + trace: { + async getThread() { + return [{ role: "system", content: "Be helpful." }]; + }, + }, + openAiApiKey: "test-api-key", + }); + expect(score.score).toBeNull(); + }); +}); diff --git a/js/llm.ts b/js/llm.ts index 41268a89..c0c15161 100644 --- a/js/llm.ts +++ b/js/llm.ts @@ -396,6 +396,12 @@ export function LLMClassifierFromTemplate({ } } + const threadCount = + Reflect.get(runtimeArgs, "thread_count") ?? threadVars.thread_count; + if (templateUsesThreadVariables(promptTemplate) && threadCount === 0) { + return { name, score: null }; + } + const prompt = promptTemplate + "\n" + (useCoT ? COT_SUFFIX : NO_COT_SUFFIX); @@ -564,3 +570,12 @@ export const TurnTaking = buildLLMClassifier< { input: { audio?: Audio; [key: string]: unknown } }, unknown >("TurnTaking", "turn_taking"); + +/** + * Test whether a voice agent correctly completed the caller's request, from the conversation + * in the trace, including its instructions, tool calls, and tool results. + */ +export const VoiceTaskSuccess = buildLLMClassifier< + Record, + unknown +>("VoiceTaskSuccess", "voice_task_success"); diff --git a/js/manifest.ts b/js/manifest.ts index c6faf632..53b50246 100644 --- a/js/manifest.ts +++ b/js/manifest.ts @@ -11,6 +11,7 @@ import { Summary, Translation, TurnTaking, + VoiceTaskSuccess, } from "./llm"; import { NumericDiff } from "./number"; import { EmbeddingSimilarity, Levenshtein } from "./string"; @@ -115,6 +116,12 @@ export const Evaluators: { "Test whether an agent takes turns well, without talking over or cutting off the caller, from a recording of the whole conversation (`input.audio`).", template: templates.turn_taking, }, + { + method: VoiceTaskSuccess, + description: + "Test whether a voice agent correctly completed the caller's request, from the conversation in the trace, including its instructions, tool calls, and tool results.", + template: templates.voice_task_success, + }, ], }, { diff --git a/js/templates.ts b/js/templates.ts index e1867094..3afa03bd 100644 --- a/js/templates.ts +++ b/js/templates.ts @@ -12,6 +12,7 @@ import sql from "../templates/sql.yaml"; import summary from "../templates/summary.yaml"; import translation from "../templates/translation.yaml"; import turn_taking from "../templates/turn_taking.yaml"; +import voice_task_success from "../templates/voice_task_success.yaml"; export const modelGradedSpecSchema = z.object({ prompt: z.string(), @@ -37,6 +38,7 @@ const templateStrings = { summary, translation, turn_taking, + voice_task_success, } as const; // eslint-disable-next-line @typescript-eslint/consistent-type-assertions diff --git a/py/autoevals/llm.py b/py/autoevals/llm.py index 04fb4b3f..30b70bee 100644 --- a/py/autoevals/llm.py +++ b/py/autoevals/llm.py @@ -507,6 +507,24 @@ async def _request_args_async(self, output, expected, **kwargs): # Thread vars come first so explicit render args can override. return super()._request_args(output, expected, **thread_vars, **kwargs) + def _run_eval_sync(self, output, expected, **kwargs): + trace = kwargs.get("trace") + if trace is not None and self._template_uses_thread_variables: + # Clear the trace so _request_args doesn't fetch the thread again. + kwargs = {**self._compute_thread_vars_sync(trace), **kwargs, "trace": None} + if self._template_uses_thread_variables and kwargs.get("thread_count") == 0: + return Score(name=self.name, score=None) + return super()._run_eval_sync(output, expected, **kwargs) + + async def _run_eval_async(self, output, expected, **kwargs): + trace = kwargs.get("trace") + if trace is not None and self._template_uses_thread_variables: + # Clear the trace so _request_args doesn't fetch the thread again. + kwargs = {**await self._compute_thread_vars_async(trace), **kwargs, "trace": None} + if self._template_uses_thread_variables and kwargs.get("thread_count") == 0: + return Score(name=self.name, score=None) + return await super()._run_eval_async(output, expected, **kwargs) + @classmethod def from_spec(cls, name: str, spec: ModelGradedSpec, client: Client | None = None, **kwargs): spec_kwargs = {} @@ -930,3 +948,24 @@ class TurnTaking(SpecFileClassifier): """ pass + + +class VoiceTaskSuccess(SpecFileClassifier): + """Rate whether a voice agent correctly completed the caller's request, from the conversation in the trace. + + The conversation comes from `trace.get_thread()`, including the agent's instructions, tool calls, and + tool results when the trace has them. An empty conversation skips the score. + + Example: + ```python + from autoevals import VoiceTaskSuccess + + result = await VoiceTaskSuccess().eval_async(output=None, trace=trace) + print(result.score) # 1 if done correctly, 0.5 if done with minor problems, 0 if not done or wrong + ``` + + Args: + trace: Trace of the voice conversation + """ + + pass diff --git a/py/autoevals/test_llm.py b/py/autoevals/test_llm.py index fc16d28a..20250be2 100644 --- a/py/autoevals/test_llm.py +++ b/py/autoevals/test_llm.py @@ -15,6 +15,7 @@ LLMClassifier, OpenAILLMClassifier, SpeechClarity, + VoiceTaskSuccess, build_classification_tools, ) from autoevals.oai import OpenAIV1Module, get_default_model @@ -816,3 +817,8 @@ def test_speech_clarity_accepts_model_override(): def test_speech_clarity_skips_without_audio(): assert SpeechClarity().eval(input={"text": "hello"}, output=None).score is None + + +def test_voice_task_success_skips_empty_thread(): + trace = _FakeTrace([{"role": "system", "content": "Be helpful."}]) + assert VoiceTaskSuccess().eval(output=None, trace=trace).score is None diff --git a/templates/voice_task_success.yaml b/templates/voice_task_success.yaml new file mode 100644 index 00000000..3ba8ce8d --- /dev/null +++ b/templates/voice_task_success.yaml @@ -0,0 +1,16 @@ +prompt: |- + You are judging a voice agent's conversation with a caller from its transcript. The transcript may include the agent's instructions, tool calls, and tool results. Here is the data: + [BEGIN DATA] + ************ + [Conversation]: {{thread_with_system}} + ************ + [END DATA] + + Judge whether the agent correctly understood and completed what the caller asked for. Check that it followed its instructions, used the right tools with the right arguments, and told the caller only what the tool results support. Saying an action succeeded is not proof that it did. If the request could not or should not be done, a clear refusal or handoff counts as completing it. Weigh correctness above conversational polish, and ignore speech quality and timing. + (A) The agent completed the request correctly, followed its instructions, and everything it told the caller is supported. + (B) The agent completed the request, but with minor problems that did not change the outcome, such as unneeded steps or repeated questions. + (C) The agent did not complete the request, took a wrong or unauthorized action, broke its instructions, or told the caller something false or unsupported. +choice_scores: + "A": 1.0 + "B": 0.5 + "C": 0.0