diff --git a/js/llm.test.ts b/js/llm.test.ts index 5863eecc..d35945d2 100644 --- a/js/llm.test.ts +++ b/js/llm.test.ts @@ -8,7 +8,9 @@ import { buildClassificationTools, LLMClassifierFromTemplate, OpenAIClassifier, + SpeechClarity, templateUsesThreadVariables, + VoiceTaskSuccess, } from "../js/llm"; import { openaiClassifierShouldEvaluateArithmeticExpressions, @@ -662,3 +664,78 @@ Issue Description: {{page_content}} init({ client }); }); }); + +describe("SpeechClarity", () => { + test("sends audio as a file to gemini-3.8-flash through chat completions", async () => { + let body: any; + server.use( + http.post( + "https://api.openai.com/v1/chat/completions", + async ({ request }) => { + body = await request.json(); + return HttpResponse.json({ + choices: [ + { + message: { + role: "assistant", + tool_calls: [ + { + id: "call_test", + type: "function", + function: { + name: "select_choice", + arguments: '{"reasons":"Clear.","choice":"A"}', + }, + }, + ], + }, + }, + ], + }); + }, + ), + ); + + const score = await SpeechClarity({ + input: { + audio: { + data: new Uint8Array([1, 2, 3]), + content_type: "audio/ogg", + }, + }, + output: undefined, + openAiApiKey: "test-api-key", + }); + + expect(score.score).toBe(1); + expect(body.model).toBe("gemini-3.8-flash"); + expect(body.messages[0].content[0].type).toBe("text"); + expect(body.messages[0].content[1].file.file_data).toBe( + "data:audio/ogg;base64,AQID", + ); + }); + + test("skips when there is no audio", async () => { + const score = await SpeechClarity({ + input: { text: "hello" }, + output: undefined, + openAiApiKey: "test-api-key", + }); + expect(score.score).toBeNull(); + }); +}); + +describe("VoiceTaskSuccess", () => { + test("skips when the conversation is empty", async () => { + const score = await VoiceTaskSuccess({ + output: undefined, + trace: { + async getThread() { + return [{ role: "system", content: "Be helpful." }]; + }, + }, + openAiApiKey: "test-api-key", + }); + expect(score.score).toBeNull(); + }); +}); diff --git a/js/llm.ts b/js/llm.ts index 1b7f318e..c0c15161 100644 --- a/js/llm.ts +++ b/js/llm.ts @@ -7,6 +7,7 @@ import { } from "./oai"; import { ModelGradedSpec, templates } from "./templates"; import { + ChatCompletionContentPart, ChatCompletionMessage, ChatCompletionMessageParam, ChatCompletionTool, @@ -62,6 +63,42 @@ function filterSystemMessagesFromThread(thread: unknown[]): unknown[] { }); } +function getPath(args: unknown, path: string): unknown { + let value = args; + for (const key of path.split(".")) { + value = + value && typeof value === "object" && !Array.isArray(value) + ? Reflect.get(value, key) + : undefined; + } + return value; +} + +export type Audio = { data: Uint8Array; content_type: string }; + +function audioPart(audio: unknown): ChatCompletionContentPart.File { + const data = Reflect.get(Object(audio), "data"); + if (!(data instanceof Uint8Array)) { + throw new TypeError( + "Audio must be an object with `data` bytes and a `content_type`", + ); + } + const contentType = String(Reflect.get(Object(audio), "content_type") ?? "") + .split(";")[0] + .trim() + .toLowerCase(); + if (!contentType.startsWith("audio/")) { + throw new Error( + `Audio must have an audio/* content type, got "${contentType}"`, + ); + } + const base64 = Buffer.from(data).toString("base64"); + return { + type: "file", + file: { file_data: `data:${contentType};base64,${base64}` }, + }; +} + const NO_COT_SUFFIX = "Answer the question by calling `select_choice` with a single choice from {{__choices}}."; @@ -142,6 +179,7 @@ export type OpenAIClassifierArgs = { messages: ChatCompletionMessageParam[]; choiceScores: Record; classificationTools: ChatCompletionTool[]; + audioFile?: ChatCompletionContentPart.File; cache?: ChatCache; } & LLMArgs & RenderArgs; @@ -174,6 +212,7 @@ export async function OpenAIClassifier( reasoningEnabled, reasoningBudget, useResponsesApi, + audioFile, cache, ...remainingRenderArgs } = remaining; @@ -212,6 +251,13 @@ export async function OpenAIClassifier( }; const messages = renderMessages(messagesArg, renderArgs); + if (audioFile) { + const last = messages[messages.length - 1]; + messages[messages.length - 1] = { + role: "user", + content: [{ type: "text", text: String(last.content) }, audioFile], + }; + } const resp = await cachedChatCompletion( { @@ -306,6 +352,7 @@ export function LLMClassifierFromTemplate({ reasoningEnabled, reasoningBudget, useResponsesApi, + audio, }: { name: string; promptTemplate: string; @@ -318,11 +365,17 @@ export function LLMClassifierFromTemplate({ reasoningEnabled?: boolean; reasoningBudget?: number; useResponsesApi?: boolean; + audio?: string; }): Scorer> { const choiceStrings = Object.keys(choiceScores); const ret = async ( runtimeArgs: ScorerArgs>, ) => { + const audioValue = audio ? getPath(runtimeArgs, audio) : undefined; + if (audio && audioValue == null) { + return { name, score: null }; + } + const useCoT = runtimeArgs.useCoT ?? useCoTArg ?? true; // Use runtime model > template model > configured default model const model = runtimeArgs.model ?? modelArg ?? getDefaultModel(); @@ -343,6 +396,12 @@ export function LLMClassifierFromTemplate({ } } + const threadCount = + Reflect.get(runtimeArgs, "thread_count") ?? threadVars.thread_count; + if (templateUsesThreadVariables(promptTemplate) && threadCount === 0) { + return { name, score: null }; + } + const prompt = promptTemplate + "\n" + (useCoT ? COT_SUFFIX : NO_COT_SUFFIX); @@ -374,6 +433,7 @@ export function LLMClassifierFromTemplate({ // Since the logic is a bit funky for computing this, include // it at the end to prevent overrides useCoT, + audioFile: audio ? audioPart(audioValue) : undefined, }; return await OpenAIClassifier(classifierArgs); @@ -398,6 +458,7 @@ export function LLMClassifierFromSpec( useCoT: spec.use_cot, temperature: spec.temperature, maxTokens: spec.max_tokens, + audio: spec.audio, }); } @@ -409,10 +470,10 @@ export function LLMClassifierFromSpecFile( return LLMClassifierFromSpec(name, doc); } -function buildLLMClassifier( +function buildLLMClassifier( name: string, templateName: keyof typeof templates, -): ScorerWithPartial> { +): ScorerWithPartial> { if (!(templateName in templates)) { throw new Error(`Model template ${name} not found`); } @@ -492,3 +553,29 @@ export const Translation = buildLLMClassifier<{ language: string; input: string; }>("Translation", "translation"); + +/** + * Test how clearly an agent speaks, from a recording of the whole conversation (`input.audio`). + */ +export const SpeechClarity = buildLLMClassifier< + { input: { audio?: Audio; [key: string]: unknown } }, + unknown +>("SpeechClarity", "speech_clarity"); + +/** + * Test whether an agent takes turns without talking over or cutting off the caller, + * from a recording of the whole conversation (`input.audio`). + */ +export const TurnTaking = buildLLMClassifier< + { input: { audio?: Audio; [key: string]: unknown } }, + unknown +>("TurnTaking", "turn_taking"); + +/** + * Test whether a voice agent correctly completed the caller's request, from the conversation + * in the trace, including its instructions, tool calls, and tool results. + */ +export const VoiceTaskSuccess = buildLLMClassifier< + Record, + unknown +>("VoiceTaskSuccess", "voice_task_success"); diff --git a/js/manifest.ts b/js/manifest.ts index c21d3703..53b50246 100644 --- a/js/manifest.ts +++ b/js/manifest.ts @@ -6,9 +6,12 @@ import { Humor, Possible, Security, + SpeechClarity, Sql, Summary, Translation, + TurnTaking, + VoiceTaskSuccess, } from "./llm"; import { NumericDiff } from "./number"; import { EmbeddingSimilarity, Levenshtein } from "./string"; @@ -82,6 +85,12 @@ export const Evaluators: { description: "Test whether an output is malicious.", template: templates.security, }, + { + method: SpeechClarity, + description: + "Test how clearly an agent's speech can be understood, from a recording of the whole conversation (`input.audio`).", + template: templates.speech_clarity, + }, { method: Sql, description: @@ -101,6 +110,18 @@ export const Evaluators: { template: templates.translation, requiresExtraParams: true, }, + { + method: TurnTaking, + description: + "Test whether an agent takes turns well, without talking over or cutting off the caller, from a recording of the whole conversation (`input.audio`).", + template: templates.turn_taking, + }, + { + method: VoiceTaskSuccess, + description: + "Test whether a voice agent correctly completed the caller's request, from the conversation in the trace, including its instructions, tool calls, and tool results.", + template: templates.voice_task_success, + }, ], }, { diff --git a/js/templates.ts b/js/templates.ts index 69f4637c..3afa03bd 100644 --- a/js/templates.ts +++ b/js/templates.ts @@ -7,9 +7,12 @@ import factuality from "../templates/factuality.yaml"; import humor from "../templates/humor.yaml"; import possible from "../templates/possible.yaml"; import security from "../templates/security.yaml"; +import speech_clarity from "../templates/speech_clarity.yaml"; import sql from "../templates/sql.yaml"; import summary from "../templates/summary.yaml"; import translation from "../templates/translation.yaml"; +import turn_taking from "../templates/turn_taking.yaml"; +import voice_task_success from "../templates/voice_task_success.yaml"; export const modelGradedSpecSchema = z.object({ prompt: z.string(), @@ -18,6 +21,7 @@ export const modelGradedSpecSchema = z.object({ use_cot: z.boolean().optional(), temperature: z.number().optional(), max_tokens: z.number().optional(), + audio: z.string().optional(), }); export type ModelGradedSpec = z.infer; @@ -29,9 +33,12 @@ const templateStrings = { humor, possible, security, + speech_clarity, sql, summary, translation, + turn_taking, + voice_task_success, } as const; // eslint-disable-next-line @typescript-eslint/consistent-type-assertions diff --git a/py/autoevals/llm.py b/py/autoevals/llm.py index b0093cb4..30b70bee 100644 --- a/py/autoevals/llm.py +++ b/py/autoevals/llm.py @@ -46,6 +46,7 @@ """ import asyncio +import base64 import inspect import json import os @@ -133,6 +134,23 @@ def build_classification_tools(useCoT, choice_strings): ] +def get_path(args, path): + value = args + for key in path.split("."): + value = value.get(key) if isinstance(value, dict) else None + return value + + +def audio_part(audio): + if not isinstance(audio, dict) or not isinstance(audio.get("data"), (bytes, bytearray)): + raise TypeError("Audio must be a dict with `data` bytes and a `content_type`") + content_type = str(audio.get("content_type", "")).split(";")[0].strip().lower() + if not content_type.startswith("audio/"): + raise ValueError(f"Audio must have an audio/* content type, got {content_type!r}") + data = base64.b64encode(audio["data"]).decode() + return {"type": "file", "file": {"file_data": f"data:{content_type};base64,{data}"}} + + class OpenAIScorer(ScorerWithPartial): def __init__( self, @@ -181,6 +199,7 @@ def __init__( reasoning_enabled=None, reasoning_budget=None, use_responses_api=None, + audio=None, engine=None, api_key=None, base_url=None, @@ -198,6 +217,7 @@ def __init__( self.model = model self.engine = engine self.messages = messages + self.audio = audio if max_tokens is not None: self.extra_args["max_tokens"] = max(max_tokens, 5) @@ -234,13 +254,21 @@ def _build_args(self, output, expected, **kwargs): def _render_messages(self, **kwargs): kwargs.update(self.render_args) - return [ + messages = [ { **m, "content": chevron.render(m["content"].strip(), kwargs, warn=True), } for m in self.messages ] + if self.audio: + text = messages[-1]["content"] + messages[-1]["content"] = [{"type": "text", "text": text}, audio_part(get_path(kwargs, self.audio))] + return messages + + def _missing_audio(self, output, expected, **kwargs): + args = {"output": output, "expected": expected, **kwargs, **self.render_args} + return self.audio and get_path(args, self.audio) is None def _request_args(self, output, expected, **kwargs): ret = { @@ -284,11 +312,15 @@ def _postprocess_response(self, resp): raise ValueError("Empty response from OpenAI") async def _run_eval_async(self, output, expected, **kwargs): + if self._missing_audio(output, expected, **kwargs): + return Score(name=self.name, score=None) return self._postprocess_response( await arun_cached_request(**(await self._request_args_async(output, expected, **kwargs))) ) def _run_eval_sync(self, output, expected, **kwargs): + if self._missing_audio(output, expected, **kwargs): + return Score(name=self.name, score=None) return self._postprocess_response(run_cached_request(**self._request_args(output, expected, **kwargs))) @@ -301,6 +333,7 @@ class ModelGradedSpec: use_cot: bool | None = None temperature: float | None = None max_tokens: int | None = None + audio: str | None = None class LLMClassifier(OpenAILLMClassifier): @@ -344,6 +377,7 @@ class LLMClassifier(OpenAILLMClassifier): reasoning_effort: Controls reasoning depth for o-series models (e.g., "low", "medium", "high"). reasoning_enabled: Enable extended thinking for supported models (e.g., Claude). Defaults to None. reasoning_budget: Token allocation for model's internal reasoning. Defaults to None. + audio: Path in the arguments (e.g. `input.audio`) to a dict with `data` bytes and an audio `content_type`, such as `audio/ogg`. Missing audio skips the score. engine: Deprecated by OpenAI. Use model instead. api_key: Deprecated. Use client instead. base_url: Deprecated. Use client instead. @@ -371,6 +405,7 @@ def __init__( reasoning_enabled=None, reasoning_budget=None, use_responses_api=None, + audio=None, engine=None, api_key=None, base_url=None, @@ -403,6 +438,7 @@ def __init__( reasoning_enabled=reasoning_enabled, reasoning_budget=reasoning_budget, use_responses_api=use_responses_api, + audio=audio, engine=engine, api_key=api_key, base_url=base_url, @@ -471,6 +507,24 @@ async def _request_args_async(self, output, expected, **kwargs): # Thread vars come first so explicit render args can override. return super()._request_args(output, expected, **thread_vars, **kwargs) + def _run_eval_sync(self, output, expected, **kwargs): + trace = kwargs.get("trace") + if trace is not None and self._template_uses_thread_variables: + # Clear the trace so _request_args doesn't fetch the thread again. + kwargs = {**self._compute_thread_vars_sync(trace), **kwargs, "trace": None} + if self._template_uses_thread_variables and kwargs.get("thread_count") == 0: + return Score(name=self.name, score=None) + return super()._run_eval_sync(output, expected, **kwargs) + + async def _run_eval_async(self, output, expected, **kwargs): + trace = kwargs.get("trace") + if trace is not None and self._template_uses_thread_variables: + # Clear the trace so _request_args doesn't fetch the thread again. + kwargs = {**await self._compute_thread_vars_async(trace), **kwargs, "trace": None} + if self._template_uses_thread_variables and kwargs.get("thread_count") == 0: + return Score(name=self.name, score=None) + return await super()._run_eval_async(output, expected, **kwargs) + @classmethod def from_spec(cls, name: str, spec: ModelGradedSpec, client: Client | None = None, **kwargs): spec_kwargs = {} @@ -484,8 +538,11 @@ def from_spec(cls, name: str, spec: ModelGradedSpec, client: Client | None = Non spec_kwargs["temperature"] = spec.temperature if spec.max_tokens is not None: spec_kwargs["max_tokens"] = spec.max_tokens + if spec.audio is not None: + spec_kwargs["audio"] = spec.audio # kwargs can override spec values - return cls(name, spec.prompt, spec.choice_scores, client=client, **spec_kwargs, **kwargs) + spec_kwargs.update(kwargs) + return cls(name, spec.prompt, spec.choice_scores, client=client, **spec_kwargs) @classmethod def from_spec_file(cls, name: str, path: str, client: Client | None = None, **kwargs): @@ -851,3 +908,64 @@ class Translation(SpecFileClassifier): """ pass + + +class SpeechClarity(SpecFileClassifier): + """Rate how clearly an agent speaks in a recording of the whole conversation. + + Example: + ```python + from pathlib import Path + from autoevals import SpeechClarity + + audio = {"data": Path("call.ogg").read_bytes(), "content_type": "audio/ogg"} + result = SpeechClarity().eval(input={"audio": audio}, output=None) + print(result.score) # 1 if clear, 0.5 if flawed but easy to follow, 0 if words are lost + ``` + + Args: + input: Dict whose `audio` is the recording, as `{"data": bytes, "content_type": "audio/ogg"}` + """ + + pass + + +class TurnTaking(SpecFileClassifier): + """Rate whether an agent takes turns without talking over or cutting off the caller. + + Example: + ```python + from pathlib import Path + from autoevals import TurnTaking + + audio = {"data": Path("call.ogg").read_bytes(), "content_type": "audio/ogg"} + result = TurnTaking().eval(input={"audio": audio}, output=None) + print(result.score) # 1 if smooth, 0.5 if brief overlaps, 0 if caller words are lost + ``` + + Args: + input: Dict whose `audio` is the recording, as `{"data": bytes, "content_type": "audio/ogg"}` + """ + + pass + + +class VoiceTaskSuccess(SpecFileClassifier): + """Rate whether a voice agent correctly completed the caller's request, from the conversation in the trace. + + The conversation comes from `trace.get_thread()`, including the agent's instructions, tool calls, and + tool results when the trace has them. An empty conversation skips the score. + + Example: + ```python + from autoevals import VoiceTaskSuccess + + result = await VoiceTaskSuccess().eval_async(output=None, trace=trace) + print(result.score) # 1 if done correctly, 0.5 if done with minor problems, 0 if not done or wrong + ``` + + Args: + trace: Trace of the voice conversation + """ + + pass diff --git a/py/autoevals/test_llm.py b/py/autoevals/test_llm.py index 8e987efa..20250be2 100644 --- a/py/autoevals/test_llm.py +++ b/py/autoevals/test_llm.py @@ -9,7 +9,15 @@ from pydantic import BaseModel from autoevals import init -from autoevals.llm import Battle, Factuality, LLMClassifier, OpenAILLMClassifier, build_classification_tools +from autoevals.llm import ( + Battle, + Factuality, + LLMClassifier, + OpenAILLMClassifier, + SpeechClarity, + VoiceTaskSuccess, + build_classification_tools, +) from autoevals.oai import OpenAIV1Module, get_default_model from autoevals.thread_utils import compute_thread_template_vars, template_uses_thread_variables @@ -767,3 +775,50 @@ async def get_thread(self): classifier.eval(output="x", expected="y", trace=trace) assert trace.calls == 0 + + +@respx.mock +def test_speech_clarity_sends_audio_as_a_file_to_chat_completions(): + route = respx.route(method="POST", path__regex=r".*/chat/completions$").respond( + json={ + "choices": [ + { + "message": { + "role": "assistant", + "tool_calls": [ + { + "id": "call_test", + "type": "function", + "function": { + "name": "select_choice", + "arguments": '{"reasons":"Clear.","choice":"A"}', + }, + } + ], + }, + } + ], + } + ) + result = SpeechClarity(base_url="https://api.openai.com/v1/", api_key="test").eval( + input={"audio": {"data": b"\x01\x02\x03", "content_type": "audio/ogg"}}, output=None + ) + + assert result.score == 1 + body = json.loads(route.calls.last.request.content) + assert body["model"] == "gemini-3.8-flash" + assert body["messages"][0]["content"][0]["type"] == "text" + assert body["messages"][0]["content"][1]["file"]["file_data"] == "data:audio/ogg;base64,AQID" + + +def test_speech_clarity_accepts_model_override(): + assert SpeechClarity(model="gpt-4o-audio-preview").model == "gpt-4o-audio-preview" + + +def test_speech_clarity_skips_without_audio(): + assert SpeechClarity().eval(input={"text": "hello"}, output=None).score is None + + +def test_voice_task_success_skips_empty_thread(): + trace = _FakeTrace([{"role": "system", "content": "Be helpful."}]) + assert VoiceTaskSuccess().eval(output=None, trace=trace).score is None diff --git a/templates/speech_clarity.yaml b/templates/speech_clarity.yaml new file mode 100644 index 00000000..9f9e23fd --- /dev/null +++ b/templates/speech_clarity.yaml @@ -0,0 +1,12 @@ +prompt: |- + Listen to the attached recording of a conversation between an agent and a caller. Judge only the agent's speech. + How easily can a listener on a phone call understand every word on first hearing? Consider garbled or cut-off words, distortion, dropouts, echo, and background noise. Do not fill in unclear words from context, and ignore accent unless it makes words hard to understand. + (A) Words are clear and understandable without effort. + (B) Some flaws are audible, but the message remains easy to understand. + (C) Important words are unclear or lost. +choice_scores: + "A": 1.0 + "B": 0.5 + "C": 0.0 +model: gemini-3.8-flash +audio: input.audio diff --git a/templates/turn_taking.yaml b/templates/turn_taking.yaml new file mode 100644 index 00000000..3471e9f6 --- /dev/null +++ b/templates/turn_taking.yaml @@ -0,0 +1,12 @@ +prompt: |- + Listen to the attached recording of a conversation between an agent and a caller. Judge only how the agent handles turn-taking. + Consider whether the agent talks over the caller, cuts the caller off before they finish, or keeps talking when the caller interrupts. Brief backchannels such as "mm-hm" are fine unless they cover the caller's words. Ignore how long the agent takes to respond. + (A) The agent takes turns smoothly: it lets the caller finish and stops promptly when interrupted. + (B) There are brief overlaps or a slow yield, but no caller words are lost. + (C) The agent talks over the caller, ignores an interruption, or cuts the caller off, so caller words are lost. +choice_scores: + "A": 1.0 + "B": 0.5 + "C": 0.0 +model: gemini-3.8-flash +audio: input.audio diff --git a/templates/voice_task_success.yaml b/templates/voice_task_success.yaml new file mode 100644 index 00000000..3ba8ce8d --- /dev/null +++ b/templates/voice_task_success.yaml @@ -0,0 +1,16 @@ +prompt: |- + You are judging a voice agent's conversation with a caller from its transcript. The transcript may include the agent's instructions, tool calls, and tool results. Here is the data: + [BEGIN DATA] + ************ + [Conversation]: {{thread_with_system}} + ************ + [END DATA] + + Judge whether the agent correctly understood and completed what the caller asked for. Check that it followed its instructions, used the right tools with the right arguments, and told the caller only what the tool results support. Saying an action succeeded is not proof that it did. If the request could not or should not be done, a clear refusal or handoff counts as completing it. Weigh correctness above conversational polish, and ignore speech quality and timing. + (A) The agent completed the request correctly, followed its instructions, and everything it told the caller is supported. + (B) The agent completed the request, but with minor problems that did not change the outcome, such as unneeded steps or repeated questions. + (C) The agent did not complete the request, took a wrong or unauthorized action, broke its instructions, or told the caller something false or unsupported. +choice_scores: + "A": 1.0 + "B": 0.5 + "C": 0.0