From 985e9f086c826da38d26f525730d508dc0685ba9 Mon Sep 17 00:00:00 2001 From: Kev Choi Date: Fri, 2 Oct 2026 19:31:56 -0400 Subject: [PATCH 1/3] feat: add voice latency and interruptions scorers Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 1 + SCORERS.md | 86 +++++++++++++++++ js/index.ts | 1 + js/voice.test.ts | 122 ++++++++++++++++++++++++ js/voice.ts | 185 +++++++++++++++++++++++++++++++++++++ py/autoevals/__init__.py | 1 + py/autoevals/test_voice.py | 92 ++++++++++++++++++ py/autoevals/voice.py | 165 +++++++++++++++++++++++++++++++++ 8 files changed, 653 insertions(+) create mode 100644 js/voice.test.ts create mode 100644 js/voice.ts create mode 100644 py/autoevals/test_voice.py create mode 100644 py/autoevals/voice.py diff --git a/AGENTS.md b/AGENTS.md index 1d1c3a7f..36ad4b96 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,6 +58,7 @@ The library maintains parallel implementations in TypeScript (`js/`) and Python - `ragas.ts` / `ragas.py` - RAG evaluation metrics (ContextRelevancy, Faithfulness, AnswerRelevancy, etc.) - `string.ts` / `string.py` - Text similarity (Levenshtein, EmbeddingSimilarity) - `json.ts` / `json.py` - JSON validation and diff +- `voice.ts` / `voice.py` - Voice call timing (VoiceLatency, VoiceInterruptions) - `oai.ts` / `oai.py` - OpenAI client wrapper with caching - `score.ts` / `score.py` - Core Score type and Scorer base class diff --git a/SCORERS.md b/SCORERS.md index e846ab44..22780538 100644 --- a/SCORERS.md +++ b/SCORERS.md @@ -9,6 +9,7 @@ Complete reference for all scorers available in Autoevals, including parameters, - [Heuristic scorers](#heuristic-scorers) - [JSON scorers](#json-scorers) - [List scorers](#list-scorers) +- [Voice scorers](#voice-scorers) --- @@ -590,6 +591,91 @@ result = scorer.eval( --- +## Voice scorers + +Deterministic scorers for the timing of a voice call. They take `utterances`, a list of `{ speaker, start_unix_ms, end_unix_ms, interrupted }` objects, where `speaker` is `"user"` or `"agent"`, times are Unix epoch milliseconds, and `interrupted` (optional) means the user cut the agent off. Build the list from your own traces. + +Both scorers group the user's speech into turns the same way: + +- Consecutive user utterances merge into one turn until the agent starts speaking. +- User speech wholly inside the agent's, like "mm-hmm", isn't a turn. +- The agent talks over a turn when it starts speaking during the turn and overlaps it by more than `minOverlapMs` (default 300). +- The agent's reply to a turn is the first agent utterance that starts after the turn starts and runs past its end. + +Utterances with a missing or invalid time, or another speaker, are skipped and counted in `skipped`. + +Timestamps should mark audible speech, on one clock: the user's end is when they stopped speaking (not when the transcript was finalized), and the agent's start is when its first audio played (not when text-to-speech was requested). Times taken on the server leave out network delay, so callers hear slightly longer gaps. + +Limits: + +- If the agent cuts in while the user pauses mid-sentence and the user carries on, it counts as a fast reply plus a barge-in, not as talking over the user. +- If the agent says nothing and the user speaks again ("hello?"), both utterances merge into one turn. +- Drop intentional agent backchannels when you build the list, or they count as talking over the user. +- The defaults (1500 ms, 300 ms) follow [Hamming's published bands](https://hamming.ai/resources/conversational-flow-measurement-voice-agents), not a standard. + +### VoiceLatency + +Measures how quickly the agent replies after the user finishes a turn. + +**Parameters:** + +- `utterances` (Utterance[], required): The call's utterances +- `maxGapMs` / `max_gap_ms` (number, optional): Longest acceptable gap before a reply (default: 1500) +- `minOverlapMs` / `min_overlap_ms` (number, optional): Overlap needed to count as talking over the user (default: 300) + +**Score Range:** 0-1, the share of replies that start within `maxGapMs`. `null` when the call has no replies. + +Turns the agent talked over aren't counted. A reply that starts slightly before the user stops counts as a gap of 0. Metadata has `replies`, `unanswered_turns`, `p50_ms`, `p95_ms`, `gaps_ms` and `skipped`; pool `gaps_ms` to get percentiles across a dataset. + +**Example:** + +```typescript +import { VoiceLatency } from "autoevals"; + +const result = await VoiceLatency({ + output: null, + utterances: [ + { speaker: "user", start_unix_ms: 0, end_unix_ms: 2000 }, + { speaker: "agent", start_unix_ms: 2800, end_unix_ms: 5000 }, + { speaker: "user", start_unix_ms: 6000, end_unix_ms: 7000 }, + { speaker: "agent", start_unix_ms: 10000, end_unix_ms: 11000 }, + ], +}); +// Score: 0.5 (replies after 0.8 s and 3 s) +``` + +### VoiceInterruptions + +Measures how often the agent talks over the user. + +**Parameters:** + +- `utterances` (Utterance[], required): The call's utterances +- `minOverlapMs` / `min_overlap_ms` (number, optional): Overlap needed to count as talking over the user (default: 300) + +**Score Range:** 0-1, the share of user turns the agent didn't talk over. `null` unless the call has timed user and agent utterances. + +Metadata has `user_turns`, `talk_overs`, `skipped`, and `barge_ins`: agent utterances marked `interrupted`, where the user cut the agent off. Barge-ins don't affect the score. + +**Example:** + +```python +from autoevals import VoiceInterruptions + +result = VoiceInterruptions().eval( + output=None, + utterances=[ + {"speaker": "user", "start_unix_ms": 0, "end_unix_ms": 3000}, + {"speaker": "agent", "start_unix_ms": 2000, "end_unix_ms": 4000}, + {"speaker": "user", "start_unix_ms": 5000, "end_unix_ms": 6000}, + {"speaker": "agent", "start_unix_ms": 6500, "end_unix_ms": 8000, "interrupted": False}, + ], +) +# Score: 0.5 (the agent talked over the first user turn) +``` + +--- + ## Custom scorers You can create custom scorers for domain-specific evaluation needs. See: diff --git a/js/index.ts b/js/index.ts index 40a2a541..b83ce6ed 100644 --- a/js/index.ts +++ b/js/index.ts @@ -39,6 +39,7 @@ export * from "./json"; export * from "./templates"; export * from "./ragas"; export * from "./value"; +export * from "./voice"; export { Evaluators } from "./manifest"; export { makePartial, ScorerWithPartial } from "./partial"; export { diff --git a/js/voice.test.ts b/js/voice.test.ts new file mode 100644 index 00000000..f2ff1c0b --- /dev/null +++ b/js/voice.test.ts @@ -0,0 +1,122 @@ +import { expect, test } from "vitest"; +import { type Utterance, VoiceInterruptions, VoiceLatency } from "./voice"; + +const user = (start: number, end: number): Utterance => ({ + speaker: "user", + start_unix_ms: start, + end_unix_ms: end, +}); +const agent = ( + start: number, + end: number, + interrupted?: boolean, +): Utterance => ({ + speaker: "agent", + start_unix_ms: start, + end_unix_ms: end, + interrupted, +}); + +const latency = (utterances: Utterance[]) => + VoiceLatency({ output: null, utterances }); +const interruptions = (utterances: Utterance[]) => + VoiceInterruptions({ output: null, utterances }); + +test("VoiceLatency scores the share of fast replies", async () => { + const result = await latency([ + user(0, 2000), + agent(2800, 5000), + user(6000, 7000), + agent(10000, 11000), + ]); + expect(result.score).toBe(0.5); + expect(result.metadata).toMatchObject({ + replies: 2, + unanswered_turns: 0, + p50_ms: 800, + p95_ms: 3000, + gaps_ms: [800, 3000], + }); +}); + +test("VoiceLatency leaves out turns the agent talked over", async () => { + const result = await latency([ + user(0, 3000), + agent(2000, 4000), + user(5000, 6000), + agent(6500, 8000), + ]); + expect(result.score).toBe(1); + expect(result.metadata?.gaps_ms).toEqual([500]); +}); + +test("VoiceLatency counts a slightly early reply as instant", async () => { + const result = await latency([user(0, 3000), agent(2800, 5000)]); + expect(result.metadata?.gaps_ms).toEqual([0]); +}); + +test("VoiceLatency doesn't take a short cut-in as the reply", async () => { + const result = await latency([ + user(0, 3000), + agent(2000, 2200), + agent(3500, 5000), + ]); + expect(result.metadata?.gaps_ms).toEqual([500]); +}); + +test("voice scorers merge user fragments and skip backchannels", async () => { + const fragmented = [user(0, 1000), user(1200, 3000), agent(3500, 5000)]; + expect((await latency(fragmented)).metadata?.gaps_ms).toEqual([500]); + expect((await interruptions(fragmented)).metadata?.user_turns).toBe(1); + + const backchannel = [ + user(0, 1000), + agent(1500, 6000), + user(3000, 3300), + agent(6200, 8000), + ]; + expect((await latency(backchannel)).metadata?.gaps_ms).toEqual([500]); + expect((await interruptions(backchannel)).metadata?.user_turns).toBe(1); +}); + +test("VoiceInterruptions scores talk-over, not barge-ins", async () => { + const result = await interruptions([ + user(0, 3000), + agent(2000, 4000), + user(5000, 6000), + agent(6500, 8000, false), + ]); + expect(result.score).toBe(0.5); + expect(result.metadata).toMatchObject({ + talk_overs: 1, + user_turns: 2, + barge_ins: 0, + }); + + const bargeIn = await interruptions([agent(0, 3000, true), user(2000, 4000)]); + expect(bargeIn.score).toBe(1); + expect(bargeIn.metadata?.barge_ins).toBe(1); +}); + +test("VoiceInterruptions ignores overlaps under minOverlapMs", async () => { + const utterances = [user(0, 3000), agent(2900, 4000)]; + expect((await interruptions(utterances)).score).toBe(1); + expect( + (await VoiceInterruptions({ output: null, utterances, minOverlapMs: 50 })) + .score, + ).toBe(0); +}); + +test("voice scorers skip invalid utterances and return null without data", async () => { + const utterances = [ + { speaker: "user" }, + { speaker: "agent", start_unix_ms: NaN, end_unix_ms: 1000 }, + { speaker: "agent", start_unix_ms: 2000, end_unix_ms: 1000 }, + { speaker: "assistant", start_unix_ms: 0, end_unix_ms: 1000 }, + ] as Utterance[]; + const result = await latency(utterances); + expect(result.score).toBeNull(); + expect(result.metadata?.skipped).toBe(4); + expect((await interruptions(utterances)).score).toBeNull(); + expect((await interruptions([user(0, 1000)])).score).toBeNull(); +}); diff --git a/js/voice.ts b/js/voice.ts new file mode 100644 index 00000000..8153a9f7 --- /dev/null +++ b/js/voice.ts @@ -0,0 +1,185 @@ +import { makePartial, ScorerWithPartial } from "./partial"; + +/** + * One stretch of speech in a voice call. Times are Unix epoch milliseconds. + * These are the timing fields of `Utterance` in Braintrust's voice trace reader. + */ +export interface Utterance { + speaker: "user" | "agent"; + start_unix_ms?: number; + end_unix_ms?: number; + /** The user cut the agent off, as reported by the voice framework. */ + interrupted?: boolean; +} + +type TimedUtterance = Utterance & { + start_unix_ms: number; + end_unix_ms: number; +}; + +type Turn = { + start: number; + end: number; + talkOver: boolean; + /** Milliseconds from the end of the turn to the agent's reply, or null without a reply. */ + gapMs: number | null; +}; + +function isTimed(u: Utterance): u is TimedUtterance { + return ( + (u.speaker === "user" || u.speaker === "agent") && + Number.isFinite(u.start_unix_ms) && + Number.isFinite(u.end_unix_ms) && + u.end_unix_ms! >= u.start_unix_ms! + ); +} + +/** + * Groups the user's speech into turns and finds the agent's reply to each. + * Both scorers use this so they count turns the same way. + */ +function userTurns( + utterances: Utterance[], + minOverlapMs: number, +): { turns: Turn[]; agents: number; skipped: number } { + const timed = utterances + .filter(isTimed) + .sort((a, b) => a.start_unix_ms - b.start_unix_ms); + const agents = timed.filter((u) => u.speaker === "agent"); + + const turns: Turn[] = []; + for (const u of timed) { + if (u.speaker !== "user") { + continue; + } + // User speech wholly inside the agent's, like "mm-hmm", isn't a turn. + if ( + agents.some( + (a) => + a.start_unix_ms <= u.start_unix_ms && u.end_unix_ms <= a.end_unix_ms, + ) + ) { + continue; + } + const last = turns[turns.length - 1]; + const agentStartedBetween = + last && + agents.some( + (a) => + a.start_unix_ms > last.start && a.start_unix_ms < u.start_unix_ms, + ); + if (last && !agentStartedBetween) { + last.end = Math.max(last.end, u.end_unix_ms); + } else { + turns.push({ + start: u.start_unix_ms, + end: u.end_unix_ms, + talkOver: false, + gapMs: null, + }); + } + } + + turns.forEach((turn, i) => { + const nextStart = i + 1 < turns.length ? turns[i + 1].start : Infinity; + turn.talkOver = agents.some( + (a) => + a.start_unix_ms > turn.start && + a.start_unix_ms < turn.end && + Math.min(a.end_unix_ms, turn.end) - a.start_unix_ms > minOverlapMs, + ); + // A reply runs past the end of the turn, so a short cut-in inside it isn't one. + const reply = agents.find( + (a) => + a.start_unix_ms > turn.start && + a.start_unix_ms < nextStart && + a.end_unix_ms >= turn.end, + ); + turn.gapMs = reply ? Math.max(0, reply.start_unix_ms - turn.end) : null; + }); + + return { + turns, + agents: agents.length, + skipped: utterances.length - timed.length, + }; +} + +function percentile(values: number[], p: number): number | null { + if (values.length === 0) { + return null; + } + const sorted = [...values].sort((a, b) => a - b); + return sorted[ + Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1) + ]; +} + +type VoiceLatencyArgs = { + utterances?: Utterance[]; + maxGapMs?: number; + minOverlapMs?: number; +}; + +/** + * The share of agent replies that start within `maxGapMs` of the user + * finishing a turn. Turns the agent talked over aren't counted. Returns a + * null score when there are no replies. + */ +export const VoiceLatency: ScorerWithPartial = + makePartial( + async ({ utterances = [], maxGapMs = 1500, minOverlapMs = 300 }) => { + const { turns, skipped } = userTurns(utterances, minOverlapMs); + const gaps = turns + .filter((t) => !t.talkOver && t.gapMs !== null) + .map((t) => t.gapMs!); + return { + name: "VoiceLatency", + score: gaps.length + ? gaps.filter((gap) => gap <= maxGapMs).length / gaps.length + : null, + metadata: { + replies: gaps.length, + unanswered_turns: turns.filter((t) => t.gapMs === null).length, + max_gap_ms: maxGapMs, + p50_ms: percentile(gaps, 50), + p95_ms: percentile(gaps, 95), + gaps_ms: gaps, + skipped, + }, + }; + }, + "VoiceLatency", + ); + +type VoiceInterruptionsArgs = { + utterances?: Utterance[]; + minOverlapMs?: number; +}; + +/** + * The share of user turns the agent let finish without talking over them. + * The agent talks over a turn when it starts speaking during the turn and + * overlaps it by more than `minOverlapMs`. Returns a null score unless the + * call has timed user and agent utterances. + */ +export const VoiceInterruptions: ScorerWithPartial< + unknown, + VoiceInterruptionsArgs +> = makePartial(async ({ utterances = [], minOverlapMs = 300 }) => { + const { turns, agents, skipped } = userTurns(utterances, minOverlapMs); + const talkOvers = turns.filter((t) => t.talkOver).length; + return { + name: "VoiceInterruptions", + score: turns.length && agents ? 1 - talkOvers / turns.length : null, + metadata: { + user_turns: turns.length, + talk_overs: talkOvers, + min_overlap_ms: minOverlapMs, + barge_ins: utterances.filter( + (u) => u.speaker === "agent" && u.interrupted === true, + ).length, + skipped, + }, + }; +}, "VoiceInterruptions"); diff --git a/py/autoevals/__init__.py b/py/autoevals/__init__.py index 0a5841ad..0cee7bae 100644 --- a/py/autoevals/__init__.py +++ b/py/autoevals/__init__.py @@ -137,3 +137,4 @@ async def evaluate_qa(): from .string import * from .thread_utils import * from .value import ExactMatch +from .voice import * diff --git a/py/autoevals/test_voice.py b/py/autoevals/test_voice.py new file mode 100644 index 00000000..012f6ccf --- /dev/null +++ b/py/autoevals/test_voice.py @@ -0,0 +1,92 @@ +import asyncio + +from autoevals import VoiceInterruptions, VoiceLatency + + +def user(start, end): + return {"speaker": "user", "start_unix_ms": start, "end_unix_ms": end} + + +def agent(start, end, interrupted=None): + return {"speaker": "agent", "start_unix_ms": start, "end_unix_ms": end, "interrupted": interrupted} + + +def latency(utterances): + return VoiceLatency().eval(output=None, utterances=utterances) + + +def interruptions(utterances): + return VoiceInterruptions().eval(output=None, utterances=utterances) + + +def test_latency_scores_share_of_fast_replies(): + result = latency([user(0, 2000), agent(2800, 5000), user(6000, 7000), agent(10000, 11000)]) + assert result.score == 0.5 + assert result.metadata["replies"] == 2 + assert result.metadata["unanswered_turns"] == 0 + assert result.metadata["p50_ms"] == 800 + assert result.metadata["p95_ms"] == 3000 + assert result.metadata["gaps_ms"] == [800, 3000] + + +def test_latency_leaves_out_turns_the_agent_talked_over(): + result = latency([user(0, 3000), agent(2000, 4000), user(5000, 6000), agent(6500, 8000)]) + assert result.score == 1 + assert result.metadata["gaps_ms"] == [500] + + +def test_latency_counts_a_slightly_early_reply_as_instant(): + assert latency([user(0, 3000), agent(2800, 5000)]).metadata["gaps_ms"] == [0] + + +def test_latency_does_not_take_a_short_cut_in_as_the_reply(): + assert latency([user(0, 3000), agent(2000, 2200), agent(3500, 5000)]).metadata["gaps_ms"] == [500] + + +def test_scorers_merge_user_fragments_and_skip_backchannels(): + fragmented = [user(0, 1000), user(1200, 3000), agent(3500, 5000)] + assert latency(fragmented).metadata["gaps_ms"] == [500] + assert interruptions(fragmented).metadata["user_turns"] == 1 + + backchannel = [user(0, 1000), agent(1500, 6000), user(3000, 3300), agent(6200, 8000)] + assert latency(backchannel).metadata["gaps_ms"] == [500] + assert interruptions(backchannel).metadata["user_turns"] == 1 + + +def test_interruptions_scores_talk_over_not_barge_ins(): + result = interruptions([user(0, 3000), agent(2000, 4000), user(5000, 6000), agent(6500, 8000, False)]) + assert result.score == 0.5 + assert result.metadata["talk_overs"] == 1 + assert result.metadata["user_turns"] == 2 + assert result.metadata["barge_ins"] == 0 + + barge_in = interruptions([agent(0, 3000, True), user(2000, 4000)]) + assert barge_in.score == 1 + assert barge_in.metadata["barge_ins"] == 1 + + +def test_interruptions_ignores_overlaps_under_min_overlap_ms(): + utterances = [user(0, 3000), agent(2900, 4000)] + assert interruptions(utterances).score == 1 + assert VoiceInterruptions().eval(output=None, utterances=utterances, min_overlap_ms=50).score == 0 + + +def test_scorers_skip_invalid_utterances_and_return_none_without_data(): + utterances = [ + {"speaker": "user"}, + {"start_unix_ms": 0, "end_unix_ms": 1000}, + {"speaker": "agent", "start_unix_ms": float("nan"), "end_unix_ms": 1000}, + {"speaker": "agent", "start_unix_ms": 2000, "end_unix_ms": 1000}, + {"speaker": "assistant", "start_unix_ms": 0, "end_unix_ms": 1000}, + ] + result = latency(utterances) + assert result.score is None + assert result.metadata["skipped"] == 5 + assert interruptions(utterances).score is None + assert interruptions([user(0, 1000)]).score is None + + +def test_partial_and_async(): + scorer = VoiceLatency.partial(max_gap_ms=5000)() + result = asyncio.run(scorer.eval_async(output=None, utterances=[user(0, 2000), agent(5000, 6000)])) + assert result.score == 1 diff --git a/py/autoevals/voice.py b/py/autoevals/voice.py new file mode 100644 index 00000000..7afde0cb --- /dev/null +++ b/py/autoevals/voice.py @@ -0,0 +1,165 @@ +"""Timing scorers for voice agents. + +This module provides scorers that read the timing of a voice call: +- VoiceLatency: The share of agent replies that start within `max_gap_ms` of the + user finishing a turn. +- VoiceInterruptions: The share of user turns the agent let finish without talking + over them. + +Both take `utterances`, a list of dicts with `speaker` ("user" or "agent"), +`start_unix_ms`, `end_unix_ms`, and optionally `interrupted` (the user cut the agent +off, as reported by the voice framework). Times are Unix epoch milliseconds. +""" + +import math +from typing import Literal, TypedDict + +from autoevals.partial import ScorerWithPartial + +from .score import Score + + +class Utterance(TypedDict, total=False): + speaker: Literal["user", "agent"] + start_unix_ms: float + end_unix_ms: float + interrupted: bool + + +def _is_time(value): + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + + +def _is_timed(u): + return ( + u.get("speaker") in ("user", "agent") + and _is_time(u.get("start_unix_ms")) + and _is_time(u.get("end_unix_ms")) + and u["end_unix_ms"] >= u["start_unix_ms"] + ) + + +def _user_turns(utterances, min_overlap_ms): + """Groups the user's speech into turns and finds the agent's reply to each. + + Both scorers use this so they count turns the same way. Each turn has `talk_over` and + `gap_ms`, the milliseconds from the end of the turn to the agent's reply (None without one). + """ + timed = sorted((u for u in utterances if _is_timed(u)), key=lambda u: u["start_unix_ms"]) + agents = [u for u in timed if u["speaker"] == "agent"] + + turns = [] + for u in timed: + if u["speaker"] != "user": + continue + # User speech wholly inside the agent's, like "mm-hmm", isn't a turn. + if any(a["start_unix_ms"] <= u["start_unix_ms"] and u["end_unix_ms"] <= a["end_unix_ms"] for a in agents): + continue + last = turns[-1] if turns else None + if last and not any(last["start"] < a["start_unix_ms"] < u["start_unix_ms"] for a in agents): + last["end"] = max(last["end"], u["end_unix_ms"]) + else: + turns.append({"start": u["start_unix_ms"], "end": u["end_unix_ms"]}) + + for i, turn in enumerate(turns): + next_start = turns[i + 1]["start"] if i + 1 < len(turns) else math.inf + turn["talk_over"] = any( + turn["start"] < a["start_unix_ms"] < turn["end"] + and min(a["end_unix_ms"], turn["end"]) - a["start_unix_ms"] > min_overlap_ms + for a in agents + ) + # A reply runs past the end of the turn, so a short cut-in inside it isn't one. + reply = next( + (a for a in agents if turn["start"] < a["start_unix_ms"] < next_start and a["end_unix_ms"] >= turn["end"]), + None, + ) + turn["gap_ms"] = max(0, reply["start_unix_ms"] - turn["end"]) if reply else None + + return turns, len(agents), len(utterances) - len(timed) + + +def _percentile(values, p): + if not values: + return None + ordered = sorted(values) + return ordered[min(len(ordered) - 1, math.ceil(p / 100 * len(ordered)) - 1)] + + +class VoiceLatency(ScorerWithPartial): + """The share of agent replies that start within `max_gap_ms` of the user finishing a turn. + + Turns the agent talked over aren't counted. + + Example: + ```python + result = VoiceLatency().eval(output=None, utterances=utterances, max_gap_ms=1500) + print(result.score) # 0.5 if half the replies were fast enough + ``` + + Args: + utterances: The call's utterances + max_gap_ms: Longest acceptable gap before a reply (default: 1500) + min_overlap_ms: Overlap needed to count as talking over the user (default: 300) + + Returns: + Score with the share of fast replies, or None when there are no replies. + """ + + def _run_eval_sync(self, output, expected=None, utterances=None, max_gap_ms=1500, min_overlap_ms=300, **kwargs): + turns, _, skipped = _user_turns(utterances or [], min_overlap_ms) + gaps = [t["gap_ms"] for t in turns if not t["talk_over"] and t["gap_ms"] is not None] + return Score( + name=self._name(), + score=sum(gap <= max_gap_ms for gap in gaps) / len(gaps) if gaps else None, + metadata={ + "replies": len(gaps), + "unanswered_turns": sum(t["gap_ms"] is None for t in turns), + "max_gap_ms": max_gap_ms, + "p50_ms": _percentile(gaps, 50), + "p95_ms": _percentile(gaps, 95), + "gaps_ms": gaps, + "skipped": skipped, + }, + ) + + +class VoiceInterruptions(ScorerWithPartial): + """The share of user turns the agent let finish without talking over them. + + The agent talks over a turn when it starts speaking during the turn and overlaps it by + more than `min_overlap_ms`. Barge-ins, where the user cuts the agent off, are counted in + metadata but don't affect the score. + + Example: + ```python + result = VoiceInterruptions().eval(output=None, utterances=utterances) + print(result.score) # 1 if the agent never talked over the user + ``` + + Args: + utterances: The call's utterances + min_overlap_ms: Overlap needed to count as talking over the user (default: 300) + + Returns: + Score with the share of user turns not talked over, or None unless the call has timed + user and agent utterances. + """ + + def _run_eval_sync(self, output, expected=None, utterances=None, min_overlap_ms=300, **kwargs): + utterances = utterances or [] + turns, agents, skipped = _user_turns(utterances, min_overlap_ms) + talk_overs = sum(t["talk_over"] for t in turns) + return Score( + name=self._name(), + score=1 - talk_overs / len(turns) if turns and agents else None, + metadata={ + "user_turns": len(turns), + "talk_overs": talk_overs, + "min_overlap_ms": min_overlap_ms, + "barge_ins": sum(u.get("speaker") == "agent" and u.get("interrupted") is True for u in utterances), + "skipped": skipped, + }, + ) + + +__all__ = ["Utterance", "VoiceLatency", "VoiceInterruptions"] From 3a01a789334d0eac3b866a5482399fd57460b6c2 Mon Sep 17 00:00:00 2001 From: Kev Choi Date: Mon, 5 Oct 2026 16:36:54 -0400 Subject: [PATCH 2/3] feat: list voice timing scorers as built-ins Co-Authored-By: Claude Opus 5.5 --- js/manifest.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/js/manifest.ts b/js/manifest.ts index cc3258c1..29c1ceed 100644 --- a/js/manifest.ts +++ b/js/manifest.ts @@ -26,6 +26,7 @@ import { ListContains } from "./list"; import { ScorerWithPartial } from "./partial"; import { Moderation } from "./moderation"; import { ExactMatch } from "./value"; +import { VoiceInterruptions, VoiceLatency } from "./voice"; import { ModelGradedSpec, templates } from "./templates"; interface AutoevalMethod { @@ -205,4 +206,19 @@ export const Evaluators: { }, ], }, + { + label: "Voice", + methods: [ + { + method: VoiceLatency, + description: + "Test how quickly a voice agent replies after the user finishes speaking. Run it with trace scope.", + }, + { + method: VoiceInterruptions, + description: + "Test how often a voice agent talks over the user. Run it with trace scope.", + }, + ], + }, ]; From d892368aa7ed918cb5dff5b3d074f4dc21f2bfab Mon Sep 17 00:00:00 2001 From: Kev Choi Date: Wed, 7 Oct 2026 11:40:17 -0400 Subject: [PATCH 3/3] fix: align voice timing docs and tests Co-Authored-By: Claude Opus 5.5 --- README.md | 5 +++++ SCORERS.md | 6 +++--- js/voice.ts | 8 ++++---- py/autoevals/test_voice.py | 11 +---------- py/autoevals/voice.py | 4 ++-- 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index bbc25677..51a4eea3 100644 --- a/README.md +++ b/README.md @@ -362,6 +362,11 @@ Eval( - Numeric difference - JSON diff +### Voice evaluations + +- Voice latency +- Voice interruptions + For detailed documentation on all scorers, including parameters, score ranges, and usage examples, see the [**Scorer Reference**](SCORERS.md). ## Custom evaluation prompts diff --git a/SCORERS.md b/SCORERS.md index 22780538..a4e7d26d 100644 --- a/SCORERS.md +++ b/SCORERS.md @@ -600,7 +600,7 @@ Both scorers group the user's speech into turns the same way: - Consecutive user utterances merge into one turn until the agent starts speaking. - User speech wholly inside the agent's, like "mm-hmm", isn't a turn. - The agent talks over a turn when it starts speaking during the turn and overlaps it by more than `minOverlapMs` (default 300). -- The agent's reply to a turn is the first agent utterance that starts after the turn starts and runs past its end. +- The agent's reply to a turn is the first agent utterance that starts after the turn starts and before the next turn, and runs past the turn's end. Utterances with a missing or invalid time, or another speaker, are skipped and counted in `skipped`. @@ -625,7 +625,7 @@ Measures how quickly the agent replies after the user finishes a turn. **Score Range:** 0-1, the share of replies that start within `maxGapMs`. `null` when the call has no replies. -Turns the agent talked over aren't counted. A reply that starts slightly before the user stops counts as a gap of 0. Metadata has `replies`, `unanswered_turns`, `p50_ms`, `p95_ms`, `gaps_ms` and `skipped`; pool `gaps_ms` to get percentiles across a dataset. +Turns the agent talked over aren't counted. A reply that starts slightly before the user stops counts as a gap of 0. Metadata has `replies`, `unanswered_turns`, `max_gap_ms`, `p50_ms`, `p95_ms`, `gaps_ms` and `skipped`; pool `gaps_ms` to get percentiles across a dataset. **Example:** @@ -655,7 +655,7 @@ Measures how often the agent talks over the user. **Score Range:** 0-1, the share of user turns the agent didn't talk over. `null` unless the call has timed user and agent utterances. -Metadata has `user_turns`, `talk_overs`, `skipped`, and `barge_ins`: agent utterances marked `interrupted`, where the user cut the agent off. Barge-ins don't affect the score. +Metadata has `user_turns`, `talk_overs`, `min_overlap_ms`, `skipped`, and `barge_ins`: agent utterances marked `interrupted`, where the user cut the agent off. Barge-ins don't affect the score. **Example:** diff --git a/js/voice.ts b/js/voice.ts index 8153a9f7..292c96ae 100644 --- a/js/voice.ts +++ b/js/voice.ts @@ -41,7 +41,7 @@ function isTimed(u: Utterance): u is TimedUtterance { function userTurns( utterances: Utterance[], minOverlapMs: number, -): { turns: Turn[]; agents: number; skipped: number } { +): { turns: Turn[]; agentCount: number; skipped: number } { const timed = utterances .filter(isTimed) .sort((a, b) => a.start_unix_ms - b.start_unix_ms); @@ -100,7 +100,7 @@ function userTurns( return { turns, - agents: agents.length, + agentCount: agents.length, skipped: utterances.length - timed.length, }; } @@ -167,11 +167,11 @@ export const VoiceInterruptions: ScorerWithPartial< unknown, VoiceInterruptionsArgs > = makePartial(async ({ utterances = [], minOverlapMs = 300 }) => { - const { turns, agents, skipped } = userTurns(utterances, minOverlapMs); + const { turns, agentCount, skipped } = userTurns(utterances, minOverlapMs); const talkOvers = turns.filter((t) => t.talkOver).length; return { name: "VoiceInterruptions", - score: turns.length && agents ? 1 - talkOvers / turns.length : null, + score: turns.length && agentCount ? 1 - talkOvers / turns.length : null, metadata: { user_turns: turns.length, talk_overs: talkOvers, diff --git a/py/autoevals/test_voice.py b/py/autoevals/test_voice.py index 012f6ccf..76d59325 100644 --- a/py/autoevals/test_voice.py +++ b/py/autoevals/test_voice.py @@ -1,5 +1,3 @@ -import asyncio - from autoevals import VoiceInterruptions, VoiceLatency @@ -74,19 +72,12 @@ def test_interruptions_ignores_overlaps_under_min_overlap_ms(): def test_scorers_skip_invalid_utterances_and_return_none_without_data(): utterances = [ {"speaker": "user"}, - {"start_unix_ms": 0, "end_unix_ms": 1000}, {"speaker": "agent", "start_unix_ms": float("nan"), "end_unix_ms": 1000}, {"speaker": "agent", "start_unix_ms": 2000, "end_unix_ms": 1000}, {"speaker": "assistant", "start_unix_ms": 0, "end_unix_ms": 1000}, ] result = latency(utterances) assert result.score is None - assert result.metadata["skipped"] == 5 + assert result.metadata["skipped"] == 4 assert interruptions(utterances).score is None assert interruptions([user(0, 1000)]).score is None - - -def test_partial_and_async(): - scorer = VoiceLatency.partial(max_gap_ms=5000)() - result = asyncio.run(scorer.eval_async(output=None, utterances=[user(0, 2000), agent(5000, 6000)])) - assert result.score == 1 diff --git a/py/autoevals/voice.py b/py/autoevals/voice.py index 7afde0cb..1ef654a3 100644 --- a/py/autoevals/voice.py +++ b/py/autoevals/voice.py @@ -147,11 +147,11 @@ class VoiceInterruptions(ScorerWithPartial): def _run_eval_sync(self, output, expected=None, utterances=None, min_overlap_ms=300, **kwargs): utterances = utterances or [] - turns, agents, skipped = _user_turns(utterances, min_overlap_ms) + turns, agent_count, skipped = _user_turns(utterances, min_overlap_ms) talk_overs = sum(t["talk_over"] for t in turns) return Score( name=self._name(), - score=1 - talk_overs / len(turns) if turns and agents else None, + score=1 - talk_overs / len(turns) if turns and agent_count else None, metadata={ "user_turns": len(turns), "talk_overs": talk_overs,