diff --git a/apps/cli/ai/eval-runner.ts b/apps/cli/ai/eval-runner.ts index 6acff923fd..e0f2c4a411 100644 --- a/apps/cli/ai/eval-runner.ts +++ b/apps/cli/ai/eval-runner.ts @@ -467,7 +467,6 @@ async function runEval( input: EvalRunnerInput ) { let success = false; let error: string | null = null; let timedOut = false; - let lastTurnEndedEmpty = false; // From the last assistant message: a provider failure ends the turn with // stopReason "error" and its message here, without throwing. let stopReason: string | null = null; @@ -586,13 +585,6 @@ async function runEval( input: EvalRunnerInput ) { numTurnsResult = numTurns; stopReason = lastAssistant?.stopReason ?? null; errorMessage = lastAssistant?.errorMessage ?? null; - lastTurnEndedEmpty = Boolean( - lastAssistant && - lastAssistant.stopReason === 'stop' && - ! lastAssistant.content.some( - ( block ) => block.type === 'toolCall' || ( block.type === 'text' && block.text.trim() ) - ) - ); } }; @@ -613,30 +605,14 @@ async function runEval( input: EvalRunnerInput ) { void query.interrupt(); }, input.timeoutMs ?? 300000 ); - // Some hosted tiers end a turn with an empty message once their reasoning - // budget runs out. STUDIO_EVAL_AUTO_CONTINUE= resumes such a turn with - // "continue" up to n times, the way a person would. - const maxAutoContinues = Number( process.env.STUDIO_EVAL_AUTO_CONTINUE ?? 0 ) || 0; - let autoContinues = 0; - - const settle = async () => { - await query.result; - while ( autoContinues < maxAutoContinues && ! timedOut && lastTurnEndedEmpty ) { - autoContinues++; - lastTurnEndedEmpty = false; - query = runTurn( 'continue' ); - await query.result; - } - }; - try { - await settle(); + await query.result; for ( const followUp of input.followUps ?? [] ) { if ( timedOut ) { break; } query = runTurn( followUp.prompt ); - await settle(); + await query.result; } } catch ( caught ) { error = caught instanceof Error ? caught.message : String( caught ); @@ -655,7 +631,6 @@ async function runEval( input: EvalRunnerInput ) { timedOut, model: resolvedModel, numTurns: numTurnsResult, - autoContinues, stopReason, errorMessage, usage, diff --git a/apps/cli/ai/runtimes/pi/index.ts b/apps/cli/ai/runtimes/pi/index.ts index fa76189104..d98cf8f723 100644 --- a/apps/cli/ai/runtimes/pi/index.ts +++ b/apps/cli/ai/runtimes/pi/index.ts @@ -37,6 +37,7 @@ import { type AiModelFamily, type AiModelId, } from '@studio/common/ai/models'; +import { findLastAssistant, isEmptyStop } from '@studio/common/ai/session-events'; import { getAiTracksIdentity } from '@studio/common/ai/tracks-identity'; import { type TSchema } from 'typebox'; import { withDesignSystemPreview } from 'cli/ai/chat-artifacts'; @@ -72,6 +73,13 @@ type ProviderConfigInput = Parameters< ModelRuntime[ 'registerProvider' ] >[ 1 ] const STUDIO_WPCOM_PROVIDER = 'studio-wpcom'; const STUDIO_AGENT_DIR = STUDIO_SITES_ROOT; +// A reply with nothing in it ends the turn without the user seeing why (see +// `isEmptyStop`). Resume it with a message the user doesn't see, up to this +// many times per turn. pi leaves a reply that has only reasoning out of the +// next request, so the message carries that reasoning; otherwise the model +// starts the same reasoning over and runs out again. +const MAX_EMPTY_STOP_CONTINUES = 2; +const EMPTY_STOP_CONTINUE_TYPE = 'studio.empty_stop_continue'; const STUDIO_COMPACTION_SETTINGS = { enabled: true, reserveTokens: 16_384, @@ -231,8 +239,28 @@ async function runAgentSessionTurn( try { session = await createStudioAgentSession( config, family, resolved.creds, payloadGuardState ); setActiveSession( session ); + let emptyStopContinues = 0; + let continueAfterEmptyStop = false; + let cutOffReasoning = ''; unsubscribe = session.subscribe( ( event ) => { updateStudioToolPayloadGuardState( event, payloadGuardState ); + if ( + event.type === 'agent_end' && + ! event.willRetry && + emptyStopContinues < MAX_EMPTY_STOP_CONTINUES + ) { + const lastAssistant = findLastAssistant( event.messages ); + if ( isEmptyStop( lastAssistant ) ) { + // Not final: consumers wait for the turn's real end. + continueAfterEmptyStop = true; + cutOffReasoning = ( lastAssistant?.content ?? [] ) + .map( ( block ) => ( block.type === 'thinking' ? block.thinking : '' ) ) + .join( '' ) + .trim(); + config.onEvent( { ...event, willRetry: true } ); + return; + } + } config.onEvent( event ); } ); @@ -251,6 +279,24 @@ async function runAgentSessionTurn( mimeType: image.mimeType, } ) ), } ); + while ( continueAfterEmptyStop ) { + continueAfterEmptyStop = false; + if ( controller.signal.aborted ) { + config.onEvent( syntheticErrorAgentEnd( 'aborted', '' ) ); + break; + } + emptyStopContinues++; + await session.sendCustomMessage( + { + customType: EMPTY_STOP_CONTINUE_TYPE, + content: cutOffReasoning + ? `Your reasoning was cut off before you replied. Continue from where it stopped:\n\n${ cutOffReasoning }` + : 'continue', + display: false, + }, + { triggerTurn: true } + ); + } await getPendingWork( config.session ).settle(); } catch ( error ) { const aborted = controller.signal.aborted; diff --git a/apps/cli/ai/tests/pi-runtime.test.ts b/apps/cli/ai/tests/pi-runtime.test.ts index 8f72c99f74..dfbb6a94ab 100644 --- a/apps/cli/ai/tests/pi-runtime.test.ts +++ b/apps/cli/ai/tests/pi-runtime.test.ts @@ -10,6 +10,9 @@ const mocks = vi.hoisted( () => ( { createAgentSession: vi.fn(), createdSessions: [] as FakeSession[], nextEvents: null as AgentSessionEvent[] | null, + continueEvents: [] as AgentSessionEvent[][], + customMessages: [] as Array< { message: unknown; options: unknown } >, + onCustomMessage: null as ( () => void ) | null, studioRoot: '/tmp/studio-ai-pi-runtime', configRoot: '/tmp/studio-ai-pi-runtime-config', } ) ); @@ -164,6 +167,14 @@ class FakeSession { } } + async sendCustomMessage( message: unknown, options: unknown ): Promise< void > { + mocks.customMessages.push( { message, options } ); + mocks.onCustomMessage?.(); + for ( const event of mocks.continueEvents.shift() ?? [] ) { + this.listener?.( event ); + } + } + async abort(): Promise< void > { this.aborted = true; } @@ -221,6 +232,9 @@ describe( 'pi runtime', () => { beforeEach( () => { mocks.createdSessions.length = 0; mocks.nextEvents = null; + mocks.continueEvents = []; + mocks.customMessages = []; + mocks.onCustomMessage = null; mocks.createAgentSession.mockReset(); mocks.createAgentSession.mockImplementation( async ( options: CreateAgentSessionOptions ) => { const session = new FakeSession( options ); @@ -524,6 +538,93 @@ describe( 'pi runtime', () => { ); } ); + describe( 'a reply with neither text nor a tool call', () => { + const emptyReply = () => assistantMessage( [ { type: 'thinking', thinking: 'reasoning…' } ] ); + const runEndingWith = ( message: MessageEndEvent ): AgentSessionEvent[] => [ + message, + { type: 'agent_end', willRetry: false, messages: [ message.message ] }, + ]; + const agentEnds = ( events: AgentSessionEvent[] ) => + events.filter( + ( event ): event is Extract< AgentSessionEvent, { type: 'agent_end' } > => + event.type === 'agent_end' + ); + const runFast = () => + runRuntime( { prompt: 'build it', env: WPCOM_ENV, model: 'fast', session: newSession() } ); + + it( 'continues the turn with a message the user does not see', async () => { + mocks.nextEvents = runEndingWith( emptyReply() ); + mocks.continueEvents = [ + runEndingWith( assistantMessage( [ { type: 'text', text: 'Done.' } ] ) ), + ]; + + const events = await runFast(); + + expect( mocks.customMessages ).toEqual( [ + { + message: { + customType: 'studio.empty_stop_continue', + content: + 'Your reasoning was cut off before you replied. Continue from where it stopped:\n\nreasoning…', + display: false, + }, + options: { triggerTurn: true }, + }, + ] ); + expect( agentEnds( events ).map( ( event ) => event.willRetry ) ).toEqual( [ true, false ] ); + expect( findAssistantText( events ) ).toBe( 'Done.' ); + } ); + + it( 'ends the turn after two continues', async () => { + mocks.nextEvents = runEndingWith( emptyReply() ); + mocks.continueEvents = [ runEndingWith( emptyReply() ), runEndingWith( emptyReply() ) ]; + + const events = await runFast(); + + expect( mocks.customMessages ).toHaveLength( 2 ); + expect( agentEnds( events ).map( ( event ) => event.willRetry ) ).toEqual( [ + true, + true, + false, + ] ); + } ); + + it( 'leaves a reply with text or a tool call alone', async () => { + for ( const content of [ + [ { type: 'text', text: 'Which colours do you like?' } ], + [ { type: 'toolCall', id: 'call-1', name: 'site_info', arguments: {} } ], + ] ) { + mocks.nextEvents = runEndingWith( assistantMessage( content ) ); + const events = await runFast(); + expect( agentEnds( events ).map( ( event ) => event.willRetry ) ).toEqual( [ false ] ); + } + expect( mocks.customMessages ).toHaveLength( 0 ); + } ); + + it( 'ends the turn as interrupted when the user stops it first', async () => { + mocks.nextEvents = runEndingWith( emptyReply() ); + const events: AgentSessionEvent[] = []; + const handle = runStudioAgentTurn( { + prompt: 'build it', + env: WPCOM_ENV, + model: 'fast', + session: newSession(), + onEvent: ( event ) => { + events.push( event ); + if ( event.type === 'agent_end' && event.willRetry ) { + void handle.interrupt(); + } + }, + } ); + await handle.result; + + expect( mocks.customMessages ).toHaveLength( 0 ); + const ends = agentEnds( events ); + expect( ends.map( ( event ) => event.willRetry ) ).toEqual( [ true, false ] ); + expect( ends[ 1 ].messages[ 0 ] ).toMatchObject( { stopReason: 'aborted' } ); + } ); + } ); + it( 'creates each AgentSession with the requested model', async () => { const session = newSession(); const otherStudioModel = 'tier-test-other' as AiModelId; diff --git a/packages/common/ai/session-events.ts b/packages/common/ai/session-events.ts index 6435a7d89a..5cb6b00cb3 100644 --- a/packages/common/ai/session-events.ts +++ b/packages/common/ai/session-events.ts @@ -14,6 +14,19 @@ export function findLastAssistant( return undefined; } +// A reply that ends the turn with neither text nor a tool call: the user +// sees nothing. Any model can do this; the fast tier does it when the model +// spends its whole reasoning budget (about 4,096 tokens) before answering. +export function isEmptyStop( message: AssistantMessage | undefined ): boolean { + return Boolean( + message && + message.stopReason === 'stop' && + ! message.content.some( + ( block ) => block.type === 'toolCall' || ( block.type === 'text' && block.text.trim() ) + ) + ); +} + export interface AgentEndTurnResult { success: boolean; interrupted: boolean; diff --git a/scripts/eval/README.md b/scripts/eval/README.md index cb274f04a7..4ee69ed4d5 100644 --- a/scripts/eval/README.md +++ b/scripts/eval/README.md @@ -39,7 +39,7 @@ Run one named test with `npm run eval -- --filter-pattern "preview sites"` (rege ## Running the runner directly -`node apps/cli/dist/cli/eval-runner.mjs "" "" '{"vars":{"timeoutMs":1800000}}'` runs one turn outside promptfoo and prints `EVAL_RUNNER_RESULT_FILE=`. Environment variables: `STUDIO_EVAL_MODEL` picks the model (`fast`, `claude-sonnet-5`, …), `STUDIO_EVAL_SESSION_DIR` sets where the session JSONL (the full transcript, kept for every run and reported as `sessionFile`) is written, by default `/studio-eval-sessions`, and `STUDIO_EVAL_AUTO_CONTINUE=` resumes a turn that ended with an empty assistant message (some hosted tiers stop that way when their reasoning budget runs out) with `continue`, up to `n` times; the result reports the count as `autoContinues`. A `followUps` var (a list of `{ "prompt": "…" }` objects; objects because promptfoo turns an array of strings into one test per element) sends each as a further user message in the same session once the previous one finishes, for cases that measure a conversation rather than a single request. +`node apps/cli/dist/cli/eval-runner.mjs "" "" '{"vars":{"timeoutMs":1800000}}'` runs one turn outside promptfoo and prints `EVAL_RUNNER_RESULT_FILE=`. Environment variables: `STUDIO_EVAL_MODEL` picks the model (`fast`, `claude-sonnet-5`, …), and `STUDIO_EVAL_SESSION_DIR` sets where the session JSONL (the full transcript, kept for every run and reported as `sessionFile`) is written, by default `/studio-eval-sessions`. A turn that ends with an empty reply is resumed by the runtime itself, as in the app, and shows up in `timing.emptyStops`. A `followUps` var (a list of `{ "prompt": "…" }` objects; objects because promptfoo turns an array of strings into one test per element) sends each as a further user message in the same session once the previous one finishes, for cases that measure a conversation rather than a single request. ## Adding tests