From 822f88136dbb2c7207a127d4eac206389b8cfaff Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Tue, 20 Oct 2026 08:15:57 -0400 Subject: [PATCH 1/3] Studio Code: Continue a turn that ends with an empty reply --- apps/cli/ai/eval-runner.ts | 29 +------ apps/cli/ai/runtimes/pi/index.ts | 38 +++++++++ apps/cli/ai/tests/pi-runtime.test.ts | 118 +++++++++++++++++++++++++++ packages/common/ai/session-events.ts | 13 +++ scripts/eval/README.md | 2 +- 5 files changed, 172 insertions(+), 28 deletions(-) diff --git a/apps/cli/ai/eval-runner.ts b/apps/cli/ai/eval-runner.ts index 6acff923fd..e0f2c4a411 100644 --- a/apps/cli/ai/eval-runner.ts +++ b/apps/cli/ai/eval-runner.ts @@ -467,7 +467,6 @@ async function runEval( input: EvalRunnerInput ) { let success = false; let error: string | null = null; let timedOut = false; - let lastTurnEndedEmpty = false; // From the last assistant message: a provider failure ends the turn with // stopReason "error" and its message here, without throwing. let stopReason: string | null = null; @@ -586,13 +585,6 @@ async function runEval( input: EvalRunnerInput ) { numTurnsResult = numTurns; stopReason = lastAssistant?.stopReason ?? null; errorMessage = lastAssistant?.errorMessage ?? null; - lastTurnEndedEmpty = Boolean( - lastAssistant && - lastAssistant.stopReason === 'stop' && - ! lastAssistant.content.some( - ( block ) => block.type === 'toolCall' || ( block.type === 'text' && block.text.trim() ) - ) - ); } }; @@ -613,30 +605,14 @@ async function runEval( input: EvalRunnerInput ) { void query.interrupt(); }, input.timeoutMs ?? 300000 ); - // Some hosted tiers end a turn with an empty message once their reasoning - // budget runs out. STUDIO_EVAL_AUTO_CONTINUE= resumes such a turn with - // "continue" up to n times, the way a person would. - const maxAutoContinues = Number( process.env.STUDIO_EVAL_AUTO_CONTINUE ?? 0 ) || 0; - let autoContinues = 0; - - const settle = async () => { - await query.result; - while ( autoContinues < maxAutoContinues && ! timedOut && lastTurnEndedEmpty ) { - autoContinues++; - lastTurnEndedEmpty = false; - query = runTurn( 'continue' ); - await query.result; - } - }; - try { - await settle(); + await query.result; for ( const followUp of input.followUps ?? [] ) { if ( timedOut ) { break; } query = runTurn( followUp.prompt ); - await settle(); + await query.result; } } catch ( caught ) { error = caught instanceof Error ? caught.message : String( caught ); @@ -655,7 +631,6 @@ async function runEval( input: EvalRunnerInput ) { timedOut, model: resolvedModel, numTurns: numTurnsResult, - autoContinues, stopReason, errorMessage, usage, diff --git a/apps/cli/ai/runtimes/pi/index.ts b/apps/cli/ai/runtimes/pi/index.ts index fa76189104..2152420c87 100644 --- a/apps/cli/ai/runtimes/pi/index.ts +++ b/apps/cli/ai/runtimes/pi/index.ts @@ -37,6 +37,7 @@ import { type AiModelFamily, type AiModelId, } from '@studio/common/ai/models'; +import { findLastAssistant, isEmptyStop } from '@studio/common/ai/session-events'; import { getAiTracksIdentity } from '@studio/common/ai/tracks-identity'; import { type TSchema } from 'typebox'; import { withDesignSystemPreview } from 'cli/ai/chat-artifacts'; @@ -72,6 +73,11 @@ type ProviderConfigInput = Parameters< ModelRuntime[ 'registerProvider' ] >[ 1 ] const STUDIO_WPCOM_PROVIDER = 'studio-wpcom'; const STUDIO_AGENT_DIR = STUDIO_SITES_ROOT; +// A reply with nothing in it ends the turn without the user seeing why (see +// `isEmptyStop`). Resume it the way a person would, with a "continue" the +// user doesn't see, up to this many times in a row. +const MAX_EMPTY_STOP_CONTINUES = 2; +const EMPTY_STOP_CONTINUE_TYPE = 'studio.empty_stop_continue'; const STUDIO_COMPACTION_SETTINGS = { enabled: true, reserveTokens: 16_384, @@ -231,8 +237,28 @@ async function runAgentSessionTurn( try { session = await createStudioAgentSession( config, family, resolved.creds, payloadGuardState ); setActiveSession( session ); + let emptyStopContinues = 0; + let continueAfterEmptyStop = false; unsubscribe = session.subscribe( ( event ) => { updateStudioToolPayloadGuardState( event, payloadGuardState ); + if ( + event.type === 'message_end' && + event.message.role === 'assistant' && + ! isEmptyStop( event.message ) + ) { + emptyStopContinues = 0; + } + if ( + event.type === 'agent_end' && + ! event.willRetry && + emptyStopContinues < MAX_EMPTY_STOP_CONTINUES && + isEmptyStop( findLastAssistant( event.messages ) ) + ) { + // Not final: consumers wait for the turn's real end. + continueAfterEmptyStop = true; + config.onEvent( { ...event, willRetry: true } ); + return; + } config.onEvent( event ); } ); @@ -251,6 +277,18 @@ async function runAgentSessionTurn( mimeType: image.mimeType, } ) ), } ); + while ( continueAfterEmptyStop ) { + continueAfterEmptyStop = false; + if ( controller.signal.aborted ) { + config.onEvent( syntheticErrorAgentEnd( 'aborted', '' ) ); + break; + } + emptyStopContinues++; + await session.sendCustomMessage( + { customType: EMPTY_STOP_CONTINUE_TYPE, content: 'continue', display: false }, + { triggerTurn: true } + ); + } await getPendingWork( config.session ).settle(); } catch ( error ) { const aborted = controller.signal.aborted; diff --git a/apps/cli/ai/tests/pi-runtime.test.ts b/apps/cli/ai/tests/pi-runtime.test.ts index 8f72c99f74..70ac7a61a0 100644 --- a/apps/cli/ai/tests/pi-runtime.test.ts +++ b/apps/cli/ai/tests/pi-runtime.test.ts @@ -10,6 +10,9 @@ const mocks = vi.hoisted( () => ( { createAgentSession: vi.fn(), createdSessions: [] as FakeSession[], nextEvents: null as AgentSessionEvent[] | null, + continueEvents: [] as AgentSessionEvent[][], + customMessages: [] as Array< { message: unknown; options: unknown } >, + onCustomMessage: null as ( () => void ) | null, studioRoot: '/tmp/studio-ai-pi-runtime', configRoot: '/tmp/studio-ai-pi-runtime-config', } ) ); @@ -164,6 +167,14 @@ class FakeSession { } } + async sendCustomMessage( message: unknown, options: unknown ): Promise< void > { + mocks.customMessages.push( { message, options } ); + mocks.onCustomMessage?.(); + for ( const event of mocks.continueEvents.shift() ?? [] ) { + this.listener?.( event ); + } + } + async abort(): Promise< void > { this.aborted = true; } @@ -221,6 +232,9 @@ describe( 'pi runtime', () => { beforeEach( () => { mocks.createdSessions.length = 0; mocks.nextEvents = null; + mocks.continueEvents = []; + mocks.customMessages = []; + mocks.onCustomMessage = null; mocks.createAgentSession.mockReset(); mocks.createAgentSession.mockImplementation( async ( options: CreateAgentSessionOptions ) => { const session = new FakeSession( options ); @@ -524,6 +538,110 @@ describe( 'pi runtime', () => { ); } ); + describe( 'a reply with neither text nor a tool call', () => { + const emptyReply = () => assistantMessage( [ { type: 'thinking', thinking: 'reasoning…' } ] ); + const runEndingWith = ( message: MessageEndEvent ): AgentSessionEvent[] => [ + message, + { type: 'agent_end', willRetry: false, messages: [ message.message ] }, + ]; + const agentEnds = ( events: AgentSessionEvent[] ) => + events.filter( + ( event ): event is Extract< AgentSessionEvent, { type: 'agent_end' } > => + event.type === 'agent_end' + ); + const runFast = () => + runRuntime( { prompt: 'build it', env: WPCOM_ENV, model: 'fast', session: newSession() } ); + + it( 'continues the turn with a message the user does not see', async () => { + mocks.nextEvents = runEndingWith( emptyReply() ); + mocks.continueEvents = [ + runEndingWith( assistantMessage( [ { type: 'text', text: 'Done.' } ] ) ), + ]; + + const events = await runFast(); + + expect( mocks.customMessages ).toEqual( [ + { + message: { + customType: 'studio.empty_stop_continue', + content: 'continue', + display: false, + }, + options: { triggerTurn: true }, + }, + ] ); + expect( agentEnds( events ).map( ( event ) => event.willRetry ) ).toEqual( [ true, false ] ); + expect( findAssistantText( events ) ).toBe( 'Done.' ); + } ); + + it( 'ends the turn after two continues', async () => { + mocks.nextEvents = runEndingWith( emptyReply() ); + mocks.continueEvents = [ runEndingWith( emptyReply() ), runEndingWith( emptyReply() ) ]; + + const events = await runFast(); + + expect( mocks.customMessages ).toHaveLength( 2 ); + expect( agentEnds( events ).map( ( event ) => event.willRetry ) ).toEqual( [ + true, + true, + false, + ] ); + } ); + + it( 'counts continues in a row, so progress in between starts the count again', async () => { + const toolCall = assistantMessage( [ + { type: 'toolCall', id: 'call-1', name: 'site_info', arguments: {} }, + ] ); + mocks.nextEvents = runEndingWith( emptyReply() ); + mocks.continueEvents = [ + runEndingWith( emptyReply() ), + [ toolCall, ...runEndingWith( emptyReply() ) ], + runEndingWith( assistantMessage( [ { type: 'text', text: 'Done.' } ] ) ), + ]; + + const events = await runFast(); + + expect( mocks.customMessages ).toHaveLength( 3 ); + expect( agentEnds( events ).at( -1 )?.willRetry ).toBe( false ); + expect( findAssistantText( events ) ).toBe( 'Done.' ); + } ); + + it( 'leaves a reply with text or a tool call alone', async () => { + for ( const content of [ + [ { type: 'text', text: 'Which colours do you like?' } ], + [ { type: 'toolCall', id: 'call-1', name: 'site_info', arguments: {} } ], + ] ) { + mocks.nextEvents = runEndingWith( assistantMessage( content ) ); + const events = await runFast(); + expect( agentEnds( events ).map( ( event ) => event.willRetry ) ).toEqual( [ false ] ); + } + expect( mocks.customMessages ).toHaveLength( 0 ); + } ); + + it( 'ends the turn as interrupted when the user stops it first', async () => { + mocks.nextEvents = runEndingWith( emptyReply() ); + const events: AgentSessionEvent[] = []; + const handle = runStudioAgentTurn( { + prompt: 'build it', + env: WPCOM_ENV, + model: 'fast', + session: newSession(), + onEvent: ( event ) => { + events.push( event ); + if ( event.type === 'agent_end' && event.willRetry ) { + void handle.interrupt(); + } + }, + } ); + await handle.result; + + expect( mocks.customMessages ).toHaveLength( 0 ); + const ends = agentEnds( events ); + expect( ends.map( ( event ) => event.willRetry ) ).toEqual( [ true, false ] ); + expect( ends[ 1 ].messages[ 0 ] ).toMatchObject( { stopReason: 'aborted' } ); + } ); + } ); + it( 'creates each AgentSession with the requested model', async () => { const session = newSession(); const otherStudioModel = 'tier-test-other' as AiModelId; diff --git a/packages/common/ai/session-events.ts b/packages/common/ai/session-events.ts index 6435a7d89a..35c208eadb 100644 --- a/packages/common/ai/session-events.ts +++ b/packages/common/ai/session-events.ts @@ -14,6 +14,19 @@ export function findLastAssistant( return undefined; } +// A reply that ends the turn with neither text nor a tool call: the user +// sees nothing. The fast tier's upstream does this when the model spends +// its whole reasoning budget (about 4,096 tokens) before answering. +export function isEmptyStop( message: AssistantMessage | undefined ): boolean { + return Boolean( + message && + message.stopReason === 'stop' && + ! message.content.some( + ( block ) => block.type === 'toolCall' || ( block.type === 'text' && block.text.trim() ) + ) + ); +} + export interface AgentEndTurnResult { success: boolean; interrupted: boolean; diff --git a/scripts/eval/README.md b/scripts/eval/README.md index cb274f04a7..4ee69ed4d5 100644 --- a/scripts/eval/README.md +++ b/scripts/eval/README.md @@ -39,7 +39,7 @@ Run one named test with `npm run eval -- --filter-pattern "preview sites"` (rege ## Running the runner directly -`node apps/cli/dist/cli/eval-runner.mjs "" "" '{"vars":{"timeoutMs":1800000}}'` runs one turn outside promptfoo and prints `EVAL_RUNNER_RESULT_FILE=`. Environment variables: `STUDIO_EVAL_MODEL` picks the model (`fast`, `claude-sonnet-5`, …), `STUDIO_EVAL_SESSION_DIR` sets where the session JSONL (the full transcript, kept for every run and reported as `sessionFile`) is written, by default `/studio-eval-sessions`, and `STUDIO_EVAL_AUTO_CONTINUE=` resumes a turn that ended with an empty assistant message (some hosted tiers stop that way when their reasoning budget runs out) with `continue`, up to `n` times; the result reports the count as `autoContinues`. A `followUps` var (a list of `{ "prompt": "…" }` objects; objects because promptfoo turns an array of strings into one test per element) sends each as a further user message in the same session once the previous one finishes, for cases that measure a conversation rather than a single request. +`node apps/cli/dist/cli/eval-runner.mjs "" "" '{"vars":{"timeoutMs":1800000}}'` runs one turn outside promptfoo and prints `EVAL_RUNNER_RESULT_FILE=`. Environment variables: `STUDIO_EVAL_MODEL` picks the model (`fast`, `claude-sonnet-5`, …), and `STUDIO_EVAL_SESSION_DIR` sets where the session JSONL (the full transcript, kept for every run and reported as `sessionFile`) is written, by default `/studio-eval-sessions`. A turn that ends with an empty reply is resumed by the runtime itself, as in the app, and shows up in `timing.emptyStops`. A `followUps` var (a list of `{ "prompt": "…" }` objects; objects because promptfoo turns an array of strings into one test per element) sends each as a further user message in the same session once the previous one finishes, for cases that measure a conversation rather than a single request. ## Adding tests From a72081fb6a61c7925c8d6797111e95c783d8ba93 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Tue, 20 Oct 2026 08:59:30 -0400 Subject: [PATCH 2/3] Studio Code: Limit empty-reply continues per turn --- apps/cli/ai/runtimes/pi/index.ts | 9 +-------- apps/cli/ai/tests/pi-runtime.test.ts | 18 ------------------ packages/common/ai/session-events.ts | 4 ++-- 3 files changed, 3 insertions(+), 28 deletions(-) diff --git a/apps/cli/ai/runtimes/pi/index.ts b/apps/cli/ai/runtimes/pi/index.ts index 2152420c87..8b76b5c0d9 100644 --- a/apps/cli/ai/runtimes/pi/index.ts +++ b/apps/cli/ai/runtimes/pi/index.ts @@ -75,7 +75,7 @@ const STUDIO_WPCOM_PROVIDER = 'studio-wpcom'; const STUDIO_AGENT_DIR = STUDIO_SITES_ROOT; // A reply with nothing in it ends the turn without the user seeing why (see // `isEmptyStop`). Resume it the way a person would, with a "continue" the -// user doesn't see, up to this many times in a row. +// user doesn't see, up to this many times per turn. const MAX_EMPTY_STOP_CONTINUES = 2; const EMPTY_STOP_CONTINUE_TYPE = 'studio.empty_stop_continue'; const STUDIO_COMPACTION_SETTINGS = { @@ -241,13 +241,6 @@ async function runAgentSessionTurn( let continueAfterEmptyStop = false; unsubscribe = session.subscribe( ( event ) => { updateStudioToolPayloadGuardState( event, payloadGuardState ); - if ( - event.type === 'message_end' && - event.message.role === 'assistant' && - ! isEmptyStop( event.message ) - ) { - emptyStopContinues = 0; - } if ( event.type === 'agent_end' && ! event.willRetry && diff --git a/apps/cli/ai/tests/pi-runtime.test.ts b/apps/cli/ai/tests/pi-runtime.test.ts index 70ac7a61a0..905e35897e 100644 --- a/apps/cli/ai/tests/pi-runtime.test.ts +++ b/apps/cli/ai/tests/pi-runtime.test.ts @@ -588,24 +588,6 @@ describe( 'pi runtime', () => { ] ); } ); - it( 'counts continues in a row, so progress in between starts the count again', async () => { - const toolCall = assistantMessage( [ - { type: 'toolCall', id: 'call-1', name: 'site_info', arguments: {} }, - ] ); - mocks.nextEvents = runEndingWith( emptyReply() ); - mocks.continueEvents = [ - runEndingWith( emptyReply() ), - [ toolCall, ...runEndingWith( emptyReply() ) ], - runEndingWith( assistantMessage( [ { type: 'text', text: 'Done.' } ] ) ), - ]; - - const events = await runFast(); - - expect( mocks.customMessages ).toHaveLength( 3 ); - expect( agentEnds( events ).at( -1 )?.willRetry ).toBe( false ); - expect( findAssistantText( events ) ).toBe( 'Done.' ); - } ); - it( 'leaves a reply with text or a tool call alone', async () => { for ( const content of [ [ { type: 'text', text: 'Which colours do you like?' } ], diff --git a/packages/common/ai/session-events.ts b/packages/common/ai/session-events.ts index 35c208eadb..5cb6b00cb3 100644 --- a/packages/common/ai/session-events.ts +++ b/packages/common/ai/session-events.ts @@ -15,8 +15,8 @@ export function findLastAssistant( } // A reply that ends the turn with neither text nor a tool call: the user -// sees nothing. The fast tier's upstream does this when the model spends -// its whole reasoning budget (about 4,096 tokens) before answering. +// sees nothing. Any model can do this; the fast tier does it when the model +// spends its whole reasoning budget (about 4,096 tokens) before answering. export function isEmptyStop( message: AssistantMessage | undefined ): boolean { return Boolean( message && From d44161bd8eee9859391aa1f7ad1a84f0ccf77675 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Tue, 20 Oct 2026 14:02:27 -0400 Subject: [PATCH 3/3] Studio Code: Carry the cut-off reasoning into the continue --- apps/cli/ai/runtimes/pi/index.ts | 33 ++++++++++++++++++++-------- apps/cli/ai/tests/pi-runtime.test.ts | 3 ++- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/apps/cli/ai/runtimes/pi/index.ts b/apps/cli/ai/runtimes/pi/index.ts index 8b76b5c0d9..d98cf8f723 100644 --- a/apps/cli/ai/runtimes/pi/index.ts +++ b/apps/cli/ai/runtimes/pi/index.ts @@ -74,8 +74,10 @@ type ProviderConfigInput = Parameters< ModelRuntime[ 'registerProvider' ] >[ 1 ] const STUDIO_WPCOM_PROVIDER = 'studio-wpcom'; const STUDIO_AGENT_DIR = STUDIO_SITES_ROOT; // A reply with nothing in it ends the turn without the user seeing why (see -// `isEmptyStop`). Resume it the way a person would, with a "continue" the -// user doesn't see, up to this many times per turn. +// `isEmptyStop`). Resume it with a message the user doesn't see, up to this +// many times per turn. pi leaves a reply that has only reasoning out of the +// next request, so the message carries that reasoning; otherwise the model +// starts the same reasoning over and runs out again. const MAX_EMPTY_STOP_CONTINUES = 2; const EMPTY_STOP_CONTINUE_TYPE = 'studio.empty_stop_continue'; const STUDIO_COMPACTION_SETTINGS = { @@ -239,18 +241,25 @@ async function runAgentSessionTurn( setActiveSession( session ); let emptyStopContinues = 0; let continueAfterEmptyStop = false; + let cutOffReasoning = ''; unsubscribe = session.subscribe( ( event ) => { updateStudioToolPayloadGuardState( event, payloadGuardState ); if ( event.type === 'agent_end' && ! event.willRetry && - emptyStopContinues < MAX_EMPTY_STOP_CONTINUES && - isEmptyStop( findLastAssistant( event.messages ) ) + emptyStopContinues < MAX_EMPTY_STOP_CONTINUES ) { - // Not final: consumers wait for the turn's real end. - continueAfterEmptyStop = true; - config.onEvent( { ...event, willRetry: true } ); - return; + const lastAssistant = findLastAssistant( event.messages ); + if ( isEmptyStop( lastAssistant ) ) { + // Not final: consumers wait for the turn's real end. + continueAfterEmptyStop = true; + cutOffReasoning = ( lastAssistant?.content ?? [] ) + .map( ( block ) => ( block.type === 'thinking' ? block.thinking : '' ) ) + .join( '' ) + .trim(); + config.onEvent( { ...event, willRetry: true } ); + return; + } } config.onEvent( event ); } ); @@ -278,7 +287,13 @@ async function runAgentSessionTurn( } emptyStopContinues++; await session.sendCustomMessage( - { customType: EMPTY_STOP_CONTINUE_TYPE, content: 'continue', display: false }, + { + customType: EMPTY_STOP_CONTINUE_TYPE, + content: cutOffReasoning + ? `Your reasoning was cut off before you replied. Continue from where it stopped:\n\n${ cutOffReasoning }` + : 'continue', + display: false, + }, { triggerTurn: true } ); } diff --git a/apps/cli/ai/tests/pi-runtime.test.ts b/apps/cli/ai/tests/pi-runtime.test.ts index 905e35897e..dfbb6a94ab 100644 --- a/apps/cli/ai/tests/pi-runtime.test.ts +++ b/apps/cli/ai/tests/pi-runtime.test.ts @@ -564,7 +564,8 @@ describe( 'pi runtime', () => { { message: { customType: 'studio.empty_stop_continue', - content: 'continue', + content: + 'Your reasoning was cut off before you replied. Continue from where it stopped:\n\nreasoning…', display: false, }, options: { triggerTurn: true },