Studio Code: Continue a turn that ends with an empty reply

apps/cli/ai/eval-runner.ts

@@ -467,7 +467,6 @@ async function runEval( input: EvalRunnerInput ) {
 	let success = false;
 	let error: string | null = null;
 	let timedOut = false;
-	let lastTurnEndedEmpty = false;
 	// From the last assistant message: a provider failure ends the turn with
 	// stopReason "error" and its message here, without throwing.
 	let stopReason: string | null = null;
@@ -586,13 +585,6 @@ async function runEval( input: EvalRunnerInput ) {
 			numTurnsResult = numTurns;
 			stopReason = lastAssistant?.stopReason ?? null;
 			errorMessage = lastAssistant?.errorMessage ?? null;
-			lastTurnEndedEmpty = Boolean(
-				lastAssistant &&
-					lastAssistant.stopReason === 'stop' &&
-					! lastAssistant.content.some(
-						( block ) => block.type === 'toolCall' || ( block.type === 'text' && block.text.trim() )
-					)
-			);
 		}
 	};
 
@@ -613,30 +605,14 @@ async function runEval( input: EvalRunnerInput ) {
 		void query.interrupt();
 	}, input.timeoutMs ?? 300000 );
 
-	// Some hosted tiers end a turn with an empty message once their reasoning
-	// budget runs out. STUDIO_EVAL_AUTO_CONTINUE=<n> resumes such a turn with
-	// "continue" up to n times, the way a person would.
-	const maxAutoContinues = Number( process.env.STUDIO_EVAL_AUTO_CONTINUE ?? 0 ) || 0;
-	let autoContinues = 0;
-
-	const settle = async () => {
-		await query.result;
-		while ( autoContinues < maxAutoContinues && ! timedOut && lastTurnEndedEmpty ) {
-			autoContinues++;
-			lastTurnEndedEmpty = false;
-			query = runTurn( 'continue' );
-			await query.result;
-		}
-	};
-
 	try {
-		await settle();
+		await query.result;
 		for ( const followUp of input.followUps ?? [] ) {
 			if ( timedOut ) {
 				break;
 			}
 			query = runTurn( followUp.prompt );
-			await settle();
+			await query.result;
 		}
 	} catch ( caught ) {
 		error = caught instanceof Error ? caught.message : String( caught );
@@ -655,7 +631,6 @@ async function runEval( input: EvalRunnerInput ) {
 		timedOut,
 		model: resolvedModel,
 		numTurns: numTurnsResult,
-		autoContinues,
 		stopReason,
 		errorMessage,
 		usage,

apps/cli/ai/runtimes/pi/index.ts

@@ -37,6 +37,7 @@ import {
 	type AiModelFamily,
 	type AiModelId,
 } from '@studio/common/ai/models';
+import { findLastAssistant, isEmptyStop } from '@studio/common/ai/session-events';
 import { getAiTracksIdentity } from '@studio/common/ai/tracks-identity';
 import { type TSchema } from 'typebox';
 import { withDesignSystemPreview } from 'cli/ai/chat-artifacts';
@@ -72,6 +73,13 @@ type ProviderConfigInput = Parameters< ModelRuntime[ 'registerProvider' ] >[ 1 ]
 
 const STUDIO_WPCOM_PROVIDER = 'studio-wpcom';
 const STUDIO_AGENT_DIR = STUDIO_SITES_ROOT;
+// A reply with nothing in it ends the turn without the user seeing why (see
+// `isEmptyStop`). Resume it with a message the user doesn't see, up to this
+// many times per turn. pi leaves a reply that has only reasoning out of the
+// next request, so the message carries that reasoning; otherwise the model
+// starts the same reasoning over and runs out again.
+const MAX_EMPTY_STOP_CONTINUES = 2;
+const EMPTY_STOP_CONTINUE_TYPE = 'studio.empty_stop_continue';
 const STUDIO_COMPACTION_SETTINGS = {
 	enabled: true,
 	reserveTokens: 16_384,
@@ -231,8 +239,28 @@ async function runAgentSessionTurn(
 	try {
 		session = await createStudioAgentSession( config, family, resolved.creds, payloadGuardState );
 		setActiveSession( session );
+		let emptyStopContinues = 0;
+		let continueAfterEmptyStop = false;
+		let cutOffReasoning = '';
 		unsubscribe = session.subscribe( ( event ) => {
 			updateStudioToolPayloadGuardState( event, payloadGuardState );
+			if (
+				event.type === 'agent_end' &&
+				! event.willRetry &&
+				emptyStopContinues < MAX_EMPTY_STOP_CONTINUES
+			) {
+				const lastAssistant = findLastAssistant( event.messages );
+				if ( isEmptyStop( lastAssistant ) ) {
+					// Not final: consumers wait for the turn's real end.
+					continueAfterEmptyStop = true;
+					cutOffReasoning = ( lastAssistant?.content ?? [] )
+						.map( ( block ) => ( block.type === 'thinking' ? block.thinking : '' ) )
+						.join( '' )
+						.trim();
+					config.onEvent( { ...event, willRetry: true } );
+					return;
+				}
+			}
 			config.onEvent( event );
 		} );
 
@@ -251,6 +279,24 @@ async function runAgentSessionTurn(
 				mimeType: image.mimeType,
 			} ) ),
 		} );
+		while ( continueAfterEmptyStop ) {
+			continueAfterEmptyStop = false;
+			if ( controller.signal.aborted ) {
+				config.onEvent( syntheticErrorAgentEnd( 'aborted', '' ) );
+				break;
+			}
+			emptyStopContinues++;
+			await session.sendCustomMessage(
+				{
+					customType: EMPTY_STOP_CONTINUE_TYPE,
+					content: cutOffReasoning
+						? `Your reasoning was cut off before you replied. Continue from where it stopped:\n\n${ cutOffReasoning }`
+						: 'continue',
+					display: false,
+				},
+				{ triggerTurn: true }
+			);
+		}
 		await getPendingWork( config.session ).settle();
 	} catch ( error ) {
 		const aborted = controller.signal.aborted;

apps/cli/ai/tests/pi-runtime.test.ts

@@ -10,6 +10,9 @@ const mocks = vi.hoisted( () => ( {
 	createAgentSession: vi.fn(),
 	createdSessions: [] as FakeSession[],
 	nextEvents: null as AgentSessionEvent[] | null,
+	continueEvents: [] as AgentSessionEvent[][],
+	customMessages: [] as Array< { message: unknown; options: unknown } >,
+	onCustomMessage: null as ( () => void ) | null,
 	studioRoot: '/tmp/studio-ai-pi-runtime',
 	configRoot: '/tmp/studio-ai-pi-runtime-config',
 } ) );
@@ -164,6 +167,14 @@ class FakeSession {
 		}
 	}
 
+	async sendCustomMessage( message: unknown, options: unknown ): Promise< void > {
+		mocks.customMessages.push( { message, options } );
+		mocks.onCustomMessage?.();
+		for ( const event of mocks.continueEvents.shift() ?? [] ) {
+			this.listener?.( event );
+		}
+	}
+
 	async abort(): Promise< void > {
 		this.aborted = true;
 	}
@@ -221,6 +232,9 @@ describe( 'pi runtime', () => {
 	beforeEach( () => {
 		mocks.createdSessions.length = 0;
 		mocks.nextEvents = null;
+		mocks.continueEvents = [];
+		mocks.customMessages = [];
+		mocks.onCustomMessage = null;
 		mocks.createAgentSession.mockReset();
 		mocks.createAgentSession.mockImplementation( async ( options: CreateAgentSessionOptions ) => {
 			const session = new FakeSession( options );
@@ -524,6 +538,93 @@ describe( 'pi runtime', () => {
 		);
 	} );
 
+	describe( 'a reply with neither text nor a tool call', () => {
+		const emptyReply = () => assistantMessage( [ { type: 'thinking', thinking: 'reasoning…' } ] );
+		const runEndingWith = ( message: MessageEndEvent ): AgentSessionEvent[] => [
+			message,
+			{ type: 'agent_end', willRetry: false, messages: [ message.message ] },
+		];
+		const agentEnds = ( events: AgentSessionEvent[] ) =>
+			events.filter(
+				( event ): event is Extract< AgentSessionEvent, { type: 'agent_end' } > =>
+					event.type === 'agent_end'
+			);
+		const runFast = () =>
+			runRuntime( { prompt: 'build it', env: WPCOM_ENV, model: 'fast', session: newSession() } );
+
+		it( 'continues the turn with a message the user does not see', async () => {
+			mocks.nextEvents = runEndingWith( emptyReply() );
+			mocks.continueEvents = [
+				runEndingWith( assistantMessage( [ { type: 'text', text: 'Done.' } ] ) ),
+			];
+
+			const events = await runFast();
+
+			expect( mocks.customMessages ).toEqual( [
+				{
+					message: {
+						customType: 'studio.empty_stop_continue',
+						content:
+							'Your reasoning was cut off before you replied. Continue from where it stopped:\n\nreasoning…',
+						display: false,
+					},
+					options: { triggerTurn: true },
+				},
+			] );
+			expect( agentEnds( events ).map( ( event ) => event.willRetry ) ).toEqual( [ true, false ] );
+			expect( findAssistantText( events ) ).toBe( 'Done.' );
+		} );
+
+		it( 'ends the turn after two continues', async () => {
+			mocks.nextEvents = runEndingWith( emptyReply() );
+			mocks.continueEvents = [ runEndingWith( emptyReply() ), runEndingWith( emptyReply() ) ];
+
+			const events = await runFast();
+
+			expect( mocks.customMessages ).toHaveLength( 2 );
+			expect( agentEnds( events ).map( ( event ) => event.willRetry ) ).toEqual( [
+				true,
+				true,
+				false,
+			] );
+		} );
+
+		it( 'leaves a reply with text or a tool call alone', async () => {
+			for ( const content of [
+				[ { type: 'text', text: 'Which colours do you like?' } ],
+				[ { type: 'toolCall', id: 'call-1', name: 'site_info', arguments: {} } ],
+			] ) {
+				mocks.nextEvents = runEndingWith( assistantMessage( content ) );
+				const events = await runFast();
+				expect( agentEnds( events ).map( ( event ) => event.willRetry ) ).toEqual( [ false ] );
+			}
+			expect( mocks.customMessages ).toHaveLength( 0 );
+		} );
+
+		it( 'ends the turn as interrupted when the user stops it first', async () => {
+			mocks.nextEvents = runEndingWith( emptyReply() );
+			const events: AgentSessionEvent[] = [];
+			const handle = runStudioAgentTurn( {
+				prompt: 'build it',
+				env: WPCOM_ENV,
+				model: 'fast',
+				session: newSession(),
+				onEvent: ( event ) => {
+					events.push( event );
+					if ( event.type === 'agent_end' && event.willRetry ) {
+						void handle.interrupt();
+					}
+				},
+			} );
+			await handle.result;
+
+			expect( mocks.customMessages ).toHaveLength( 0 );
+			const ends = agentEnds( events );
+			expect( ends.map( ( event ) => event.willRetry ) ).toEqual( [ true, false ] );
+			expect( ends[ 1 ].messages[ 0 ] ).toMatchObject( { stopReason: 'aborted' } );
+		} );
+	} );
+
 	it( 'creates each AgentSession with the requested model', async () => {
 		const session = newSession();
 		const otherStudioModel = 'tier-test-other' as AiModelId;

packages/common/ai/session-events.ts

@@ -14,6 +14,19 @@ export function findLastAssistant(
 	return undefined;
 }
 
+// A reply that ends the turn with neither text nor a tool call: the user
+// sees nothing. Any model can do this; the fast tier does it when the model
+// spends its whole reasoning budget (about 4,096 tokens) before answering.
+export function isEmptyStop( message: AssistantMessage | undefined ): boolean {
+	return Boolean(
+		message &&
+			message.stopReason === 'stop' &&
+			! message.content.some(
+				( block ) => block.type === 'toolCall' || ( block.type === 'text' && block.text.trim() )
+			)
+	);
+}
+
 export interface AgentEndTurnResult {
 	success: boolean;
 	interrupted: boolean;

scripts/eval/README.md

@@ -39,7 +39,7 @@ Run one named test with `npm run eval -- --filter-pattern "preview sites"` (rege
 
 ## Running the runner directly
 
-`node apps/cli/dist/cli/eval-runner.mjs "<prompt>" "" '{"vars":{"timeoutMs":1800000}}'` runs one turn outside promptfoo and prints `EVAL_RUNNER_RESULT_FILE=<path>`. Environment variables: `STUDIO_EVAL_MODEL` picks the model (`fast`, `claude-sonnet-5`, …), `STUDIO_EVAL_SESSION_DIR` sets where the session JSONL (the full transcript, kept for every run and reported as `sessionFile`) is written, by default `<tmpdir>/studio-eval-sessions`, and `STUDIO_EVAL_AUTO_CONTINUE=<n>` resumes a turn that ended with an empty assistant message (some hosted tiers stop that way when their reasoning budget runs out) with `continue`, up to `n` times; the result reports the count as `autoContinues`. A `followUps` var (a list of `{ "prompt": "…" }` objects; objects because promptfoo turns an array of strings into one test per element) sends each as a further user message in the same session once the previous one finishes, for cases that measure a conversation rather than a single request.
+`node apps/cli/dist/cli/eval-runner.mjs "<prompt>" "" '{"vars":{"timeoutMs":1800000}}'` runs one turn outside promptfoo and prints `EVAL_RUNNER_RESULT_FILE=<path>`. Environment variables: `STUDIO_EVAL_MODEL` picks the model (`fast`, `claude-sonnet-5`, …), and `STUDIO_EVAL_SESSION_DIR` sets where the session JSONL (the full transcript, kept for every run and reported as `sessionFile`) is written, by default `<tmpdir>/studio-eval-sessions`. A turn that ends with an empty reply is resumed by the runtime itself, as in the app, and shows up in `timing.emptyStops`. A `followUps` var (a list of `{ "prompt": "…" }` objects; objects because promptfoo turns an array of strings into one test per element) sends each as a further user message in the same session once the previous one finishes, for cases that measure a conversation rather than a single request.
 
 ## Adding tests