From 0a5e2506940c47f26b435a6043f3a98ca6affa01 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Tue, 20 Oct 2026 11:10:00 -0400 Subject: [PATCH 1/3] Studio Code evals: Script the user's answers to the agent's questions --- apps/cli/ai/eval-runner.ts | 30 +++++++++++++++++++++++++++++- scripts/eval/README.md | 4 ++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/apps/cli/ai/eval-runner.ts b/apps/cli/ai/eval-runner.ts index 6acff923fd..342c414141 100644 --- a/apps/cli/ai/eval-runner.ts +++ b/apps/cli/ai/eval-runner.ts @@ -51,7 +51,7 @@ import { STUDIO_SITES_ROOT } from 'cli/lib/site-paths'; import { Logger } from 'cli/logger'; import type { AgentSessionEvent } from '@earendil-works/pi-coding-agent'; import type { AiProviderId } from 'cli/ai/providers'; -import type { SiteInfo } from 'cli/ai/types'; +import type { AskUserHandler, SiteInfo } from 'cli/ai/types'; // Optional fixtures a test can pre-seed before the agent turn, so flows that // depend on connected remote sites and/or preview sites can be exercised @@ -93,6 +93,11 @@ interface EvalRunnerInput { // Objects rather than strings: promptfoo expands an array of strings into // one test case per element. followUps?: Array< { prompt: string } >; + // Scripted replies for the questions the agent asks the user, in order: + // each question takes the next label. With answers set, the turn runs with + // Studio chat's ask-the-user handler, so AskUserQuestion and tools that + // confirm with the user work; once they run out, the user cancels. + answers?: Array< { label: string } >; timeoutMs?: number; model?: AiModelId; seed?: EvalSeed; @@ -409,9 +414,14 @@ function readInput(): EvalRunnerInput { ? z.array( z.object( { prompt: z.string() } ) ).parse( vars.followUps ) : undefined; + const answers = vars.answers + ? z.array( z.object( { label: z.string() } ) ).parse( vars.answers ) + : undefined; + return { prompt: ( vars.prompt as string ) ?? prompt, followUps, + answers, timeoutMs: typeof vars.timeoutMs === 'number' ? vars.timeoutMs : undefined, model, seed, @@ -456,6 +466,22 @@ async function runEval( input: EvalRunnerInput ) { const toolEvents: ToolEvent[] = []; const requests: ModelRequestRecord[] = []; const textSegments: string[] = []; + const questions: Array< { question: string; options: string[]; answer: string } > = []; + const pendingAnswers = [ ...( input.answers ?? [] ) ]; + const onAskUser: AskUserHandler | undefined = input.answers + ? async ( asked ) => + Object.fromEntries( + asked.map( ( { question, options } ) => { + const answer = pendingAnswers.shift()?.label ?? 'Cancel'; + questions.push( { + question, + options: options.map( ( option ) => option.label ), + answer, + } ); + return [ question, answer ]; + } ) + ) + : undefined; const toolNameById = new Map< string, string >(); const toolEventById = new Map< string, ToolEvent >(); let firstToolError: FirstToolError | null = null; @@ -603,6 +629,7 @@ async function runEval( input: EvalRunnerInput ) { session, activeSite: seeded?.activeSite, onEvent: handleEvent, + onAskUser, ...( input.model ? { model: input.model } : {} ), } ); let query = runTurn( prompt ); @@ -669,6 +696,7 @@ async function runEval( input: EvalRunnerInput ) { toolEvents, firstToolError, textSegments, + questions, inspections, }; } diff --git a/scripts/eval/README.md b/scripts/eval/README.md index cb274f04a7..01e0d0f084 100644 --- a/scripts/eval/README.md +++ b/scripts/eval/README.md @@ -39,11 +39,11 @@ Run one named test with `npm run eval -- --filter-pattern "preview sites"` (rege ## Running the runner directly -`node apps/cli/dist/cli/eval-runner.mjs "" "" '{"vars":{"timeoutMs":1800000}}'` runs one turn outside promptfoo and prints `EVAL_RUNNER_RESULT_FILE=`. Environment variables: `STUDIO_EVAL_MODEL` picks the model (`fast`, `claude-sonnet-5`, …), `STUDIO_EVAL_SESSION_DIR` sets where the session JSONL (the full transcript, kept for every run and reported as `sessionFile`) is written, by default `/studio-eval-sessions`, and `STUDIO_EVAL_AUTO_CONTINUE=` resumes a turn that ended with an empty assistant message (some hosted tiers stop that way when their reasoning budget runs out) with `continue`, up to `n` times; the result reports the count as `autoContinues`. A `followUps` var (a list of `{ "prompt": "…" }` objects; objects because promptfoo turns an array of strings into one test per element) sends each as a further user message in the same session once the previous one finishes, for cases that measure a conversation rather than a single request. +`node apps/cli/dist/cli/eval-runner.mjs "" "" '{"vars":{"timeoutMs":1800000}}'` runs one turn outside promptfoo and prints `EVAL_RUNNER_RESULT_FILE=`. Environment variables: `STUDIO_EVAL_MODEL` picks the model (`fast`, `claude-sonnet-5`, …), `STUDIO_EVAL_SESSION_DIR` sets where the session JSONL (the full transcript, kept for every run and reported as `sessionFile`) is written, by default `/studio-eval-sessions`, and `STUDIO_EVAL_AUTO_CONTINUE=` resumes a turn that ended with an empty assistant message (some hosted tiers stop that way when their reasoning budget runs out) with `continue`, up to `n` times; the result reports the count as `autoContinues`. A `followUps` var (a list of `{ "prompt": "…" }` objects; objects because promptfoo turns an array of strings into one test per element) sends each as a further user message in the same session once the previous one finishes, for cases that measure a conversation rather than a single request. An `answers` var (a list of `{ "label": "…" }` objects) scripts the user's replies: the turn then runs with Studio chat's ask-the-user handler, so `AskUserQuestion` and tools that confirm with the user work, each question taking the next label in order, and the user cancels once they run out. Without it the agent has no way to ask the user, unlike the app and `studio ai`, which always give it one. ## Adding tests -Tests live in `promptfoo.config.yaml`. The runner returns raw JSON (`toolCalls`, `toolResults`, `toolEvents`, `textSegments`, `questions`, `turnDurationsMs`, `usage` (input, output, cache read, cache write and total tokens, summed over every assistant message), `stopReason` and `errorMessage` (from the last assistant message, so a provider failure shows its cause), `sessionFile`, `requests` (one per model request: start, end and duration, context tokens sent, output tokens, `stopReason` and any `errorMessage`, and whether it carried text or tool calls), `timing` (model time, tool wall time with parallel calls counted once, peak context, empty stops — a `stop` with neither text nor a tool call — and per-tool calls, total and max time and output size), and `inspections` when the seed asks for them) — write assertions in the YAML, not in the runner. Each `toolEvents` entry also carries `outputChars`, the length of the result's text. Each `toolResults` entry carries the result's text block plus any image content blocks (`images`, base64) and structured `details` (e.g. `studioArtifacts`). +Tests live in `promptfoo.config.yaml`. The runner returns raw JSON (`toolCalls`, `toolResults`, `toolEvents`, `textSegments`, `questions` (each question the agent asked the user, its option labels and the scripted answer; see `answers`), `turnDurationsMs`, `usage` (input, output, cache read, cache write and total tokens, summed over every assistant message), `stopReason` and `errorMessage` (from the last assistant message, so a provider failure shows its cause), `sessionFile`, `requests` (one per model request: start, end and duration, context tokens sent, output tokens, `stopReason` and any `errorMessage`, and whether it carried text or tool calls), `timing` (model time, tool wall time with parallel calls counted once, peak context, empty stops — a `stop` with neither text nor a tool call — and per-tool calls, total and max time and output size), and `inspections` when the seed asks for them) — write assertions in the YAML, not in the runner. Each `toolEvents` entry also carries `outputChars`, the length of the result's text. Each `toolResults` entry carries the result's text block plus any image content blocks (`images`, base64) and structured `details` (e.g. `studioArtifacts`). ### Seeding fixtures From 2ae5501644f43ee896bec01db81b39e2c2a9abde Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Tue, 20 Oct 2026 11:18:55 -0400 Subject: [PATCH 2/3] Studio Code evals: Connect WordPress.com sites to the seeded active site --- apps/cli/ai/eval-runner.ts | 9 +++++++++ scripts/eval/README.md | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/apps/cli/ai/eval-runner.ts b/apps/cli/ai/eval-runner.ts index 342c414141..8a55bde3f9 100644 --- a/apps/cli/ai/eval-runner.ts +++ b/apps/cli/ai/eval-runner.ts @@ -81,6 +81,9 @@ const evalSeedSchema = z.object( { files: z.record( z.string(), z.string() ).optional(), wpCli: z.array( z.array( z.string() ) ).optional(), inspect: z.array( z.array( z.string() ) ).optional(), + // WordPress.com sites connected to this site, whose id is only known + // once it exists. + connectedWpcomSites: z.array( syncSiteSchema.omit( { localSiteId: true } ) ).optional(), } ) .optional(), } ); @@ -235,6 +238,12 @@ async function seedFixtures( seed: EvalSeed ): Promise< SeededFixtures > { path: activeSitePath, running: false, }; + const { id: localSiteId } = await getSiteByFolder( activeSitePath ); + for ( const connection of activeSite.connectedWpcomSites ?? [] ) { + const site = { ...connection, localSiteId }; + await addConnectedWpcomSite( localSiteId, site ); + seededConnections.push( site ); + } } const inspect = async (): Promise< Inspection[] > => { diff --git a/scripts/eval/README.md b/scripts/eval/README.md index 01e0d0f084..d141014a69 100644 --- a/scripts/eval/README.md +++ b/scripts/eval/README.md @@ -47,7 +47,7 @@ Tests live in `promptfoo.config.yaml`. The runner returns raw JSON (`toolCalls`, ### Seeding fixtures -Set a `seed` var on a test to pre-populate config files before the agent turn — useful for flows that depend on connected remote sites or preview sites without making a real WordPress.com connection or preview. The seed accepts `localSite` (written to `cli.json`), `activeSite` (creates a real stopped site under `~/Studio/`, with optional `files` written into it by path relative to the site root, and makes it the session's active site, which the system prompt states in its `## Active site` section, as it does for the site picked in the app or CLI; the site is deleted after the turn. `wpCli` is a list of WP-CLI argv arrays run on it after the files are written, for state that lives in the database, such as activating a theme or saving Site Editor customizations; a command fails the seed when it exits non-zero or prints `SEED_ERROR`, since `wp eval` exits 0 on PHP warnings. `inspect` is a list of WP-CLI argv arrays run after the turn, before the site is deleted; their output lands in the result's `inspections` for assertions on the final site state), `connectedWpcomSites` (written to `shared.json`, requires `studio auth login`), `snapshots` (written to `cli.json`), and `globalInstructions` (written to `~/.studio/knowledge/instructions.md`; the file's prior content — or absence — is restored after the turn). Everything seeded is removed automatically after the turn so reruns start clean. +Set a `seed` var on a test to pre-populate config files before the agent turn — useful for flows that depend on connected remote sites or preview sites without making a real WordPress.com connection or preview. The seed accepts `localSite` (written to `cli.json`), `activeSite` (creates a real stopped site under `~/Studio/`, with optional `files` written into it by path relative to the site root, and makes it the session's active site, which the system prompt states in its `## Active site` section, as it does for the site picked in the app or CLI; the site is deleted after the turn. `wpCli` is a list of WP-CLI argv arrays run on it after the files are written, for state that lives in the database, such as activating a theme or saving Site Editor customizations; a command fails the seed when it exits non-zero or prints `SEED_ERROR`, since `wp eval` exits 0 on PHP warnings. `inspect` is a list of WP-CLI argv arrays run after the turn, before the site is deleted; their output lands in the result's `inspections` for assertions on the final site state. `connectedWpcomSites` connects WordPress.com sites to it, as below without `localSiteId`, which is the new site's), `connectedWpcomSites` (written to `shared.json`, requires `studio auth login`), `snapshots` (written to `cli.json`), and `globalInstructions` (written to `~/.studio/knowledge/instructions.md`; the file's prior content — or absence — is restored after the turn). Everything seeded is removed automatically after the turn so reruns start clean. ### Graded assertions From 0163412f51b60527fcb444a5510aa1d6551694fa Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Tue, 20 Oct 2026 20:13:45 -0400 Subject: [PATCH 3/3] Studio Code evals: Make one of the user's WordPress.com sites the active site --- apps/cli/ai/eval-runner.ts | 18 +++++++++++++++++- scripts/eval/README.md | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/apps/cli/ai/eval-runner.ts b/apps/cli/ai/eval-runner.ts index dba675d0ca..d9520a28af 100644 --- a/apps/cli/ai/eval-runner.ts +++ b/apps/cli/ai/eval-runner.ts @@ -86,6 +86,15 @@ const evalSeedSchema = z.object( { connectedWpcomSites: z.array( syncSiteSchema.omit( { localSiteId: true } ) ).optional(), } ) .optional(), + // One of the user's WordPress.com sites, active for the turn as when it is + // picked in the app or CLI. Nothing is created; the site must exist. + remoteActiveSite: z + .object( { + name: z.string(), + url: z.string(), + wpcomSiteId: z.number(), + } ) + .optional(), } ); type EvalSeed = z.infer< typeof evalSeedSchema >; @@ -133,6 +142,7 @@ async function seedFixtures( seed: EvalSeed ): Promise< SeededFixtures > { snapshots = [], globalInstructions, activeSite, + remoteActiveSite, } = seed; let restoreInstructions: ( () => Promise< void > ) | null = null; @@ -297,7 +307,13 @@ async function seedFixtures( seed: EvalSeed ): Promise< SeededFixtures > { } } }; - return { cleanup, inspect, activeSite: seededSite }; + return { + cleanup, + inspect, + activeSite: remoteActiveSite + ? { ...remoteActiveSite, path: '', running: false, remote: true } + : seededSite, + }; } function extractToolCalls( event: AgentSessionEvent ) { diff --git a/scripts/eval/README.md b/scripts/eval/README.md index 653cd69e6d..b1924e267e 100644 --- a/scripts/eval/README.md +++ b/scripts/eval/README.md @@ -47,7 +47,7 @@ Tests live in `promptfoo.config.yaml`. The runner returns raw JSON (`toolCalls`, ### Seeding fixtures -Set a `seed` var on a test to pre-populate config files before the agent turn — useful for flows that depend on connected remote sites or preview sites without making a real WordPress.com connection or preview. The seed accepts `localSite` (written to `cli.json`), `activeSite` (creates a real stopped site under `~/Studio/`, with optional `files` written into it by path relative to the site root, and makes it the session's active site, which the system prompt states in its `## Active site` section, as it does for the site picked in the app or CLI; the site is deleted after the turn. `wpCli` is a list of WP-CLI argv arrays run on it after the files are written, for state that lives in the database, such as activating a theme or saving Site Editor customizations; a command fails the seed when it exits non-zero or prints `SEED_ERROR`, since `wp eval` exits 0 on PHP warnings. `inspect` is a list of WP-CLI argv arrays run after the turn, before the site is deleted; their output lands in the result's `inspections` for assertions on the final site state. `connectedWpcomSites` connects WordPress.com sites to it, as below without `localSiteId`, which is the new site's), `connectedWpcomSites` (written to `shared.json`, requires `studio auth login`), `snapshots` (written to `cli.json`), and `globalInstructions` (written to `~/.studio/knowledge/instructions.md`; the file's prior content — or absence — is restored after the turn). Everything seeded is removed automatically after the turn so reruns start clean. +Set a `seed` var on a test to pre-populate config files before the agent turn — useful for flows that depend on connected remote sites or preview sites without making a real WordPress.com connection or preview. The seed accepts `localSite` (written to `cli.json`), `activeSite` (creates a real stopped site under `~/Studio/`, with optional `files` written into it by path relative to the site root, and makes it the session's active site, which the system prompt states in its `## Active site` section, as it does for the site picked in the app or CLI; the site is deleted after the turn. `wpCli` is a list of WP-CLI argv arrays run on it after the files are written, for state that lives in the database, such as activating a theme or saving Site Editor customizations; a command fails the seed when it exits non-zero or prints `SEED_ERROR`, since `wp eval` exits 0 on PHP warnings. `inspect` is a list of WP-CLI argv arrays run after the turn, before the site is deleted; their output lands in the result's `inspections` for assertions on the final site state. `connectedWpcomSites` connects WordPress.com sites to it, as below without `localSiteId`, which is the new site's), `remoteActiveSite` (`name`, `url` and `wpcomSiteId` of one of the user's existing WordPress.com sites, made the session's active site as when it is picked in the app or CLI; nothing is created or deleted, so any change the agent makes stays on that site), `connectedWpcomSites` (written to `shared.json`, requires `studio auth login`), `snapshots` (written to `cli.json`), and `globalInstructions` (written to `~/.studio/knowledge/instructions.md`; the file's prior content — or absence — is restored after the turn). Everything seeded is removed automatically after the turn so reruns start clean. ### Graded assertions