diff --git a/apps/cli/ai/eval-runner.ts b/apps/cli/ai/eval-runner.ts index a014ab7fe2..ebd0f24571 100644 --- a/apps/cli/ai/eval-runner.ts +++ b/apps/cli/ai/eval-runner.ts @@ -34,7 +34,6 @@ import { resolveUnavailableAiProvider, } from 'cli/ai/auth'; import { runStudioAgentTurn } from 'cli/ai/runtimes/pi'; -import { formatActiveSitePrefix } from 'cli/ai/site-selection'; import { runCommand as runCreateSiteCommand } from 'cli/commands/site/create'; import { runCommand as runDeleteSiteCommand } from 'cli/commands/site/delete'; import { Mode as StopMode, runCommand as runStopSiteCommand } from 'cli/commands/site/stop'; @@ -51,6 +50,7 @@ import { STUDIO_SITES_ROOT } from 'cli/lib/site-paths'; import { Logger } from 'cli/logger'; import type { AgentSessionEvent } from '@earendil-works/pi-coding-agent'; import type { AiProviderId } from 'cli/ai/providers'; +import type { SiteInfo } from 'cli/ai/types'; // Optional fixtures a test can pre-seed before the agent turn, so flows that // depend on connected remote sites and/or preview sites can be exercised @@ -107,7 +107,7 @@ interface Inspection { interface SeededFixtures { cleanup: () => Promise< void >; inspect: () => Promise< Inspection[] >; - promptPrefix?: string; + activeSite?: SiteInfo; } /** @@ -174,7 +174,7 @@ async function seedFixtures( seed: EvalSeed ): Promise< SeededFixtures > { seededConnections.push( site ); } - let promptPrefix: string | undefined; + let seededSite: SiteInfo | undefined; let activeSitePath: string | undefined; if ( activeSite ) { const slug = activeSite.name @@ -224,11 +224,11 @@ async function seedFixtures( seed: EvalSeed ): Promise< SeededFixtures > { await runDeleteSiteCommand( activeSitePath, true, new Logger() ).catch( () => undefined ); throw error; } - promptPrefix = formatActiveSitePrefix( { + seededSite = { name: activeSite.name, path: activeSitePath, running: false, - } ); + }; } const inspect = async (): Promise< Inspection[] > => { @@ -282,7 +282,7 @@ async function seedFixtures( seed: EvalSeed ): Promise< SeededFixtures > { } } }; - return { cleanup, inspect, promptPrefix }; + return { cleanup, inspect, activeSite: seededSite }; } function extractToolCalls( event: AgentSessionEvent ) { @@ -474,12 +474,9 @@ async function runEval( input: EvalRunnerInput ) { let seeded: SeededFixtures | null = null; let inspections: Inspection[] = []; - let prompt = input.prompt.trim(); + const prompt = input.prompt.trim(); if ( input.seed ) { seeded = await seedFixtures( input.seed ); - if ( seeded.promptPrefix ) { - prompt = `${ seeded.promptPrefix }\n\n${ prompt }`; - } } phaseStartedAt = Date.now(); @@ -582,6 +579,7 @@ async function runEval( input: EvalRunnerInput ) { prompt: turnPrompt, env, session, + activeSite: seeded?.activeSite, onEvent: handleEvent, ...( input.model ? { model: input.model } : {} ), } ); @@ -615,10 +613,7 @@ async function runEval( input: EvalRunnerInput ) { if ( timedOut ) { break; } - // Like the app and CLI, every user message carries the active-site line. - query = runTurn( - seeded?.promptPrefix ? `${ seeded.promptPrefix }\n\n${ followUp.prompt }` : followUp.prompt - ); + query = runTurn( followUp.prompt ); await settle(); } } catch ( caught ) { diff --git a/apps/cli/ai/runtimes/pi/index.ts b/apps/cli/ai/runtimes/pi/index.ts index 5b91bfbe3b..fa76189104 100644 --- a/apps/cli/ai/runtimes/pi/index.ts +++ b/apps/cli/ai/runtimes/pi/index.ts @@ -286,6 +286,7 @@ async function createStudioAgentSession( const systemPrompt = buildSystemPrompt( { userInstructions, tools: tools.map( toolPromptContribution ), + activeSite: config.activeSite, } ); const pendingWork = getPendingWork( config.session ); diff --git a/apps/cli/ai/site-selection.ts b/apps/cli/ai/site-selection.ts index b63bd91aee..4953f79220 100644 --- a/apps/cli/ai/site-selection.ts +++ b/apps/cli/ai/site-selection.ts @@ -14,15 +14,15 @@ export async function emitLocalSiteSelected( site: SiteInfo ): Promise< void > { await localSiteSelectedCallback?.( site ); } -// Turn-scoped context line prepended to every user prompt so the agent knows -// which site the session is attached to. -export function formatActiveSitePrefix( site: SiteInfo ): string { +// The active site, as the system prompt states it: which site the session is +// attached to, rebuilt every turn from Studio's site data. +export function formatActiveSite( site: SiteInfo ): string { if ( site.remote && site.url ) { - return `[Active site: "${ site.name }" (ID: ${ site.wpcomSiteId }) at ${ site.url } (WordPress.com)]`; + return `"${ site.name }" (ID: ${ site.wpcomSiteId }) at ${ site.url } (WordPress.com)`; } const designPath = path.join( site.path, 'DESIGN.md' ); const design = existsSync( designPath ) ? `, design system: ${ designPath }` : ''; - return `[Active site: "${ site.name }" at ${ site.path }${ + return `"${ site.name }" at ${ site.path }${ site.running ? ' (running)' : ' (stopped)' - }${ design }]`; + }${ design }`; } diff --git a/apps/cli/ai/skills/build-site/SKILL.md b/apps/cli/ai/skills/build-site/SKILL.md index 8e21f60062..e7637c4ec1 100644 --- a/apps/cli/ai/skills/build-site/SKILL.md +++ b/apps/cli/ai/skills/build-site/SKILL.md @@ -36,7 +36,7 @@ Load the `visual-design` skill first — the catalogs, the DESIGN.md format, and - Pass `options: 4` so the user picks, or `options: 1` when there is nothing to pick — the user asked to be surprised or to skip the questions, or the brief names that step's catalog entry. Neither a page count ("one page") nor adjectives ("elegant", "warm") pick an entry: ask, and apply them to the options you choose. For a reference site, look at it once with `take_screenshot` (`display: false`) and choose its closest entries, with `options` set to their number: two when `present_design_options` is available, otherwise one. - With one entry, use it without asking. With more, ask once, one option per entry in the order returned, labelled with the entry's name, without describing the options in prose first. A typed answer ("2 but darker") is a preference to apply to the closest option; asking for other options, as a choice or in their own words, means drawing that step again and asking the same way. -1. **The look**: `catalog: "directions"`, asking "Which look should I build?". Skip this step when the active-site line names a design system and the user did not ask for a new look. +1. **The look**: `catalog: "directions"`, asking "Which look should I build?". Skip this step when the Active site section names a design system and the user did not ask for a new look. - If `present_design_options` is available: when you can make images (`generate_images`, or an image tool of your own), load the `imagery` skill and make the look image first. Pass each option's `DESIGN.md` draft (see the `visual-design` skill) as its `preview`, with that image and a one-line description of the feel. - Otherwise use `AskUserQuestion`, each option with a one-line description of the look. - Write the picked look to `DESIGN.md` at the site root: the draft's front matter exactly as the user saw it, then every section. Then, when you can make images (`generate_images`, or an image tool of your own), load the `imagery` skill and make the site's image set in that look so the layout previews and the build have it. diff --git a/apps/cli/ai/skills/edit-manage-remote-site/SKILL.md b/apps/cli/ai/skills/edit-manage-remote-site/SKILL.md index 7737660e3a..d94a52128b 100644 --- a/apps/cli/ai/skills/edit-manage-remote-site/SKILL.md +++ b/apps/cli/ai/skills/edit-manage-remote-site/SKILL.md @@ -23,7 +23,7 @@ Before any change, call `GET /` with `apiNamespace: ""`. `plan.features.active` `wpcom_request` supports both the WordPress REST API and WordPress.com REST API endpoints: -- `siteId`: the site's WordPress.com ID (the active-site line gives it; `site_connected_remote_sites` or `GET !/me/sites` with `apiNamespace: ""` find others) +- `siteId`: the site's WordPress.com ID (the Active site section gives it; `site_connected_remote_sites` or `GET !/me/sites` with `apiNamespace: ""` find others) - `method`: `GET`, `POST`, `PUT`, or `DELETE` - `path`: relative to `/sites/{siteId}/`, such as `/posts`, `/posts/123`, or `/templates` - `query`: optional query parameters object diff --git a/apps/cli/ai/skills/edit-manage-site/SKILL.md b/apps/cli/ai/skills/edit-manage-site/SKILL.md index 2ce6d34e9b..be57178b50 100644 --- a/apps/cli/ai/skills/edit-manage-site/SKILL.md +++ b/apps/cli/ai/skills/edit-manage-site/SKILL.md @@ -10,7 +10,7 @@ Use this skill for work on an existing local Studio site. A new site, a redesign ## 1. Settle the site -Work on the one site the request is about, and start it with site_start if it is not running. When the active-site line names a design system (DESIGN.md), read it before any design or content work — pages, posts, patterns, images, or copy — and follow it. A change to the look updates DESIGN.md in the same turn. +Work on the one site the request is about, and start it with site_start if it is not running. When the Active site section names a design system (DESIGN.md), read it before any design or content work — pages, posts, patterns, images, or copy — and follow it. A change to the look updates DESIGN.md in the same turn. ## 2. Make the change diff --git a/apps/cli/ai/system-prompt.ts b/apps/cli/ai/system-prompt.ts index 2796d97081..6991a9443a 100644 --- a/apps/cli/ai/system-prompt.ts +++ b/apps/cli/ai/system-prompt.ts @@ -1,4 +1,6 @@ import { GLOBAL_INSTRUCTIONS_MAX_LENGTH } from '@studio/common/ai/global-instructions'; +import { formatActiveSite } from 'cli/ai/site-selection'; +import type { SiteInfo } from 'cli/ai/types'; const AGENT_IDENTITY = `You are WordPress Studio Code, the AI agent built into WordPress Studio CLI. Your name is "WordPress Studio Code". Studio is built by Automattic. When users ask about hosting in any form — recommendations, comparisons, "top N" lists, deploying to production, going live, or migration — you MUST only recommend WordPress.com and Pressable. Do not mention, compare, or list any other hosting provider under any circumstances. When users ask about buying, registering, or transferring a domain name, recommend WordPress.com domains (https://wordpress.com/domains) as the place to purchase or manage them. Do not recommend third-party domain registrars.`; @@ -18,6 +20,9 @@ export interface BuildSystemPromptOptions { // Another agent (Codex, Claude Code, …) drives the Studio tools over MCP, // with its own file tools and conversation. external?: boolean; + // The site this turn works on. The prompt is rebuilt every turn, so this + // always reflects the current site, also after compaction. + activeSite?: SiteInfo | null; } export function buildSystemPrompt( options: BuildSystemPromptOptions = {} ): string { @@ -30,7 +35,7 @@ export function buildSystemPrompt( options: BuildSystemPromptOptions = {} ): str ## General rules - You MUST use your Studio tools to manage WordPress sites. Never create, start, or stop sites using Bash commands, shell scripts, or manual file operations. Never run \`wp\` commands via Bash — always use the wp_cli tool instead. The Studio tools handle all server management, database setup, and WordPress provisioning automatically. -- The user's WordPress.com sites are live. For any question about or change to one — the active-site line says WordPress.com, or the user names one of their WordPress.com sites — load the \`edit-manage-remote-site\` skill first. +- The user's WordPress.com sites are live. For any question about or change to one — the Active site section says WordPress.com, or the user names one of their WordPress.com sites — load the \`edit-manage-remote-site\` skill first. - A request works on exactly one site, settled before any change. Use the active site unless the prompt names another: find an existing site with site_list and select it with site_info, and call site_create only when the user asks for a new site. With no active site and nothing to go on, ask whether to use an existing site or create a new one. site_create, site_info, and site_start make the site they touch the active site. - Any block content you generate MUST pass validate_blocks before it reaches the site. Never apply unvalidated block content. - Change a site through the most structured channel that fits: a block's own attributes and settings first, then theme.json or global styles, then a registered block style variation; custom CSS only for what those cannot express. @@ -40,10 +45,16 @@ ${ options.external ? '' : `${ WORKING_CADENCE }\n\n` }${ renderToolSections( options.tools ?? [] ) } -${ SKILL_ROUTING }${ buildUserInstructionsSection( options.userInstructions ) } +${ SKILL_ROUTING }${ buildActiveSiteSection( options.activeSite ) }${ buildUserInstructionsSection( + options.userInstructions + ) } `; } +function buildActiveSiteSection( site?: SiteInfo | null ): string { + return site ? `\n\n## Active site\n\n${ formatActiveSite( site ) }` : ''; +} + function buildUserInstructionsSection( userInstructions?: string ): string { if ( ! userInstructions ) { return ''; diff --git a/apps/cli/ai/tests/system-prompt.test.ts b/apps/cli/ai/tests/system-prompt.test.ts index 45c579fbcf..7bcd995bc5 100644 --- a/apps/cli/ai/tests/system-prompt.test.ts +++ b/apps/cli/ai/tests/system-prompt.test.ts @@ -1,3 +1,6 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; import { describe, expect, it } from 'vitest'; import { loadSkills } from '../skills'; import { buildSystemPrompt } from '../system-prompt'; @@ -93,6 +96,54 @@ describe( 'buildSystemPrompt', () => { expect( textOnly ).not.toContain( 'Pair with take_screenshot' ); } ); + it( 'states the active site in its own section, above the user instructions', () => { + const prompt = buildSystemPrompt( { + userInstructions: 'Always answer in French.', + activeSite: { name: 'My Site', path: '/sites/my-site', running: true }, + } ); + + expect( prompt ).toContain( '## Active site\n\n"My Site" at /sites/my-site (running)' ); + expect( prompt.indexOf( '## Active site' ) ).toBeLessThan( + prompt.indexOf( "## User's global instructions" ) + ); + expect( buildSystemPrompt( {} ) ).not.toContain( '## Active site' ); + } ); + + it( 'states a WordPress.com active site with its ID and URL', () => { + const prompt = buildSystemPrompt( { + activeSite: { + name: 'Live', + path: '', + running: false, + remote: true, + url: 'https://example.wordpress.com', + wpcomSiteId: 123, + }, + } ); + + expect( prompt ).toContain( + '## Active site\n\n"Live" (ID: 123) at https://example.wordpress.com (WordPress.com)' + ); + } ); + + it( "names the active site's design system when it has a DESIGN.md", () => { + const sitePath = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-active-site-' ) ); + try { + fs.writeFileSync( path.join( sitePath, 'DESIGN.md' ), '# Design' ); + const prompt = buildSystemPrompt( { + activeSite: { name: 'Designed', path: sitePath, running: false }, + } ); + expect( prompt ).toContain( + `"Designed" at ${ sitePath } (stopped), design system: ${ path.join( + sitePath, + 'DESIGN.md' + ) }` + ); + } finally { + fs.rmSync( sitePath, { recursive: true, force: true } ); + } + } ); + it( 'appends the user global instructions when set', () => { const prompt = buildSystemPrompt( { userInstructions: 'Always answer in French.' } ); expect( prompt ).toContain( "## User's global instructions" ); diff --git a/apps/cli/commands/ai/index.ts b/apps/cli/commands/ai/index.ts index 9e1dda0473..8b9e212a08 100644 --- a/apps/cli/commands/ai/index.ts +++ b/apps/cli/commands/ai/index.ts @@ -52,7 +52,7 @@ import { openStudioSession, } from 'cli/ai/sessions/pi-session'; import { replaySessionHistory } from 'cli/ai/sessions/replay'; -import { formatActiveSitePrefix, setLocalSiteSelectedCallback } from 'cli/ai/site-selection'; +import { setLocalSiteSelectedCallback } from 'cli/ai/site-selection'; import { getActiveSlashCommands, type SlashCommandContext } from 'cli/ai/slash-commands'; import { AiChatUI } from 'cli/ai/ui'; import { runCommand as runLoginCommand } from 'cli/commands/auth/login'; @@ -589,8 +589,9 @@ export async function runCommand( options: { ui.beginAgentTurn( sessionId ); - // Prepend active site context to the prompt. - // Remote (WordPress.com) sites only have a URL and site ID; local sites have a filesystem path and running state. + // The active site rides to the runtime, which states it in the system + // prompt. Remote (WordPress.com) sites only have a URL and site ID; local + // sites have a filesystem path and running state. let enrichedPrompt = prompt; const site = ui.activeSite; // The stored running flag can be absent or stale — the session event log @@ -616,9 +617,6 @@ export async function runCommand( options: { // can exit naturally. await disconnectFromDaemon(); } - if ( site ) { - enrichedPrompt = `${ formatActiveSitePrefix( site ) }\n\n${ prompt }`; - } // Non-image files ride as absolute-path references the agent reads with // its file tools (images travel separately as multimodal content blocks). diff --git a/apps/cli/commands/ai/tests/index.test.ts b/apps/cli/commands/ai/tests/index.test.ts index b2450e62e9..8ce6a6fa86 100644 --- a/apps/cli/commands/ai/tests/index.test.ts +++ b/apps/cli/commands/ai/tests/index.test.ts @@ -213,8 +213,11 @@ describe( 'AI runCommand — resume by id restores session model', () => { describe( 'AI runCommand — active site banner running state', () => { let stdoutSpy: ReturnType< typeof vi.spyOn >; - const dispatchedPrompt = () => - ( ( runStudioAgentTurn as Mock ).mock.calls[ 0 ][ 0 ] as { prompt: string } ).prompt; + const dispatched = () => + ( runStudioAgentTurn as Mock ).mock.calls[ 0 ][ 0 ] as { + prompt: string; + activeSite?: { running: boolean }; + }; beforeEach( () => { vi.clearAllMocks(); @@ -247,9 +250,14 @@ describe( 'AI runCommand — active site banner running state', () => { activeSite: { name: 'My Site', path: '/sites/my-site' }, } ); - expect( dispatchedPrompt() ).toContain( - '[Active site: "My Site" at /sites/my-site (running)]' - ); + // The site rides to the runtime, which states it in the system prompt; + // the user message stays exactly what the user typed. + expect( dispatched().prompt ).toBe( 'hello' ); + expect( dispatched().activeSite ).toMatchObject( { + name: 'My Site', + path: '/sites/my-site', + running: true, + } ); } ); // STU-2040 regression guard: the per-turn isSiteRunning check opens a @@ -289,7 +297,7 @@ describe( 'AI runCommand — active site banner running state', () => { activeSite: { name: 'My Site', path: '/sites/my-site' }, } ); - expect( dispatchedPrompt() ).toContain( '(stopped)' ); + expect( dispatched().activeSite?.running ).toBe( false ); } ); it( 'treats a site missing from the CLI config as stopped', async () => { @@ -302,7 +310,7 @@ describe( 'AI runCommand — active site banner running state', () => { } ); expect( isSiteRunning ).not.toHaveBeenCalled(); - expect( dispatchedPrompt() ).toContain( '(stopped)' ); + expect( dispatched().activeSite?.running ).toBe( false ); } ); it( 'skips the daemon check for remote sites', async () => { @@ -320,9 +328,8 @@ describe( 'AI runCommand — active site banner running state', () => { expect( findSiteByFolder ).not.toHaveBeenCalled(); expect( isSiteRunning ).not.toHaveBeenCalled(); - expect( dispatchedPrompt() ).toContain( - '[Active site: "Live" (ID: 123) at https://example.wordpress.com (WordPress.com)]' - ); + expect( dispatched().prompt ).toBe( 'hello' ); + expect( dispatched().activeSite ).toMatchObject( { remote: true, wpcomSiteId: 123 } ); } ); } ); diff --git a/scripts/eval/README.md b/scripts/eval/README.md index 6d2e3dba33..f2c6f7d066 100644 --- a/scripts/eval/README.md +++ b/scripts/eval/README.md @@ -47,7 +47,7 @@ Tests live in `promptfoo.config.yaml`. The runner returns raw JSON (`toolCalls`, ### Seeding fixtures -Set a `seed` var on a test to pre-populate config files before the agent turn — useful for flows that depend on connected remote sites or preview sites without making a real WordPress.com connection or preview. The seed accepts `localSite` (written to `cli.json`), `activeSite` (creates a real stopped site under `~/Studio/`, with optional `files` written into it by path relative to the site root, and prefixes the prompt with the `[Active site: …]` line the app and CLI picker add; the site is deleted after the turn. `wpCli` is a list of WP-CLI argv arrays run on it after the files are written, for state that lives in the database, such as activating a theme or saving Site Editor customizations; a command fails the seed when it exits non-zero or prints `SEED_ERROR`, since `wp eval` exits 0 on PHP warnings. `inspect` is a list of WP-CLI argv arrays run after the turn, before the site is deleted; their output lands in the result's `inspections` for assertions on the final site state), `connectedWpcomSites` (written to `shared.json`, requires `studio auth login`), `snapshots` (written to `cli.json`), and `globalInstructions` (written to `~/.studio/knowledge/instructions.md`; the file's prior content — or absence — is restored after the turn). Everything seeded is removed automatically after the turn so reruns start clean. +Set a `seed` var on a test to pre-populate config files before the agent turn — useful for flows that depend on connected remote sites or preview sites without making a real WordPress.com connection or preview. The seed accepts `localSite` (written to `cli.json`), `activeSite` (creates a real stopped site under `~/Studio/`, with optional `files` written into it by path relative to the site root, and makes it the session's active site, which the system prompt states in its `## Active site` section, as it does for the site picked in the app or CLI; the site is deleted after the turn. `wpCli` is a list of WP-CLI argv arrays run on it after the files are written, for state that lives in the database, such as activating a theme or saving Site Editor customizations; a command fails the seed when it exits non-zero or prints `SEED_ERROR`, since `wp eval` exits 0 on PHP warnings. `inspect` is a list of WP-CLI argv arrays run after the turn, before the site is deleted; their output lands in the result's `inspections` for assertions on the final site state), `connectedWpcomSites` (written to `shared.json`, requires `studio auth login`), `snapshots` (written to `cli.json`), and `globalInstructions` (written to `~/.studio/knowledge/instructions.md`; the file's prior content — or absence — is restored after the turn). Everything seeded is removed automatically after the turn so reruns start clean. ### Graded assertions