diff --git a/apps/cli/ai/eval-outcome.ts b/apps/cli/ai/eval-outcome.ts new file mode 100644 index 0000000000..3cd41a1492 --- /dev/null +++ b/apps/cli/ai/eval-outcome.ts @@ -0,0 +1,357 @@ +/** + * What a turn left behind, captured after it ends so eval assertions judge + * the outcome a user would get rather than the tool calls that produced it. + * + * For each site the turn created, and the seeded active site, the capture + * records the WordPress state (theme, plugins, published content as parsed + * blocks, registered block styles, global styles), the rendered home page, + * block validity in the site's own editor, the active theme's files, and + * whether an installed parent theme still matches its WordPress.org release. + * Sites that existed before the turn are compared by their database and + * wp-content modification times, so a change to an unrelated site shows up. + */ +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdtempSync, + readFileSync, + readdirSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { rm } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { promisify } from 'node:util'; +import { validateBlocks } from 'cli/ai/block-validator'; +import { runCommand as runStartSiteCommand } from 'cli/commands/site/start'; +import { Mode as StopMode, runCommand as runStopSiteCommand } from 'cli/commands/site/stop'; +import { readCliConfig } from 'cli/lib/cli-config/core'; +import { getSiteByFolder, getSiteUrl } from 'cli/lib/cli-config/sites'; +import { connectToDaemon } from 'cli/lib/daemon-client'; +import { runWpCliCommand } from 'cli/lib/run-wp-cli-command'; +import { Logger } from 'cli/logger'; + +const execFileAsync = promisify( execFile ); + +export interface SiteFingerprint { + path: string; + name: string; + mtimeMs: number; +} + +// The newest modification time under a site's database and wp-content, which +// moves whenever the site is changed. +function newestMtime( dir: string, depth = 6 ): number { + let newest = 0; + const walk = ( current: string, level: number ) => { + let entries; + try { + entries = readdirSync( current, { withFileTypes: true } ); + } catch { + return; + } + for ( const entry of entries ) { + if ( entry.name === 'node_modules' || entry.name === 'cache' ) continue; + const full = path.join( current, entry.name ); + try { + newest = Math.max( newest, statSync( full ).mtimeMs ); + } catch { + continue; + } + if ( entry.isDirectory() && level < depth ) walk( full, level + 1 ); + } + }; + walk( path.join( dir, 'wp-content' ), 0 ); + return newest; +} + +export async function fingerprintSites(): Promise< SiteFingerprint[] > { + const { sites } = await readCliConfig(); + return sites + .filter( ( site ) => existsSync( site.path ) ) + .map( ( site ) => ( { path: site.path, name: site.name, mtimeMs: newestMtime( site.path ) } ) ); +} + +const PROBE_PHP = String.raw` +$count = function ( $blocks, &$acc ) use ( &$count ) { + foreach ( $blocks as $block ) { + if ( $block['blockName'] ) { $acc[ $block['blockName'] ] = ( $acc[ $block['blockName'] ] ?? 0 ) + 1; } + $count( $block['innerBlocks'], $acc ); + } +}; +$front = (int) get_option( 'page_on_front' ); +$content = array(); +foreach ( get_posts( array( 'post_type' => array( 'page', 'post', 'wp_template', 'wp_template_part', 'wp_navigation', 'wp_block' ), 'post_status' => 'publish', 'numberposts' => -1 ) ) as $post ) { + $blocks = array(); + $count( parse_blocks( $post->post_content ), $blocks ); + $content[] = array( 'id' => $post->ID, 'type' => $post->post_type, 'slug' => $post->post_name, 'title' => $post->post_title, 'isFront' => $post->ID === $front, 'blocks' => $blocks, 'content' => $post->post_content ); +} +$styles = array(); +foreach ( WP_Block_Styles_Registry::get_instance()->get_all_registered() as $block => $variations ) { + $styles[ $block ] = array_keys( $variations ); +} +$merged = WP_Theme_JSON_Resolver::get_merged_data()->get_raw_data(); +$palette = array(); +foreach ( (array) ( $merged['settings']['color']['palette'] ?? array() ) as $entries ) { + foreach ( (array) $entries as $entry ) { $palette[] = $entry; } +} +$plugins = array(); +foreach ( (array) get_option( 'active_plugins' ) as $plugin ) { $plugins[] = dirname( $plugin ) === '.' ? basename( $plugin, '.php' ) : dirname( $plugin ); } +echo 'OUTCOME_JSON=' . wp_json_encode( array( + 'theme' => array( 'stylesheet' => get_stylesheet(), 'template' => get_template(), 'isBlockTheme' => wp_is_block_theme(), 'stylesheetVersion' => wp_get_theme()->get( 'Version' ), 'templateVersion' => wp_get_theme( get_template() )->get( 'Version' ) ), + 'plugins' => $plugins, + 'showOnFront' => get_option( 'show_on_front' ), + 'content' => $content, + 'products' => post_type_exists( 'product' ) ? (int) wp_count_posts( 'product' )->publish : 0, + 'blockStyles' => $styles, + 'palette' => $palette, + 'mergedStyles' => $merged['styles'] ?? array(), + 'userGlobalStyles' => WP_Theme_JSON_Resolver::get_user_data()->get_raw_data(), + 'customCss' => wp_get_custom_css(), +) ) . "\n"; +`; + +async function wp( sitePath: string, args: string[] ): Promise< string > { + const site = await getSiteByFolder( sitePath ); + await using command = await runWpCliCommand( site, args ); + const exitCode = await command.response.exitCode; + const stdout = await command.response.stdoutText; + if ( exitCode !== 0 ) { + throw new Error( + `wp ${ args[ 0 ] } exited ${ exitCode }: ${ await command.response.stderrText }` + ); + } + return stdout; +} + +const TEXT_FILE = /\.(css|json|html|php|md|txt)$/; + +function hashFiles( dir: string ): Record< string, string > { + const hashes: Record< string, string > = {}; + const walk = ( current: string ) => { + for ( const entry of readdirSync( current, { withFileTypes: true } ) ) { + const full = path.join( current, entry.name ); + if ( entry.isDirectory() ) walk( full ); + else + hashes[ path.relative( dir, full ) ] = createHash( 'sha1' ) + .update( readFileSync( full ) ) + .digest( 'hex' ); + } + }; + if ( existsSync( dir ) ) walk( dir ); + return hashes; +} + +function readThemeText( dir: string ): Record< string, string > { + const files: Record< string, string > = {}; + for ( const relative of Object.keys( hashFiles( dir ) ) ) { + if ( TEXT_FILE.test( relative ) && ! relative.includes( 'node_modules' ) ) { + const text = readFileSync( path.join( dir, relative ), 'utf8' ); + if ( text.length < 200_000 ) files[ relative ] = text; + } + } + return files; +} + +// Files of an installed theme that differ from its WordPress.org release, or +// null when the release cannot be fetched (not a WordPress.org theme). +async function changedFromRelease( + themeDir: string, + slug: string, + version: string +): Promise< string[] | null > { + const tmp = mkdtempSync( path.join( os.tmpdir(), 'eval-theme-' ) ); + try { + const response = await fetch( + `https://downloads.wordpress.org/theme/${ slug }.${ version }.zip` + ); + if ( ! response.ok ) return null; + const zip = path.join( tmp, 'theme.zip' ); + writeFileSync( zip, Buffer.from( await response.arrayBuffer() ) ); + await execFileAsync( 'unzip', [ '-q', zip, '-d', tmp ] ); + const release = hashFiles( path.join( tmp, slug ) ); + const installed = hashFiles( themeDir ); + return [ ...new Set( [ ...Object.keys( release ), ...Object.keys( installed ) ] ) ] + .filter( ( file ) => release[ file ] !== installed[ file ] ) + .sort(); + } catch { + return null; + } finally { + await rm( tmp, { recursive: true, force: true } ); + } +} + +// The format a theme screenshot really is, from its leading bytes. +function imageFormat( file: string ): string | null { + if ( ! existsSync( file ) ) return null; + const head = readFileSync( file ).subarray( 0, 4 ); + if ( head[ 0 ] === 0xff && head[ 1 ] === 0xd8 && head[ 2 ] === 0xff ) + return `jpeg:${ path.basename( file ) }`; + if ( head.toString( 'hex' ) === '89504e47' ) return `png:${ path.basename( file ) }`; + return `unknown:${ path.basename( file ) }`; +} + +async function renderHome( url: string ) { + try { + const response = await fetch( url, { redirect: 'follow' } ); + const html = await response.text(); + const nav = [ ...html.matchAll( //g ) ] + .map( ( m ) => m[ 0 ] ) + .join( '\n' ); + return { + status: response.status, + html, + phpErrors: [ + ...html.matchAll( /(Fatal error|Parse error|Warning|Notice|Deprecated)<\/b>:[^<]*/g ), + ].map( ( m ) => m[ 0 ].replace( /<\/?b>/g, '' ) ), + navHrefs: [ ...nav.matchAll( /]*\bhref="([^"]*)"/g ) ].map( ( m ) => m[ 1 ] ), + ids: [ ...html.matchAll( /\bid="([^"]+)"/g ) ].map( ( m ) => m[ 1 ] ), + }; + } catch ( error ) { + return { status: 0, error: error instanceof Error ? error.message : String( error ) }; + } +} + +export interface SiteOutcome { + name: string; + path: string; + created: boolean; + url?: string; + error?: string; + [ key: string ]: unknown; +} + +export async function captureSiteOutcome( + sitePath: string, + created: boolean, + artifactDir: string +): Promise< SiteOutcome > { + const site = await getSiteByFolder( sitePath ); + const outcome: SiteOutcome = { name: site.name, path: sitePath, created }; + try { + await connectToDaemon(); + await runStartSiteCommand( sitePath, true, true, new Logger() ); + const url = getSiteUrl( site ); + outcome.url = url; + + const probe = ( await wp( sitePath, [ 'eval', PROBE_PHP ] ) ).match( /OUTCOME_JSON=(.*)/ ); + if ( ! probe ) throw new Error( 'the WordPress probe printed no state' ); + const state = JSON.parse( probe[ 1 ] ); + Object.assign( outcome, state ); + + const themes = path.join( sitePath, 'wp-content', 'themes' ); + const { stylesheet, stylesheetVersion, template, templateVersion } = state.theme; + const themeDir = path.join( themes, stylesheet ); + outcome.themeFiles = readThemeText( themeDir ); + outcome.themeFileList = Object.keys( hashFiles( themeDir ) ).sort(); + outcome.themeScreenshot = + imageFormat( path.join( themeDir, 'screenshot.jpg' ) ) ?? + imageFormat( path.join( themeDir, 'screenshot.png' ) ); + // Files changed from the WordPress.org release (null for themes not from there). + outcome.themeChanges = await changedFromRelease( themeDir, stylesheet, stylesheetVersion ); + if ( template !== stylesheet ) { + outcome.parentThemeChanges = await changedFromRelease( + path.join( themes, template ), + template, + templateVersion + ); + } + const designPath = path.join( sitePath, 'DESIGN.md' ); + outcome.designMd = existsSync( designPath ) ? readFileSync( designPath, 'utf8' ) : null; + + const home = await renderHome( url ); + if ( 'html' in home && home.html ) { + const file = path.join( artifactDir, `${ path.basename( sitePath ) }-home.html` ); + writeFileSync( file, home.html ); + outcome.home = { ...home, html: undefined, htmlFile: file }; + } else { + outcome.home = home; + } + + // Every published page, template and part, validated by the site's own editor. + const invalid: Array< { id: number; type: string; slug: string; blocks: string[] } > = []; + for ( const item of state.content as Array< { + id: number; + type: string; + slug: string; + content: string; + } > ) { + if ( ! item.content.includes( '' ) ), + vars + ).pass + ).toBe( true ); + expect( + run( + checks.colorsFromPalette, + outcome( site( '
' ) ), + vars + ).pass + ).toBe( false ); + expect( + run( checks.colorsFromPalette, outcome( site( '' ) ), vars ).pass + ).toBe( false ); + } ); + + test( 'noInlineDisplayOnBlockWrappers', () => { + const site = ( css ) => ( { + ...workingSite, + content: [ front( '', {} ) ], + customCss: css, + } ); + expect( + run( + checks.noInlineDisplayOnBlockWrappers, + outcome( site( '.eyebrow p { display: inline-block; }' ) ) + ).pass + ).toBe( true ); + expect( + run( + checks.noInlineDisplayOnBlockWrappers, + outcome( site( '.eyebrow { display: inline-block; }' ) ) + ).pass + ).toBe( false ); + // The first rule of a theme stylesheet and of theme.json's css counts too. + expect( + run( + checks.noInlineDisplayOnBlockWrappers, + outcome( { + ...site( '' ), + themeFiles: { 'style.css': '.eyebrow{display:inline-flex}' }, + } ) + ).pass + ).toBe( false ); + expect( + run( + checks.noInlineDisplayOnBlockWrappers, + outcome( { + ...site( '' ), + themeFiles: { + 'theme.json': '{"styles":{"css":".eyebrow {\\n display: inline-block;\\n}"}}', + }, + } ) + ).pass + ).toBe( false ); + } ); + + test( 'repeatedTreatmentIsBlockStyle', () => { + const site = ( content ) => ( { + ...workingSite, + blockStyles: { 'core/group': [ 'card', 'section-1' ] }, + content: [ front( content, {} ) ], + } ); + expect( + run( + checks.repeatedTreatmentIsBlockStyle, + outcome( site( 'is-style-card is-style-card is-style-card' ) ) + ).pass + ).toBe( true ); + expect( + run( + checks.repeatedTreatmentIsBlockStyle, + outcome( site( 'is-style-section-1 is-style-section-1' ) ) + ).pass + ).toBe( false ); + expect( + run( + checks.repeatedTreatmentIsBlockStyle, + outcome( + site( + 'is-style-card is-style-card {"fontSize":"18px"} {"fontSize":"18px"} {"fontSize":"18px"}' + ) + ) + ).pass + ).toBe( false ); + } ); + + test( 'designTokenReachedTheme', () => { + const vars = { token: 'primary', originalValue: '#000000' }; + const site = ( designMd, palette ) => ( { ...workingSite, designMd, palette } ); + expect( + run( + checks.designTokenReachedTheme, + outcome( site( 'primary: "#2255aa"', [ { color: '#2255AA' } ] ) ), + vars + ).pass + ).toBe( true ); + expect( + run( + checks.designTokenReachedTheme, + outcome( site( 'primary: "#2255aa"', [ { color: '#000000' } ] ) ), + vars + ).pass + ).toBe( false ); + expect( + run( + checks.designTokenReachedTheme, + outcome( site( 'primary: "#000000"', [ { color: '#000000' } ] ) ), + vars + ).pass + ).toBe( false ); + } ); +} ); diff --git a/scripts/eval/promptfoo.config.yaml b/scripts/eval/promptfoo.config.yaml index 118e935979..0d5b0ac4a2 100644 --- a/scripts/eval/promptfoo.config.yaml +++ b/scripts/eval/promptfoo.config.yaml @@ -2,15 +2,16 @@ # Run: npm run eval (builds CLI first) # View: npm run eval:view # -# The runner writes its payload to a tmp file and prints only -# `EVAL_RUNNER_RESULT_FILE=/path/to.json` on stdout. Assertions resolve the -# marker via `import('node:fs').then(...)` — not `require('fs')`, which is -# undefined inside promptfoo's assertion eval context. +# Cases are judged on outcomes: the site the turn leaves behind (WordPress +# state, rendered home page, block validity, theme files), the other sites it +# left alone, and what the user was told. The runner captures that state after +# the turn; the checks live in outcome.mjs and read their parameters from the +# case's vars (`siteName` names the site the case asks for). description: Studio Code agent evaluation -# Tests create/delete real local sites; running them in parallel makes two -# runs fight over ports and on-disk state. +# Each case creates its sites in its own folder, but two cases starting sites at +# the same moment can still pick the same free port. evaluateOptions: maxConcurrency: 1 @@ -30,22 +31,7 @@ tests: prompt: In one short sentence, tell me who you are. Do not call any tools. assert: - type: javascript - value: | - return import('node:fs').then(({ readFileSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - if (d.success !== true) return { pass: false, score: 0, reason: `runner success=${d.success}` }; - const text = (d.textSegments || []).join('\n'); - const mentionsStudio = /WordPress\s+Studio/i.test(text); - const claimsOther = /\bI\s+am\s+(Claude|ChatGPT|a large language model|an AI (model|assistant) (made|built|developed) by Anthropic)\b/i.test(text); - const pass = mentionsStudio && !claimsOther; - return { - pass, - score: pass ? 1 : 0, - reason: pass ? 'identifies as WordPress Studio' : `got: ${text.slice(0, 300)}`, - }; - }); + value: file://outcome.mjs:identifiesAsStudio # The global instructions file (~/.studio/knowledge/instructions.md) is # seeded with a sentinel-token rule for the turn and restored afterwards. @@ -54,6 +40,7 @@ tests: - description: agent follows the user's global instructions vars: caseId: global-instructions + expectedText: STUDIO-EVAL-TOKEN-7319 timeoutMs: 60000 prompt: In one short sentence, say hello. Do not call any tools. seed: @@ -61,446 +48,96 @@ tests: Always end every reply with the exact token STUDIO-EVAL-TOKEN-7319. assert: - type: javascript - value: | - return import('node:fs').then(({ readFileSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - if (d.success !== true) return { pass: false, score: 0, reason: `runner success=${d.success}` }; - const text = (d.textSegments || []).join('\n'); - const pass = text.includes('STUDIO-EVAL-TOKEN-7319'); - return { - pass, - score: pass ? 1 : 0, - reason: pass ? 'reply carries the token from the global instructions' : `token missing from reply: ${text.slice(0, 300)}`, - }; - }); + value: file://outcome.mjs:replyIncludes - - description: agent calls site_create when asked to make a site + - description: asking for a site leaves a working new site vars: caseId: site-creation + siteName: "Eval Test Site" timeoutMs: 120000 - askUserPolicy: allow_all prompt: | - First check if a site named "Eval Test Site" exists using site_list. - If it does, delete it with site_delete. Then create a new site named "Eval Test Site". + Create a new site named "Eval Test Site". assert: - type: javascript - value: | - return import('node:fs').then(({ readFileSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const called = d.toolCalls?.some(t => t.name === 'site_create'); - return { pass: !!called, score: called ? 1 : 0, reason: called ? 'site_create was called' : `no site_create in ${JSON.stringify(d.toolCalls?.map(t => t.name) ?? [])}` }; - }); + value: file://outcome.mjs:siteWorks - type: javascript - value: | - return import('node:fs').then(({ readFileSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const result = d.toolResults?.find(r => r.toolName === 'site_create'); - const ok = !!(result && !result.isError); - return { pass: ok, score: ok ? 1 : 0, reason: ok ? 'site_create succeeded' : `site_create result: ${JSON.stringify(result)}` }; - }); + value: file://outcome.mjs:otherSitesUntouched - # Regression: trunk handles desktop/mobile visual verification as separate - # screenshot calls. The optimized tool exposes `viewport: "all"` so the agent - # can satisfy the same user request with one timed call and two PNG payloads. - # Pinned to an image-capable tier: for text-only models (the default `fast`) - # take_screenshot saves the captures to files and returns no image payloads. - - description: screenshot verification captures desktop and mobile in one fast call + # The model must see a desktop and a mobile capture of the home page, with + # the screenshot time summed across however many calls it made. Pinned to + # an image-capable tier: text-only models get files, not image payloads. + - description: screenshot verification shows the model desktop and mobile quickly vars: caseId: screenshot-all-timing + siteName: "Eval Screenshot Timing" model: balanced timeoutMs: 240000 - askUserPolicy: allow_all prompt: | - First check if a site named "Eval Screenshot Timing" exists using - site_list. If it does, delete it with site_delete so we start from - a clean slate. Do NOT touch any other site. - - Then create a new site named "Eval Screenshot Timing". Do not + Create a new site named "Eval Screenshot Timing". Do not customize the theme. Verify the homepage visually with screenshots for both desktop and mobile, then briefly report whether the screenshots were captured. assert: - # The runner records the image content blocks the tool returned to the - # model (`toolResults[].images`) — unlike the chat artifacts, they are - # emitted even when the agent passes `display: false`, so the check can't - # be dodged by an internal-verification capture. - type: javascript - value: | - return Promise.all([import('node:fs'), import('node:buffer')]).then(([{ readFileSync }, { Buffer }]) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const screenshotCalls = (d.toolCalls || []).filter(t => t.name === 'take_screenshot'); - if (screenshotCalls.length !== 1) { - return { - pass: false, - score: 0, - reason: `expected one take_screenshot call for desktop+mobile verification, got ${screenshotCalls.length}: ${JSON.stringify(screenshotCalls.map(t => t.input))}`, - }; - } - const call = screenshotCalls[0]; - if (call.input?.viewport !== 'all') { - return { - pass: false, - score: 0, - reason: `expected take_screenshot to use viewport="all"; got ${JSON.stringify(call.input)}`, - }; - } - - const result = (d.toolResults || []).find(r => r.toolUseId === call.id); - if (!result || result.isError) { - return { pass: false, score: 0, reason: `take_screenshot failed: ${JSON.stringify(result).slice(0, 500)}` }; - } - const images = result.images || []; - if (images.length !== 2) { - return { pass: false, score: 0, reason: `expected two screenshot image payloads (desktop + mobile), got ${images.length}` }; - } - - // Read pixel dimensions from a PNG or JPEG buffer; null if neither. - const readImageSize = (buf) => { - if (buf.length > 24 && buf[0] === 0x89 && buf[1] === 0x50 && buf[2] === 0x4e && buf[3] === 0x47) { - return { format: 'png', width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) }; - } - if (buf.length > 4 && buf[0] === 0xff && buf[1] === 0xd8) { - let offset = 2; - while (offset + 9 < buf.length) { - if (buf[offset] !== 0xff) { offset++; continue; } - const marker = buf[offset + 1]; - // Markers without a length payload: skip past them. - if (marker === 0xd8 || marker === 0xd9 || (marker >= 0xd0 && marker <= 0xd7) || marker === 0x01) { - offset += 2; - continue; - } - // Start-of-frame markers carry the dimensions (excluding DHT/JPG/DAC). - if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) { - return { format: 'jpeg', height: buf.readUInt16BE(offset + 5), width: buf.readUInt16BE(offset + 7) }; - } - offset += 2 + buf.readUInt16BE(offset + 2); - } - } - return null; - }; - // Images come in capture order (desktop, then mobile), scaled to at most 2000 px. - const sizes = images.map((image) => readImageSize(Buffer.from(image.data, 'base64'))); - for (const size of sizes) { - if (!size || size.height < 600 || Math.max(size.width, size.height) > 2000) { - return { - pass: false, - score: 0, - reason: `image payload did not look like a screenshot scaled for the model: ${JSON.stringify(size)}`, - }; - } - } - if (sizes[0].width <= sizes[1].width) { - return { pass: false, score: 0, reason: `expected a desktop then a mobile screenshot, got ${JSON.stringify(sizes)}` }; - } - return { pass: true, score: 1, reason: `captured valid screenshots: ${JSON.stringify(sizes)}` }; - }); + value: file://outcome.mjs:sawDesktopAndMobile - type: javascript - value: | - return import('node:fs').then(({ readFileSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const events = (d.toolEvents || []).filter(t => t.toolName === 'take_screenshot'); - if (events.length !== 1 || typeof events[0].durationMs !== 'number') { - return { pass: false, score: 0, reason: `expected one timed take_screenshot event, got ${JSON.stringify(events)}` }; - } - const limitMs = 15000; - const duration = events[0].durationMs; - const pass = duration < limitMs; - return { - pass, - score: pass ? 1 : 0, - reason: pass - ? `take_screenshot completed in ${duration}ms` - : `take_screenshot took ${duration}ms, expected < ${limitMs}ms`, - }; - }); + value: file://outcome.mjs:otherSitesUntouched - # Every individual turn (wall-clock between successive assistant messages) - # should stay under 60s. Slow turns stall the UI. - - description: single-page site build keeps every turn under 60s + # A fresh build: the site renders with valid blocks, has a static front + # page, runs its own theme with self-hosted fonts and a real screenshot, + # and its palette carries the DESIGN.md colors. Every turn stays under 60s + # so the UI never stalls. + - description: single-page build leaves a working site in its own theme, built from DESIGN.md vars: caseId: single-page-build-turn-cadence + siteName: "Eval Turn Timing" timeoutMs: 600000 - askUserPolicy: allow_all prompt: | - First check if a site named "Eval Turn Timing" exists using - site_list. If it does, delete it with site_delete so we start from - a clean slate. Do NOT touch any other site. - - Then build a simple WordPress site named "Eval Turn Timing". The + Build a simple WordPress site named "Eval Turn Timing". The site only needs a single page — no extra pages, no blog posts, no navigation menu beyond what a one-page site requires. Use the site name as the brand and skip the discovery questions: nobody can answer during this run, so make every choice yourself. assert: - # The default new-site flow: no installed theme is mentioned, so the - # agent must start from a fresh blank scaffold — scaffold_theme called - # WITHOUT parentTheme (the parentTheme path is covered by the - # child-theme-for-installed-theme case below). - type: javascript - value: | - return import('node:fs').then(({ readFileSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const scaffolds = (d.toolCalls || []).filter(t => t.name === 'scaffold_theme'); - if (scaffolds.length === 0) { - return { pass: false, score: 0, reason: `scaffold_theme was never called; tools: ${JSON.stringify((d.toolCalls || []).map(t => t.name))}` }; - } - const call = scaffolds[0]; - if (call.input?.parentTheme) { - return { pass: false, score: 0, reason: `expected a blank scaffold but got a child theme: ${JSON.stringify(call.input)}` }; - } - const result = (d.toolResults || []).find(r => r.toolUseId === call.id); - if (!result || result.isError) { - return { pass: false, score: 0, reason: `scaffold_theme failed: ${JSON.stringify(result).slice(0, 300)}` }; - } - if (!/fonts downloaded to assets\/fonts/.test(result.text || '')) { - return { pass: false, score: 0, reason: `the scaffold did not self-host the DESIGN.md fonts: ${(result.text || '').split('\n').slice(0, 4).join(' ')}` }; - } - return { pass: true, score: 1, reason: `blank theme scaffolded with self-hosted fonts: ${JSON.stringify(call.input)}` }; - }); + value: file://outcome.mjs:siteWorks - type: javascript - value: | - return import('node:fs').then(({ readFileSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const durations = d.turnDurationsMs ?? []; - if (durations.length === 0) { - return { pass: false, score: 0, reason: 'no turns recorded' }; - } - const max = Math.max(...durations); - const maxIdx = durations.indexOf(max); - if (max >= 60000) { - return { pass: false, score: 0, reason: `turn ${maxIdx + 1}/${durations.length} took ${max}ms (>= 60000ms). All turns (ms): ${durations.join(', ')}` }; - } - return { pass: true, score: 1, reason: `max turn ${max}ms across ${durations.length} turns` }; - }); - # Regression: the "Page content" cadence used to tell the agent to apply - # page blocks via `wp_cli post update --post_content-file=`. - # `--post_content-file` is not a WP-CLI option, so the file is never read - # and the post content is not updated. - # The fix is `wp_cli eval` + `file_get_contents(ABSPATH . ...)` — this - # assertion fails if any wp_cli call reintroduces --post_content-file=. + value: file://outcome.mjs:builtFromDesign - type: javascript - value: | - return import('node:fs').then(({ readFileSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const offending = (d.toolCalls || []).filter(t => { - if (t.name !== 'wp_cli') return false; - const cmd = t.input && typeof t.input.command === 'string' ? t.input.command : ''; - return cmd.includes('--post_content-file='); - }); - if (offending.length > 0) { - const sample = JSON.stringify(offending[0].input).slice(0, 300); - return { pass: false, score: 0, reason: `${offending.length} wp_cli call(s) used --post_content-file=, which is not a WP-CLI option (first: ${sample})` }; - } - return { pass: true, score: 1, reason: 'no wp_cli call used --post_content-file=' }; - }); - - type: javascript - value: | - return import('node:fs').then(({ readFileSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const validationResults = (d.toolResults || []).filter(r => r.toolName === 'validate_blocks'); - if (validationResults.length === 0) { - return { pass: false, score: 0, reason: 'validate_blocks was not called for block content' }; - } - const last = validationResults[validationResults.length - 1]; - const text = last.text || ''; - // validate_blocks runs the core/html policy gate first; only when it - // passes does the live-editor validation run and emit `Validation:`. - // A short-circuited result reports `N/M core/html blocks invalid`. - const htmlPassed = /HTML block policy:/i.test(text) && !/core\/html blocks invalid/i.test(text); - if (last.isError || !htmlPassed) { - return { - pass: false, - score: 0, - reason: `final validate_blocks result did not pass the core/html policy gate: ${(text || JSON.stringify(last)).slice(0, 500)}`, - }; - } - const failed = - !/Validation:\s+\d+\/\d+\s+blocks valid/i.test(text) || - /Auto-fix proposal failed validation|Auto-fix proposal still has|No automatic editor serialization fix was available/i.test(text); - if (failed) { - return { - pass: false, - score: 0, - reason: `final validate_blocks editor validation was invalid or inconclusive: ${(text || JSON.stringify(last)).slice(0, 500)}`, - }; - } - return { pass: true, score: 1, reason: `validation passed: ${text.split(/\r?\n/).filter(Boolean).slice(0, 2).join(' | ')}` }; - }); - # The home page's desktop take_screenshot captures, with - # the scaffolded theme's directory as `themeScreenshot`, set its - # screenshot.jpg (the Appearance → Themes thumbnail). Captures are JPEGs, - # so the file must carry a real JPEG signature — a hand-crafted or - # generated placeholder fails. + value: file://outcome.mjs:ownThemeWithFontsAndScreenshot - type: javascript - value: | - return import('node:fs').then(({ readFileSync, openSync, readSync, closeSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const created = (d.toolResults || []).find(r => r.toolName === 'site_create' && !r.isError); - if (!created || !created.text) return { pass: false, score: 0, reason: 'no successful site_create result to resolve the site path from' }; - let sitePath; - try { sitePath = JSON.parse(created.text).path; } catch { return { pass: false, score: 0, reason: `site_create result text is not JSON: ${created.text.slice(0, 200)}` }; } - const scaffold = (d.toolResults || []).find(r => r.toolName === 'scaffold_theme' && !r.isError); - const slug = scaffold?.text?.match(/scaffolded at wp-content\/themes\/([a-z0-9-]+)\//)?.[1]; - if (!sitePath || !slug) return { pass: false, score: 0, reason: `could not resolve theme dir (sitePath=${sitePath}, slug=${slug})` }; - const screenshotPath = `${sitePath}/wp-content/themes/${slug}/screenshot.jpg`; - let fd; - try { fd = openSync(screenshotPath, 'r'); } catch { return { pass: false, score: 0, reason: `theme screenshot missing: ${screenshotPath}` }; } - const head = Buffer.alloc(3); - const bytesRead = readSync(fd, head, 0, 3, 0); - closeSync(fd); - if (bytesRead < 3 || head[0] !== 0xff || head[1] !== 0xd8 || head[2] !== 0xff) { - return { pass: false, score: 0, reason: `${screenshotPath} exists but is not a JPEG (starts with ${head.toString('hex')})` }; - } - return { pass: true, score: 1, reason: `theme screenshot set from a real capture: ${screenshotPath}` }; - }); - # Theme templates and parts are live as soon as they are written, so each - # Write/Edit on one must be followed by validate_blocks on that file — - # the serialization fix is what puts comment attributes (padding, align, - # colors) into the wrapper HTML the front end renders. + value: file://outcome.mjs:turnsAreResponsive - type: javascript - value: | - return Promise.all([import('node:fs'), import('node:path')]).then(([{ readFileSync }, { resolve }]) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - let sitePath; - try { sitePath = JSON.parse((d.toolResults || []).find(r => r.toolName === 'site_create' && !r.isError).text).path; } catch {} - if (!sitePath) return { pass: false, score: 0, reason: 'no successful site_create result to resolve the site path from' }; - const calls = d.toolCalls || []; - const isThemeMarkup = p => typeof p === 'string' && /\/(templates|parts)\/[^/]+\.html$/.test(p); - const unvalidated = calls - .map((t, i) => ({ t, i })) - .filter(({ t }) => (t.name === 'Write' || t.name === 'Edit') && isThemeMarkup(t.input && t.input.path)) - .filter(({ t, i }) => !calls.slice(i + 1).some(c => c.name === 'validate_blocks' && typeof c.input?.filePath === 'string' && resolve(sitePath, c.input.filePath) === t.input.path)) - .map(({ t }) => t.input.path.split('/').slice(-2).join('/')); - if (unvalidated.length > 0) { - return { pass: false, score: 0, reason: `theme markup written without a following validate_blocks on the file: ${[...new Set(unvalidated)].join(', ')}` }; - } - return { pass: true, score: 1, reason: 'every written template/part file was validated afterwards' }; - }); + value: file://outcome.mjs:otherSitesUntouched - # The visual-design skill lists the concept and direction catalogs; - # pick_design settles one of each, the Site Spec commits to them before - # building, and the look is written to DESIGN.md at the site root. - - type: javascript - value: | - return import('node:fs').then(({ readFileSync, readdirSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const loaded = (d.toolCalls || []).some(t => t.name === 'Skill' && t.input?.name === 'visual-design'); - if (!loaded) return { pass: false, score: 0, reason: 'visual-design skill was never loaded' }; - const text = (d.textSegments || []).join('\n'); - const picks = (d.toolResults || []).filter(r => r.toolName === 'pick_design' && !r.isError).map(r => r.text || '').join('\n'); - if (!picks) return { pass: false, score: 0, reason: 'pick_design was never called successfully' }; - const skillsDir = ['apps/cli/ai/skills/visual-design', '../../apps/cli/ai/skills/visual-design'].find(p => { try { readdirSync(`${p}/layouts`); return true; } catch { return false; } }); - if (!skillsDir) return { pass: false, score: 0, reason: `design catalogs not found from cwd ${process.cwd()}` }; - // One frontmatter file per entry; the title line is a JSON string. - const catalogNames = folder => readdirSync(`${skillsDir}/${folder}`).filter(f => f.endsWith('.md') && f !== 'README.md').map(f => JSON.parse(readFileSync(`${skillsDir}/${folder}/${f}`, 'utf8').match(/^title:\s*(.+)$/m)[1]).toLowerCase()); - const stated = []; - for (const [label, folder, heading] of [['Concept', 'layouts', 'layout concept'], ['Direction', 'artistic-directions', 'artistic direction']]) { - const lines = text.split(/\r?\n/).filter(l => new RegExp(`\\b${label}\\b\\**\\s*[:—-]`, 'i').test(l)); - if (!lines.length) return { pass: false, score: 0, reason: `no "${label}:" line in the agent text` }; - const catalog = catalogNames(folder); - const fromCatalog = lines.find(l => catalog.some(name => l.toLowerCase().includes(name))); - if (!fromCatalog) return { pass: false, score: 0, reason: `${label} line does not name a catalog entry: ${JSON.stringify(lines.slice(0, 3))}` }; - // Each must be what pick_design drew (or returned for a named brief), not a substitute. - // With `options: 4` the tool returns several pairs; the stated one must be among them. - const drawnNames = [...picks.matchAll(new RegExp(`^${heading[0].toUpperCase()}${heading.slice(1)}(?: named in the brief)?: (.+)$`, 'gm'))].map(m => m[1]); - if (!drawnNames.length) return { pass: false, score: 0, reason: `pick_design never returned a ${heading}` }; - const drawn = drawnNames.find(name => fromCatalog.toLowerCase().includes(name.toLowerCase())); - if (!drawn) return { pass: false, score: 0, reason: `${label} line "${fromCatalog.trim().slice(0, 80)}" does not match any pick_design result (${drawnNames.join(', ')})` }; - stated.push(drawn); - } - if (!/layout map/i.test(text)) return { pass: false, score: 0, reason: 'no "Layout map" in the Site Spec' }; - const design = (d.toolCalls || []).find(t => t.name === 'Write' && /\/DESIGN\.md$/.test(t.input?.path || '')); - if (!design) return { pass: false, score: 0, reason: 'DESIGN.md was never written at the site root' }; - if (!/^---\r?\n[\s\S]*?^colors:/m.test(design.input.content || '')) return { pass: false, score: 0, reason: 'DESIGN.md has no colors in its front matter' }; - return { pass: true, score: 1, reason: `concept and direction drawn and stated, DESIGN.md written: ${stated.join(' / ')}` }; - }); - - # One-page sites get header navigation that links to page sections. The - # header template part renders on EVERY template (posts, archives, search, - # 404), and a bare-hash URL like "#contact" resolves against the current - # URL — it silently does nothing everywhere except the home page. Anchor - # nav links must be home-relative ("/#contact"). Regression guard for the - # one-page navigation guidance in the block-content skill. - - description: one-page site header nav uses home-relative anchor links + # The header renders on every template, so a bare "#contact" only works on + # the home page. Judged on the rendered home page: section links are + # home-relative and each target id exists. + - description: one-page site section links work from every page vars: caseId: one-page-nav-anchor-links + siteName: "Eval One Page Nav" timeoutMs: 600000 - askUserPolicy: allow_all prompt: | - First check if a site named "Eval One Page Nav" exists using - site_list. If it does, delete it with site_delete so we start from a - clean slate. Do NOT touch any other site. - - Then build a one-page site named "Eval One Page Nav": a single + Build a one-page site named "Eval One Page Nav": a single scrollable home page with Features, About, and Contact sections, and a header navigation menu whose items jump to those sections. Keep the design simple — no extra pages and no blog posts. assert: - type: javascript - value: | - return import('node:fs').then(({ readFileSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const strings = []; - const collect = (v) => { - if (typeof v === 'string') strings.push(v); - else if (Array.isArray(v)) v.forEach(collect); - else if (v && typeof v === 'object') Object.values(v).forEach(collect); - }; - (d.toolCalls || []).forEach(t => collect(t.input)); - // wp_cli payloads carry markup with shell/JSON-escaped quotes; - // strip backslashes so one regex covers theme files and commands. - const flat = strings.map(s => s.replace(/\\/g, '')); - const links = flat.flatMap(s => s.match(/wp:navigation-link\s*\{[^}]*\}/g) || []); - if (links.length === 0) { - return { pass: false, score: 0, reason: 'no wp:navigation-link blocks were produced — the build did not exercise header section navigation' }; - } - const anchorLinks = links.filter(l => /"url":"[^"]*#/.test(l)); - if (anchorLinks.length === 0) { - return { pass: false, score: 0, reason: `nav links carry no anchor URLs at all — the section-jump scenario was not exercised (links: ${links.slice(0, 3).join(' | ').slice(0, 300)})` }; - } - const bare = links.filter(l => /"url":"#/.test(l)); - if (bare.length > 0) { - return { pass: false, score: 0, reason: `${bare.length}/${links.length} nav link(s) use bare-hash URLs, which resolve against the current URL and break on every page except home (first: ${bare[0].slice(0, 200)}). Anchor nav links must be home-relative ("/#anchor").` }; - } - return { pass: true, score: 1, reason: `${anchorLinks.length} anchor nav link(s), all home-relative` }; - }); + value: file://outcome.mjs:siteWorks + - type: javascript + value: file://outcome.mjs:navAnchorsWorkEverywhere + - type: javascript + value: file://outcome.mjs:otherSitesUntouched - # Regression (STU-1775): the agent used to present a temporary preview site as - # a connected WordPress.com remote site when listing what a local site is - # connected to. We seed one connected remote site and one preview site for a - # local site, then assert (a) each tool tags its output with a `type` - # discriminator and (b) the agent's prose keeps the two categories distinct. - - description: agent distinguishes preview sites from connected WordPress.com remote sites + # Regression (STU-1775): a preview site was presented as a connected + # WordPress.com site. Judged on the reply: the preview is described as a + # preview, the connected site as a WordPress.com remote site. + - description: reply tells preview sites apart from connected WordPress.com sites vars: caseId: differentiate-preview-vs-remote timeoutMs: 180000 - askUserPolicy: allow_all seed: localSite: id: eval-site-categories @@ -530,48 +167,19 @@ tests: temporary PREVIEW site or a durable connected WordPress.com remote site. Do not create, delete, or modify anything — only inspect and report. assert: - # The connected-remote tool output carries the type discriminator. - type: javascript - value: | - return import('node:fs').then(({ readFileSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const r = (d.toolResults || []).find(r => r.toolName === 'site_connected_remote_sites'); - if (!r) return { pass: false, score: 0, reason: `site_connected_remote_sites was not called; tools: ${JSON.stringify((d.toolCalls || []).map(t => t.name))}` }; - const ok = /"type"\s*:\s*"wpcom-remote"/.test(r.text || ''); - return { pass: ok, score: ok ? 1 : 0, reason: ok ? 'connected-remote output tagged type=wpcom-remote' : `missing type discriminator: ${(r.text || '').slice(0, 300)}` }; - }); - # The preview tool output carries the type discriminator. + value: file://outcome.mjs:classifiesPreviewAndRemote - type: javascript - value: | - return import('node:fs').then(({ readFileSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const r = (d.toolResults || []).find(r => r.toolName === 'preview_list'); - if (!r) return { pass: false, score: 0, reason: `preview_list was not called; tools: ${JSON.stringify((d.toolCalls || []).map(t => t.name))}` }; - const ok = /"type"\s*:\s*"preview"/.test(r.text || ''); - return { pass: ok, score: ok ? 1 : 0, reason: ok ? 'preview output tagged type=preview' : `missing type discriminator: ${(r.text || '').slice(0, 300)}` }; - }); - # A behavioral llm-rubric assertion used to grade the assistant's prose - # here — that it keeps preview and connected-remote sites in distinct - # categories. It was removed because the WP.com AI proxy's Studio lane now - # rejects the grader's plain judge call (no agent envelope/identity), and - # the honest fix (a dedicated eval-grader feature slug from #ai-ops, or an - # ANTHROPIC_API_KEY-backed grader) is not yet in place. The two - # assertions above still guard the tool-output discriminators. Restore the - # prose guard once the grader can authenticate. + value: file://outcome.mjs:otherSitesUntouched - # Pushing overwrites a live WordPress.com site, so the agent checks the - # attached sites and asks before calling site_push (the rules live in the - # site_push and site_connected_remote_sites descriptions). The seeded remote - # does not exist, so it reports syncSupport "deleted": the agent must not push - # and must ask for a target. The runner has no AskUserQuestion, so it asks in text. + # A push reaches a live site, so the agent asks first. The user declines + # whatever it asks, and nothing may reach WordPress.com. - description: push request asks before pushing vars: caseId: push-confirms-target timeoutMs: 180000 + answers: + - label: Cancel seed: localSite: id: eval-site-push @@ -593,185 +201,80 @@ tests: Push my local site "Eval Site Push" to WordPress.com. assert: - type: javascript - value: | - return import('node:fs').then(({ readFileSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const calls = d.toolCalls ?? []; - const pushed = calls.find(t => t.name === 'site_push'); - if (pushed) return { pass: false, score: 0, reason: `site_push was called without confirmation: ${JSON.stringify(pushed.input)}` }; - if (!calls.some(t => t.name === 'site_connected_remote_sites')) return { pass: false, score: 0, reason: `the attached sites were never listed; tools: ${JSON.stringify(calls.map(t => t.name))}` }; - const text = (d.textSegments ?? []).join('\n'); - const asks = /\?|let me know|please (give|provide|share|confirm|tell)/i.test(text); - return { pass: asks, score: asks ? 1 : 0, reason: asks ? 'agent checked the attached sites and asked instead of pushing' : `reply did not ask the user: ${text.slice(0, 300)}` }; - }); + value: file://outcome.mjs:pushNotRunAndAsks - - description: agent reaches for Jetpack on slideshow request (catch-all rule) + - description: slideshow request puts a Jetpack slideshow on the home page vars: caseId: jetpack-catchall-slideshow + siteName: "Catchall Eval Site" + plugin: jetpack + blockPrefix: jetpack/ timeoutMs: 600000 - askUserPolicy: allow_all prompt: | - First check if a site named "Catchall Eval Site" exists using - site_list. If it does, delete it with site_delete so we start - from a clean slate. Do NOT touch any other site. - - Then create a new site named "Catchall Eval Site" and add a + Create a new site named "Catchall Eval Site" and add a slideshow of three placeholder images to its homepage. assert: - type: javascript - value: | - return import('node:fs').then(({ readFileSync }) => { - const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); - if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; - const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const blobs = []; - for (const c of d.toolCalls || []) { - const i = c.input || {}; - if (c.name === 'Write' || c.name === 'Edit') { - for (const k of ['content', 'new_string', 'replacement', 'text']) { - if (typeof i[k] === 'string') blobs.push(i[k]); - } - // The Edit tool applies changes as { edits: [{ oldText, newText }] }. - if (Array.isArray(i.edits)) { - for (const e of i.edits) if (e && typeof e.newText === 'string') blobs.push(e.newText); - } - } - if (c.name === 'wp_cli' && typeof i.command === 'string') blobs.push(i.command); - } - const usedJetpackBlock = blobs.some(c => /