From 50175bd8d243b8a1a2cb2d994dfd3a78cf7f6e5c Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Fri, 16 Oct 2026 19:53:05 -0400 Subject: [PATCH 1/5] Eval: judge each case on the outcome a user would see, against an isolated site registry --- apps/cli/ai/eval-outcome.ts | 357 +++++++++++ apps/cli/ai/eval-runner.ts | 148 ++++- scripts/eval/README.md | 58 +- scripts/eval/outcome.mjs | 494 +++++++++++++++ scripts/eval/promptfoo.config.yaml | 962 ++++------------------------- 5 files changed, 1159 insertions(+), 860 deletions(-) create mode 100644 apps/cli/ai/eval-outcome.ts create mode 100644 scripts/eval/outcome.mjs diff --git a/apps/cli/ai/eval-outcome.ts b/apps/cli/ai/eval-outcome.ts new file mode 100644 index 0000000000..3cd41a1492 --- /dev/null +++ b/apps/cli/ai/eval-outcome.ts @@ -0,0 +1,357 @@ +/** + * What a turn left behind, captured after it ends so eval assertions judge + * the outcome a user would get rather than the tool calls that produced it. + * + * For each site the turn created, and the seeded active site, the capture + * records the WordPress state (theme, plugins, published content as parsed + * blocks, registered block styles, global styles), the rendered home page, + * block validity in the site's own editor, the active theme's files, and + * whether an installed parent theme still matches its WordPress.org release. + * Sites that existed before the turn are compared by their database and + * wp-content modification times, so a change to an unrelated site shows up. + */ +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdtempSync, + readFileSync, + readdirSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { rm } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { promisify } from 'node:util'; +import { validateBlocks } from 'cli/ai/block-validator'; +import { runCommand as runStartSiteCommand } from 'cli/commands/site/start'; +import { Mode as StopMode, runCommand as runStopSiteCommand } from 'cli/commands/site/stop'; +import { readCliConfig } from 'cli/lib/cli-config/core'; +import { getSiteByFolder, getSiteUrl } from 'cli/lib/cli-config/sites'; +import { connectToDaemon } from 'cli/lib/daemon-client'; +import { runWpCliCommand } from 'cli/lib/run-wp-cli-command'; +import { Logger } from 'cli/logger'; + +const execFileAsync = promisify( execFile ); + +export interface SiteFingerprint { + path: string; + name: string; + mtimeMs: number; +} + +// The newest modification time under a site's database and wp-content, which +// moves whenever the site is changed. +function newestMtime( dir: string, depth = 6 ): number { + let newest = 0; + const walk = ( current: string, level: number ) => { + let entries; + try { + entries = readdirSync( current, { withFileTypes: true } ); + } catch { + return; + } + for ( const entry of entries ) { + if ( entry.name === 'node_modules' || entry.name === 'cache' ) continue; + const full = path.join( current, entry.name ); + try { + newest = Math.max( newest, statSync( full ).mtimeMs ); + } catch { + continue; + } + if ( entry.isDirectory() && level < depth ) walk( full, level + 1 ); + } + }; + walk( path.join( dir, 'wp-content' ), 0 ); + return newest; +} + +export async function fingerprintSites(): Promise< SiteFingerprint[] > { + const { sites } = await readCliConfig(); + return sites + .filter( ( site ) => existsSync( site.path ) ) + .map( ( site ) => ( { path: site.path, name: site.name, mtimeMs: newestMtime( site.path ) } ) ); +} + +const PROBE_PHP = String.raw` +$count = function ( $blocks, &$acc ) use ( &$count ) { + foreach ( $blocks as $block ) { + if ( $block['blockName'] ) { $acc[ $block['blockName'] ] = ( $acc[ $block['blockName'] ] ?? 0 ) + 1; } + $count( $block['innerBlocks'], $acc ); + } +}; +$front = (int) get_option( 'page_on_front' ); +$content = array(); +foreach ( get_posts( array( 'post_type' => array( 'page', 'post', 'wp_template', 'wp_template_part', 'wp_navigation', 'wp_block' ), 'post_status' => 'publish', 'numberposts' => -1 ) ) as $post ) { + $blocks = array(); + $count( parse_blocks( $post->post_content ), $blocks ); + $content[] = array( 'id' => $post->ID, 'type' => $post->post_type, 'slug' => $post->post_name, 'title' => $post->post_title, 'isFront' => $post->ID === $front, 'blocks' => $blocks, 'content' => $post->post_content ); +} +$styles = array(); +foreach ( WP_Block_Styles_Registry::get_instance()->get_all_registered() as $block => $variations ) { + $styles[ $block ] = array_keys( $variations ); +} +$merged = WP_Theme_JSON_Resolver::get_merged_data()->get_raw_data(); +$palette = array(); +foreach ( (array) ( $merged['settings']['color']['palette'] ?? array() ) as $entries ) { + foreach ( (array) $entries as $entry ) { $palette[] = $entry; } +} +$plugins = array(); +foreach ( (array) get_option( 'active_plugins' ) as $plugin ) { $plugins[] = dirname( $plugin ) === '.' ? basename( $plugin, '.php' ) : dirname( $plugin ); } +echo 'OUTCOME_JSON=' . wp_json_encode( array( + 'theme' => array( 'stylesheet' => get_stylesheet(), 'template' => get_template(), 'isBlockTheme' => wp_is_block_theme(), 'stylesheetVersion' => wp_get_theme()->get( 'Version' ), 'templateVersion' => wp_get_theme( get_template() )->get( 'Version' ) ), + 'plugins' => $plugins, + 'showOnFront' => get_option( 'show_on_front' ), + 'content' => $content, + 'products' => post_type_exists( 'product' ) ? (int) wp_count_posts( 'product' )->publish : 0, + 'blockStyles' => $styles, + 'palette' => $palette, + 'mergedStyles' => $merged['styles'] ?? array(), + 'userGlobalStyles' => WP_Theme_JSON_Resolver::get_user_data()->get_raw_data(), + 'customCss' => wp_get_custom_css(), +) ) . "\n"; +`; + +async function wp( sitePath: string, args: string[] ): Promise< string > { + const site = await getSiteByFolder( sitePath ); + await using command = await runWpCliCommand( site, args ); + const exitCode = await command.response.exitCode; + const stdout = await command.response.stdoutText; + if ( exitCode !== 0 ) { + throw new Error( + `wp ${ args[ 0 ] } exited ${ exitCode }: ${ await command.response.stderrText }` + ); + } + return stdout; +} + +const TEXT_FILE = /\.(css|json|html|php|md|txt)$/; + +function hashFiles( dir: string ): Record< string, string > { + const hashes: Record< string, string > = {}; + const walk = ( current: string ) => { + for ( const entry of readdirSync( current, { withFileTypes: true } ) ) { + const full = path.join( current, entry.name ); + if ( entry.isDirectory() ) walk( full ); + else + hashes[ path.relative( dir, full ) ] = createHash( 'sha1' ) + .update( readFileSync( full ) ) + .digest( 'hex' ); + } + }; + if ( existsSync( dir ) ) walk( dir ); + return hashes; +} + +function readThemeText( dir: string ): Record< string, string > { + const files: Record< string, string > = {}; + for ( const relative of Object.keys( hashFiles( dir ) ) ) { + if ( TEXT_FILE.test( relative ) && ! relative.includes( 'node_modules' ) ) { + const text = readFileSync( path.join( dir, relative ), 'utf8' ); + if ( text.length < 200_000 ) files[ relative ] = text; + } + } + return files; +} + +// Files of an installed theme that differ from its WordPress.org release, or +// null when the release cannot be fetched (not a WordPress.org theme). +async function changedFromRelease( + themeDir: string, + slug: string, + version: string +): Promise< string[] | null > { + const tmp = mkdtempSync( path.join( os.tmpdir(), 'eval-theme-' ) ); + try { + const response = await fetch( + `https://downloads.wordpress.org/theme/${ slug }.${ version }.zip` + ); + if ( ! response.ok ) return null; + const zip = path.join( tmp, 'theme.zip' ); + writeFileSync( zip, Buffer.from( await response.arrayBuffer() ) ); + await execFileAsync( 'unzip', [ '-q', zip, '-d', tmp ] ); + const release = hashFiles( path.join( tmp, slug ) ); + const installed = hashFiles( themeDir ); + return [ ...new Set( [ ...Object.keys( release ), ...Object.keys( installed ) ] ) ] + .filter( ( file ) => release[ file ] !== installed[ file ] ) + .sort(); + } catch { + return null; + } finally { + await rm( tmp, { recursive: true, force: true } ); + } +} + +// The format a theme screenshot really is, from its leading bytes. +function imageFormat( file: string ): string | null { + if ( ! existsSync( file ) ) return null; + const head = readFileSync( file ).subarray( 0, 4 ); + if ( head[ 0 ] === 0xff && head[ 1 ] === 0xd8 && head[ 2 ] === 0xff ) + return `jpeg:${ path.basename( file ) }`; + if ( head.toString( 'hex' ) === '89504e47' ) return `png:${ path.basename( file ) }`; + return `unknown:${ path.basename( file ) }`; +} + +async function renderHome( url: string ) { + try { + const response = await fetch( url, { redirect: 'follow' } ); + const html = await response.text(); + const nav = [ ...html.matchAll( //g ) ] + .map( ( m ) => m[ 0 ] ) + .join( '\n' ); + return { + status: response.status, + html, + phpErrors: [ + ...html.matchAll( /(Fatal error|Parse error|Warning|Notice|Deprecated)<\/b>:[^<]*/g ), + ].map( ( m ) => m[ 0 ].replace( /<\/?b>/g, '' ) ), + navHrefs: [ ...nav.matchAll( /]*\bhref="([^"]*)"/g ) ].map( ( m ) => m[ 1 ] ), + ids: [ ...html.matchAll( /\bid="([^"]+)"/g ) ].map( ( m ) => m[ 1 ] ), + }; + } catch ( error ) { + return { status: 0, error: error instanceof Error ? error.message : String( error ) }; + } +} + +export interface SiteOutcome { + name: string; + path: string; + created: boolean; + url?: string; + error?: string; + [ key: string ]: unknown; +} + +export async function captureSiteOutcome( + sitePath: string, + created: boolean, + artifactDir: string +): Promise< SiteOutcome > { + const site = await getSiteByFolder( sitePath ); + const outcome: SiteOutcome = { name: site.name, path: sitePath, created }; + try { + await connectToDaemon(); + await runStartSiteCommand( sitePath, true, true, new Logger() ); + const url = getSiteUrl( site ); + outcome.url = url; + + const probe = ( await wp( sitePath, [ 'eval', PROBE_PHP ] ) ).match( /OUTCOME_JSON=(.*)/ ); + if ( ! probe ) throw new Error( 'the WordPress probe printed no state' ); + const state = JSON.parse( probe[ 1 ] ); + Object.assign( outcome, state ); + + const themes = path.join( sitePath, 'wp-content', 'themes' ); + const { stylesheet, stylesheetVersion, template, templateVersion } = state.theme; + const themeDir = path.join( themes, stylesheet ); + outcome.themeFiles = readThemeText( themeDir ); + outcome.themeFileList = Object.keys( hashFiles( themeDir ) ).sort(); + outcome.themeScreenshot = + imageFormat( path.join( themeDir, 'screenshot.jpg' ) ) ?? + imageFormat( path.join( themeDir, 'screenshot.png' ) ); + // Files changed from the WordPress.org release (null for themes not from there). + outcome.themeChanges = await changedFromRelease( themeDir, stylesheet, stylesheetVersion ); + if ( template !== stylesheet ) { + outcome.parentThemeChanges = await changedFromRelease( + path.join( themes, template ), + template, + templateVersion + ); + } + const designPath = path.join( sitePath, 'DESIGN.md' ); + outcome.designMd = existsSync( designPath ) ? readFileSync( designPath, 'utf8' ) : null; + + const home = await renderHome( url ); + if ( 'html' in home && home.html ) { + const file = path.join( artifactDir, `${ path.basename( sitePath ) }-home.html` ); + writeFileSync( file, home.html ); + outcome.home = { ...home, html: undefined, htmlFile: file }; + } else { + outcome.home = home; + } + + // Every published page, template and part, validated by the site's own editor. + const invalid: Array< { id: number; type: string; slug: string; blocks: string[] } > = []; + for ( const item of state.content as Array< { + id: number; + type: string; + slug: string; + content: string; + } > ) { + if ( ! item.content.includes( '' ) ), + vars + ).pass + ).toBe( true ); + expect( + run( + checks.colorsFromPalette, + outcome( site( '
' ) ), + vars + ).pass + ).toBe( false ); + expect( + run( checks.colorsFromPalette, outcome( site( '' ) ), vars ).pass + ).toBe( false ); + } ); + + test( 'noInlineDisplayOnBlockWrappers', () => { + const site = ( css ) => ( { + ...workingSite, + content: [ front( '', {} ) ], + customCss: css, + } ); + expect( + run( + checks.noInlineDisplayOnBlockWrappers, + outcome( site( '.eyebrow p { display: inline-block; }' ) ) + ).pass + ).toBe( true ); + expect( + run( + checks.noInlineDisplayOnBlockWrappers, + outcome( site( '.eyebrow { display: inline-block; }' ) ) + ).pass + ).toBe( false ); + // The first rule of a theme stylesheet and of theme.json's css counts too. + expect( + run( + checks.noInlineDisplayOnBlockWrappers, + outcome( { + ...site( '' ), + themeFiles: { 'style.css': '.eyebrow{display:inline-flex}' }, + } ) + ).pass + ).toBe( false ); + expect( + run( + checks.noInlineDisplayOnBlockWrappers, + outcome( { + ...site( '' ), + themeFiles: { + 'theme.json': '{"styles":{"css":".eyebrow {\\n display: inline-block;\\n}"}}', + }, + } ) + ).pass + ).toBe( false ); + } ); + + test( 'repeatedTreatmentIsBlockStyle', () => { + const site = ( content ) => ( { + ...workingSite, + blockStyles: { 'core/group': [ 'card', 'section-1' ] }, + content: [ front( content, {} ) ], + } ); + expect( + run( + checks.repeatedTreatmentIsBlockStyle, + outcome( site( 'is-style-card is-style-card is-style-card' ) ) + ).pass + ).toBe( true ); + expect( + run( + checks.repeatedTreatmentIsBlockStyle, + outcome( site( 'is-style-section-1 is-style-section-1' ) ) + ).pass + ).toBe( false ); + expect( + run( + checks.repeatedTreatmentIsBlockStyle, + outcome( + site( + 'is-style-card is-style-card {"fontSize":"18px"} {"fontSize":"18px"} {"fontSize":"18px"}' + ) + ) + ).pass + ).toBe( false ); + } ); + + test( 'designTokenReachedTheme', () => { + const vars = { token: 'primary', originalValue: '#000000' }; + const site = ( designMd, palette ) => ( { ...workingSite, designMd, palette } ); + expect( + run( + checks.designTokenReachedTheme, + outcome( site( 'primary: "#2255aa"', [ { color: '#2255AA' } ] ) ), + vars + ).pass + ).toBe( true ); + expect( + run( + checks.designTokenReachedTheme, + outcome( site( 'primary: "#2255aa"', [ { color: '#000000' } ] ) ), + vars + ).pass + ).toBe( false ); + expect( + run( + checks.designTokenReachedTheme, + outcome( site( 'primary: "#000000"', [ { color: '#000000' } ] ) ), + vars + ).pass + ).toBe( false ); + } ); +} );