diff --git a/apps/cli/ai/active-code.ts b/apps/cli/ai/active-code.ts new file mode 100644 index 0000000000..b574af54a5 --- /dev/null +++ b/apps/cli/ai/active-code.ts @@ -0,0 +1,75 @@ +/** + * The code a site runs: its active plugins and themes, read from the options + * WordPress loads them by (active_plugins, template and stylesheet), and the + * files in their folders, as one version a check compares to see whether that + * code changed. Dependencies a build installs (node_modules) are not the + * site's code and are not read. + */ +import { readdir, stat } from 'fs/promises'; +import path from 'path'; +import { readWordPressSite } from 'cli/ai/site-database'; + +// A file as its path, modification time and size. +async function version( file: string, from: string ): Promise< string > { + const { mtimeMs, size } = await stat( file ).catch( () => ( { mtimeMs: 0, size: -1 } ) ); + return `${ path.relative( from, file ) }:${ mtimeMs }:${ size }`; +} + +async function files( dir: string, wanted: ( name: string ) => boolean ): Promise< string[] > { + const entries = await readdir( dir, { withFileTypes: true } ).catch( () => [] ); + const found = await Promise.all( + entries.map( ( entry ) => { + const at = path.join( dir, entry.name ); + if ( entry.isDirectory() ) { + return entry.name === 'node_modules' ? [] : files( at, wanted ); + } + return entry.isFile() && wanted( entry.name ) ? [ at ] : []; + } ) + ); + return found.flat(); +} + +/** + * The version of each of the site's active plugins and themes, by + * `plugin ` (as active_plugins names it) and `theme `, from the + * files in them `wanted` keeps. + */ +export async function activeCode( + sitePath: string, + wanted: ( name: string ) => boolean +): Promise< Record< string, string > > { + const content = path.join( sitePath, 'wp-content' ); + const active = await readWordPressSite( sitePath, ( site ) => { + const plugins = site.option( 'active_plugins' ); + return { + plugins: plugins && typeof plugins === 'object' ? Object.values( plugins ).map( String ) : [], + themes: [ ...new Set( [ site.option( 'template' ), site.option( 'stylesheet' ) ] ) ].filter( + ( theme ): theme is string => typeof theme === 'string' && theme !== '' + ), + }; + } ); + // A plugin in a folder is the folder; a single-file plugin is its file. + const dirVersion = async ( dir: string ) => + ( await Promise.all( ( await files( dir, wanted ) ).map( ( file ) => version( file, dir ) ) ) ) + .sort() + .join( '|' ); + const plugins = active.plugins.map( async ( plugin ) => [ + `plugin ${ plugin }`, + plugin.includes( '/' ) + ? await dirVersion( path.join( content, 'plugins', path.dirname( plugin ) ) ) + : await version( path.join( content, 'plugins', plugin ), content ), + ] ); + const themes = active.themes.map( async ( theme ) => [ + `theme ${ theme }`, + await dirVersion( path.join( content, 'themes', theme ) ), + ] ); + return Object.fromEntries( await Promise.all( [ ...plugins, ...themes ] ) ); +} + +/** The site's active plugins and themes, and the files in them `wanted` keeps, as one version. */ +export async function activeCodeVersion( + sitePath: string, + wanted: ( name: string ) => boolean +): Promise< string > { + return JSON.stringify( await activeCode( sitePath, wanted ) ); +} diff --git a/apps/cli/ai/admin-capabilities.ts b/apps/cli/ai/admin-capabilities.ts new file mode 100644 index 0000000000..3472882cfc --- /dev/null +++ b/apps/cli/ai/admin-capabilities.ts @@ -0,0 +1,341 @@ +/** + * The capabilities the administrator role grants that the site's code takes + * from its administrator. WordPress resolves a capability through + * map_meta_cap() and the user_has_cap filter (wp-includes/capabilities.php), + * which plugins and themes extend, so code that maps a capability to + * do_not_allow denies it to every user and every screen that checks it. A + * post type that names a primitive capability, such as edit_posts, as its + * edit_post, read_post or delete_post makes WordPress map that capability as + * the meta capability for every check (_post_type_meta_capabilities() in + * wp-includes/post.php). The meta capabilities a post type names from its own + * capability_type (get_post_type_capabilities()) need a post by design, so a + * role that holds them, as WooCommerce gives the administrator edit_product, + * is not reported. + * + * When the PHP of a site's active plugins or themes changes, or the plugins and theme active change, Studio asks WordPress + * which of the role's capabilities user_can() denies the first administrator, + * with the site's code loaded and with its plugins and themes skipped: the + * ones denied only with the code loaded are the code's. + * + * In the same runs, and one with only the changed plugins or theme skipped, + * Studio states what the changed code registers with WordPress: what the + * registries hold only with it loaded. + * + * Site fact: what all the site's code registers, by registry, from the same + * registries with the code loaded and skipped, and where WordPress lists each. + */ +import { activeCode } from 'cli/ai/active-code'; +import { runWpCli } from 'cli/ai/tools/utils'; +import { findSiteByFolder } from 'cli/lib/cli-config/sites'; +import { connectToDaemon, disconnectFromDaemon } from 'cli/lib/daemon-client'; +import type { SiteChangeSource, Versions } from 'cli/ai/site-changes'; +import type { WordPressSite } from 'cli/ai/site-database'; +import type { SiteData } from 'cli/lib/cli-config/core'; + +const KEY = 'administrator capabilities'; + +// The registries a plugin or theme adds to, by the function that adds to each. +const REGISTRIES: Record< string, string > = { + 'post types': 'register_post_type()', + taxonomies: 'register_taxonomy()', + 'REST routes': 'register_rest_route()', + 'block types': 'register_block_type()', + abilities: 'wp_register_ability(), in wp-includes/abilities-api.php', + shortcodes: 'add_shortcode()', +}; + +const REGISTERED = `echo 'REGISTERED=' . wp_json_encode( array( + 'post types' => array_keys( get_post_types() ), + 'taxonomies' => array_keys( get_taxonomies() ), + 'REST routes' => array_keys( rest_get_server()->get_routes() ), + 'block types' => array_keys( WP_Block_Type_Registry::get_instance()->get_all_registered() ), + 'abilities' => function_exists( 'wp_get_abilities' ) ? array_map( function ( $ability ) { return $ability->get_name(); }, array_values( wp_get_abilities() ) ) : array(), + 'shortcodes' => array_keys( $GLOBALS['shortcode_tags'] ), +) ) . "\\n"; +echo 'OUT_OF_REST=' . wp_json_encode( array( + 'post types' => array_keys( get_post_types( array( 'show_in_rest' => false ) ) ), + 'taxonomies' => array_keys( get_taxonomies( array( 'show_in_rest' => false ) ) ), + 'abilities' => function_exists( 'wp_get_abilities' ) ? array_values( array_filter( array_map( function ( $ability ) { return $ability->get_meta_item( 'show_in_rest' ) ? null : $ability->get_name(); }, array_values( wp_get_abilities() ) ) ) ) : array(), +) ) . "\\n";`; + +// The setting that keeps a registry's item out of WordPress's REST API, which +// is how agents and the block editor reach it. +const IN_REST: Record< string, string > = { + 'post types': 'show_in_rest false', + taxonomies: 'show_in_rest false', + abilities: + 'meta show_in_rest false, so not listed by wp-includes/rest-api/endpoints/class-wp-rest-abilities-v1-list-controller.php', +}; + +type Registered = Record< string, string[] >; + +interface Denied { + user: number; + caps: Record< string, { mapped: string[]; types: Array< [ string, string ] > } >; +} + +const DENIED = `global $post_type_meta_caps; +$admins = get_users( array( 'role' => 'administrator', 'number' => 1, 'fields' => 'ID', 'orderby' => 'ID' ) ); +$role = get_role( 'administrator' ); +$denied = array( 'user' => $admins ? (int) $admins[0] : 0, 'caps' => new stdClass() ); +foreach ( $admins && $role ? array_keys( array_filter( $role->capabilities ) ) : array() as $cap ) { + if ( user_can( $denied['user'], $cap ) ) { + continue; + } + $types = array(); + $own = false; + foreach ( isset( $post_type_meta_caps[ $cap ] ) ? get_post_types( array( 'map_meta_cap' => true ), 'objects' ) : array() as $type ) { + foreach ( array( 'read_post', 'edit_post', 'delete_post' ) as $meta ) { + if ( $type->cap->$meta === $cap ) { + // The meta capability get_post_type_capabilities() names from the + // type's capability_type, as WooCommerce's edit_product, needs a post + // by design, whatever roles hold its name. + $singular = ( (array) $type->capability_type )[0]; + $own = $own || $cap === str_replace( '_post', '_' . $singular, $meta ); + $types[] = array( $type->name, $meta ); + } + } + } + if ( $own ) { + continue; + } + $denied['caps']->$cap = array( 'mapped' => map_meta_cap( $cap, $denied['user'] ), 'types' => $types ); +} +echo 'DENIED_CAPS=' . wp_json_encode( $denied );`; + +// What WordPress prints after a marker, on its line. +const printed = ( output: string, marker: string ) => + JSON.parse( output.slice( output.indexOf( marker ) + marker.length ).split( '\n' )[ 0 ] ); + +async function askWordPress( + site: SiteData, + skip: string[] +): Promise< { denied: Denied; registered: Registered; outOfRest: Registered } > { + const output = await runWpCli( site, [ 'eval', REGISTERED + DENIED, ...skip ] ); + return { + denied: printed( output, 'DENIED_CAPS=' ), + registered: printed( output, 'REGISTERED=' ), + outOfRest: printed( output, 'OUT_OF_REST=' ), + }; +} + +// The site's active code, by plugin and theme: which are active, and their +// PHP files. Must-use plugins load with plugins skipped too. +async function readVersions( sitePath: string ): Promise< Versions > { + return activeCode( sitePath, ( name ) => name.endsWith( '.php' ) ); +} + +// The WP-CLI flags that skip the given plugins and themes: a plugin by its +// folder, or a single-file plugin by its name (WP_CLI\Utils\get_plugin_name()). +function skipping( keys: string[] ): string[] { + const plugins = keys + .filter( ( key ) => key.startsWith( 'plugin ' ) ) + .map( ( key ) => + key + .slice( 'plugin '.length ) + .split( '/' )[ 0 ] + .replace( /\.php$/, '' ) + ); + return [ + ...( plugins.length ? [ `--skip-plugins=${ plugins.join( ',' ) }` ] : [] ), + ...( keys.some( ( key ) => key.startsWith( 'theme ' ) ) ? [ '--skip-themes' ] : [] ), + ]; +} + +const list = ( items: string[] ) => + items.length < 2 + ? items.join( '' ) + : `${ items.slice( 0, -1 ).join( ', ' ) } and ${ items.at( -1 ) }`; + +const MAX_NAMES = 12; + +function describeRegistered( + site: SiteData, + changed: string[], + added: Registered, + outOfRest: Registered +): string { + const names = ( items: string[] ) => + items.length > MAX_NAMES + ? `${ items.slice( 0, MAX_NAMES ).join( ', ' ) } and ${ items.length - MAX_NAMES } more` + : items.join( ', ' ); + const holds = Object.keys( REGISTRIES ).filter( ( registry ) => added[ registry ]?.length ); + const none = Object.keys( REGISTRIES ).filter( ( registry ) => ! added[ registry ]?.length ); + return `${ site.name }: with ${ list( changed ) } loaded, WordPress registers ${ [ + ...holds.map( ( registry ) => { + const hidden = added[ registry ].filter( + ( name ) => outOfRest[ registry ]?.includes( name ) + ); + return `${ registry } ${ names( added[ registry ] ) } (${ REGISTRIES[ registry ] }${ + hidden.length + ? `; with ${ IN_REST[ registry ] }, out of the REST API: ${ names( hidden ) }` + : '' + })`; + } ), + ...( none.length + ? [ + `no ${ list( + none.map( ( registry ) => `${ registry } (${ REGISTRIES[ registry ] })` ) + ) } of its own`, + ] + : [] ), + ].join( '; ' ) }.`; +} + +function describeDenied( site: SiteData, user: number, denied: Denied[ 'caps' ] ): string { + const caps = Object.keys( denied ); + const mapped = [ ...new Set( caps.flatMap( ( cap ) => denied[ cap ].mapped ) ) ]; + const byType = new Map< string, string[] >(); + for ( const cap of caps ) { + for ( const [ type, meta ] of denied[ cap ].types ) { + byType.set( type, [ ...( byType.get( type ) ?? [] ), `${ cap } as its ${ meta }` ] ); + } + } + const them = caps.length === 1 ? 'it' : 'them'; + return [ + `${ site.name }: user_can() denies its administrator (user ${ user }) ${ list( + caps + ) }, which the administrator role grants and WordPress grants with the site's plugins and theme skipped; map_meta_cap() (wp-includes/capabilities.php) maps ${ them } to ${ list( + mapped + ) }.`, + ...[ ...byType ].map( + ( [ type, named ] ) => + `Post type ${ type } names ${ list( named ) } capability, so WordPress checks ${ + named.length === 1 + ? 'that capability as the meta capability, which needs' + : 'those capabilities as the meta capabilities, which need' + } a post (_post_type_meta_capabilities() in wp-includes/post.php).` + ), + ...( byType.size + ? [ + "register_post_type()'s capability_type names a post type's capabilities from its own name, as core's page post type has edit_page and edit_pages.", + ] + : [] ), + ].join( ' ' ); +} + +// Where WordPress's REST API lists a registry's items, and the registries whose +// items are named by namespace (namespace/name). +const LISTED_AT: Record< string, string > = { + 'post types': '/wp/v2/types', + taxonomies: '/wp/v2/taxonomies', + 'REST routes': '/wp-json/', + 'block types': '/wp/v2/block-types/', + abilities: '/wp-abilities/v1/abilities', +}; +const NAMESPACED = [ 'block types', 'abilities' ]; + +function describeSiteCode( added: Registered ): string | undefined { + const holds = Object.keys( REGISTRIES ).filter( ( registry ) => added[ registry ]?.length ); + if ( ! holds.length ) { + return undefined; + } + const counts = holds.map( ( registry ) => { + const namespaces = [ + ...new Set( added[ registry ].map( ( name ) => name.split( '/' )[ 0 ] ) ), + ]; + return `${ registry } ${ added[ registry ].length }${ + NAMESPACED.includes( registry ) ? ` (namespace ${ list( namespaces ) })` : '' + }`; + } ); + const listed = holds.filter( ( registry ) => LISTED_AT[ registry ] ); + return `With the active plugins and theme loaded, WordPress registers, beyond core: ${ list( + counts + ) }. Its REST API lists those shown in it: ${ listed + .map( ( registry ) => `${ registry } at ${ LISTED_AT[ registry ] }` ) + .join( ', ' ) }.`; +} + +// By site, the site's code it was read for and the fact, read again only when that code changes. +const siteCodeFacts = new Map< string, { code: string; fact: Promise< string | undefined > } >(); + +async function readSiteCodeFact( sitePath: string ): Promise< string | undefined > { + const site = await findSiteByFolder( sitePath ); + if ( ! site ) { + return undefined; + } + try { + await connectToDaemon(); + const [ withCode, core ] = [ + await askWordPress( site, [] ), + await askWordPress( site, [ '--skip-plugins', '--skip-themes' ] ), + ]; + return describeSiteCode( + Object.fromEntries( + Object.entries( withCode.registered ).map( ( [ registry, names ] ) => [ + registry, + names.filter( ( name ) => ! core.registered[ registry ]?.includes( name ) ), + ] ) + ) + ); + } catch { + // Code that does not load is reported by what runs it. + return undefined; + } finally { + await disconnectFromDaemon(); + } +} + +export async function siteCodeFact( site: WordPressSite ): Promise< string | undefined > { + const code = JSON.stringify( await readVersions( site.root ) ); + const known = siteCodeFacts.get( site.root ); + if ( known?.code === code ) { + return known.fact; + } + const fact = readSiteCodeFact( site.root ); + siteCodeFacts.set( site.root, { code, fact } ); + return fact; +} + +export const adminCapabilities: SiteChangeSource = { + name: 'admin-capabilities', + read: readVersions, + async check( site, before, after ) { + const changed = Object.keys( after ).filter( ( key ) => before[ key ] !== after[ key ] ); + const removed = Object.keys( before ).some( ( key ) => ! ( key in after ) ); + if ( ! changed.length && ! removed ) { + return { reports: [] }; + } + let withCode: Awaited< ReturnType< typeof askWordPress > >; + let core: Awaited< ReturnType< typeof askWordPress > >; + let without: Awaited< ReturnType< typeof askWordPress > >; + try { + await connectToDaemon(); + withCode = await askWordPress( site, [] ); + core = await askWordPress( site, [ '--skip-plugins', '--skip-themes' ] ); + without = changed.length ? await askWordPress( site, skipping( changed ) ) : withCode; + } catch { + // Code that does not load is reported by what runs it. + return { reports: [] }; + } finally { + await disconnectFromDaemon(); + } + const denied = Object.fromEntries( + Object.entries( withCode.denied.caps ).filter( ( [ cap ] ) => ! ( cap in core.denied.caps ) ) + ); + const added = Object.fromEntries( + Object.entries( withCode.registered ).map( ( [ registry, names ] ) => [ + registry, + names.filter( ( name ) => ! without.registered[ registry ]?.includes( name ) ), + ] ) + ); + return { + reports: [ + { + key: KEY, + problem: Object.keys( denied ).length + ? describeDenied( site, withCode.denied.user, denied ) + : undefined, + }, + ...( changed.length + ? [ + { + key: 'registered', + summary: describeRegistered( site, changed, added, withCode.outOfRest ), + }, + ] + : [] ), + ], + }; + }, +}; diff --git a/apps/cli/ai/block-alignment.ts b/apps/cli/ai/block-alignment.ts new file mode 100644 index 0000000000..30fa7cf3b7 --- /dev/null +++ b/apps/cli/ai/block-alignment.ts @@ -0,0 +1,179 @@ +/** + * Full-width blocks that cannot reach the page edges. A block spans the page + * with `align: full` only inside containers that span it: a constrained layout + * (wp-includes/block-supports/layout.php) holds each other child to the + * content width, and the full-width blocks inside that child with it. + */ + +// A block's opening, closing, or self-closing comment in serialized markup. +const BLOCK_DELIMITER = + //g; + +interface BlockAttributes { + align?: unknown; + className?: unknown; + layout?: { type?: unknown; inherit?: unknown; contentSize?: unknown; wideSize?: unknown }; +} + +interface OpenBlock { + label: string; + name: string; + attributes: BlockAttributes; + constrains: boolean; + reachesEdges: boolean; + /** Where its opening comment starts and ends in the markup. */ + at: number; + end: number; + selfClosing: boolean; +} + +interface Trap { + subject: string; + stop: OpenBlock; + /** The block that holds the stop to the content width. */ + holder: OpenBlock; +} + +// Which layout WordPress applies: a `layout` without a type that inherits or +// sets a content size is the constrained layout. +function constrains( { layout }: BlockAttributes ): boolean { + return ( + layout?.type === 'constrained' || + ( layout?.type === undefined && + ( layout?.inherit === true || layout?.contentSize !== undefined ) ) + ); +} + +function labelOf( name: string, attributes: BlockAttributes ): string { + const block = name.includes( '/' ) ? name : `core/${ name }`; + return typeof attributes.className === 'string' + ? `${ block } (${ attributes.className + .split( /\s+/ ) + .map( ( c ) => `.${ c }` ) + .join( '' ) })` + : block; +} + +// The page content: its align full blocks span the page only when it does. +const POST_CONTENT = 'core/post-content'; + +// What keeps each full-width block in the markup, and the page content, from +// the page edges, taking its top-level blocks to span the page: those of a +// template do, and those of content do unless the template holds them. +function traps( markup: string ): Trap[] { + const open: OpenBlock[] = []; + const found: Trap[] = []; + for ( const match of markup.matchAll( BLOCK_DELIMITER ) ) { + const [ delimiter, closing, name, json, selfClosing ] = match; + if ( closing ) { + open.pop(); + continue; + } + let attributes: BlockAttributes = {}; + try { + attributes = json ? JSON.parse( json ) : {}; + } catch { + // Invalid attributes are the block check's to report. + } + const parent = open.at( -1 ); + const full = attributes.align === 'full'; + const block: OpenBlock = { + label: labelOf( name, attributes ), + name: name.includes( '/' ) ? name : `core/${ name }`, + attributes, + constrains: constrains( attributes ), + reachesEdges: ! parent || ( parent.reachesEdges && ( full || ! parent.constrains ) ), + at: match.index, + end: match.index + delimiter.length, + selfClosing: Boolean( selfClosing ), + }; + const isPostContent = block.name === POST_CONTENT; + if ( ( full || isPostContent ) && ! block.reachesEdges ) { + // The outermost block that stops short: the one align full would free. + const chain = [ ...open, block ]; + const index = chain.findIndex( ( { reachesEdges } ) => ! reachesEdges ); + found.push( { + subject: isPostContent + ? `${ POST_CONTENT }, and so no align full block in a page's content,` + : `${ block.label }, align full,`, + stop: chain[ index ], + holder: chain[ index - 1 ], + } ); + } + if ( ! selfClosing ) { + open.push( block ); + } + } + return found; +} + +/** What keeps each full-width block in this markup, and the page content, from the page edges. */ +export function trappedFullWidthNotes( markup: string ): string[] { + return [ + ...new Set( + traps( markup ).map( + ( { subject, stop } ) => + `${ subject } cannot reach the page edges: ${ stop.label } is not align full, and the constrained layout around it holds it to the content width (wp-includes/block-supports/layout.php). Give ${ stop.label } align full.` + ) + ), + ]; +} + +// The attributes that give a block no look of its own: what it is and how it +// lays out its blocks. +const STRUCTURAL = new Set( [ 'layout', 'tagName', 'lock', 'metadata', 'anchor' ] ); + +const ownSizes = ( { layout }: BlockAttributes ) => + layout?.contentSize !== undefined || layout?.wideSize !== undefined; + +/** + * The markup with each block that keeps full-width blocks from the page edges + * made align full, where that changes nothing else: a block with no look of + * its own whose constrained layout holds its other blocks to the content width + * the block around it held them to. `supportsFull` says whether a block type + * supports align full (block.json supports.align). + */ +export function freeTrappedBlocks( + markup: string, + supportsFull: ( name: string ) => boolean +): { markup: string; freed: string[] } { + const stops = new Map< number, OpenBlock >(); + for ( const { stop, holder } of traps( markup ) ) { + if ( + stop.constrains && + ! ownSizes( stop.attributes ) && + ! ownSizes( holder.attributes ) && + Object.keys( stop.attributes ).every( ( key ) => STRUCTURAL.has( key ) ) && + supportsFull( stop.name ) + ) { + stops.set( stop.at, stop ); + } + } + let freed = markup; + const labels: string[] = []; + // From the last, so each place stays where the scan found it. + for ( const stop of [ ...stops.values() ].sort( ( a, b ) => b.at - a.at ) ) { + // A saved block carries the alignfull class on its wrapper, as the + // align support adds it (wp-includes/block-supports/align.php). + const wrapper = stop.selfClosing + ? undefined + : /^\s*<[a-z][a-z0-9]*\b[^>]*?\bclass="([^"]*)"/i.exec( freed.slice( stop.end ) ); + if ( ! stop.selfClosing && ! wrapper ) { + continue; + } + const comment = freed + .slice( stop.at, stop.end ) + .replace( + /^()$/, + `$1 ${ JSON.stringify( { ...stop.attributes, align: 'full' } ) }$2` + ); + const after = wrapper + ? freed + .slice( stop.end ) + .replace( wrapper[ 0 ], `${ wrapper[ 0 ].slice( 0, -1 ) } alignfull"` ) + : freed.slice( stop.end ); + freed = freed.slice( 0, stop.at ) + comment + after; + labels.push( stop.label ); + } + return { markup: freed, freed: labels.reverse() }; +} diff --git a/apps/cli/ai/block-content-policy.ts b/apps/cli/ai/block-content-policy.ts index 8104a6f962..1809b4ed5e 100644 --- a/apps/cli/ai/block-content-policy.ts +++ b/apps/cli/ai/block-content-policy.ts @@ -7,7 +7,8 @@ const PLUGIN_RECOMMENDATION_HTML_BLOCK_POLICIES = [ { htmlPatterns: [ /<(form|input|select|textarea|fieldset)\b/i ], htmlPolicyMessage: - 'core/html contains form markup. Load the plugin-recommendations skill and use editable plugin blocks such as Jetpack Forms. This keeps forms editable in the block editor and handles submission without custom backend code.', + "core/html contains form markup. A form plugin's block, such as Jetpack Forms (jetpack/contact-form), stays editable in the block editor and handles submission; the plugin-recommendations skill names the plugin for each feature.", + block: 'jetpack/contact-form', }, ]; @@ -16,6 +17,8 @@ interface HtmlBlockPolicyResult { line: number; content: string; issues: string[]; + /** The block a plugin registers for it, when one is recommended. */ + block?: string; } export interface HtmlBlockPolicyReport { @@ -45,7 +48,7 @@ export function getHtmlBlockPolicyIssues( content: string ): string[] { } return [ - 'core/html contains markup that should use editable core blocks. Load the block-content skill and use core/group, core/columns, core/heading, core/paragraph, core/list, core/image, core/buttons, and theme CSS instead. Keep core/html only for inline SVG, interaction markup with no block equivalent (marquee, cursor), or a single script block.', + 'core/html contains markup that should use editable core blocks. Use core/group, core/columns, core/heading, core/paragraph, core/list, core/image, core/buttons, and theme CSS instead. Keep core/html only for inline SVG, interaction markup with no block equivalent (marquee, cursor), or a single script block.', ]; } @@ -61,11 +64,16 @@ export function validateHtmlBlockPolicy( content: string ): HtmlBlockPolicyRepor continue; } + const html = blockContent.replace( BLOCK_COMMENT_PATTERN, '' ); + const block = PLUGIN_RECOMMENDATION_HTML_BLOCK_POLICIES.find( ( recommendation ) => + recommendation.htmlPatterns.some( ( pattern ) => pattern.test( html ) ) + )?.block; invalidHtmlBlocks.push( { blockNumber: totalHtmlBlocks, line: content.slice( 0, match.index ?? 0 ).split( '\n' ).length, content: blockContent.trim(), issues, + ...( block && { block } ), } ); } diff --git a/apps/cli/ai/block-editor-scripts.ts b/apps/cli/ai/block-editor-scripts.ts new file mode 100644 index 0000000000..b22f73d70a --- /dev/null +++ b/apps/cli/ai/block-editor-scripts.ts @@ -0,0 +1,88 @@ +/** + * The blocks a site's own code registers from a block.json that WordPress + * gives nothing to register them in the block editor. A block type has the + * editor scripts its block.json "editorScript" names + * (register_block_type_from_metadata() in wp-includes/blocks.php fills + * WP_Block_Type::$editor_script_handles from it), which the block editor + * loads for it; a block whose "supports" has "autoRegister" and that renders + * on the server needs none, as the editor registers it itself + * (_wp_enqueue_auto_register_blocks() in wp-includes/blocks.php). + * + * When the block.json, PHP or JavaScript files of a site's active plugins or + * themes change, Studio asks WordPress, with the site's code loaded, which + * block types named by a block.json in them have neither. + */ +import { activeCodeVersion } from 'cli/ai/active-code'; +import { runWpCli } from 'cli/ai/tools/utils'; +import { connectToDaemon, disconnectFromDaemon } from 'cli/lib/daemon-client'; +import type { SiteChangeSource } from 'cli/ai/site-changes'; + +const KEY = 'block editor scripts'; + +const WITHOUT = `$dirs = array_map( 'dirname', array_filter( wp_get_active_and_valid_plugins(), fn ( $file ) => dirname( $file ) !== WP_PLUGIN_DIR ) ); +$dirs = array_unique( array_merge( $dirs, array( get_template_directory(), get_stylesheet_directory() ) ) ); +$without = array(); +foreach ( $dirs as $dir ) { + $files = new RecursiveIteratorIterator( + new RecursiveCallbackFilterIterator( + new RecursiveDirectoryIterator( $dir, FilesystemIterator::SKIP_DOTS ), + fn ( $file ) => 'node_modules' !== $file->getFilename() + ) + ); + foreach ( $files as $file ) { + if ( 'block.json' !== $file->getFilename() ) { + continue; + } + $metadata = wp_json_file_decode( $file->getPathname(), array( 'associative' => true ) ); + $type = is_array( $metadata ) && isset( $metadata['name'] ) && is_string( $metadata['name'] ) + ? WP_Block_Type_Registry::get_instance()->get_registered( $metadata['name'] ) + : null; + if ( ! $type || $type->editor_script_handles || ( ! empty( $type->supports['autoRegister'] ) && $type->render_callback ) ) { + continue; + } + $without[ $type->name ][] = $file->getPathname(); + } +} +echo 'WITHOUT_EDITOR_SCRIPT=' . wp_json_encode( (object) $without );`; + +export const blockEditorScripts: SiteChangeSource = { + name: KEY, + async read( sitePath ) { + return { + code: await activeCodeVersion( sitePath, ( name ) => /\.(php|js|json)$/.test( name ) ), + }; + }, + async check( site, before, after ) { + if ( before.code === after.code ) { + return { reports: [] }; + } + let without: Record< string, string[] >; + try { + await connectToDaemon(); + const output = await runWpCli( site, [ 'eval', WITHOUT ] ); + without = JSON.parse( + output.slice( output.indexOf( 'WITHOUT_EDITOR_SCRIPT=' ) + 'WITHOUT_EDITOR_SCRIPT='.length ) + ) as Record< string, string[] >; + } catch { + // Code that does not load is reported by what runs it. + return { reports: [] }; + } finally { + await disconnectFromDaemon(); + } + const names = Object.keys( without ); + return { + reports: [ + { + key: KEY, + problem: names.length + ? `${ site.name }: WordPress registers ${ names + .map( ( name ) => `${ name } (${ without[ name ].join( ', ' ) })` ) + .join( + ', ' + ) } with no editor script. register_block_type_from_metadata() (wp-includes/blocks.php) gives a block type the scripts its block.json "editorScript" names, and the block editor loads them for it (WP_Block_Type::$editor_script_handles). "editorScript" takes a handle the plugin registers before the block, or "file:./index.js", which register_block_script_handle() registers with the dependencies and version of the index.asset.php beside it, as @wordpress/scripts writes it. A block that renders on the server can instead have "supports": { "autoRegister": true }, and the editor registers it itself (_wp_enqueue_auto_register_blocks() in wp-includes/blocks.php).` + : undefined, + }, + ], + }; + }, +}; diff --git a/apps/cli/ai/block-render-check.ts b/apps/cli/ai/block-render-check.ts new file mode 100644 index 0000000000..eeed7666ed --- /dev/null +++ b/apps/cli/ai/block-render-check.ts @@ -0,0 +1,277 @@ +/** + * The blocks of a post or page that WordPress renders as nothing on the front + * end. Validation compares the markup written with what the block saves, so a + * block whose save is empty, as a slideshow written without its slides is, + * passes it and still shows nothing. Studio renders the content as the front + * end does, through the_content, whose do_blocks() renders each block with + * render_block() (wp-includes/blocks.php), and names the blocks whose output + * is empty. + * + * A template, template part or pattern renders inside a page, so when one + * changes Studio renders the front page as a visitor who is not logged in + * gets it, and states which of its blocks rendered as nothing. Blocks render + * as nothing there by design, as a query's pagination does with one page of + * posts, so that is what WordPress shows, not a problem. + */ +import { readFile, stat } from 'fs/promises'; +import path from 'path'; +import { readPosts, readVersions } from 'cli/ai/post-content-check'; +import { readWordPressSite } from 'cli/ai/site-database'; +import { themeMarkupFiles } from 'cli/ai/site-markup'; +import { runWpCli } from 'cli/ai/tools/utils'; +import { connectToDaemon, disconnectFromDaemon } from 'cli/lib/daemon-client'; +import type { ChangeReport, SiteChangeSource, Versions } from 'cli/ai/site-changes'; +import type { SiteData } from 'cli/lib/cli-config/core'; + +// The post types whose content the front end renders through the_content. +const FRONT_END_TYPES = new Set( [ 'post', 'page' ] ); + +const RENDER = ( ids: number[] ) => ` +$empty = array(); +foreach ( ${ JSON.stringify( ids ) } as $id ) { + $post = get_post( $id ); + if ( ! $post ) { + continue; + } + $GLOBALS['post'] = $post; + setup_postdata( $post ); + $found = array(); + // A dynamic block renders its inner blocks into its own output, as a + // form field does its label, so their output alone is not what shows. + $parent = function ( $parsed, $source, $parent_block ) { + $parsed['studioInDynamicBlock'] = $parent_block && $parent_block->block_type && $parent_block->block_type->is_dynamic(); + return $parsed; + }; + $record = function ( $html, $block ) use ( &$found ) { + if ( ! empty( $block['blockName'] ) && empty( $block['studioInDynamicBlock'] ) && '' === trim( $html ) ) { + $type = WP_Block_Type_Registry::get_instance()->get_registered( $block['blockName'] ); + $sourced = array(); + foreach ( $type ? (array) $type->attributes : array() as $name => $attribute ) { + if ( isset( $attribute['source'] ) && 'meta' !== $attribute['source'] ) { + $sourced[] = $name; + } + } + // The file a block.json "render" names, which + // register_block_type_from_metadata() requires inside an output buffer. + $render = null; + if ( $type && $type->render_callback instanceof Closure ) { + $used = ( new ReflectionFunction( $type->render_callback ) )->getStaticVariables(); + $render = isset( $used['template_path'] ) && is_string( $used['template_path'] ) ? $used['template_path'] : null; + } + $found[] = array( + 'name' => $block['blockName'], + 'saved' => '' !== trim( implode( '', array_filter( $block['innerContent'], 'is_string' ) ) ), + 'sourced' => $sourced, + 'render' => $render, + ); + } + return $html; + }; + add_filter( 'render_block_data', $parent, PHP_INT_MAX, 3 ); + add_filter( 'render_block', $record, PHP_INT_MAX, 2 ); + apply_filters( 'the_content', $post->post_content ); + remove_filter( 'render_block', $record, PHP_INT_MAX ); + remove_filter( 'render_block_data', $parent, PHP_INT_MAX ); + wp_reset_postdata(); + $empty[ $id ] = $found; +} +echo 'EMPTY_BLOCKS=' . wp_json_encode( (object) $empty );`; + +// The front page as template-loader.php resolves it for '/', rendered by +// get_the_block_template_html() (wp-includes/block-template.php). The blocks a +// dynamic block renders into its own output, and the content core/post-content +// renders, which the posts' own check covers, are not named. +const RENDER_FRONT_PAGE = ` +$_SERVER['REQUEST_URI'] = '/'; +wp(); +$names = array(); +$content = 0; +add_filter( 'render_block_data', function ( $parsed, $source, $parent_block ) use ( &$content ) { + $parsed['studioSkipped'] = $content > 0 || ( $parent_block && $parent_block->block_type && $parent_block->block_type->is_dynamic() ); + if ( 'core/post-content' === $parsed['blockName'] ) { + ++$content; + } + return $parsed; +}, PHP_INT_MAX, 3 ); +add_filter( 'render_block', function ( $html, $block ) use ( &$names, &$content ) { + if ( 'core/post-content' === $block['blockName'] ) { + --$content; + } + if ( ! empty( $block['blockName'] ) && empty( $block['studioSkipped'] ) && '' === trim( $html ) ) { + $names[] = $block['blockName']; + } + return $html; +}, PHP_INT_MAX, 2 ); +$template = false; +foreach ( array( 'is_front_page' => 'get_front_page_template', 'is_home' => 'get_home_template', 'is_page' => 'get_page_template', 'is_singular' => 'get_singular_template' ) as $tag => $getter ) { + if ( $tag() && ( $template = $getter() ) ) { + break; + } +} +if ( ! $template ) { + get_index_template(); +} +if ( ! empty( $GLOBALS['_wp_current_template_content'] ) ) { + get_the_block_template_html(); +} +echo 'FRONT_EMPTY=' . wp_json_encode( array_values( array_unique( $names ) ) );`; + +/** The blocks of the front page that render as nothing for a visitor who is not logged in. */ +async function frontPageEmptyBlocks( site: SiteData ): Promise< string[] > { + try { + await connectToDaemon(); + const output = await runWpCli( site, [ 'eval', RENDER_FRONT_PAGE ] ); + return JSON.parse( + output.slice( output.indexOf( 'FRONT_EMPTY=' ) + 'FRONT_EMPTY='.length ) + ) as string[]; + } finally { + await disconnectFromDaemon(); + } +} + +const FILE = 'file '; + +// The posts the front end renders, and the active theme's template, template +// part and pattern files, by modification time and size. +async function readRenderedVersions( sitePath: string ): Promise< Versions > { + const themes = await readWordPressSite( sitePath, ( site ) => [ + ...new Set( [ site.option( 'template' ), site.option( 'stylesheet' ) ] ), + ] ); + const files = ( + await Promise.all( + themes + .filter( ( theme ): theme is string => typeof theme === 'string' && theme !== '' ) + .map( ( theme ) => + themeMarkupFiles( path.join( sitePath, 'wp-content', 'themes', theme ) ) + ) + ) + ).flat(); + const versions = await Promise.all( + files.map( async ( file ) => { + const { mtimeMs, size } = await stat( file ); + return [ FILE + file, `${ mtimeMs }:${ size }` ]; + } ) + ); + return { ...( await readVersions( sitePath ) ), ...Object.fromEntries( versions ) }; +} + +// The blocks a piece of markup holds, by name, as the block parser names them. +const blockNames = ( markup: string ) => + [ ...markup.matchAll( //g; + +/** + * Block markup with `is-style-` beside the class on each block that + * carries it, in its attributes and its element's class, or undefined if a + * block's element has no class to add it to. + */ +export function withBlockStyle( markup: string, name: string ): string | undefined { + const style = `is-style-${ name }`; + let out = ''; + let last = 0; + for ( const match of markup.matchAll( BLOCK ) ) { + const [ comment, , json, selfClosing ] = match; + let className: unknown; + try { + className = JSON.parse( json ).className; + } catch { + continue; + } + const classes = String( className ?? '' ).split( /\s+/ ); + if ( ! classes.includes( name ) || classes.includes( style ) ) { + continue; + } + const attribute = `"className":${ JSON.stringify( className ) }`; + if ( ! json.includes( attribute ) ) { + return undefined; + } + const updated = comment.replace( + attribute, + `"className":${ JSON.stringify( `${ className } ${ style }` ) }` + ); + const after = match.index + comment.length; + out += markup.slice( last, match.index ) + updated; + last = after; + if ( selfClosing ) { + continue; + } + // The block's element is the first tag after its comment. + const tag = markup.slice( after ).match( /^\s*<[a-z][a-z0-9-]*\b[^>]*>/i )?.[ 0 ]; + const classAttribute = tag?.match( /\sclass="([^"]*)"/ ); + if ( ! tag || ! classAttribute || ! classAttribute[ 1 ].split( /\s+/ ).includes( name ) ) { + return undefined; + } + out += tag.replace( classAttribute[ 0 ], ` class="${ classAttribute[ 1 ] } ${ style }"` ); + last = after + tag.length; + } + return out + markup.slice( last ); +} + +// Whether an element carries the class without the style: one that is not a +// block's, as in a core/html block. +function carriedWithoutStyle( markup: string, name: string ): boolean { + return [ ...markup.matchAll( /\sclass="([^"]*)"/g ) ].some( ( [ , classes ] ) => { + const list = classes.split( /\s+/ ); + return list.includes( name ) && ! list.includes( `is-style-${ name }` ); + } ); +} + +// Whether something Studio does not move names the class: another theme file, +// whose code or markup may print or style it, or CSS the site saved, which +// WordPress prints after the theme's (Additional CSS, Global Styles). +async function namedElsewhere( + siteRoot: string, + themeDir: string, + name: string, + moved: string[] +): Promise< boolean > { + const word = new RegExp( `(^|[^A-Za-z0-9_-])${ name }(?![A-Za-z0-9_-])` ); + const entries = await readdir( themeDir, { recursive: true, withFileTypes: true } ); + const files = entries + .filter( ( entry ) => entry.isFile() && /\.(php|js|html|json)$/.test( entry.name ) ) + .map( ( entry ) => path.join( entry.parentPath, entry.name ) ) + .filter( + ( file ) => + ! moved.includes( file ) && ! file.includes( `${ path.sep }node_modules${ path.sep }` ) + ); + if ( files.some( ( file ) => word.test( readFileSync( file, 'utf8' ) ) ) ) { + return true; + } + const saved = await readWordPressSite( siteRoot, ( site ) => + site.query< { content: string } >( + `SELECT post_content AS content FROM ${ site.prefix }posts WHERE post_type IN ('custom_css', 'wp_global_styles')` + ) + ); + return saved.some( ( { content } ) => token( name ).test( content ) ); +} + +const title = ( name: string ) => + name + .split( /[-_]+/ ) + .filter( Boolean ) + .map( ( word ) => word[ 0 ].toUpperCase() + word.slice( 1 ) ) + .join( ' ' ); + +/** + * Moves a class's look into a block style of the theme for the block types + * that carry it, and says what Studio did; undefined, changing nothing, if it + * cannot be moved as it renders. + */ +export async function moveToBlockStyle( + siteRoot: string, + themeDir: string, + name: string, + blockTypes: string[], + stylesheets: string[], + markup: Markup[] +): Promise< string | undefined > { + const styleFile = path.join( themeDir, 'styles', 'blocks', `${ name }.json` ); + if ( + existsSync( styleFile ) || + ( await isUpdatableTheme( siteRoot, path.basename( themeDir ) ) ) + ) { + return undefined; + } + const sheets = stylesheets.map( ( file ) => ( { file, css: readFileSync( file, 'utf8' ) } ) ); + const css = movableCss( + sheets.map( ( sheet ) => sheet.css ), + name + ); + const styled = markup.map( ( item ) => ( { + ...item, + styled: withBlockStyle( item.content, name ), + } ) ); + if ( + ! css || + styled.some( + ( item ) => item.styled === undefined || carriedWithoutStyle( item.styled, name ) + ) || + ( await namedElsewhere( siteRoot, themeDir, name, [ + ...stylesheets, + ...markup.flatMap( ( item ) => item.file ?? [] ), + ] ) ) + ) { + return undefined; + } + const changed = styled.filter( ( item ) => item.styled !== item.content ); + // The posts first: one Studio cannot save leaves the stylesheet as it was. + for ( const item of changed.filter( ( candidate ) => candidate.post ) ) { + try { + await saveMarkup( siteRoot, item, item.styled! ); + } catch { + return undefined; + } + } + for ( const item of changed.filter( ( candidate ) => candidate.file ) ) { + await writeFile( item.file!, item.styled!, 'utf8' ); + } + await mkdir( path.dirname( styleFile ), { recursive: true } ); + await writeFile( + styleFile, + JSON.stringify( + { + $schema: 'https://schemas.wp.org/trunk/theme.json', + version: 3, + title: title( name ), + slug: name, + blockTypes, + styles: { css }, + }, + null, + '\t' + ) + '\n', + 'utf8' + ); + const moved = []; + for ( const sheet of sheets ) { + const rest = withoutRules( sheet.css, name ); + if ( rest !== sheet.css ) { + await writeFile( sheet.file, rest, 'utf8' ); + moved.push( path.relative( themeDir, sheet.file ) ); + } + } + const places = changed.map( ( item ) => placeOf( item, themeDir ) ); + return `Studio moved the look \`.${ name }\` gives ${ blockTypes.join( + ', ' + ) } blocks into a block style, WordPress's form for a look blocks of one type share: ${ styleFile }, which WordPress registers for its "blockTypes" (WP_Theme_JSON_Resolver::get_style_variations() in wp-includes/class-wp-theme-json-resolver.php), the block's Styles panel offers, and Global Styles edits. Its rules moved from ${ moved.join( + ', ' + ) } into the style's "styles": { "css" }, as they were, and the blocks in ${ places.join( + ', ' + ) } carry "is-style-${ name }" beside "${ name }", so they render as before. A block takes the look as "className":"is-style-${ name }"; the style's "styles" can hold each property at its path (WP_Theme_JSON::PROPERTIES_METADATA in wp-includes/class-wp-theme-json.php).`; +} diff --git a/apps/cli/ai/block-theme.ts b/apps/cli/ai/block-theme.ts new file mode 100644 index 0000000000..dcad7cd92c --- /dev/null +++ b/apps/cli/ai/block-theme.ts @@ -0,0 +1,137 @@ +/** + * Whether a theme made during the session is a block theme, as WordPress's + * default theme is. WordPress treats a theme as a block theme when it has + * templates/index.html (WP_Theme::is_block_theme() in + * wp-includes/class-wp-theme.php), and only a block theme's templates, parts + * and styles are edited in the Site Editor, around page content that is blocks. + * + * A theme the site already had keeps the kind it is. One made during the + * session is classic only when the user chooses it, asked before the write + * that starts it (classicThemeQuestion); one that WordPress lists as broken + * for having neither index yet stays an open problem while it is. + */ +import { readFile, readdir, stat } from 'fs/promises'; +import path from 'path'; +import { resolveSiteFile } from 'cli/ai/site-file-guards'; +import { readThemeHeader, isUpdatableTheme } from 'cli/ai/themes'; +import type { SiteChangeSource, Versions } from 'cli/ai/site-changes'; + +const CREATED = 'created:'; +const exists = ( file: string ) => stat( file ).then( Boolean, () => false ); + +/** + * A standalone theme's kind: block, with a block template index + * (WP_Theme::is_block_theme()); classic, with PHP templates at its root + * instead, as index.php, header.php or page.php are, which WordPress's + * default theme has none of. Its one root PHP file is functions.php, which + * WordPress loads for every theme (wp-settings.php). A child theme is its + * parent's kind, so it is left to its parent. + */ +async function themeKind( root: string, theme: string ): Promise< string > { + const dir = path.join( root, 'wp-content', 'themes', theme ); + if ( await readThemeHeader( root, theme, 'Template' ).catch( () => '' ) ) { + return 'child'; + } + if ( await exists( path.join( dir, 'templates/index.html' ) ) ) { + return 'block'; + } + return ( await rootTemplates( dir ) ).length ? 'classic' : 'unfinished'; +} + +async function rootTemplates( dir: string ): Promise< string[] > { + const files = await readdir( dir ).catch( () => [] as string[] ); + return files.filter( ( file ) => file.endsWith( '.php' ) && file !== 'functions.php' ).sort(); +} + +/** WP_DEFAULT_THEME, the theme core falls back to (wp-includes/default-constants.php). */ +async function defaultTheme( root: string ): Promise< string | undefined > { + const php = await readFile( + path.join( root, 'wp-includes/default-constants.php' ), + 'utf8' + ).catch( () => '' ); + return php.match( /define\(\s*'WP_DEFAULT_THEME',\s*'([^']+)'/ )?.[ 1 ]; +} + +/** + * The question a write that starts a classic theme asks the user first: a + * root PHP template, such as index.php, header.php or page.php, in a theme + * made without one, with no block template index. A classic theme is the + * user's choice: its templates are edited only in code. Themes the site has + * keep their kind, so a classic theme's further templates, a block theme's + * PHP and a child theme (its parent's kind) ask nothing, nor do themes + * WordPress updates, whose files Studio does not write. + */ +export async function classicThemeQuestion( + filePath: string +): Promise< { question: string; consequence: string; declined: string } | undefined > { + const file = resolveSiteFile( filePath ); + const theme = file?.theme; + if ( + ! file || + ! theme || + ! /^[^/]+\.php$/.test( theme.file ) || + theme.file === 'functions.php' || + ( await themeKind( file.siteRoot, theme.slug ) ) !== 'unfinished' || + ( await isUpdatableTheme( file.siteRoot, theme.slug ) ) + ) { + return undefined; + } + const fallback = await defaultTheme( file.siteRoot ); + return { + question: `Make "${ theme.slug }" a classic theme?`, + consequence: `Its templates are PHP files, edited only in code: the Site Editor cannot edit its templates, header, footer or styles.`, + declined: `"${ theme.slug }" is to be a block theme, as WordPress's default theme${ + fallback ? `, ${ fallback } (WP_DEFAULT_THEME),` : '' + } is: its templates are block markup in templates/*.html, with templates/index.html (WP_Theme::is_block_theme() in wp-includes/class-wp-theme.php), and its parts in parts/*.html, all edited in the Site Editor.`, + }; +} + +// A theme WordPress reports broken, with neither index (WP_Theme::__construct()). +async function missingTemplateProblem( root: string, theme: string ): Promise< string > { + const fallback = await defaultTheme( root ); + const named = fallback ? `, ${ fallback } (WP_DEFAULT_THEME),` : ''; + return `${ theme } (wp-content/themes/${ theme }), made this session, has neither templates/index.html nor index.php, so WordPress lists it as broken: "Template is missing. Standalone themes need to have a templates/index.html or index.php template file." (WP_Theme::__construct() in wp-includes/class-wp-theme.php). WordPress's default theme${ named } is a block theme with templates/index.html. Give ${ theme } its templates/index.html.`; +} + +export const blockTheme: SiteChangeSource = { + name: 'block-theme', + async read( sitePath, last ) { + const dir = path.join( sitePath, 'wp-content', 'themes' ); + const themes = await readdir( dir ).catch( () => [] as string[] ); + const versions: Versions = {}; + for ( const theme of themes ) { + if ( ! ( await exists( path.join( dir, theme, 'style.css' ) ) ) ) { + continue; + } + const kind = await themeKind( sitePath, theme ); + versions[ theme ] = ( last?.[ theme ]?.startsWith( CREATED ) ? CREATED : '' ) + kind; + } + return versions; + }, + async check( site, before, after ) { + const next: Versions = {}; + const reports = []; + for ( const [ theme, version ] of Object.entries( after ) ) { + const created = ! ( theme in before ) || version.startsWith( CREATED ); + next[ theme ] = created && ! version.startsWith( CREATED ) ? CREATED + version : version; + if ( ! created || before[ theme ] === next[ theme ] ) { + continue; + } + const broken = + next[ theme ] === `${ CREATED }unfinished` && + ! ( await isUpdatableTheme( site.path, theme ) ); + reports.push( { + key: `block theme ${ theme }`, + problem: broken ? await missingTemplateProblem( site.path, theme ) : undefined, + atTurnEnd: true, + } ); + } + // A theme made this session and deleted since leaves no problem. + for ( const theme of Object.keys( before ) ) { + if ( before[ theme ].startsWith( CREATED ) && ! ( theme in after ) ) { + reports.push( { key: `block theme ${ theme }`, problem: undefined } ); + } + } + return { reports, next }; + }, +}; diff --git a/apps/cli/ai/block-validation-gate.ts b/apps/cli/ai/block-validation-gate.ts new file mode 100644 index 0000000000..e895d6c5ac --- /dev/null +++ b/apps/cli/ai/block-validation-gate.ts @@ -0,0 +1,530 @@ +/** + * Block validation as a gate the harness runs, not a tool the model must + * remember to call. + * + * Writing block markup (a theme template or part, or any file that opens with a + * block, as a page draft does wherever it is written) queues that file. Validation starts in the background at once, + * and what puts content on the site or renders it (a shell command, which + * runs WP-CLI, or take_screenshot) waits for it first. A fix the block editor + * can make losslessly is applied to the file silently, once validation has + * settled and only while the file still holds the content validated, so an + * edit made meanwhile is never overwritten; the model hears only about what it + * has to change itself. + */ +import { createHash } from 'crypto'; +import { readFileSync } from 'fs'; +import { readFile, writeFile } from 'fs/promises'; +import path from 'path'; +import { validateHtmlBlockPolicy } from 'cli/ai/block-content-policy'; +import { + bindNavigationLinks, + blockNames, + insertedTemplates, + storePresetValues, + dropCommentsBetweenBlocks, + validateBlocks, + type PresetValues, + type PresetValuesResult, + type ValidationReportBase, +} from 'cli/ai/block-validator'; +import { getOpenProblems, type OpenProblems } from 'cli/ai/open-problems'; +import { readWordPressSite } from 'cli/ai/site-database'; +import { themePresets } from 'cli/ai/theme-presets'; +import { getSiteByFolder, getSiteUrl } from 'cli/lib/cli-config/sites'; + +const THEME_MARKUP = + /[\\/]wp-content[\\/]themes[\\/][^\\/]+[\\/](templates|parts)[\\/][^\\/]+\.html$/; +// A theme's patterns are the PHP files in its patterns folder, which WordPress +// includes for what they output (WP_Theme::get_block_patterns(), +// WP_Block_Patterns_Registry::get_content()). +const THEME_PATTERN = /[\\/]wp-content[\\/]themes[\\/][^\\/]+[\\/]patterns[\\/].+\.php$/; +// The header comment a pattern opens with, and the newline PHP drops after its closing tag. +const PATTERN_HEADER = /^\s*<\?php\s*\/\*[\s\S]*?\*\/\s*\?>\n?/; + +/** + * A theme pattern's header and the block markup it outputs, when the header + * is its only PHP, so the file outputs the markup as written. + */ +function splitPattern( content: string ): { header: string; markup: string } | undefined { + const header = content.match( PATTERN_HEADER )?.[ 0 ]; + if ( header === undefined ) { + return undefined; + } + const markup = content.slice( header.length ); + return markup.includes( ' length ? `${ flat.slice( 0, length ) }…` : flat; +} + +/** + * Each invalid block with every mismatch the editor found, and the HTML the + * block saves from its comment attributes — the markup that validates. + */ +function describeInvalidBlocks( + report: ValidationReportBase, + lost = new Map< string, string >() +): string[] { + return report.results + .filter( ( result ) => ( lost.size ? lost.has( result.place ?? '' ) : ! result.isValid ) ) + .slice( 0, 5 ) + .flatMap( ( result ) => [ + ` - ${ result.blockName }: ${ + lost.has( result.place ?? '' ) + ? `the editor's fix would ${ lost.get( result.place ?? '' ) }; ` + : '' + }${ result.issues.slice( 0, 3 ).join( '; ' ) || 'does not match what the block saves' }`, + ...( result.originalContent + ? [ ` written: ${ oneLine( result.originalContent, 700 ) }` ] + : [] ), + ...( result.expectedContent + ? [ ` the block saves: ${ oneLine( result.expectedContent, 700 ) }` ] + : [] ), + ] ); +} + +interface FileValidation { + /** The hash of the content validated. */ + contentHash: string; + summary: string; + problem?: string; + /** What the block editor would save instead, losing nothing. */ + fixedContent?: string; +} + +interface MarkupValidation { + summary: string; + problem?: string; + /** What the block editor would save instead, when that loses nothing. */ + fixedContent?: string; + invalidBlocks?: number; + /** The palette colors the fix stores as presets. */ + paletteColors?: PresetValuesResult[ 'colors' ]; + /** The font sizes the fix stores as presets. */ + fontSizes?: PresetValuesResult[ 'fontSizes' ]; + /** The spacing values the fix stores as presets. */ + spacingSizes?: PresetValuesResult[ 'spacingSizes' ]; + /** The navigation links the fix binds to the entity they name. */ + boundLinks?: string[]; + /** How many runs of HTML comments between blocks the save drops. */ + droppedComments?: number; +} + +/** What saving through the editor changes, for a note. */ +export function describeEditorSave( result: MarkupValidation, fixes: string ): string[] { + return [ + result.invalidBlocks ? `fixes ${ result.invalidBlocks } block(s)${ fixes }` : undefined, + result.droppedComments + ? `drops ${ result.droppedComments } HTML comment(s) between blocks, which the editor would turn into Classic blocks (normalizeRawBlock in @wordpress/blocks wraps text outside a block in paragraphs) and which render nothing` + : undefined, + result.paletteColors?.length ? describePaletteColors( result.paletteColors ) : undefined, + result.fontSizes?.length ? describeFontSizes( result.fontSizes ) : undefined, + result.spacingSizes?.length ? describeSpacingSizes( result.spacingSizes ) : undefined, + result.boundLinks?.length ? describeBoundLinks( result.boundLinks ) : undefined, + ].filter( ( note ): note is string => Boolean( note ) ); +} + +/** How the editor links a navigation link to the post or term it names, for a note. */ +function describeBoundLinks( links: string[] ): string { + return `binds the url of the navigation link(s) ${ links + .map( ( label ) => `"${ label }"` ) + .join( + ', ' + ) }, which named a page or term by id with no url, to its permalink, as the editor's link picker does (useEntityBinding in @wordpress/block-library): core prints a link's href only from its url (wp-includes/blocks/navigation-link.php)`; +} + +/** What the editor inserts for the blocks named, for a note to write them from. */ +async function describeTemplates( names: string[], siteUrl: string ): Promise< string[] > { + const templates = await insertedTemplates( names, siteUrl ).catch( () => [] ); + return templates.map( + ( { name, markup } ) => + `What the site's block editor inserts for ${ name }, from its default inserter variation (createBlock in @wordpress/blocks), to write it from:\n${ markup }` + ); +} + +/** + * What the editor stores for the markup: comments between blocks dropped, palette + * colors as presets, entity links bound. `invalidBefore` counts the problems in + * the content it replaces. + */ +async function storeAsTheEditorDoes( + content: string, + siteUrl: string, + presetValues: PresetValues, + invalidBefore: () => Promise< number > +) { + const between = await dropCommentsBetweenBlocks( content, siteUrl ); + const presets = await storePresetValues( between.content, siteUrl, presetValues ); + const bound = await bindNavigationLinks( presets.content, siteUrl ); + const stored = { + content: bound.content, + changed: + between.comments > 0 || + presets.colors.length > 0 || + presets.fontSizes.length > 0 || + presets.spacingSizes.length > 0 || + bound.links.length > 0, + ...( between.comments && { droppedComments: between.comments } ), + ...( presets.colors.length && { paletteColors: presets.colors } ), + ...( presets.fontSizes.length && { fontSizes: presets.fontSizes } ), + ...( presets.spacingSizes.length && { spacingSizes: presets.spacingSizes } ), + ...( bound.links.length && { boundLinks: bound.links } ), + }; + // A save stands only when the editor finds no more problems in it than in + // what it replaces, whatever the pass that changed it. + if ( + stored.changed && + ( await validateBlocks( stored.content, siteUrl ) ).invalidBlocks > ( await invalidBefore() ) + ) { + return { content, changed: false }; + } + return stored; +} + +/** How the editor stores colors written as palette values, for a note. */ +function describePaletteColors( colors: PresetValuesResult[ 'colors' ] ): string { + const each = [ + ...new Set( colors.map( ( { color, slug } ) => `${ color } as the palette's ${ slug }` ) ), + ]; + return `stores ${ each.join( + ', ' + ) }, as the editor stores a palette color (encodeColorValueWithPalette in @wordpress/block-editor), so it follows the palette`; +} + +/** How the editor stores font sizes written as preset sizes, for a note. */ +function describeFontSizes( sizes: PresetValuesResult[ 'fontSizes' ] ): string { + const each = [ + ...new Set( sizes.map( ( { size, slug } ) => `${ size } as the font size ${ slug }` ) ), + ]; + return `stores ${ each.join( + ', ' + ) }, as the editor stores a font size preset (getFontSizeObjectByValue in @wordpress/block-editor), so it follows the preset`; +} + +/** How the editor stores spacing written as preset sizes, for a note. */ +function describeSpacingSizes( sizes: PresetValuesResult[ 'spacingSizes' ] ): string { + const each = [ + ...new Set( sizes.map( ( { size, slug } ) => `${ size } as the spacing size ${ slug }` ) ), + ]; + return `stores ${ each.join( + ', ' + ) }, as the editor stores a spacing size preset (getPresetValueFromCustomValue in @wordpress/block-editor), so it follows the preset`; +} + +/** + * The palette colors, font sizes and spacing sizes markup in \`theme\` renders with (the + * active theme's by default), as WordPress resolves them. + */ +export async function sitePresetValues( + siteRoot: string, + theme?: string +): Promise< PresetValues > { + const presets = await readWordPressSite( siteRoot, ( site ) => + themePresets( site, theme ) + ).catch( () => undefined ); + return { + colors: presets?.colors ?? [], + fontSizes: presets?.fontSizes ?? [], + spacingSizes: presets?.spacingSizes ?? [], + }; +} + +const NO_PRESETS: PresetValues = { colors: [], fontSizes: [], spacingSizes: [] }; + +/** Validates one block markup file, with the lossless fix for the caller to save. */ +export async function validateBlockMarkupFile( + filePath: string, + siteUrl: string, + presetValues: PresetValues = NO_PRESETS +): Promise< FileValidation > { + const label = filePath.split( path.sep ).slice( -2 ).join( '/' ); + const content = await readFile( filePath, 'utf8' ); + // A pattern is validated for the markup it outputs, and saved under its header. + const pattern = THEME_PATTERN.test( filePath ) ? splitPattern( content ) : undefined; + if ( THEME_PATTERN.test( filePath ) && ! pattern ) { + return { contentHash: hash( content ), summary: `${ label }: not validated, as it holds PHP` }; + } + + const result = await validateBlockMarkup( + pattern?.markup ?? content, + label, + siteUrl, + presetValues + ); + if ( result.fixedContent === undefined ) { + return { contentHash: hash( content ), summary: result.summary, problem: result.problem }; + } + const notes = describeEditorSave( result, '' ); + return { + contentHash: hash( content ), + summary: `${ result.summary }; saved through the block editor, which ${ notes.join( + ' and ' + ) }`, + ...( result.problem && { problem: result.problem } ), + fixedContent: ( pattern?.header ?? '' ) + result.fixedContent, + }; +} + +/** + * Validates block markup against what each block saves on the site, and + * against the core/html policy. A lossless editor fix is returned for the + * caller to save where the markup lives. + */ +export async function validateBlockMarkup( + content: string, + label: string, + siteUrl: string, + presetValues: PresetValues = NO_PRESETS +): Promise< MarkupValidation > { + const policy = validateHtmlBlockPolicy( content ); + if ( policy.invalidHtmlBlocks.length > 0 ) { + return { + summary: `${ label }: core/html policy failed`, + problem: [ + `${ label }: ${ policy.invalidHtmlBlocks.length } core/html block(s) must become editable blocks:`, + ...policy.invalidHtmlBlocks.map( + ( block ) => ` - line ${ block.line }: ${ block.issues[ 0 ] }` + ), + ...( await describeTemplates( + [ ...new Set( policy.invalidHtmlBlocks.flatMap( ( block ) => block.block ?? [] ) ) ], + siteUrl + ) ), + ].join( '\n' ), + }; + } + + const report = await validateBlocks( content, siteUrl ); + if ( report.error ) { + return { + summary: `${ label }: not validated (${ report.error })`, + }; + } + const valid = `${ label }: ${ report.totalBlocks }/${ report.totalBlocks } blocks valid`; + if ( report.invalidBlocks === 0 ) { + const { + content: stored, + changed, + ...notes + } = await storeAsTheEditorDoes( content, siteUrl, presetValues, async () => 0 ); + return changed ? { summary: valid, fixedContent: stored, ...notes } : { summary: valid }; + } + + const fix = report.proposedFix; + if ( ! fix || fix.report.error || fix.report.invalidBlocks > 0 ) { + return { + summary: `${ label }: ${ report.invalidBlocks } invalid block(s)`, + problem: [ + `${ label }: ${ report.invalidBlocks } invalid block(s), which the block editor cannot fix:`, + ...describeInvalidBlocks( report ), + ...( await describeTemplates( + blockNames( + report.results + .filter( ( result ) => ! result.isValid ) + .map( ( result ) => result.originalContent ) + .join( '\n' ) + ), + siteUrl + ) ), + ].join( '\n' ), + }; + } + + // A block stays as written when the fix loses something from it or from a + // block it is inside; the editor's fix is saved for the rest. + const lost = new Map( fix.losses.map( ( loss ) => [ loss.place, loss.lost ] ) ); + const keptAsWritten = ( place = '' ) => + [ ...lost.keys() ].some( ( at ) => place === at || place.startsWith( `${ at }/` ) ); + const fixedBlocks = report.results.filter( + ( result ) => ! result.isValid && ! keptAsWritten( result.place ) + ).length; + const stored = fixedBlocks + ? await storeAsTheEditorDoes( fix.fixedContent, siteUrl, presetValues, async () => + lost.size ? ( await validateBlocks( fix.fixedContent, siteUrl ) ).invalidBlocks : 0 + ) + : undefined; + const saved = stored && { + fixedContent: stored.content, + invalidBlocks: fixedBlocks, + ...( stored.paletteColors && { paletteColors: stored.paletteColors } ), + ...( stored.fontSizes && { fontSizes: stored.fontSizes } ), + ...( stored.spacingSizes && { spacingSizes: stored.spacingSizes } ), + ...( stored.boundLinks && { boundLinks: stored.boundLinks } ), + ...( stored.droppedComments && { droppedComments: stored.droppedComments } ), + }; + if ( lost.size === 0 ) { + return { summary: valid, ...saved }; + } + return { + summary: `${ label }: ${ report.invalidBlocks - fixedBlocks } invalid block(s)`, + problem: [ + `${ label }: ${ lost.size } block(s) do not match what the block saves, and the editor's fix would lose something from them. Change them to what the block saves, or change the comment attributes the block saves them from (className and style carry extra classes and styles):`, + ...describeInvalidBlocks( report, lost ), + ].join( '\n' ), + ...saved, + }; +} + +interface Tracked { + siteRoot: string; + result: Promise< FileValidation >; + reported?: string; +} + +export class BlockValidationGate { + private readonly files = new Map< string, Tracked >(); + + constructor( + private readonly validateFile: ( + filePath: string, + siteRoot: string + ) => Promise< FileValidation > = validateWithSite, + private readonly open?: OpenProblems + ) {} + + /** Queues a written file and starts validating it in the background. */ + track( filePath: string, siteRoot: string ): void { + if ( isBlockMarkupFile( filePath ) ) { + this.files.set( filePath, { siteRoot, result: this.validate( filePath, siteRoot ) } ); + } + } + + /** Waits for every queued file and returns what is new to report. */ + async settle(): Promise< { summaries: string[]; problems: string[] } > { + const summaries: string[] = []; + const problems: string[] = []; + for ( const [ filePath, tracked ] of this.files ) { + const result = await this.current( filePath, tracked.siteRoot ).then( + ( current ) => current && this.applyFix( filePath, current ) + ); + this.open?.record( tracked.siteRoot, filePath, result?.problem ); + if ( ! result ) { + this.files.delete( filePath ); + continue; + } + const reportKey = `${ result.contentHash } ${ result.summary }`; + if ( reportKey !== tracked.reported ) { + tracked.reported = reportKey; + summaries.push( result.summary ); + problems.push( ...( result.problem ? [ result.problem ] : [] ) ); + } + } + return { summaries, problems }; + } + + /** + * Saves a validation's lossless fix, now that it holds the file's current + * content, and returns the validation of the fixed file. + */ + private async applyFix( + filePath: string, + result: FileValidation + ): Promise< FileValidation | undefined > { + const { fixedContent, ...validation } = result; + if ( fixedContent === undefined ) { + return result; + } + const content = await readFile( filePath, 'utf8' ).catch( () => undefined ); + if ( content === undefined || hash( content ) !== result.contentHash ) { + // Written again while it was validated: validate what it holds now. + const current = await this.current( filePath, this.files.get( filePath )!.siteRoot ); + return current && this.applyFix( filePath, current ); + } + await writeFile( filePath, fixedContent, 'utf8' ); + const fixed = { ...validation, contentHash: hash( fixedContent ) }; + this.files.get( filePath )!.result = Promise.resolve( fixed ); + return fixed; + } + + /** + * The validation of a file's current content, reusing the last one while + * the file is unchanged; undefined once the file is gone. + */ + private async current( + filePath: string, + siteRoot: string + ): Promise< FileValidation | undefined > { + const content = await readFile( filePath, 'utf8' ).catch( () => undefined ); + if ( content === undefined ) { + return undefined; + } + const tracked = this.files.get( filePath ) ?? { + siteRoot, + result: this.validate( filePath, siteRoot ), + }; + let result = await tracked.result; + if ( result.contentHash !== hash( content ) ) { + tracked.result = this.validate( filePath, siteRoot ); + result = await tracked.result; + } + this.files.set( filePath, tracked ); + return result; + } + + /** Validates a file; a validation that cannot run is reported, never skipped. */ + private validate( filePath: string, siteRoot: string ): Promise< FileValidation > { + return this.validateFile( filePath, siteRoot ).catch( async ( error: unknown ) => ( { + contentHash: hash( ( await readFile( filePath, 'utf8' ).catch( () => '' ) ) ?? '' ), + summary: `${ filePath.split( path.sep ).slice( -2 ).join( '/' ) }: not validated (${ + error instanceof Error ? error.message : String( error ) + })`, + } ) ); + } +} + +// The theme a file is in, by its folder under wp-content/themes. +const themeOf = ( siteRoot: string, filePath: string ) => + path + .relative( siteRoot, filePath ) + .split( path.sep ) + .join( '/' ) + .match( /^wp-content\/themes\/([^/]+)\// )?.[ 1 ]; + +async function validateWithSite( filePath: string, siteRoot: string ): Promise< FileValidation > { + return validateBlockMarkupFile( + filePath, + getSiteUrl( await getSiteByFolder( siteRoot ) ), + await sitePresetValues( siteRoot, themeOf( siteRoot, filePath ) ) + ); +} + +const gates = new WeakMap< object, BlockValidationGate >(); + +export function getBlockValidationGate( session: object ): BlockValidationGate { + let gate = gates.get( session ); + if ( ! gate ) { + gate = new BlockValidationGate( validateWithSite, getOpenProblems( session ) ); + gates.set( session, gate ); + } + return gate; +} diff --git a/apps/cli/ai/block-validator.ts b/apps/cli/ai/block-validator.ts index c6fe9695e9..8ed8125eb0 100644 --- a/apps/cli/ai/block-validator.ts +++ b/apps/cli/ai/block-validator.ts @@ -2,6 +2,8 @@ import { EditorPage } from 'cli/ai/browser-utils'; interface BlockValidationResult { blockName: string; + /** The block's place in the content, as `/0/2` for the third block in the first. */ + place?: string; isValid: boolean; issues: string[]; originalContent: string; @@ -17,8 +19,18 @@ export interface ValidationReportBase { } interface BlockFixProposal { + /** What the editor saves for every block that loses nothing; the rest stay as written. */ fixedContent: string; report: ValidationReportBase; + /** The blocks the fix loses something from, and what. */ + losses: BlockLoss[]; +} + +export interface BlockLoss { + place: string; + blockName: string; + /** What the fix loses, as `drop the styles position:sticky`. */ + lost: string; } export interface ValidationReport extends ValidationReportBase { @@ -38,6 +50,39 @@ function getEditorPage( siteUrl: string ): EditorPage { return ep; } +/** The block names that block comments in content name, `core/` for those without a namespace. */ +export function blockNames( content: string ): string[] { + return [ + ...new Set( + [ ...content.matchAll( //g, '' ) + .replace( /<\/?p>|/gi, '' ) + .trim(); + const blocks = wpBlocks.parse( html ); + const kept = blocks.filter( ( block: any ) => ! onlyComments( block ) ); + return kept.length === blocks.length + ? { content: html, comments: 0 } + : { content: wpBlocks.serialize( kept ), comments: blocks.length - kept.length }; + /* eslint-enable @typescript-eslint/no-explicit-any */ + }, content ); +} + +/** + * Binds the url of each navigation link that names a post or term by its id + * but has no url to that entity's link, as the editor's link picker does + * (useEntityBinding in @wordpress/block-library): core prints a link's href + * only from its url (wp-includes/blocks/navigation-link.php), and fills the + * url from the binding with the entity's permalink (core/post-data and + * core/term-data in wp-includes/block-bindings). Content the editor would + * save otherwise is returned as is. + */ +export async function bindNavigationLinks( + content: string, + siteUrl: string +): Promise< NavigationLinksResult > { + if ( ! /([\s\S]*?)/g; + +/** The note on the buttons in `markup` that link nowhere, if any. */ +export function buttonsWithoutLinkNotes( markup: string ): string[] { + const texts = new Map< string, number >(); + for ( const [ , inner ] of markup.matchAll( BUTTON ) ) { + const link = inner.match( /]*>/ ); + if ( ! link ) { + continue; + } + const href = link[ 0 ].match( /\shref="([^"]*)"/ )?.[ 1 ]; + if ( href && href !== '#' ) { + continue; + } + const text = inner.replace( /<[^>]*>/g, '' ).trim() || '(no text)'; + texts.set( text, ( texts.get( text ) ?? 0 ) + 1 ); + } + if ( ! texts.size ) { + return []; + } + const named = [ ...texts ] + .map( ( [ text, count ] ) => ( count > 1 ? `"${ text }" ×${ count }` : `"${ text }"` ) ) + .join( ', ' ); + return [ + `core/button blocks that open nothing: ${ named }. A button's link is its "url", saved as the href of its (wp-includes/blocks/button/block.json); without one the browser gives the link no action, and "#" opens the top of the page. Give each the URL of what it offers.`, + ]; +} diff --git a/apps/cli/ai/chat-artifacts.ts b/apps/cli/ai/chat-artifacts.ts index 3c6beb1ba1..f2c33f95ec 100644 --- a/apps/cli/ai/chat-artifacts.ts +++ b/apps/cli/ai/chat-artifacts.ts @@ -1,8 +1,5 @@ import { randomUUID } from 'node:crypto'; -import { existsSync } from 'node:fs'; -import path from 'node:path'; import { - DESIGN_SYSTEM_PREVIEW_PATH, STUDIO_CHAT_ARTIFACT_VERSION, type StudioChatArtifactData, type StudioChatArtifactWidgetDraft, @@ -56,33 +53,3 @@ function cloneChatArtifactWidgets( widgetProps: { ...widget.widgetProps }, } ) ); } - -interface FileWritingTool { - execute( toolCallId: string, params: { path?: unknown }, ...rest: unknown[] ): Promise< unknown >; -} - -/** - * Wraps a file-writing tool so that writing a site's DESIGN.md opens the design - * system in the preview. - */ -export function withDesignSystemPreview< TTool extends FileWritingTool >( - tool: TTool, - cwd: string -): TTool { - return { - ...tool, - async execute( toolCallId: string, params: { path?: unknown }, ...rest: unknown[] ) { - const result = await tool.execute( toolCallId, params, ...rest ); - const filePath = path.resolve( cwd, String( params.path ) ); - if ( - path.basename( filePath ) === 'DESIGN.md' && - existsSync( path.join( path.dirname( filePath ), 'wp-content' ) ) - ) { - await emitChatArtifactWidgets( [ - { type: 'site-preview', widgetProps: { path: DESIGN_SYSTEM_PREVIEW_PATH } }, - ] ).catch( () => undefined ); - } - return result; - }, - }; -} diff --git a/apps/cli/ai/child-theme-copies.ts b/apps/cli/ai/child-theme-copies.ts new file mode 100644 index 0000000000..be42292e64 --- /dev/null +++ b/apps/cli/ai/child-theme-copies.ts @@ -0,0 +1,113 @@ +/** + * Files a child theme holds unchanged from its parent, by any tool or script. + * WordPress already reads a file the child lacks from the parent + * (get_theme_file_path(), get_theme_file_uri() for theme.json's `file:./` + * fonts, locate_template(), the block template and pattern resolvers) and merges the child's theme.json over the parent's + * (WP_Theme_JSON_Resolver::get_theme_data()), so a copy only stops following + * the parent's updates; both functions.php files load (wp-settings.php), so a + * copied one redeclares the parent's functions. A child theme holds the files + * it changes (https://developer.wordpress.org/themes/advanced-topics/child-themes/). + * + * A screenshot is the child's own: WP_Theme::get_screenshot() reads only the + * child's folder. + * + * Each file's version is both files' modified time and size and whether their + * content is the same, compared again only when a time or size moves. + */ +import { readdir, readFile, stat } from 'fs/promises'; +import path from 'path'; +import { readThemeHeader } from 'cli/ai/themes'; +import type { SiteChangeSource, Versions } from 'cli/ai/site-changes'; + +const SCREENSHOT = /^screenshot\.(png|gif|jpe?g|webp|avif)$/; +const SAME = ' same'; + +const stampOf = async ( file: string ) => { + const { mtimeMs, size } = await stat( file ); + return { stamp: `${ mtimeMs }:${ size }`, size }; +}; + +async function readCopies( + themesDir: string, + child: string, + parent: string, + last: Versions +): Promise< Versions > { + const versions: Versions = {}; + const childDir = path.join( themesDir, child ); + for ( const entry of await readdir( childDir, { recursive: true, withFileTypes: true } ) ) { + const relative = path + .relative( childDir, path.join( entry.parentPath, entry.name ) ) + .split( path.sep ) + .join( '/' ); + if ( ! entry.isFile() || SCREENSHOT.test( relative ) ) { + continue; + } + const childFile = path.join( childDir, relative ); + const parentFile = path.join( themesDir, parent, relative ); + const [ own, inherited ] = await Promise.all( [ + stampOf( childFile ), + stampOf( parentFile ).catch( () => undefined ), + ] ); + if ( ! inherited ) { + continue; + } + const key = `${ child }/${ relative }`; + const stamps = `${ own.stamp }|${ inherited.stamp }`; + const same = last[ key ]?.startsWith( `${ stamps } ` ) + ? last[ key ].endsWith( SAME ) + : own.size === inherited.size && + ( await readFile( childFile ) ).equals( await readFile( parentFile ) ); + versions[ key ] = `${ stamps }${ same ? SAME : ' own' }`; + } + return versions; +} + +const copiesOf = ( versions: Versions, child: string ) => + Object.entries( versions ) + .filter( ( [ key, version ] ) => key.startsWith( `${ child }/` ) && version.endsWith( SAME ) ) + .map( ( [ key ] ) => key.slice( child.length + 1 ) ) + .sort(); + +function problem( child: string, parent: string, files: string[] ): string { + const named = files.length > 5 ? `${ files.slice( 0, 5 ).join( ', ' ) }, …` : files.join( ', ' ); + return `The child theme ${ child } holds files of its parent ${ parent } unchanged: ${ named }. WordPress reads a file the child lacks from the parent and merges the child's theme.json over the parent's, so a copy only stops following ${ parent }'s updates, and a copied functions.php loads beside the parent's. Delete the copies, so ${ child } holds only the files it changes (https://developer.wordpress.org/themes/advanced-topics/child-themes/).`; +} + +export const childThemeCopies: SiteChangeSource = { + name: 'child-themes', + async read( sitePath, last = {} ) { + const themesDir = path.join( sitePath, 'wp-content', 'themes' ); + const themes = ( await readdir( themesDir, { withFileTypes: true } ).catch( () => [] ) ) + .filter( ( entry ) => entry.isDirectory() ) + .map( ( entry ) => entry.name ); + const versions: Versions = {}; + for ( const child of themes ) { + const parent = await readThemeHeader( sitePath, child, 'Template' ).catch( () => '' ); + if ( parent && parent !== child ) { + versions[ child ] = parent; + Object.assign( versions, await readCopies( themesDir, child, parent, last ) ); + } + } + return versions; + }, + async check( _site, before, after ) { + const reports = []; + const children = new Set( + [ ...Object.keys( before ), ...Object.keys( after ) ].filter( + ( key ) => ! key.includes( '/' ) + ) + ); + for ( const child of children ) { + const was = copiesOf( before, child ); + const now = copiesOf( after, child ); + if ( was.join( '\n' ) !== now.join( '\n' ) ) { + reports.push( { + key: `child theme ${ child }`, + problem: now.length ? problem( child, after[ child ], now ) : undefined, + } ); + } + } + return { reports }; + }, +}; diff --git a/apps/cli/ai/design-board.ts b/apps/cli/ai/design-board.ts index f473350d74..31a2f4fa74 100644 --- a/apps/cli/ai/design-board.ts +++ b/apps/cli/ai/design-board.ts @@ -2,9 +2,9 @@ import { designSheet, fontFamilyName as fontFamily, googleFontsUrl, - parseDesignMd, - Style, -} from '@studio/design-md'; + themeJsonDesign, + type Style, +} from '@studio/common/sites/design-sheet'; const escapeHtml = ( value: unknown ) => String( value ).replace( /[&<>"']/g, ( char ) => `&#${ char.charCodeAt( 0 ) };` ); @@ -30,8 +30,23 @@ function fontLinks( styles: Style[] ): string { return url ? `` : ''; } -export function renderDesignBoard( design: string, image?: string ): string { - const tokens = parseDesignMd( design ); +/** Renders a look's theme.json as a design-system board. */ +export function renderDesignBoard( themeJson: string, image?: string ): string { + let parsed: Record< string, unknown >; + try { + parsed = JSON.parse( themeJson ); + } catch ( error ) { + throw new Error( `The look's theme.json is not valid JSON: ${ ( error as Error ).message }` ); + } + const tokens = themeJsonDesign( parsed ); + if ( + Object.keys( tokens.colors ?? {} ).length < 2 || + ! Object.values( tokens.typography ?? {} ).some( ( style ) => style.fontFamily ) + ) { + throw new Error( + "The look's theme.json needs at least two hex colors in settings.color.palette, and a font family in styles.typography." + ); + } const { palette, background, @@ -79,8 +94,6 @@ export function renderDesignBoard( design: string, image?: string ): string { `padding:${ dimension( button.padding ) }`, fontCss( button.typography ), ].join( ';' ); - const treatment = tokens.imagery ?? {}; - const overlay = [ 'multiply', 'screen' ].find( ( mode ) => mode === treatment.overlay ); return ` @@ -141,16 +154,7 @@ figcaption{margin-top:12px;font-size:13px;opacity:.65} .picture{position:relative;flex:1;overflow:hidden;border-radius:${ dimension( shape ) };background-color:${ css( primary ) };color:${ css( ink( primary ) ) }} -.picture img{width:100%;height:100%;object-fit:cover;display:block;filter:${ css( - treatment.filter ?? 'none' - ) }} -${ - overlay - ? `.picture img+i{position:absolute;inset:0;background-color:${ css( - primary - ) };mix-blend-mode:${ overlay }}` - : '' -} +.picture img{width:100%;height:100%;object-fit:cover;display:block} .pattern{background-image:radial-gradient(currentColor 20%,transparent 21%);background-size:32px 32px} .strip{position:absolute;left:0;right:0;bottom:0;height:12px;display:flex} @@ -175,7 +179,7 @@ ${ }

Components

ButtonButtonLink
Input
TagTag
CardCard title

Imagery

${ - image ? `${ overlay ? '' : '' }` : '' + image ? `` : '' }
${ strip }
diff --git a/apps/cli/ai/eval-answers.ts b/apps/cli/ai/eval-answers.ts new file mode 100644 index 0000000000..ca205df03d --- /dev/null +++ b/apps/cli/ai/eval-answers.ts @@ -0,0 +1,45 @@ +import { z } from 'zod'; +import type { AskUserHandler } from 'cli/ai/types'; + +// A case's scripted user. Each answer is a reply the user gives to any +// question it fits: one whose text matches `when` (a regex, case-insensitive; +// every question without it), and that takes the reply: a typed answer where +// the question allows one, otherwise one of its option labels. The first +// answer that fits is given, so answers do not depend on the order the agent +// asks in; a question none fits is dismissed, as a user does who closes the +// picker, and is left out of the answers. +export const evalAnswersSchema = z.array( + z.object( { label: z.string(), when: z.string().optional() } ) +); + +export type EvalAnswers = z.infer< typeof evalAnswersSchema >; + +// The answer recorded for a question the scripted user dismissed. +export const DISMISSED = '(dismissed)'; + +export interface AskedQuestion { + question: string; + options: string[]; + answer: string; +} + +export function createScriptedUser( answers: EvalAnswers ): { + onAskUser: AskUserHandler; + questions: AskedQuestion[]; +} { + const questions: AskedQuestion[] = []; + const onAskUser: AskUserHandler = async ( asked ) => + Object.fromEntries( + asked.flatMap( ( { question, options, allowFreeForm } ) => { + const labels = options.map( ( option ) => option.label ); + const answer = answers.find( + ( { label, when } ) => + ( ! when || new RegExp( when, 'i' ).test( question ) ) && + ( allowFreeForm || labels.includes( label ) ) + )?.label; + questions.push( { question, options: labels, answer: answer ?? DISMISSED } ); + return answer === undefined ? [] : [ [ question, answer ] ]; + } ) + ); + return { onAskUser, questions }; +} diff --git a/apps/cli/ai/eval-outcome.ts b/apps/cli/ai/eval-outcome.ts index 52600224dd..aadbca0a29 100644 --- a/apps/cli/ai/eval-outcome.ts +++ b/apps/cli/ai/eval-outcome.ts @@ -5,8 +5,10 @@ * For each site the turn created, and the seeded active site, the capture * records the WordPress state (theme, plugins, published content as parsed * blocks, registered block styles, global styles), the rendered home page, - * block validity in the site's own editor, the active theme's files, and - * whether an installed parent theme still matches its WordPress.org release. + * block validity in the site's own editor, the active theme's files, the + * code (themes, plugins, mu-plugins) the turn added or changed on the active + * site, and whether an installed parent theme still matches its WordPress.org + * release. * Sites that existed before the turn are compared by their database and * wp-content modification times, so a change to an unrelated site shows up. */ @@ -25,6 +27,7 @@ import os from 'node:os'; import path from 'node:path'; import { promisify } from 'node:util'; import { validateBlocks } from 'cli/ai/block-validator'; +import { getSharedBrowser } from 'cli/ai/browser-utils'; import { runCommand as runStartSiteCommand } from 'cli/commands/site/start'; import { Mode as StopMode, runCommand as runStopSiteCommand } from 'cli/commands/site/stop'; import { readCliConfig } from 'cli/lib/cli-config/core'; @@ -39,10 +42,15 @@ export interface SiteFingerprint { path: string; name: string; mtimeMs: number; + // Content hashes of the site's code (themes, plugins, mu-plugins), keyed by + // path relative to wp-content, so the files a turn adds or changes show up. + code: Record< string, string >; } // The newest modification time under a site's database and wp-content, which -// moves whenever the site is changed. +// moves whenever the site is changed. A read of the SQLite database moves its +// -shm file, the WAL index, which holds none of the site +// (https://sqlite.org/walformat.html). function newestMtime( dir: string, depth = 6 ): number { let newest = 0; const walk = ( current: string, level: number ) => { @@ -53,10 +61,20 @@ function newestMtime( dir: string, depth = 6 ): number { return; } for ( const entry of entries ) { - if ( entry.name === 'node_modules' || entry.name === 'cache' ) continue; + if ( [ 'node_modules', 'cache' ].includes( entry.name ) || entry.name.endsWith( '-shm' ) ) + continue; const full = path.join( current, entry.name ); try { - newest = Math.max( newest, statSync( full ).mtimeMs ); + const stat = statSync( full ); + // SQLite's WAL files: a read-only read creates the WAL index (-shm) + // and an empty WAL, and cannot remove them, so it changes the + // database folder; a write adds frames to the WAL or, on the last + // close, checkpoints them into the database + // (https://sqlite.org/walformat.html). + const readOnlyTrace = + ( level === 0 && entry.name === 'database' ) || + ( entry.name.endsWith( '-wal' ) && ! stat.size ); + if ( ! readOnlyTrace ) newest = Math.max( newest, stat.mtimeMs ); } catch { continue; } @@ -71,7 +89,12 @@ export async function fingerprintSites(): Promise< SiteFingerprint[] > { const { sites } = await readCliConfig(); return sites .filter( ( site ) => existsSync( site.path ) ) - .map( ( site ) => ( { path: site.path, name: site.name, mtimeMs: newestMtime( site.path ) } ) ); + .map( ( site ) => ( { + path: site.path, + name: site.name, + mtimeMs: newestMtime( site.path ), + code: hashCode( site.path ), + } ) ); } const PROBE_PHP = String.raw` @@ -86,13 +109,22 @@ $content = array(); foreach ( get_posts( array( 'post_type' => array( 'page', 'post', 'wp_template', 'wp_template_part', 'wp_navigation', 'wp_block' ), 'post_status' => 'publish', 'numberposts' => -1 ) ) as $post ) { $blocks = array(); $count( parse_blocks( $post->post_content ), $blocks ); - $content[] = array( 'id' => $post->ID, 'type' => $post->post_type, 'slug' => $post->post_name, 'title' => $post->post_title, 'isFront' => $post->ID === $front, 'blocks' => $blocks, 'content' => $post->post_content ); + $content[] = array( 'id' => $post->ID, 'type' => $post->post_type, 'slug' => $post->post_name, 'title' => $post->post_title, 'renderedTitle' => get_the_title( $post ), 'link' => get_permalink( $post ), 'categories' => 'post' === $post->post_type ? wp_get_post_categories( $post->ID, array( 'fields' => 'names' ) ) : array(), 'isFront' => $post->ID === $front, 'blocks' => $blocks, 'content' => $post->post_content ); +} +$block_types = array(); +foreach ( WP_Block_Type_Registry::get_instance()->get_all_registered() as $name => $type ) { + if ( str_starts_with( $name, 'core/' ) ) { continue; } + $sources = array(); + foreach ( (array) $type->editor_script_handles as $handle ) { $sources[] = wp_scripts()->registered[ $handle ]->src ?? $handle; } + $block_types[] = array( 'name' => $name, 'editorScripts' => $sources ); } +// Reading the theme's data registers the block styles of its styles/blocks/*.json +// files (WP_Theme_JSON_Resolver::get_theme_data), so it comes first. +$merged = WP_Theme_JSON_Resolver::get_merged_data()->get_raw_data(); $styles = array(); foreach ( WP_Block_Styles_Registry::get_instance()->get_all_registered() as $block => $variations ) { $styles[ $block ] = array_keys( $variations ); } -$merged = WP_Theme_JSON_Resolver::get_merged_data()->get_raw_data(); $palette = array(); foreach ( (array) ( $merged['settings']['color']['palette'] ?? array() ) as $entries ) { foreach ( (array) $entries as $entry ) { $palette[] = $entry; } @@ -104,8 +136,10 @@ echo 'OUTCOME_JSON=' . wp_json_encode( array( 'plugins' => $plugins, 'showOnFront' => get_option( 'show_on_front' ), 'content' => $content, - 'products' => post_type_exists( 'product' ) ? (int) wp_count_posts( 'product' )->publish : 0, + 'products' => function_exists( 'wc_get_products' ) ? array_map( function ( $product ) { return array( 'name' => $product->get_name(), 'price' => $product->get_price(), 'purchasable' => $product->is_purchasable(), 'visible' => $product->is_visible() ); }, wc_get_products( array( 'status' => 'publish', 'limit' => -1 ) ) ) : array(), + 'shopPage' => function_exists( 'wc_get_page_id' ) ? get_post_status( wc_get_page_id( 'shop' ) ) : null, 'blockStyles' => $styles, + 'blockTypes' => $block_types, 'palette' => $palette, 'mergedStyles' => $merged['styles'] ?? array(), 'userGlobalStyles' => WP_Theme_JSON_Resolver::get_user_data()->get_raw_data(), @@ -144,6 +178,54 @@ function hashFiles( dir: string ): Record< string, string > { return hashes; } +const CODE_DIRS = [ 'themes', 'plugins', 'mu-plugins' ]; +const CODE_TEXT_FILE = /\.(css|json|html|php|js|mjs|ts|tsx|jsx|md|txt)$/; + +function hashCode( sitePath: string ): Record< string, string > { + const hashes: Record< string, string > = {}; + for ( const dir of CODE_DIRS ) { + for ( const [ file, hash ] of Object.entries( + hashFiles( path.join( sitePath, 'wp-content', dir ) ) + ) ) { + hashes[ `${ dir }/${ file }` ] = hash; + } + } + return hashes; +} + +interface CodeChange { + file: string; + status: 'added' | 'changed' | 'removed'; + text?: string; +} + +// The code files (relative to wp-content) a turn added, changed or removed, +// with the text of the added and changed ones. +function diffCode( + sitePath: string, + before: Record< string, string >, + after: Record< string, string > +): CodeChange[] { + const changes: CodeChange[] = []; + for ( const file of [ + ...new Set( [ ...Object.keys( before ), ...Object.keys( after ) ] ), + ].sort() ) { + if ( before[ file ] === after[ file ] ) continue; + const status = ! after[ file ] ? 'removed' : ! before[ file ] ? 'added' : 'changed'; + const change: CodeChange = { file, status }; + if ( status !== 'removed' && CODE_TEXT_FILE.test( file ) ) { + try { + const text = readFileSync( path.join( sitePath, 'wp-content', file ), 'utf8' ); + if ( text.length < 200_000 ) change.text = text; + } catch { + // Removed after the hashes were taken. + } + } + changes.push( change ); + } + return changes; +} + function readThemeText( dir: string ): Record< string, string > { const files: Record< string, string > = {}; for ( const relative of Object.keys( hashFiles( dir ) ) ) { @@ -197,16 +279,43 @@ async function renderHome( url: string ) { try { const response = await fetch( url, { redirect: 'follow' } ); const html = await response.text(); + // Each image's address, and the status the site answers it with (null + // for images from another origin, which are not requested). + const images = await Promise.all( + [ ...html.matchAll( /]*\bsrc="([^"]+)"/g ) ].map( async ( m ) => { + const src = new URL( m[ 1 ].replace( /&/g, '&' ), url ); + const sameOrigin = src.origin === new URL( url ).origin; + const status = sameOrigin + ? await fetch( src, { method: 'HEAD' } ).then( + ( r ) => r.status, + () => 0 + ) + : null; + return { src: src.href, status }; + } ) + ); const nav = [ ...html.matchAll( //g ) ] .map( ( m ) => m[ 0 ] ) .join( '\n' ); + const navHrefs = [ ...nav.matchAll( /]*\bhref="([^"]*)"/g ) ].map( ( m ) => m[ 1 ] ); + // Where each of the site's own navigation links leads. + const navTargets: Record< string, { status: number; url: string } > = {}; + for ( const href of new Set( navHrefs ) ) { + const target = new URL( href.replace( /�?38;|&/g, '&' ), url ); + target.hash = ''; + if ( target.origin !== new URL( url ).origin ) continue; + const page = await fetch( target, { redirect: 'follow' } ).catch( () => undefined ); + navTargets[ href ] = { status: page?.status ?? 0, url: page?.url ?? target.href }; + } return { status: response.status, html, phpErrors: [ ...html.matchAll( /(Fatal error|Parse error|Warning|Notice|Deprecated)<\/b>:[^<]*/g ), ].map( ( m ) => m[ 0 ].replace( /<\/?b>/g, '' ) ), - navHrefs: [ ...nav.matchAll( /]*\bhref="([^"]*)"/g ) ].map( ( m ) => m[ 1 ] ), + navHrefs, + navTargets, + images, ids: [ ...html.matchAll( /\bid="([^"]+)"/g ) ].map( ( m ) => m[ 1 ] ), }; } catch ( error ) { @@ -214,6 +323,78 @@ async function renderHome( url: string ) { } } +/** + * The home page's image frames: each element holding several images, with how + * many it shows at once. Measured in a browser as tall as the page, so every + * image is in view: an image is shown when its center is inside every + * ancestor that clips it and nothing in its chain is transparent or hidden, + * so slides faded out or slid out of a clipped frame, as slideshows and + * carousels hold them, are not. + */ +async function homeImageFrames( url: string ) { + try { + const browser = await getSharedBrowser(); + const page = await browser.newPage( { viewport: { width: 1280, height: 900 } } ); + try { + await page.goto( url, { waitUntil: 'load', timeout: 30000 } ); + const height = await page.evaluate( () => document.documentElement.scrollHeight ); + await page.setViewportSize( { + width: 1280, + height: Math.min( Math.max( height, 900 ), 8000 ), + } ); + await page.waitForTimeout( 1500 ); + return await page.evaluate( () => { + const images = [ ...document.querySelectorAll( 'img' ) ]; + // Shown: its center inside every ancestor that clips it, and + // nothing in its chain transparent or hidden. + const shown = ( img: HTMLImageElement ) => { + const box = img.getBoundingClientRect(); + if ( ! box.width || ! box.height ) return false; + const x = box.x + box.width / 2; + const y = box.y + box.height / 2; + if ( getComputedStyle( img ).visibility === 'hidden' ) return false; + for ( let el: Element | null = img; el; el = el.parentElement ) { + const style = getComputedStyle( el ); + if ( Number( style.opacity ) === 0 ) return false; + if ( + el !== img && + ( style.overflowX !== 'visible' || style.overflowY !== 'visible' ) + ) { + const clip = el.getBoundingClientRect(); + if ( x < clip.left || x > clip.right || y < clip.top || y > clip.bottom ) + return false; + } + } + return true; + }; + const frames = new Map< Element, { images: number; shown: number } >(); + for ( const img of images ) { + let frame = img.parentElement; + while ( frame && frame.querySelectorAll( 'img' ).length < 3 ) frame = frame.parentElement; + if ( ! frame || frame === document.body ) continue; + const entry = frames.get( frame ) ?? { images: 0, shown: 0 }; + entry.images++; + entry.shown += shown( img ) ? 1 : 0; + frames.set( frame, entry ); + } + return [ ...frames ].map( ( [ el, entry ] ) => { + const box = el.getBoundingClientRect(); + return { + element: `${ el.tagName.toLowerCase() }.${ [ ...el.classList ].join( '.' ) }`, + width: Math.round( box.width ), + height: Math.round( box.height ), + ...entry, + }; + } ); + } ); + } finally { + await page.close(); + } + } catch ( error ) { + return { error: error instanceof Error ? error.message : String( error ) }; + } +} + interface SiteOutcome { name: string; path: string; @@ -226,10 +407,13 @@ interface SiteOutcome { async function captureSiteOutcome( sitePath: string, created: boolean, - artifactDir: string + artifactDir: string, + code?: { before: Record< string, string >; after: Record< string, string > } ): Promise< SiteOutcome > { const site = await getSiteByFolder( sitePath ); const outcome: SiteOutcome = { name: site.name, path: sitePath, created }; + // Code files the turn added, changed or removed (null for sites it created). + outcome.codeChanges = code ? diffCode( sitePath, code.before, code.after ) : null; try { await connectToDaemon(); await runStartSiteCommand( sitePath, true, true, new Logger() ); @@ -258,10 +442,7 @@ async function captureSiteOutcome( templateVersion ); } - const designPath = path.join( sitePath, 'DESIGN.md' ); - outcome.designMd = existsSync( designPath ) ? readFileSync( designPath, 'utf8' ) : null; - - const home = await renderHome( url ); + const home = { ...( await renderHome( url ) ), imageFrames: await homeImageFrames( url ) }; if ( 'html' in home && home.html ) { const file = path.join( artifactDir, `${ path.basename( sitePath ) }-home.html` ); writeFileSync( file, home.html ); @@ -289,10 +470,14 @@ async function captureSiteOutcome( } ); } } + // A file as its WordPress.org release ships it is the theme author's, and + // may hold blocks of a plugin the site does not run. + const released = ( file: string ) => + Array.isArray( outcome.themeChanges ) && ! outcome.themeChanges.includes( file ); for ( const [ file, text ] of Object.entries( outcome.themeFiles as Record< string, string > ) ) { - if ( ! /^(templates|parts)\/.+\.html$/.test( file ) ) continue; + if ( ! /^(templates|parts)\/.+\.html$/.test( file ) || released( file ) ) continue; const report = await validateBlocks( text, url ); if ( report.invalidBlocks > 0 ) { invalid.push( { @@ -338,8 +523,16 @@ export async function captureTurnOutcome( const others = before.filter( ( site ) => site.path !== requestSitePath ); const sites: SiteOutcome[] = []; - if ( requestSitePath && after.has( requestSitePath ) ) { - sites.push( await captureSiteOutcome( requestSitePath, false, artifactDir ) ); + const requestAfter = requestSitePath ? after.get( requestSitePath ) : undefined; + if ( requestSitePath && requestAfter ) { + sites.push( + await captureSiteOutcome( + requestSitePath, + false, + artifactDir, + requestBefore ? { before: requestBefore.code, after: requestAfter.code } : undefined + ) + ); } for ( const site of created ) { sites.push( await captureSiteOutcome( site.path, true, artifactDir ) ); diff --git a/apps/cli/ai/eval-port-range.ts b/apps/cli/ai/eval-port-range.ts new file mode 100644 index 0000000000..a9e26d925d --- /dev/null +++ b/apps/cli/ai/eval-port-range.ts @@ -0,0 +1,31 @@ +import net from 'node:net'; + +const FIRST_PORT = 20000; +const PORTS_PER_RANGE = 100; +const RANGES = 100; +// The port that holds each range, below the ranges themselves. +const FIRST_HOLD_PORT = FIRST_PORT - RANGES; + +function hold( port: number ): Promise< net.Server | undefined > { + return new Promise( ( resolve ) => { + const server = net.createServer(); + server.once( 'error', () => resolve( undefined ) ); + server.listen( port, '127.0.0.1', () => resolve( server.unref() ) ); + } ); +} + +/** + * Claims a range of ports for this process's sites and returns its first + * port. Concurrent cases each create sites, and a case cannot see the port + * another assigns after it looked, so each takes a range of its own. A + * range is held by listening on a port of its own, which one process at a + * time can do and which ends with the process. + */ +export async function claimPortRange(): Promise< number > { + for ( let range = 0; range < RANGES; range++ ) { + if ( await hold( FIRST_HOLD_PORT + range ) ) { + return FIRST_PORT + range * PORTS_PER_RANGE; + } + } + throw new Error( `All ${ RANGES } eval port ranges are held by running processes.` ); +} diff --git a/apps/cli/ai/eval-runner-entry.ts b/apps/cli/ai/eval-runner-entry.ts index dad790194b..642e7cead1 100644 --- a/apps/cli/ai/eval-runner-entry.ts +++ b/apps/cli/ai/eval-runner-entry.ts @@ -1,6 +1,7 @@ import { mkdtempSync, realpathSync } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { claimPortRange } from './eval-port-range'; // Each case creates its sites in a folder of its own, so concurrent cases, // including repeats of one case, never share or delete each other's sites. @@ -10,4 +11,9 @@ process.env.E2E_SITES_ROOT = realpathSync( mkdtempSync( path.join( os.tmpdir(), 'studio-eval-sites-' ) ) ); -void import( './eval-runner' ); +// Set before the port finder loads: it reads its first port then. A port the +// user picked stays theirs. +void ( async () => { + process.env.STUDIO_BASE_PORT ??= String( await claimPortRange() ); + await import( './eval-runner' ); +} )(); diff --git a/apps/cli/ai/eval-runner.ts b/apps/cli/ai/eval-runner.ts index e1ad759a0e..54406d0f5c 100644 --- a/apps/cli/ai/eval-runner.ts +++ b/apps/cli/ai/eval-runner.ts @@ -41,14 +41,13 @@ import { resolveInitialAiProvider, resolveUnavailableAiProvider, } from 'cli/ai/auth'; +import { createScriptedUser, evalAnswersSchema, type EvalAnswers } from 'cli/ai/eval-answers'; import { captureTurnOutcome, fingerprintSites, type TurnOutcome } from 'cli/ai/eval-outcome'; -import { matchesStopRule, stopRuleSchema, type StopRule } from 'cli/ai/eval-stop'; import { summarizeTiming, type ModelRequestRecord } from 'cli/ai/eval-timing'; import { runStudioAgentTurn } from 'cli/ai/runtimes/pi'; -import { addAssistantMessageUsage, createTokenUsage } from 'cli/ai/token-usage'; +import { addAssistantMessageUsage, addToolCallUsage, createTokenUsage } from 'cli/ai/token-usage'; import { runCommand as runCreateSiteCommand } from 'cli/commands/site/create'; import { runCommand as runDeleteSiteCommand } from 'cli/commands/site/delete'; -import { Mode as StopMode, runCommand as runStopSiteCommand } from 'cli/commands/site/stop'; import { lockCliConfig, readCliConfig, @@ -116,17 +115,13 @@ interface EvalRunnerInput { // Objects rather than strings: promptfoo expands an array of strings into // one test case per element. followUps?: Array< { prompt: string } >; - // Scripted replies for the questions the agent asks the user, in order: - // each question takes the next label. With answers set, the turn runs with - // Studio chat's ask-the-user handler, so AskUserQuestion and tools that - // confirm with the user work; once they run out, the user cancels. - answers?: Array< { label: string } >; + // The scripted user's replies (eval-answers.ts). With answers set, the turn + // runs with Studio chat's ask-the-user handler, so AskUserQuestion and + // tools that confirm with the user work. + answers?: EvalAnswers; timeoutMs?: number; model?: AiModelId; seed?: EvalSeed; - // Ends the run at the first tool call matching one of these rules, before - // any follow-ups; the result's `stoppedAt` names it. - stopAt?: StopRule[]; } interface Inspection { @@ -285,8 +280,8 @@ async function seedFixtures( seed: EvalSeed ): Promise< SeededFixtures > { let activeSitePath: string | undefined; if ( activeSite ) { activeSitePath = path.join( STUDIO_SITES_ROOT, siteSlug( activeSite.name ) ); - // WordPress is installed on first start, so WP-CLI seeds need a started site. - const needsInstall = !! activeSite.wpCli?.length; + // WordPress is installed on first start; the site stays running, as a + // site the user has open in Studio is. await runCreateSiteCommand( activeSitePath, { @@ -295,7 +290,7 @@ async function seedFixtures( seed: EvalSeed ): Promise< SeededFixtures > { phpVersion: DEFAULT_PHP_VERSION, fileAccess: SITE_FILE_ACCESS_SITE_DIRECTORY, enableHttps: false, - noStart: ! needsInstall, + noStart: false, skipBrowser: true, skipLogDetails: true, }, @@ -320,9 +315,6 @@ async function seedFixtures( seed: EvalSeed ): Promise< SeededFixtures > { ); } } - if ( needsInstall ) { - await runStopSiteCommand( StopMode.STOP_SINGLE_SITE, activeSitePath, new Logger() ); - } } catch ( error ) { await runDeleteSiteCommand( activeSitePath, true, new Logger() ).catch( () => undefined ); throw error; @@ -330,7 +322,7 @@ async function seedFixtures( seed: EvalSeed ): Promise< SeededFixtures > { seededSite = { name: activeSite.name, path: activeSitePath, - running: false, + running: true, }; const { id: localSiteId } = await getSiteByFolder( activeSitePath ); for ( const connection of activeSite.connectedWpcomSites ?? [] ) { @@ -396,7 +388,13 @@ async function seedFixtures( seed: EvalSeed ): Promise< SeededFixtures > { inspect, activeSitePath, activeSite: remoteActiveSite - ? { ...remoteActiveSite, path: '', running: false, remote: true } + ? { + name: remoteActiveSite.name, + url: remoteActiveSite.url, + path: '', + running: false, + remote: { host: 'wpcom', id: remoteActiveSite.wpcomSiteId }, + } : seededSite, }; } @@ -524,16 +522,13 @@ function readInput(): EvalRunnerInput { ? z.array( z.object( { prompt: z.string() } ) ).parse( vars.followUps ) : undefined; - const answers = vars.answers - ? z.array( z.object( { label: z.string() } ) ).parse( vars.answers ) - : undefined; + const answers = vars.answers ? evalAnswersSchema.parse( vars.answers ) : undefined; return { prompt: ( vars.prompt as string ) ?? prompt, followUps, answers, timeoutMs: typeof vars.timeoutMs === 'number' ? vars.timeoutMs : undefined, - stopAt: vars.stopAt ? z.array( stopRuleSchema ).parse( vars.stopAt ) : undefined, model, seed, }; @@ -577,34 +572,21 @@ async function runEval( input: EvalRunnerInput ) { const toolEvents: ToolEvent[] = []; const requests: ModelRequestRecord[] = []; const textSegments: string[] = []; - const questions: Array< { question: string; options: string[]; answer: string } > = []; - const pendingAnswers = [ ...( input.answers ?? [] ) ]; - const onAskUser: AskUserHandler | undefined = input.answers - ? async ( asked ) => - Object.fromEntries( - asked.map( ( { question, options } ) => { - const answer = pendingAnswers.shift()?.label ?? 'Cancel'; - questions.push( { - question, - options: options.map( ( option ) => option.label ), - answer, - } ); - return [ question, answer ]; - } ) - ) - : undefined; + const scriptedUser = input.answers ? createScriptedUser( input.answers ) : undefined; + const onAskUser: AskUserHandler | undefined = scriptedUser?.onAskUser; const toolNameById = new Map< string, string >(); const toolEventById = new Map< string, ToolEvent >(); let firstToolError: FirstToolError | null = null; // Wall-clock per turn, measured between successive assistant messages. const turnDurationsMs: number[] = []; + // Each turn's system prompt, so a result shows the facts the agent was given. + const systemPrompts: string[] = []; let turnIndex = 0; let numTurns = 0; let numTurnsResult: number | null = null; let success = false; let error: string | null = null; let timedOut = false; - let stoppedAt: { toolUseId: string; toolName: string; index: number } | null = null; // From the last assistant message: a provider failure ends the turn with // stopReason "error" and its message here, without throwing. let stopReason: string | null = null; @@ -663,10 +645,6 @@ async function runEval( input: EvalRunnerInput ) { turnStart = now; } for ( const tc of extractToolCalls( event ) ) { - if ( input.stopAt && ! stoppedAt && matchesStopRule( tc, input.stopAt ) ) { - stoppedAt = { toolUseId: tc.id, toolName: tc.name, index: toolCalls.length }; - void query.interrupt(); - } toolCalls.push( tc ); toolNameById.set( tc.id, tc.name ); const evt: ToolEvent = { @@ -682,6 +660,7 @@ async function runEval( input: EvalRunnerInput ) { textSegments.push( ...extractTextSegments( event ) ); if ( event.type === 'tool_execution_end' ) { + addToolCallUsage( usage, event ); const tr = extractToolResult( event ); if ( tr ) { const id = tr.toolUseId; @@ -749,9 +728,10 @@ async function runEval( input: EvalRunnerInput ) { try { await query.result; for ( const followUp of input.followUps ?? [] ) { - if ( timedOut || stoppedAt ) { + if ( timedOut ) { break; } + systemPrompts.push( query.systemPrompt() ?? '' ); query = runTurn( followUp.prompt ); await query.result; } @@ -759,6 +739,7 @@ async function runEval( input: EvalRunnerInput ) { error = caught instanceof Error ? caught.message : String( caught ); } finally { clearTimeout( timeout ); + systemPrompts.push( query.systemPrompt() ?? '' ); outcome = await captureTurnOutcome( sitesBeforeTurn, seeded?.activeSitePath, @@ -778,7 +759,6 @@ async function runEval( input: EvalRunnerInput ) { success, error, timedOut, - stoppedAt, model: resolvedModel, numTurns: numTurnsResult, stopReason, @@ -789,12 +769,13 @@ async function runEval( input: EvalRunnerInput ) { timing: summarizeTiming( requests, toolEvents ), requests, turnDurationsMs, + systemPrompts, toolCalls, toolResults, toolEvents, firstToolError, textSegments, - questions, + questions: scriptedUser?.questions ?? [], inspections, outcome, }; diff --git a/apps/cli/ai/eval-stop.ts b/apps/cli/ai/eval-stop.ts deleted file mode 100644 index 4b975818bb..0000000000 --- a/apps/cli/ai/eval-stop.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { z } from 'zod'; - -// A tool call that ends an eval run: the tool's name and, optionally, a regex -// for each input field to match. Cases that judge what the agent did before -// starting the work (for example, which skills it loaded) stop there instead -// of paying for the work itself. -export const stopRuleSchema = z.object( { - tool: z.string(), - input: z.record( z.string(), z.string() ).optional(), -} ); -export type StopRule = z.infer< typeof stopRuleSchema >; - -function fieldText( value: unknown ): string { - return typeof value === 'string' ? value : JSON.stringify( value ) ?? ''; -} - -export function matchesStopRule( - call: { name: string; input: unknown }, - rules: StopRule[] -): boolean { - const input = ( call.input ?? {} ) as Record< string, unknown >; - return rules.some( - ( rule ) => - rule.tool === call.name && - Object.entries( rule.input ?? {} ).every( ( [ field, pattern ] ) => - new RegExp( pattern ).test( fieldText( input[ field ] ) ) - ) - ); -} diff --git a/apps/cli/ai/front-page-template.ts b/apps/cli/ai/front-page-template.ts new file mode 100644 index 0000000000..158688e544 --- /dev/null +++ b/apps/cli/ai/front-page-template.ts @@ -0,0 +1,104 @@ +/** + * A theme's front-page template, which WordPress renders at the site's front + * page whatever Settings → Reading picks (wp-includes/template.php, + * get_front_page_template). It shows the page Reading picks only through a + * post-content block, so a theme must render one there: the Theme Directory + * requires a theme to show the content the front page setting picks. + */ +import { readdir, readFile } from 'fs/promises'; +import path from 'path'; +import type { WordPressSite } from 'cli/ai/site-database'; + +const REQUIREMENT = 'https://make.wordpress.org/themes/handbook/review/required/'; +const POST_CONTENT = //g; + +/** The front-page template's path, relative to its theme. */ +export const FRONT_PAGE_TEMPLATE = 'templates/front-page.html'; + +// The theme's patterns, by slug, from the Slug header of patterns/*.php. +async function themePatterns( themeDir: string ): Promise< Map< string, string > > { + const dir = path.join( themeDir, 'patterns' ); + const files = await readdir( dir ).catch( () => [] as string[] ); + const patterns = new Map< string, string >(); + for ( const file of files.filter( ( name ) => name.endsWith( '.php' ) ) ) { + const content = await readFile( path.join( dir, file ), 'utf8' ).catch( () => '' ); + const slug = content.match( /^[\s*]*Slug:\s*(\S+)/im )?.[ 1 ]; + if ( slug ) { + patterns.set( slug, content ); + } + } + return patterns; +} + +/** + * Whether a front-page template renders a page's content, with the theme's + * patterns it includes; undefined when it includes a pattern the theme does + * not hold, which may render it. + */ +export async function rendersPageContent( + template: string, + themeDirs: string[] +): Promise< boolean | undefined > { + const slugs = [ ...template.matchAll( PATTERN ) ].map( ( match ) => { + try { + return String( JSON.parse( match[ 1 ] ).slug ?? '' ); + } catch { + return ''; + } + } ); + if ( POST_CONTENT.test( template ) ) { + return true; + } + const patterns = new Map< string, string >(); + for ( const dir of [ ...themeDirs ].reverse() ) { + for ( const [ slug, content ] of await themePatterns( dir ) ) { + patterns.set( slug, content ); + } + } + const included = slugs.map( ( slug ) => patterns.get( slug ) ); + if ( included.some( ( content ) => content === undefined ) ) { + return undefined; + } + return included.some( ( content ) => POST_CONTENT.test( content! ) ); +} + +/** What a front-page template without a page's content makes WordPress do. */ +export function frontPageTemplateProblem( source: string, frontPage?: string ): string { + const shows = frontPage + ? `so ${ frontPage }, the front page Settings → Reading picks, never shows, and edits to it change nothing on the site` + : "so no page set as the homepage (Settings → Reading, or the editor's Set as homepage) can show, and the content lives in the theme instead of a page the site owner edits"; + return `WordPress renders ${ source } at the front page whatever Settings → Reading picks (wp-includes/template.php, get_front_page_template), and it has no core/post-content block, the only block that renders the page shown (wp-includes/blocks/post-content.php), ${ shows }. The Theme Directory requires a theme to show the content the front page setting picks (${ REQUIREMENT }). Put the content in a page, set it as the homepage (show_on_front=page, page_on_front), and render it with in the template, or remove the template so page.html renders the page.`; +} + +/** + * What Settings → Reading shows at the homepage and the posts page, as a site + * fact. WP_Query::parse_query() shows page_on_front whenever show_on_front is + * `page` and it names one, so a page that is gone or unpublished is stated + * as such. + */ +export function homepageFact( site: WordPressSite ): string | undefined { + const page = ( option: string ) => { + const id = Number( site.option( option ) ); + if ( ! id ) { + return undefined; + } + const row = site.query< { title: string; status: string } >( + `SELECT post_title AS title, post_status AS status FROM ${ site.prefix }posts WHERE ID = ? AND post_type = 'page'`, + id + )[ 0 ]; + if ( ! row ) { + return `page ${ id }, which does not exist`; + } + return `page ${ id } "${ row.title }"${ row.status === 'publish' ? '' : ` (${ row.status })` }`; + }; + const showOnFront = site.option( 'show_on_front' ); + const front = showOnFront === 'page' ? page( 'page_on_front' ) : undefined; + if ( ! front ) { + return showOnFront === undefined ? undefined : 'Homepage: the latest posts (show_on_front).'; + } + const posts = page( 'page_for_posts' ); + return `Homepage: ${ front } (show_on_front, page_on_front)${ + posts ? `; posts page: ${ posts } (page_for_posts)` : '' + }.`; +} diff --git a/apps/cli/ai/header-menu.ts b/apps/cli/ai/header-menu.ts new file mode 100644 index 0000000000..5d23344c50 --- /dev/null +++ b/apps/cli/ai/header-menu.ts @@ -0,0 +1,57 @@ +/** + * A header that links to several of the site's pages, or of the home page's + * sections, with no navigation. The + * block that renders a menu is core/navigation: a