diff --git a/scripts/eval/README.md b/scripts/eval/README.md index cf78e99c90..e9faea6021 100644 --- a/scripts/eval/README.md +++ b/scripts/eval/README.md @@ -48,7 +48,7 @@ The checks in `outcome.mjs` read that state. A case names the site it asks for i - **design-system-annotation** — `DESIGN.md`'s primary color changed, and the active theme's palette carries the new value. - **template-part-db-override** — Regression for STU-2155. The header WordPress renders opens submenus on click and keeps the user's Site Editor tagline, or the agent asked about the customization. - **global-styles-db-override** — Regression for STU-2155. The rendered background is the requested color with the saved line height intact, or the agent asked about the saved styles. -- **routing-\*** — Routing: the skills in `skills` were loaded before the agent started the work. These are the one kind of case judged on the route rather than the outcome, since which runbook the agent read is the thing under test. Each run stops at its first call that does the work (`stopAt`), so it costs only the turns before it. Run just these with `npm run eval -- --filter-pattern "^routing:"`. +- **routing-\*** — Routing: the skills in `skills` (a list of `{ "name": "…" }` objects, like `followUps`, so promptfoo keeps them in one test) were loaded before the agent started the work. These are the one kind of case judged on the route rather than the outcome, since which runbook the agent read is the thing under test. Each run stops at its first call that does the work (`stopAt`), so it costs only the turns before it. Run just these with `npm run eval -- --filter-pattern "^routing:"`. ## Running the runner directly diff --git a/scripts/eval/outcome.mjs b/scripts/eval/outcome.mjs index 67af91c9a6..b39ee788e1 100644 --- a/scripts/eval/outcome.mjs +++ b/scripts/eval/outcome.mjs @@ -538,7 +538,10 @@ export const skillsLoadedFirst = check( ( d, vars ) => { ? ( d.toolCalls || [] ).slice( 0, d.stoppedAt.index ) : d.toolCalls || []; const loaded = calls.filter( ( c ) => c.name === 'Skill' ).map( ( c ) => c.input?.name ); - const missing = ( vars.skills || [] ).filter( ( name ) => ! loaded.includes( name ) ); + // Objects, not strings: promptfoo would expand an array of strings into one test per element. + const missing = ( vars.skills || [] ) + .map( ( skill ) => skill.name ) + .filter( ( name ) => ! loaded.includes( name ) ); const before = d.stoppedAt ? `before ${ d.stoppedAt.toolName }` : 'in the whole run'; return missing.length ? fail( diff --git a/scripts/eval/outcome.test.mjs b/scripts/eval/outcome.test.mjs index f5c977420e..a66a1cf901 100644 --- a/scripts/eval/outcome.test.mjs +++ b/scripts/eval/outcome.test.mjs @@ -2,6 +2,7 @@ import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { describe, expect, test } from 'vitest'; +import { parse } from 'yaml'; import * as checks from './outcome.mjs'; // Each check reads the runner's result file named on stdout, as promptfoo @@ -484,7 +485,7 @@ describe( 'site checks', () => { describe( 'routing checks', () => { const skill = ( name ) => ( { name: 'Skill', input: { name } } ); - const vars = { skills: [ 'block-content', 'imagery' ] }; + const vars = { skills: [ { name: 'block-content' }, { name: 'imagery' } ] }; test( 'skillsLoadedFirst', () => { const calls = [ skill( 'block-content' ), { name: 'Write', input: {} }, skill( 'imagery' ) ]; @@ -505,3 +506,20 @@ describe( 'routing checks', () => { expect( run( checks.skillsLoadedFirst, { toolCalls: calls }, vars ).pass ).toBe( true ); } ); } ); + +// promptfoo expands a var that is an array of plain values into one test per +// element, so a list a check reads must be a list of objects. +test( 'list vars in the config are lists of objects', () => { + const config = parse( + readFileSync( path.join( import.meta.dirname, 'promptfoo.config.yaml' ), 'utf8' ) + ); + const scalarLists = config.tests.flatMap( ( { vars = {} } ) => + Object.entries( vars ) + .filter( + ( [ , value ] ) => + Array.isArray( value ) && value.some( ( item ) => typeof item !== 'object' ) + ) + .map( ( [ name ] ) => `${ vars.caseId }.${ name }` ) + ); + expect( scalarLists ).toEqual( [] ); +} ); diff --git a/scripts/eval/promptfoo.config.yaml b/scripts/eval/promptfoo.config.yaml index bb94b1f4fe..e5c055a4d7 100644 --- a/scripts/eval/promptfoo.config.yaml +++ b/scripts/eval/promptfoo.config.yaml @@ -655,7 +655,7 @@ tests: vars: caseId: routing-new-site-make timeoutMs: 300000 - skills: [build-site] + skills: [{ name: build-site }] stopAt: - tool: site_create - tool: scaffold_theme @@ -673,7 +673,7 @@ tests: vars: caseId: routing-new-site-contact-form timeoutMs: 300000 - skills: [build-site, plugin-recommendations] + skills: [{ name: build-site }, { name: plugin-recommendations }] stopAt: &stopAtWork - tool: Write - tool: Edit @@ -689,7 +689,7 @@ tests: vars: caseId: routing-new-site-slideshow timeoutMs: 300000 - skills: [build-site, plugin-recommendations] + skills: [{ name: build-site }, { name: plugin-recommendations }] stopAt: *stopAtWork prompt: Create a new site named "Routing Slideshow Site" and add a slideshow of three placeholder images to its homepage. assert: @@ -699,7 +699,7 @@ tests: vars: caseId: routing-form timeoutMs: 300000 - skills: [plugin-recommendations] + skills: [{ name: plugin-recommendations }] stopAt: *stopAtWork seed: activeSite: @@ -712,7 +712,7 @@ tests: vars: caseId: routing-carousel timeoutMs: 300000 - skills: [plugin-recommendations] + skills: [{ name: plugin-recommendations }] stopAt: *stopAtWork seed: activeSite: @@ -725,7 +725,7 @@ tests: vars: caseId: routing-newsletter timeoutMs: 300000 - skills: [plugin-recommendations] + skills: [{ name: plugin-recommendations }] stopAt: *stopAtWork seed: activeSite: @@ -738,7 +738,7 @@ tests: vars: caseId: routing-poll timeoutMs: 300000 - skills: [plugin-recommendations] + skills: [{ name: plugin-recommendations }] stopAt: *stopAtWork seed: activeSite: @@ -751,7 +751,7 @@ tests: vars: caseId: routing-events timeoutMs: 300000 - skills: [plugin-recommendations] + skills: [{ name: plugin-recommendations }] stopAt: *stopAtWork seed: activeSite: @@ -764,7 +764,7 @@ tests: vars: caseId: routing-typography timeoutMs: 300000 - skills: [visual-design] + skills: [{ name: visual-design }] stopAt: *stopAtWork seed: activeSite: @@ -777,7 +777,7 @@ tests: vars: caseId: routing-redesign timeoutMs: 300000 - skills: [visual-design] + skills: [{ name: visual-design }] stopAt: *stopAtWork seed: activeSite: @@ -790,7 +790,7 @@ tests: vars: caseId: routing-section timeoutMs: 300000 - skills: [block-content] + skills: [{ name: block-content }] stopAt: *stopAtWork seed: activeSite: @@ -803,7 +803,7 @@ tests: vars: caseId: routing-template-part timeoutMs: 300000 - skills: [block-content] + skills: [{ name: block-content }] stopAt: *stopAtWork seed: activeSite: @@ -816,7 +816,7 @@ tests: vars: caseId: routing-hero-image timeoutMs: 300000 - skills: [block-content, imagery] + skills: [{ name: block-content }, { name: imagery }] stopAt: *stopAtWork seed: activeSite: @@ -829,7 +829,7 @@ tests: vars: caseId: routing-polish timeoutMs: 300000 - skills: [visual-polish] + skills: [{ name: visual-polish }] stopAt: *stopAtWork seed: activeSite: @@ -850,7 +850,7 @@ tests: vars: caseId: routing-plan-question timeoutMs: 300000 - skills: [hosting-plans-helper] + skills: [{ name: hosting-plans-helper }] stopAt: *stopAtWork prompt: Which WordPress.com plan do I need to install my own plugins? assert: