Eval: Keep each routing case's skills in one test

scripts/eval/README.md

@@ -48,7 +48,7 @@ The checks in `outcome.mjs` read that state. A case names the site it asks for i
 - **design-system-annotation** — `DESIGN.md`'s primary color changed, and the active theme's palette carries the new value.
 - **template-part-db-override** — Regression for STU-2155. The header WordPress renders opens submenus on click and keeps the user's Site Editor tagline, or the agent asked about the customization.
 - **global-styles-db-override** — Regression for STU-2155. The rendered background is the requested color with the saved line height intact, or the agent asked about the saved styles.
-- **routing-\*** — Routing: the skills in `skills` were loaded before the agent started the work. These are the one kind of case judged on the route rather than the outcome, since which runbook the agent read is the thing under test. Each run stops at its first call that does the work (`stopAt`), so it costs only the turns before it. Run just these with `npm run eval -- --filter-pattern "^routing:"`.
+- **routing-\*** — Routing: the skills in `skills` (a list of `{ "name": "…" }` objects, like `followUps`, so promptfoo keeps them in one test) were loaded before the agent started the work. These are the one kind of case judged on the route rather than the outcome, since which runbook the agent read is the thing under test. Each run stops at its first call that does the work (`stopAt`), so it costs only the turns before it. Run just these with `npm run eval -- --filter-pattern "^routing:"`.
 
 ## Running the runner directly
 

scripts/eval/outcome.mjs

@@ -538,7 +538,10 @@ export const skillsLoadedFirst = check( ( d, vars ) => {
 		? ( d.toolCalls || [] ).slice( 0, d.stoppedAt.index )
 		: d.toolCalls || [];
 	const loaded = calls.filter( ( c ) => c.name === 'Skill' ).map( ( c ) => c.input?.name );
-	const missing = ( vars.skills || [] ).filter( ( name ) => ! loaded.includes( name ) );
+	// Objects, not strings: promptfoo would expand an array of strings into one test per element.
+	const missing = ( vars.skills || [] )
+		.map( ( skill ) => skill.name )
+		.filter( ( name ) => ! loaded.includes( name ) );
 	const before = d.stoppedAt ? `before ${ d.stoppedAt.toolName }` : 'in the whole run';
 	return missing.length
 		? fail(

scripts/eval/outcome.test.mjs

@@ -2,6 +2,7 @@ import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { describe, expect, test } from 'vitest';
+import { parse } from 'yaml';
 import * as checks from './outcome.mjs';
 
 // Each check reads the runner's result file named on stdout, as promptfoo
@@ -484,7 +485,7 @@ describe( 'site checks', () => {
 
 describe( 'routing checks', () => {
 	const skill = ( name ) => ( { name: 'Skill', input: { name } } );
-	const vars = { skills: [ 'block-content', 'imagery' ] };
+	const vars = { skills: [ { name: 'block-content' }, { name: 'imagery' } ] };
 
 	test( 'skillsLoadedFirst', () => {
 		const calls = [ skill( 'block-content' ), { name: 'Write', input: {} }, skill( 'imagery' ) ];
@@ -505,3 +506,20 @@ describe( 'routing checks', () => {
 		expect( run( checks.skillsLoadedFirst, { toolCalls: calls }, vars ).pass ).toBe( true );
 	} );
 } );
+
+// promptfoo expands a var that is an array of plain values into one test per
+// element, so a list a check reads must be a list of objects.
+test( 'list vars in the config are lists of objects', () => {
+	const config = parse(
+		readFileSync( path.join( import.meta.dirname, 'promptfoo.config.yaml' ), 'utf8' )
+	);
+	const scalarLists = config.tests.flatMap( ( { vars = {} } ) =>
+		Object.entries( vars )
+			.filter(
+				( [ , value ] ) =>
+					Array.isArray( value ) && value.some( ( item ) => typeof item !== 'object' )
+			)
+			.map( ( [ name ] ) => `${ vars.caseId }.${ name }` )
+	);
+	expect( scalarLists ).toEqual( [] );
+} );

scripts/eval/promptfoo.config.yaml

@@ -655,7 +655,7 @@ tests:
     vars:
       caseId: routing-new-site-make
       timeoutMs: 300000
-      skills: [build-site]
+      skills: [{ name: build-site }]
       stopAt:
         - tool: site_create
         - tool: scaffold_theme
@@ -673,7 +673,7 @@ tests:
     vars:
       caseId: routing-new-site-contact-form
       timeoutMs: 300000
-      skills: [build-site, plugin-recommendations]
+      skills: [{ name: build-site }, { name: plugin-recommendations }]
       stopAt: &stopAtWork
         - tool: Write
         - tool: Edit
@@ -689,7 +689,7 @@ tests:
     vars:
       caseId: routing-new-site-slideshow
       timeoutMs: 300000
-      skills: [build-site, plugin-recommendations]
+      skills: [{ name: build-site }, { name: plugin-recommendations }]
       stopAt: *stopAtWork
       prompt: Create a new site named "Routing Slideshow Site" and add a slideshow of three placeholder images to its homepage.
     assert:
@@ -699,7 +699,7 @@ tests:
     vars:
       caseId: routing-form
       timeoutMs: 300000
-      skills: [plugin-recommendations]
+      skills: [{ name: plugin-recommendations }]
       stopAt: *stopAtWork
       seed:
         activeSite:
@@ -712,7 +712,7 @@ tests:
     vars:
       caseId: routing-carousel
       timeoutMs: 300000
-      skills: [plugin-recommendations]
+      skills: [{ name: plugin-recommendations }]
       stopAt: *stopAtWork
       seed:
         activeSite:
@@ -725,7 +725,7 @@ tests:
     vars:
       caseId: routing-newsletter
       timeoutMs: 300000
-      skills: [plugin-recommendations]
+      skills: [{ name: plugin-recommendations }]
       stopAt: *stopAtWork
       seed:
         activeSite:
@@ -738,7 +738,7 @@ tests:
     vars:
       caseId: routing-poll
       timeoutMs: 300000
-      skills: [plugin-recommendations]
+      skills: [{ name: plugin-recommendations }]
       stopAt: *stopAtWork
       seed:
         activeSite:
@@ -751,7 +751,7 @@ tests:
     vars:
       caseId: routing-events
       timeoutMs: 300000
-      skills: [plugin-recommendations]
+      skills: [{ name: plugin-recommendations }]
       stopAt: *stopAtWork
       seed:
         activeSite:
@@ -764,7 +764,7 @@ tests:
     vars:
       caseId: routing-typography
       timeoutMs: 300000
-      skills: [visual-design]
+      skills: [{ name: visual-design }]
       stopAt: *stopAtWork
       seed:
         activeSite:
@@ -777,7 +777,7 @@ tests:
     vars:
       caseId: routing-redesign
       timeoutMs: 300000
-      skills: [visual-design]
+      skills: [{ name: visual-design }]
       stopAt: *stopAtWork
       seed:
         activeSite:
@@ -790,7 +790,7 @@ tests:
     vars:
       caseId: routing-section
       timeoutMs: 300000
-      skills: [block-content]
+      skills: [{ name: block-content }]
       stopAt: *stopAtWork
       seed:
         activeSite:
@@ -803,7 +803,7 @@ tests:
     vars:
       caseId: routing-template-part
       timeoutMs: 300000
-      skills: [block-content]
+      skills: [{ name: block-content }]
       stopAt: *stopAtWork
       seed:
         activeSite:
@@ -816,7 +816,7 @@ tests:
     vars:
       caseId: routing-hero-image
       timeoutMs: 300000
-      skills: [block-content, imagery]
+      skills: [{ name: block-content }, { name: imagery }]
       stopAt: *stopAtWork
       seed:
         activeSite:
@@ -829,7 +829,7 @@ tests:
     vars:
       caseId: routing-polish
       timeoutMs: 300000
-      skills: [visual-polish]
+      skills: [{ name: visual-polish }]
       stopAt: *stopAtWork
       seed:
         activeSite:
@@ -850,7 +850,7 @@ tests:
     vars:
       caseId: routing-plan-question
       timeoutMs: 300000
-      skills: [hosting-plans-helper]
+      skills: [{ name: hosting-plans-helper }]
       stopAt: *stopAtWork
       prompt: Which WordPress.com plan do I need to install my own plugins?
     assert: