Report the skills a turn loads in studio_code_turn_completed

apps/cli/ai/tests/turn-stats.test.ts

@@ -47,12 +47,53 @@ function propsFor( events: AgentSessionEvent[], outcome: 'success' | 'error' = '
 	return tracker.getProps( { outcome, site: null, hasTool } );
 }
 
+function skillStart( args: unknown ): AgentSessionEvent {
+	return {
+		type: 'tool_execution_start',
+		toolCallId: 'Skill-id',
+		toolName: 'Skill',
+		args,
+	} as AgentSessionEvent;
+}
+
 describe( 'createTurnStatsTracker', () => {
+	it.each( [
+		{
+			name: 'two skills, one loaded twice',
+			events: [
+				skillStart( { name: 'visual-design' } ),
+				skillStart( { name: 'block-content' } ),
+				skillStart( { name: 'visual-design' } ),
+			],
+			expected: 'block-content,visual-design',
+		},
+		{
+			name: 'a name Studio ships no skill for',
+			events: [ skillStart( { name: '/Users/me/notes' } ), skillStart( {} ) ],
+			expected: '',
+		},
+		{
+			name: 'a site named after a skill',
+			events: [
+				{
+					type: 'tool_execution_start',
+					toolCallId: 'site_create-id',
+					toolName: 'site_create',
+					args: { name: 'liberate' },
+				} as AgentSessionEvent,
+			],
+			expected: '',
+		},
+	] )( 'reports the skills loaded in a turn: $name', ( { events, expected } ) => {
+		expect( propsFor( events ).skills_loaded ).toBe( expected );
+	} );
+
 	it( 'reports zeroes for a turn with no tools or model responses', () => {
 		expect( propsFor( [] ) ).toEqual( {
 			tool_calls: 0,
 			tool_errors: 0,
 			failed_tools: '',
+			skills_loaded: '',
 			edits: 0,
 			model_requests: 0,
 			input_tokens: 0,

apps/cli/ai/turn-stats.ts

@@ -4,6 +4,7 @@ import {
 	isOutOfCreditsError,
 	isUsageCapError,
 } from '@studio/common/ai/json-events';
+import { findSkill } from 'cli/ai/skills';
 import { addAssistantMessageUsage, createTokenUsage } from 'cli/ai/token-usage';
 import type { AgentSessionEvent } from '@earendil-works/pi-coding-agent';
 import type { TurnStatus } from '@studio/common/ai/sessions/types';
@@ -16,6 +17,8 @@ const FILE_CHANGE_TOOL_NAMES: ReadonlySet< string > = new Set( [ 'Write', 'Edit'
 
 const UNKNOWN_TOOL = 'unknown';
 
+const SKILL_TOOL_NAME = 'Skill';
+
 export type TurnErrorKind =
 	| 'out_of_credits'
 	| 'auth'
@@ -30,6 +33,7 @@ export interface TurnStatsProps {
 	tool_calls: number;
 	tool_errors: number;
 	failed_tools: string;
+	skills_loaded: string;
 	edits: number;
 	model_requests: number;
 	input_tokens: number;
@@ -146,6 +150,7 @@ export function createTurnStatsTracker(): TurnStatsTracker {
 	let toolCalls = 0;
 	let toolErrors = 0;
 	const failedTools = new Set< string >();
+	const skillsLoaded = new Set< string >();
 	let edits = 0;
 	let modelRequests = 0;
 	const usage = createTokenUsage();
@@ -157,6 +162,14 @@ export function createTurnStatsTracker(): TurnStatsTracker {
 				addAssistantMessageUsage( usage, event );
 				return;
 			}
+			// Only the name of a skill Studio ships is kept, never other text the model wrote.
+			if ( event.type === 'tool_execution_start' && event.toolName === SKILL_TOOL_NAME ) {
+				const name = ( event.args as { name?: unknown } | undefined )?.name;
+				if ( typeof name === 'string' && findSkill( name ) ) {
+					skillsLoaded.add( name );
+				}
+				return;
+			}
 			if ( event.type !== 'tool_execution_end' ) {
 				return;
 			}
@@ -180,6 +193,7 @@ export function createTurnStatsTracker(): TurnStatsTracker {
 				tool_calls: toolCalls,
 				tool_errors: toolErrors,
 				failed_tools: [ ...failedToolNames ].sort().join( ',' ),
+				skills_loaded: [ ...skillsLoaded ].sort().join( ',' ),
 				edits,
 				model_requests: modelRequests,
 				input_tokens: usage.input,

docs/design-docs/analytics-tracks.md

@@ -470,7 +470,7 @@ AI identity props described under "Property vocabulary" above. Filter by `channe
 | Event | Emitted from | Event-specific props |
 |---|---|---|
 | `studio_code_message_sent` | CLI `runAgentTurn` | `provider` (`wpcom`/`anthropic-api-key` — the gateway serving the request, not the model vendor), `model` (e.g. `claude-sonnet-5`), `model_family` (`anthropic`/`openai`), `ability_name` (predefined skill, absent for an ordinary message), `has_images`, `has_files` (booleans). One per user turn dispatched. |
-| `studio_code_turn_completed` | CLI `runAgentTurn` | `outcome` (`success`/`error`/`interrupted`/`max_turns`), `duration_ms`, plus the same `provider`/`model`/`model_family`. What happened inside the turn, accumulated from the agent events by `apps/cli/ai/turn-stats.ts` (attempts that pi retries count toward the same turn): `tool_calls` (tool executions that finished), `tool_errors` (of those, how many errored), `failed_tools` (sorted, de-duplicated, comma-separated names of the tools that errored, empty when none; a call to a tool the session doesn't register, which keeps whatever name the model wrote, is reported as `unknown`, so these are only Studio's own tool names or `unknown`), `edits` (successful `Write`/`Edit` calls), `model_requests` (assistant model responses), `input_tokens`/`output_tokens`/`cache_read_tokens`/`cache_write_tokens` (their summed usage, by kind; a long session's tokens are mostly cache reads, so add all four for its size), `error_kind` (only when `outcome=error`: `out_of_credits` — the 402 `studio_out_of_credits` or the monthly cost cap —, `auth`, `model_unavailable`, `rate_limited`, `network`, `other`; classified from the failure, the raw text is never sent), `site_kind` (`local`/`wpcom`/`none`). These props must be registered in the Tracks Registration tool. One per turn finishing. Partially overlaps the MC Stats `recordAgentRun` bump (`packages/common/ai/agent-stats.ts`), which is a bare counter with no model or duration breakdown. |
+| `studio_code_turn_completed` | CLI `runAgentTurn` | `outcome` (`success`/`error`/`interrupted`/`max_turns`), `duration_ms`, plus the same `provider`/`model`/`model_family`. What happened inside the turn, accumulated from the agent events by `apps/cli/ai/turn-stats.ts` (attempts that pi retries count toward the same turn): `tool_calls` (tool executions that finished), `tool_errors` (of those, how many errored), `failed_tools` (sorted, de-duplicated, comma-separated names of the tools that errored, empty when none; a call to a tool the session doesn't register, which keeps whatever name the model wrote, is reported as `unknown`, so these are only Studio's own tool names or `unknown`), `skills_loaded` (sorted, de-duplicated, comma-separated names of the skills the agent loaded with the `Skill` tool, empty when none; only names of skills Studio ships, so a slash-command start shows here as well as in `studio_code_message_sent`'s `ability_name`), `edits` (successful `Write`/`Edit` calls), `model_requests` (assistant model responses), `input_tokens`/`output_tokens`/`cache_read_tokens`/`cache_write_tokens` (their summed usage, by kind; a long session's tokens are mostly cache reads, so add all four for its size), `error_kind` (only when `outcome=error`: `out_of_credits` — the 402 `studio_out_of_credits` or the monthly cost cap —, `auth`, `model_unavailable`, `rate_limited`, `network`, `other`; classified from the failure, the raw text is never sent), `site_kind` (`local`/`wpcom`/`none`). These props must be registered in the Tracks Registration tool. One per turn finishing. Partially overlaps the MC Stats `recordAgentRun` bump (`packages/common/ai/agent-stats.ts`), which is a bare counter with no model or duration breakdown. |
 | `studio_code_session_created` | Desktop Main (`createAiSession`) | `has_site` (boolean — whether the session is bound to a site; the site name and path are **never** sent). Emitted only when a session is actually created, not when an empty draft is reused. |
 | `studio_code_design_option_proposed` | CLI `pick_design` | `catalog`, `option` (the catalog entry name), `position` (1-based display order), `is_chosen` (boolean — the model picked it itself rather than it being drawn at random; always true for directions), `options_count` (1 when the model settles the entry without asking, up to 4 when the user will pick), `is_redraw` (boolean — a further draw in the same turn after the user asked for other options), `draw_id`. One row **per entry shown**, so a four-option draw is four rows sharing a `draw_id`. |
 | `studio_code_design_option_picked` | CLI `present_design_options` | `catalog` (passed by the model, the same value it gave `pick_design`), `options_count`, `answer_type`, `picked` (the entry name, only when `answer_type=picked`), `pick_index` (1-based position, only when picked). One per question shown. |