diff --git a/apps/cli/ai/tests/turn-stats.test.ts b/apps/cli/ai/tests/turn-stats.test.ts index f1b0b7aea7..4b5ba88429 100644 --- a/apps/cli/ai/tests/turn-stats.test.ts +++ b/apps/cli/ai/tests/turn-stats.test.ts @@ -47,12 +47,53 @@ function propsFor( events: AgentSessionEvent[], outcome: 'success' | 'error' = ' return tracker.getProps( { outcome, site: null, hasTool } ); } +function skillStart( args: unknown ): AgentSessionEvent { + return { + type: 'tool_execution_start', + toolCallId: 'Skill-id', + toolName: 'Skill', + args, + } as AgentSessionEvent; +} + describe( 'createTurnStatsTracker', () => { + it.each( [ + { + name: 'two skills, one loaded twice', + events: [ + skillStart( { name: 'visual-design' } ), + skillStart( { name: 'block-content' } ), + skillStart( { name: 'visual-design' } ), + ], + expected: 'block-content,visual-design', + }, + { + name: 'a name Studio ships no skill for', + events: [ skillStart( { name: '/Users/me/notes' } ), skillStart( {} ) ], + expected: '', + }, + { + name: 'a site named after a skill', + events: [ + { + type: 'tool_execution_start', + toolCallId: 'site_create-id', + toolName: 'site_create', + args: { name: 'liberate' }, + } as AgentSessionEvent, + ], + expected: '', + }, + ] )( 'reports the skills loaded in a turn: $name', ( { events, expected } ) => { + expect( propsFor( events ).skills_loaded ).toBe( expected ); + } ); + it( 'reports zeroes for a turn with no tools or model responses', () => { expect( propsFor( [] ) ).toEqual( { tool_calls: 0, tool_errors: 0, failed_tools: '', + skills_loaded: '', edits: 0, model_requests: 0, input_tokens: 0, diff --git a/apps/cli/ai/turn-stats.ts b/apps/cli/ai/turn-stats.ts index a35ac4c34a..9f1c36f7b7 100644 --- a/apps/cli/ai/turn-stats.ts +++ b/apps/cli/ai/turn-stats.ts @@ -4,6 +4,7 @@ import { isOutOfCreditsError, isUsageCapError, } from '@studio/common/ai/json-events'; +import { findSkill } from 'cli/ai/skills'; import { addAssistantMessageUsage, createTokenUsage } from 'cli/ai/token-usage'; import type { AgentSessionEvent } from '@earendil-works/pi-coding-agent'; import type { TurnStatus } from '@studio/common/ai/sessions/types'; @@ -16,6 +17,8 @@ const FILE_CHANGE_TOOL_NAMES: ReadonlySet< string > = new Set( [ 'Write', 'Edit' const UNKNOWN_TOOL = 'unknown'; +const SKILL_TOOL_NAME = 'Skill'; + export type TurnErrorKind = | 'out_of_credits' | 'auth' @@ -30,6 +33,7 @@ export interface TurnStatsProps { tool_calls: number; tool_errors: number; failed_tools: string; + skills_loaded: string; edits: number; model_requests: number; input_tokens: number; @@ -146,6 +150,7 @@ export function createTurnStatsTracker(): TurnStatsTracker { let toolCalls = 0; let toolErrors = 0; const failedTools = new Set< string >(); + const skillsLoaded = new Set< string >(); let edits = 0; let modelRequests = 0; const usage = createTokenUsage(); @@ -157,6 +162,14 @@ export function createTurnStatsTracker(): TurnStatsTracker { addAssistantMessageUsage( usage, event ); return; } + // Only the name of a skill Studio ships is kept, never other text the model wrote. + if ( event.type === 'tool_execution_start' && event.toolName === SKILL_TOOL_NAME ) { + const name = ( event.args as { name?: unknown } | undefined )?.name; + if ( typeof name === 'string' && findSkill( name ) ) { + skillsLoaded.add( name ); + } + return; + } if ( event.type !== 'tool_execution_end' ) { return; } @@ -180,6 +193,7 @@ export function createTurnStatsTracker(): TurnStatsTracker { tool_calls: toolCalls, tool_errors: toolErrors, failed_tools: [ ...failedToolNames ].sort().join( ',' ), + skills_loaded: [ ...skillsLoaded ].sort().join( ',' ), edits, model_requests: modelRequests, input_tokens: usage.input, diff --git a/docs/design-docs/analytics-tracks.md b/docs/design-docs/analytics-tracks.md index d749899380..057b1091e5 100644 --- a/docs/design-docs/analytics-tracks.md +++ b/docs/design-docs/analytics-tracks.md @@ -470,7 +470,7 @@ AI identity props described under "Property vocabulary" above. Filter by `channe | Event | Emitted from | Event-specific props | |---|---|---| | `studio_code_message_sent` | CLI `runAgentTurn` | `provider` (`wpcom`/`anthropic-api-key` — the gateway serving the request, not the model vendor), `model` (e.g. `claude-sonnet-5`), `model_family` (`anthropic`/`openai`), `ability_name` (predefined skill, absent for an ordinary message), `has_images`, `has_files` (booleans). One per user turn dispatched. | -| `studio_code_turn_completed` | CLI `runAgentTurn` | `outcome` (`success`/`error`/`interrupted`/`max_turns`), `duration_ms`, plus the same `provider`/`model`/`model_family`. What happened inside the turn, accumulated from the agent events by `apps/cli/ai/turn-stats.ts` (attempts that pi retries count toward the same turn): `tool_calls` (tool executions that finished), `tool_errors` (of those, how many errored), `failed_tools` (sorted, de-duplicated, comma-separated names of the tools that errored, empty when none; a call to a tool the session doesn't register, which keeps whatever name the model wrote, is reported as `unknown`, so these are only Studio's own tool names or `unknown`), `edits` (successful `Write`/`Edit` calls), `model_requests` (assistant model responses), `input_tokens`/`output_tokens`/`cache_read_tokens`/`cache_write_tokens` (their summed usage, by kind; a long session's tokens are mostly cache reads, so add all four for its size), `error_kind` (only when `outcome=error`: `out_of_credits` — the 402 `studio_out_of_credits` or the monthly cost cap —, `auth`, `model_unavailable`, `rate_limited`, `network`, `other`; classified from the failure, the raw text is never sent), `site_kind` (`local`/`wpcom`/`none`). These props must be registered in the Tracks Registration tool. One per turn finishing. Partially overlaps the MC Stats `recordAgentRun` bump (`packages/common/ai/agent-stats.ts`), which is a bare counter with no model or duration breakdown. | +| `studio_code_turn_completed` | CLI `runAgentTurn` | `outcome` (`success`/`error`/`interrupted`/`max_turns`), `duration_ms`, plus the same `provider`/`model`/`model_family`. What happened inside the turn, accumulated from the agent events by `apps/cli/ai/turn-stats.ts` (attempts that pi retries count toward the same turn): `tool_calls` (tool executions that finished), `tool_errors` (of those, how many errored), `failed_tools` (sorted, de-duplicated, comma-separated names of the tools that errored, empty when none; a call to a tool the session doesn't register, which keeps whatever name the model wrote, is reported as `unknown`, so these are only Studio's own tool names or `unknown`), `skills_loaded` (sorted, de-duplicated, comma-separated names of the skills the agent loaded with the `Skill` tool, empty when none; only names of skills Studio ships, so a slash-command start shows here as well as in `studio_code_message_sent`'s `ability_name`), `edits` (successful `Write`/`Edit` calls), `model_requests` (assistant model responses), `input_tokens`/`output_tokens`/`cache_read_tokens`/`cache_write_tokens` (their summed usage, by kind; a long session's tokens are mostly cache reads, so add all four for its size), `error_kind` (only when `outcome=error`: `out_of_credits` — the 402 `studio_out_of_credits` or the monthly cost cap —, `auth`, `model_unavailable`, `rate_limited`, `network`, `other`; classified from the failure, the raw text is never sent), `site_kind` (`local`/`wpcom`/`none`). These props must be registered in the Tracks Registration tool. One per turn finishing. Partially overlaps the MC Stats `recordAgentRun` bump (`packages/common/ai/agent-stats.ts`), which is a bare counter with no model or duration breakdown. | | `studio_code_session_created` | Desktop Main (`createAiSession`) | `has_site` (boolean — whether the session is bound to a site; the site name and path are **never** sent). Emitted only when a session is actually created, not when an empty draft is reused. | | `studio_code_design_option_proposed` | CLI `pick_design` | `catalog`, `option` (the catalog entry name), `position` (1-based display order), `is_chosen` (boolean — the model picked it itself rather than it being drawn at random; always true for directions), `options_count` (1 when the model settles the entry without asking, up to 4 when the user will pick), `is_redraw` (boolean — a further draw in the same turn after the user asked for other options), `draw_id`. One row **per entry shown**, so a four-option draw is four rows sharing a `draw_id`. | | `studio_code_design_option_picked` | CLI `present_design_options` | `catalog` (passed by the model, the same value it gave `pick_design`), `options_count`, `answer_type`, `picked` (the entry name, only when `answer_type=picked`), `pick_index` (1-based position, only when picked). One per question shown. |