From 38c1e5725c7f6fab9ccce5fcb5885bb1821411f2 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 27 Oct 2026 11:07:56 -0400 Subject: [PATCH 1/2] fix(pi): align agent instrumentation with tracing spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > šŸ¤– **This text was AI-generated.** Pi instrumentation did not always reflect what happened during a prompt: - Listening to agent events missed tools invoked through the session, so nested tool calls and their application spans could be absent or misplaced. - Tool start and end events could be emitted for a call blocked before its executor ran, but the trace marked that call as approved. - An installed tool wrapper could retain one SDK copy's tracing state after another copy became active, losing context and approval information. - Some error, attachment, usage, and request metadata handling did not fully match the instrumentation specification. Queued streaming prompts could also produce misleading prompt spans. - Several Pi version scenarios used fabricated responses, leaving their behavior against real provider responses unverified. Subscribe to session events and route tool callbacks through a shared invocation hook. This captures the tool lifecycle and places application spans under the tool that created them: ``` prompt ā”œā”€ā”€ model calls ā””ā”€ā”€ tool execution ā””ā”€ā”€ application spans ``` Record approval only when the executor actually runs; leave it unset when execution cannot be observed. Preserve tool context across SDK copies and plugin teardown, and update errors, attachments, usage metrics, and metadata to follow the instrumentation specification. Avoid creating prompt spans for work only queued while Pi is streaming. Replace fabricated responses with recorded provider responses for pinned and latest Pi versions, update the span-tree snapshots, and combine redundant tests while preserving coverage of the instrumentation behavior. --- .../pi-coding-agent-v0-latest.cassette.json | 271 +++++ .../pi-coding-agent-v0.cassette.json | 80 +- .../pi-coding-agent-v081-latest.cassette.json | 269 +++++ .../pi-coding-agent-v081.cassette.json | 269 +++++ .../pi-coding-agent-v1-latest.cassette.json | 120 +- .../pi-coding-agent-v1.cassette.json | 120 +- ...i-coding-agent-v0-auto-hook.span-tree.json | 123 +- ...pi-coding-agent-v0-auto-hook.span-tree.txt | 119 +- ...g-agent-v0-latest-auto-hook.span-tree.json | 139 ++- ...ng-agent-v0-latest-auto-hook.span-tree.txt | 135 ++- ...ing-agent-v0-latest-wrapped.span-tree.json | 138 ++- ...ding-agent-v0-latest-wrapped.span-tree.txt | 134 ++- .../pi-coding-agent-v0-wrapped.span-tree.json | 122 +- .../pi-coding-agent-v0-wrapped.span-tree.txt | 118 +- ...coding-agent-v081-auto-hook.span-tree.json | 133 ++- ...-coding-agent-v081-auto-hook.span-tree.txt | 129 ++- ...agent-v081-latest-auto-hook.span-tree.json | 133 ++- ...-agent-v081-latest-auto-hook.span-tree.txt | 129 ++- ...g-agent-v081-latest-wrapped.span-tree.json | 132 ++- ...ng-agent-v081-latest-wrapped.span-tree.txt | 128 ++- ...i-coding-agent-v081-wrapped.span-tree.json | 132 ++- ...pi-coding-agent-v081-wrapped.span-tree.txt | 128 ++- ...i-coding-agent-v1-auto-hook.span-tree.json | 129 ++- ...pi-coding-agent-v1-auto-hook.span-tree.txt | 125 ++- ...g-agent-v1-latest-auto-hook.span-tree.json | 129 ++- ...ng-agent-v1-latest-auto-hook.span-tree.txt | 125 ++- ...ing-agent-v1-latest-wrapped.span-tree.json | 128 ++- ...ding-agent-v1-latest-wrapped.span-tree.txt | 124 +- .../pi-coding-agent-v1-wrapped.span-tree.json | 128 ++- .../pi-coding-agent-v1-wrapped.span-tree.txt | 124 +- .../assertions.ts | 192 +--- .../scenario.impl.mjs | 55 +- .../scenario.test.ts | 149 +-- .../configs/pi-coding-agent.test.ts | 18 - .../plugins/pi-coding-agent-context.test.ts | 71 +- .../plugins/pi-coding-agent-interceptors.ts | 3 + .../plugins/pi-coding-agent-plugin.test.ts | 1000 ++++++++++++----- .../plugins/pi-coding-agent-plugin.ts | 541 ++++++--- js/src/vendor-sdk-types/pi-coding-agent.ts | 18 +- js/src/wrappers/pi-coding-agent.test.ts | 33 +- 40 files changed, 4973 insertions(+), 1320 deletions(-) create mode 100644 e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v0-latest.cassette.json create mode 100644 e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v081-latest.cassette.json create mode 100644 e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v081.cassette.json diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v0-latest.cassette.json b/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v0-latest.cassette.json new file mode 100644 index 000000000..c328b145b --- /dev/null +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v0-latest.cassette.json @@ -0,0 +1,271 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "5625e915bcb72d05", + "matchKey": "POST api.anthropic.com/v1/messages", + "recordedAt": "2026-10-27T00:29:34.010Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_tokens": 64000, + "messages": [ + { + "content": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-haiku-4-5", + "stream": true, + "system": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- You can inspect PI_* environment variables for current model and session details.\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/run-Y5pgi3/scenarios/pi-coding-agent-instrumentation", + "type": "text" + } + ], + "thinking": { + "type": "disabled" + }, + "tools": [ + { + "cache_control": { + "type": "ephemeral" + }, + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "eager_input_streaming": true, + "input_schema": { + "properties": { + "command": { + "description": "Shell command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": ["command"], + "type": "object" + }, + "name": "bash" + } + ] + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.anthropic.com/v1/messages" + }, + "response": { + "body": { + "chunks": [ + "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CgRoAfADfwQ2T1NQ6BDoJ\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"container\":null,\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1517,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":22,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\",\"speed\":\"standard\"},\"diagnostics\":null}}", + "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01D7Jj2MKzsuduZw8PNWLdnn\",\"name\":\"bash\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", + "event: ping\ndata: {\"type\":\"ping\"}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"command\\\": \\\"printf\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\" pi_tool_\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"ok\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", + "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", + "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"container\":null},\"usage\":{\"input_tokens\":1517,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":57}}", + "event: message_stop\ndata: {\"type\":\"message_stop\"}" + ], + "kind": "sse" + }, + "headers": { + "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", + "anthropic-ratelimit-input-tokens-limit": "10000000", + "anthropic-ratelimit-input-tokens-remaining": "9999000", + "anthropic-ratelimit-input-tokens-reset": "2026-10-27T00:29:33Z", + "anthropic-ratelimit-output-tokens-limit": "2000000", + "anthropic-ratelimit-output-tokens-remaining": "2000000", + "anthropic-ratelimit-output-tokens-reset": "2026-10-27T00:29:33Z", + "anthropic-ratelimit-requests-limit": "20000", + "anthropic-ratelimit-requests-remaining": "19999", + "anthropic-ratelimit-requests-reset": "2026-10-27T00:29:33Z", + "anthropic-ratelimit-tokens-limit": "12000000", + "anthropic-ratelimit-tokens-remaining": "11999000", + "anthropic-ratelimit-tokens-reset": "2026-10-27T00:29:33Z", + "anthropic-workspace-id": "wrkspc_018bFG1FgNVWqx5pthtdGTzS", + "cache-control": "no-cache", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a50d964b4d2539d2-YYZ", + "content-encoding": "gzip", + "content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "content-type": "text/event-stream; charset=utf-8", + "date": "Tue, 27 Oct 2026 00:29:33 GMT", + "request-id": "req_011CgRoAex5LoeYvT3gAoUqs", + "server": "cloudflare", + "server-timing": "x-originResponse;dur=100", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "traceresponse": "00-b5c9279b0bbaf41285c3d43d6567c1de-ba1de76677eca3a6-01", + "vary": "Accept-Encoding", + "x-robots-tag": "none" + }, + "status": 200 + } + }, + { + "callIndex": 1, + "id": "0c74acf5b76aad36", + "matchKey": "POST api.anthropic.com/v1/messages", + "recordedAt": "2026-10-27T00:29:34.622Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_tokens": 64000, + "messages": [ + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "id": "toolu_01D7Jj2MKzsuduZw8PNWLdnn", + "input": { + "command": "printf pi_tool_ok" + }, + "name": "bash", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "cache_control": { + "type": "ephemeral" + }, + "content": "pi_tool_ok", + "is_error": false, + "tool_use_id": "toolu_01D7Jj2MKzsuduZw8PNWLdnn", + "type": "tool_result" + } + ], + "role": "user" + } + ], + "model": "claude-haiku-4-5", + "stream": true, + "system": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- You can inspect PI_* environment variables for current model and session details.\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/run-Y5pgi3/scenarios/pi-coding-agent-instrumentation", + "type": "text" + } + ], + "thinking": { + "type": "disabled" + }, + "tools": [ + { + "cache_control": { + "type": "ephemeral" + }, + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "eager_input_streaming": true, + "input_schema": { + "properties": { + "command": { + "description": "Shell command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": ["command"], + "type": "object" + }, + "name": "bash" + } + ] + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.anthropic.com/v1/messages" + }, + "response": { + "body": { + "chunks": [ + "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CgRoAiZc2juDBqHM7TR7G\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"container\":null,\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1591,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\",\"speed\":\"standard\"},\"diagnostics\":null}}", + "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", + "event: ping\ndata: {\"type\":\"ping\"}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"PI\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"_CODING_\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"AGENT_OK\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"\\n\\nCommand\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" output:\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" `\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"pi\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"_tool_ok\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"`\"}}", + "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", + "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"container\":null},\"usage\":{\"input_tokens\":1591,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":23}}", + "event: message_stop\ndata: {\"type\":\"message_stop\"}" + ], + "kind": "sse" + }, + "headers": { + "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", + "anthropic-ratelimit-input-tokens-limit": "10000000", + "anthropic-ratelimit-input-tokens-remaining": "9999000", + "anthropic-ratelimit-input-tokens-reset": "2026-10-27T00:29:34Z", + "anthropic-ratelimit-output-tokens-limit": "2000000", + "anthropic-ratelimit-output-tokens-remaining": "2000000", + "anthropic-ratelimit-output-tokens-reset": "2026-10-27T00:29:34Z", + "anthropic-ratelimit-requests-limit": "20000", + "anthropic-ratelimit-requests-remaining": "19999", + "anthropic-ratelimit-requests-reset": "2026-10-27T00:29:34Z", + "anthropic-ratelimit-tokens-limit": "12000000", + "anthropic-ratelimit-tokens-remaining": "11999000", + "anthropic-ratelimit-tokens-reset": "2026-10-27T00:29:34Z", + "anthropic-workspace-id": "wrkspc_018bFG1FgNVWqx5pthtdGTzS", + "cache-control": "no-cache", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a50d965048dd39d2-YYZ", + "content-encoding": "gzip", + "content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "content-type": "text/event-stream; charset=utf-8", + "date": "Tue, 27 Oct 2026 00:29:34 GMT", + "request-id": "req_011CgRoAiNSf847kjJfNii7U", + "server": "cloudflare", + "server-timing": "x-originResponse;dur=86", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "traceresponse": "00-413bafc569e115c371d70312b0dd72ee-842c6ca646645f82-01", + "vary": "Accept-Encoding", + "x-robots-tag": "none" + }, + "status": 200 + } + } + ], + "meta": { + "createdAt": "2026-10-27T00:28:35.559Z" + } +} diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v0.cassette.json b/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v0.cassette.json index dfd8c6fe7..bf30d8978 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v0.cassette.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v0.cassette.json @@ -4,7 +4,7 @@ "callIndex": 0, "id": "5625e915bcb72d05", "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-07-06T18:58:08.198Z", + "recordedAt": "2026-10-27T00:29:26.996Z", "request": { "body": { "kind": "json", @@ -31,7 +31,7 @@ "cache_control": { "type": "ephemeral" }, - "text": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /workspace/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-a0beda3ff342f9cd/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /workspace/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-a0beda3ff342f9cd/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /workspace/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-a0beda3ff342f9cd/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\nā”œā”€ā”€ js/ # Main `braintrust` package\nā”œā”€ā”€ integrations/ # Integration packages (@braintrust/*)\nā”œā”€ā”€ e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\nā”œā”€ā”€ docs/ # Docs and reference material\nā””ā”€ā”€ internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** ā€” there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: 2026-07-06\nCurrent working directory: /workspace/e2e/.bt-tmp/run-65wF9N/scenarios/pi-coding-agent-instrumentation", + "text": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: 2026-10-26\nCurrent working directory: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/run-Y5pgi3/scenarios/pi-coding-agent-instrumentation", "type": "text" } ], @@ -71,14 +71,15 @@ "response": { "body": { "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_019WGRRz5ny5C3mCcd36RMbC\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":2261,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":57,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_015FEcLpdNm5dDJCRYpiqTPj\",\"name\":\"bash\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", + "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CgRoA9MLPbh7ZBCTQXMgD\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"container\":null,\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1501,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":21,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\",\"speed\":\"standard\"},\"diagnostics\":null}}", + "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01XSw7Y1eGefio6LgmwKFSw8\",\"name\":\"bash\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", "event: ping\ndata: {\"type\":\"ping\"}", "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"command\\\": \\\"printf pi_tool_ok\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"command\\\": \\\"printf pi_tool\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"_ok\"}}", "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":2261,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":57}}", + "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"container\":null},\"usage\":{\"input_tokens\":1501,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":57}}", "event: message_stop\ndata: {\"type\":\"message_stop\"}" ], "kind": "sse" @@ -87,41 +88,40 @@ "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", "anthropic-ratelimit-input-tokens-limit": "10000000", "anthropic-ratelimit-input-tokens-remaining": "9999000", - "anthropic-ratelimit-input-tokens-reset": "2026-07-06T18:58:07Z", + "anthropic-ratelimit-input-tokens-reset": "2026-10-27T00:29:26Z", "anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000", - "anthropic-ratelimit-output-tokens-reset": "2026-07-06T18:58:07Z", + "anthropic-ratelimit-output-tokens-reset": "2026-10-27T00:29:26Z", "anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-07-06T18:58:07Z", + "anthropic-ratelimit-requests-reset": "2026-10-27T00:29:26Z", "anthropic-ratelimit-tokens-limit": "12000000", "anthropic-ratelimit-tokens-remaining": "11999000", - "anthropic-ratelimit-tokens-reset": "2026-07-06T18:58:07Z", + "anthropic-ratelimit-tokens-reset": "2026-10-27T00:29:26Z", + "anthropic-workspace-id": "wrkspc_018bFG1FgNVWqx5pthtdGTzS", "cache-control": "no-cache", "cf-cache-status": "DYNAMIC", - "cf-ray": "a170d6ca09f8cb7b-VIE", - "connection": "keep-alive", + "cf-ray": "a50d961fb83939d2-YYZ", "content-encoding": "gzip", "content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 06 Jul 2026 18:58:08 GMT", - "request-id": "req_011CcmKJCn7wFd54uu1SxCzF", + "date": "Tue, 27 Oct 2026 00:29:26 GMT", + "request-id": "req_011CgRoA9BAd1zvAWf248Hs2", "server": "cloudflare", + "server-timing": "x-originResponse;dur=89", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-6393f554e55b24340bab14a0b500ace9-0391e1d18f0eb979-01", - "transfer-encoding": "chunked", + "traceresponse": "00-bdb44bc722a8355cf33fdbcb5fae6cd6-60df90060053429f-01", "vary": "Accept-Encoding", "x-robots-tag": "none" }, - "status": 200, - "statusText": "OK" + "status": 200 } }, { "callIndex": 1, "id": "0c74acf5b76aad36", "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-07-06T18:58:09.570Z", + "recordedAt": "2026-10-27T00:29:27.658Z", "request": { "body": { "kind": "json", @@ -140,7 +140,7 @@ { "content": [ { - "id": "toolu_015FEcLpdNm5dDJCRYpiqTPj", + "id": "toolu_01XSw7Y1eGefio6LgmwKFSw8", "input": { "command": "printf pi_tool_ok" }, @@ -158,7 +158,7 @@ }, "content": "pi_tool_ok", "is_error": false, - "tool_use_id": "toolu_015FEcLpdNm5dDJCRYpiqTPj", + "tool_use_id": "toolu_01XSw7Y1eGefio6LgmwKFSw8", "type": "tool_result" } ], @@ -172,7 +172,7 @@ "cache_control": { "type": "ephemeral" }, - "text": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /workspace/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-a0beda3ff342f9cd/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /workspace/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-a0beda3ff342f9cd/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /workspace/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-a0beda3ff342f9cd/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\nā”œā”€ā”€ js/ # Main `braintrust` package\nā”œā”€ā”€ integrations/ # Integration packages (@braintrust/*)\nā”œā”€ā”€ e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\nā”œā”€ā”€ docs/ # Docs and reference material\nā””ā”€ā”€ internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** ā€” there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: 2026-07-06\nCurrent working directory: /workspace/e2e/.bt-tmp/run-65wF9N/scenarios/pi-coding-agent-instrumentation", + "text": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: 2026-10-26\nCurrent working directory: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/run-Y5pgi3/scenarios/pi-coding-agent-instrumentation", "type": "text" } ], @@ -212,13 +212,20 @@ "response": { "body": { "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_01XGoGZGY6NLAfyzKALmg6nJ\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":2335,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", + "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CgRoACeXDKGoyf2QJB3rN\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"container\":null,\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1575,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\",\"speed\":\"standard\"},\"diagnostics\":null}}", "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", "event: ping\ndata: {\"type\":\"ping\"}", "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"PI\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"_CODING_AGENT_OK\\n\\nCommand output: `pi_tool_ok`\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"_CODING_\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"AGENT_OK\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"\\n\\nCommand\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" output:\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" `\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"pi\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"_tool_ok\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"`\"}}", "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":2335,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":23}}", + "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"container\":null},\"usage\":{\"input_tokens\":1575,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":23}}", "event: message_stop\ndata: {\"type\":\"message_stop\"}" ], "kind": "sse" @@ -227,34 +234,33 @@ "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", "anthropic-ratelimit-input-tokens-limit": "10000000", "anthropic-ratelimit-input-tokens-remaining": "9999000", - "anthropic-ratelimit-input-tokens-reset": "2026-07-06T18:58:08Z", + "anthropic-ratelimit-input-tokens-reset": "2026-10-27T00:29:27Z", "anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000", - "anthropic-ratelimit-output-tokens-reset": "2026-07-06T18:58:08Z", + "anthropic-ratelimit-output-tokens-reset": "2026-10-27T00:29:27Z", "anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-07-06T18:58:08Z", + "anthropic-ratelimit-requests-reset": "2026-10-27T00:29:27Z", "anthropic-ratelimit-tokens-limit": "12000000", "anthropic-ratelimit-tokens-remaining": "11999000", - "anthropic-ratelimit-tokens-reset": "2026-07-06T18:58:08Z", + "anthropic-ratelimit-tokens-reset": "2026-10-27T00:29:27Z", + "anthropic-workspace-id": "wrkspc_018bFG1FgNVWqx5pthtdGTzS", "cache-control": "no-cache", "cf-cache-status": "DYNAMIC", - "cf-ray": "a170d6d20dbdcb7b-VIE", - "connection": "keep-alive", + "cf-ray": "a50d96248b0539d2-YYZ", "content-encoding": "gzip", "content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 06 Jul 2026 18:58:08 GMT", - "request-id": "req_011CcmKJJ66tPfgsxWNTwjup", + "date": "Tue, 27 Oct 2026 00:29:27 GMT", + "request-id": "req_011CgRoACSNnevgc42U73tok", "server": "cloudflare", + "server-timing": "x-originResponse;dur=91", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-cd3deb39b87618ac8715904e54233712-85010038df6bc187-01", - "transfer-encoding": "chunked", + "traceresponse": "00-f857f803164c581875ea43900f1e95fb-dad4e44465b0cff9-01", "vary": "Accept-Encoding", "x-robots-tag": "none" }, - "status": 200, - "statusText": "OK" + "status": 200 } } ], diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v081-latest.cassette.json b/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v081-latest.cassette.json new file mode 100644 index 000000000..c15a5c20f --- /dev/null +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v081-latest.cassette.json @@ -0,0 +1,269 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "5625e915bcb72d05", + "matchKey": "POST api.anthropic.com/v1/messages", + "recordedAt": "2026-10-27T00:29:47.051Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_tokens": 64000, + "messages": [ + { + "content": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-haiku-4-5", + "stream": true, + "system": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/run-Y5pgi3/scenarios/pi-coding-agent-instrumentation", + "type": "text" + } + ], + "thinking": { + "type": "disabled" + }, + "tools": [ + { + "cache_control": { + "type": "ephemeral" + }, + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "eager_input_streaming": true, + "input_schema": { + "properties": { + "command": { + "description": "Bash command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": ["command"], + "type": "object" + }, + "name": "bash" + } + ] + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.anthropic.com/v1/messages" + }, + "response": { + "body": { + "chunks": [ + "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CgRoBcprVdjxx3eTrUun6\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"container\":null,\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1490,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":21,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\",\"speed\":\"standard\"},\"diagnostics\":null}}", + "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01BjLXaEfTY4AuyNnEoaq3gn\",\"name\":\"bash\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", + "event: ping\ndata: {\"type\":\"ping\"}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"command\\\": \\\"printf pi_tool\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"_ok\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", + "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", + "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"container\":null},\"usage\":{\"input_tokens\":1490,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":57}}", + "event: message_stop\ndata: {\"type\":\"message_stop\"}" + ], + "kind": "sse" + }, + "headers": { + "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", + "anthropic-ratelimit-input-tokens-limit": "10000000", + "anthropic-ratelimit-input-tokens-remaining": "9999000", + "anthropic-ratelimit-input-tokens-reset": "2026-10-27T00:29:46Z", + "anthropic-ratelimit-output-tokens-limit": "2000000", + "anthropic-ratelimit-output-tokens-remaining": "2000000", + "anthropic-ratelimit-output-tokens-reset": "2026-10-27T00:29:46Z", + "anthropic-ratelimit-requests-limit": "20000", + "anthropic-ratelimit-requests-remaining": "19999", + "anthropic-ratelimit-requests-reset": "2026-10-27T00:29:46Z", + "anthropic-ratelimit-tokens-limit": "12000000", + "anthropic-ratelimit-tokens-remaining": "11999000", + "anthropic-ratelimit-tokens-reset": "2026-10-27T00:29:46Z", + "anthropic-workspace-id": "wrkspc_018bFG1FgNVWqx5pthtdGTzS", + "cache-control": "no-cache", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a50d969c98e439d2-YYZ", + "content-encoding": "gzip", + "content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "content-type": "text/event-stream; charset=utf-8", + "date": "Tue, 27 Oct 2026 00:29:46 GMT", + "request-id": "req_011CgRoBcaiKikXLVoLoEwCD", + "server": "cloudflare", + "server-timing": "x-originResponse;dur=97", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "traceresponse": "00-5c3803118c4730505578cd212f6cd4b5-da6c8a7399c5d8b2-01", + "vary": "Accept-Encoding", + "x-robots-tag": "none" + }, + "status": 200 + } + }, + { + "callIndex": 1, + "id": "0c74acf5b76aad36", + "matchKey": "POST api.anthropic.com/v1/messages", + "recordedAt": "2026-10-27T00:29:47.660Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_tokens": 64000, + "messages": [ + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "id": "toolu_01BjLXaEfTY4AuyNnEoaq3gn", + "input": { + "command": "printf pi_tool_ok" + }, + "name": "bash", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "cache_control": { + "type": "ephemeral" + }, + "content": "pi_tool_ok", + "is_error": false, + "tool_use_id": "toolu_01BjLXaEfTY4AuyNnEoaq3gn", + "type": "tool_result" + } + ], + "role": "user" + } + ], + "model": "claude-haiku-4-5", + "stream": true, + "system": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/run-Y5pgi3/scenarios/pi-coding-agent-instrumentation", + "type": "text" + } + ], + "thinking": { + "type": "disabled" + }, + "tools": [ + { + "cache_control": { + "type": "ephemeral" + }, + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "eager_input_streaming": true, + "input_schema": { + "properties": { + "command": { + "description": "Bash command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": ["command"], + "type": "object" + }, + "name": "bash" + } + ] + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.anthropic.com/v1/messages" + }, + "response": { + "body": { + "chunks": [ + "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CgRoBgNR4wNZe9UdQehi2\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"container\":null,\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1564,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\",\"speed\":\"standard\"},\"diagnostics\":null}}", + "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", + "event: ping\ndata: {\"type\":\"ping\"}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"PI\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"_CODING_\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"AGENT_OK\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"\\n\\nCommand\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" output:\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" `\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"pi_tool_\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"ok`\"}}", + "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", + "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"container\":null},\"usage\":{\"input_tokens\":1564,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":23}}", + "event: message_stop\ndata: {\"type\":\"message_stop\"}" + ], + "kind": "sse" + }, + "headers": { + "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", + "anthropic-ratelimit-input-tokens-limit": "10000000", + "anthropic-ratelimit-input-tokens-remaining": "9999000", + "anthropic-ratelimit-input-tokens-reset": "2026-10-27T00:29:47Z", + "anthropic-ratelimit-output-tokens-limit": "2000000", + "anthropic-ratelimit-output-tokens-remaining": "2000000", + "anthropic-ratelimit-output-tokens-reset": "2026-10-27T00:29:47Z", + "anthropic-ratelimit-requests-limit": "20000", + "anthropic-ratelimit-requests-remaining": "19999", + "anthropic-ratelimit-requests-reset": "2026-10-27T00:29:47Z", + "anthropic-ratelimit-tokens-limit": "12000000", + "anthropic-ratelimit-tokens-remaining": "11999000", + "anthropic-ratelimit-tokens-reset": "2026-10-27T00:29:47Z", + "anthropic-workspace-id": "wrkspc_018bFG1FgNVWqx5pthtdGTzS", + "cache-control": "no-cache", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a50d96a1cd1039d2-YYZ", + "content-encoding": "gzip", + "content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "content-type": "text/event-stream; charset=utf-8", + "date": "Tue, 27 Oct 2026 00:29:47 GMT", + "request-id": "req_011CgRoBg92UVnqU8NJCkz9z", + "server": "cloudflare", + "server-timing": "x-originResponse;dur=105", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "traceresponse": "00-6faa290eac7b93f25af08fe538f019b4-6e0d99feba2cdc10-01", + "vary": "Accept-Encoding", + "x-robots-tag": "none" + }, + "status": 200 + } + } + ], + "meta": { + "createdAt": "2026-10-27T00:28:40.210Z" + } +} diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v081.cassette.json b/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v081.cassette.json new file mode 100644 index 000000000..4ac06b02b --- /dev/null +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v081.cassette.json @@ -0,0 +1,269 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "5625e915bcb72d05", + "matchKey": "POST api.anthropic.com/v1/messages", + "recordedAt": "2026-10-27T00:29:39.615Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_tokens": 64000, + "messages": [ + { + "content": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-haiku-4-5", + "stream": true, + "system": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/run-Y5pgi3/scenarios/pi-coding-agent-instrumentation", + "type": "text" + } + ], + "thinking": { + "type": "disabled" + }, + "tools": [ + { + "cache_control": { + "type": "ephemeral" + }, + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "eager_input_streaming": true, + "input_schema": { + "properties": { + "command": { + "description": "Bash command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": ["command"], + "type": "object" + }, + "name": "bash" + } + ] + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.anthropic.com/v1/messages" + }, + "response": { + "body": { + "chunks": [ + "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CgRoB5EKhUC27bPHCWqda\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"container\":null,\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1490,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":21,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\",\"speed\":\"standard\"},\"diagnostics\":null}}", + "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01RqWb7uUvhPmfKf6Bq4ox5x\",\"name\":\"bash\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", + "event: ping\ndata: {\"type\":\"ping\"}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"command\\\": \\\"printf pi_tool\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"_ok\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", + "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", + "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"container\":null},\"usage\":{\"input_tokens\":1490,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":57}}", + "event: message_stop\ndata: {\"type\":\"message_stop\"}" + ], + "kind": "sse" + }, + "headers": { + "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", + "anthropic-ratelimit-input-tokens-limit": "10000000", + "anthropic-ratelimit-input-tokens-remaining": "9999000", + "anthropic-ratelimit-input-tokens-reset": "2026-10-27T00:29:39Z", + "anthropic-ratelimit-output-tokens-limit": "2000000", + "anthropic-ratelimit-output-tokens-remaining": "2000000", + "anthropic-ratelimit-output-tokens-reset": "2026-10-27T00:29:39Z", + "anthropic-ratelimit-requests-limit": "20000", + "anthropic-ratelimit-requests-remaining": "19999", + "anthropic-ratelimit-requests-reset": "2026-10-27T00:29:39Z", + "anthropic-ratelimit-tokens-limit": "12000000", + "anthropic-ratelimit-tokens-remaining": "11999000", + "anthropic-ratelimit-tokens-reset": "2026-10-27T00:29:39Z", + "anthropic-workspace-id": "wrkspc_018bFG1FgNVWqx5pthtdGTzS", + "cache-control": "no-cache", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a50d966e6b2f39d2-YYZ", + "content-encoding": "gzip", + "content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "content-type": "text/event-stream; charset=utf-8", + "date": "Tue, 27 Oct 2026 00:29:39 GMT", + "request-id": "req_011CgRoB52QrAnDGe3mfofFm", + "server": "cloudflare", + "server-timing": "x-originResponse;dur=97", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "traceresponse": "00-997927d3a39357f6267a956b016843f4-e15cace116a03d92-01", + "vary": "Accept-Encoding", + "x-robots-tag": "none" + }, + "status": 200 + } + }, + { + "callIndex": 1, + "id": "0c74acf5b76aad36", + "matchKey": "POST api.anthropic.com/v1/messages", + "recordedAt": "2026-10-27T00:29:40.306Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_tokens": 64000, + "messages": [ + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "id": "toolu_01RqWb7uUvhPmfKf6Bq4ox5x", + "input": { + "command": "printf pi_tool_ok" + }, + "name": "bash", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "cache_control": { + "type": "ephemeral" + }, + "content": "pi_tool_ok", + "is_error": false, + "tool_use_id": "toolu_01RqWb7uUvhPmfKf6Bq4ox5x", + "type": "tool_result" + } + ], + "role": "user" + } + ], + "model": "claude-haiku-4-5", + "stream": true, + "system": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-4f3d5e05469eaabc/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/run-Y5pgi3/scenarios/pi-coding-agent-instrumentation", + "type": "text" + } + ], + "thinking": { + "type": "disabled" + }, + "tools": [ + { + "cache_control": { + "type": "ephemeral" + }, + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "eager_input_streaming": true, + "input_schema": { + "properties": { + "command": { + "description": "Bash command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": ["command"], + "type": "object" + }, + "name": "bash" + } + ] + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.anthropic.com/v1/messages" + }, + "response": { + "body": { + "chunks": [ + "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CgRoB8p7tu3WfibUMqmjY\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"container\":null,\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1564,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\",\"speed\":\"standard\"},\"diagnostics\":null}}", + "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", + "event: ping\ndata: {\"type\":\"ping\"}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"PI\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"_CODING_\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"AGENT_OK\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"\\n\\nCommand\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" output:\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" `\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"pi_tool_\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"ok`\"}}", + "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", + "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"container\":null},\"usage\":{\"input_tokens\":1564,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":23}}", + "event: message_stop\ndata: {\"type\":\"message_stop\"}" + ], + "kind": "sse" + }, + "headers": { + "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", + "anthropic-ratelimit-input-tokens-limit": "10000000", + "anthropic-ratelimit-input-tokens-remaining": "9999000", + "anthropic-ratelimit-input-tokens-reset": "2026-10-27T00:29:39Z", + "anthropic-ratelimit-output-tokens-limit": "2000000", + "anthropic-ratelimit-output-tokens-remaining": "2000000", + "anthropic-ratelimit-output-tokens-reset": "2026-10-27T00:29:39Z", + "anthropic-ratelimit-requests-limit": "20000", + "anthropic-ratelimit-requests-remaining": "19999", + "anthropic-ratelimit-requests-reset": "2026-10-27T00:29:39Z", + "anthropic-ratelimit-tokens-limit": "12000000", + "anthropic-ratelimit-tokens-remaining": "11999000", + "anthropic-ratelimit-tokens-reset": "2026-10-27T00:29:39Z", + "anthropic-workspace-id": "wrkspc_018bFG1FgNVWqx5pthtdGTzS", + "cache-control": "no-cache", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a50d9673cf6939d2-YYZ", + "content-encoding": "gzip", + "content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "content-type": "text/event-stream; charset=utf-8", + "date": "Tue, 27 Oct 2026 00:29:40 GMT", + "request-id": "req_011CgRoB8eT7JCu4c9NijW8T", + "server": "cloudflare", + "server-timing": "x-originResponse;dur=89", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "traceresponse": "00-5d85db2af12f54ae2b37d82de347746a-f0a346c56583ae0c-01", + "vary": "Accept-Encoding", + "x-robots-tag": "none" + }, + "status": 200 + } + } + ], + "meta": { + "createdAt": "2026-10-27T00:28:37.931Z" + } +} diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v1-latest.cassette.json b/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v1-latest.cassette.json index 81842a2f5..b95737f67 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v1-latest.cassette.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v1-latest.cassette.json @@ -2,9 +2,9 @@ "entries": [ { "callIndex": 0, - "id": "9d4c6e540841b9c5", + "id": "732705731b290aa3", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-10-26T13:06:16.355Z", + "recordedAt": "2026-10-27T00:29:58.839Z", "request": { "body": { "kind": "json", @@ -12,7 +12,7 @@ "include": ["reasoning.encrypted_content"], "input": [ { - "content": "You are a coding assistant. Use tools when requested.\n\n\n/Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/resolve-issue-red-green-73219c51/e2e/.bt-tmp/run-0w90vN/scenarios/pi-coding-agent-instrumentation\n", + "content": "You are a coding assistant. Use tools when requested.\n\n\n/Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/run-Y5pgi3/scenarios/pi-coding-agent-instrumentation\n", "role": "developer" }, { @@ -73,27 +73,27 @@ "response": { "body": { "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0b86321b42b1ad5e016adf5047b89487d1aa5c6e63418ce6f2\",\"object\":\"response\",\"created_at\":1793019975,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17851-9740-712e-af1b-3238a14569b1\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0b86321b42b1ad5e016adf5047b89487d1aa5c6e63418ce6f2\",\"object\":\"response\",\"created_at\":1793019975,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17851-9740-712e-af1b-3238a14569b1\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"rs_0b86321b42b1ad5e016adf50481f5487d1816244e8f64a915c\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq31BIw6YQtpfi2G8QHtq0DFuRnEcQbf3XHRA5qhUTYjmLoVmcH4NdHW4t-KYmk_KU80_Dcls3k863t3oYOfMiHrMm84gE4k3EMeyHayRGJWNxoGEDmp1mNDGZrQdjfQKUTwkkbx9wB260m_X5Vywp3uKKzOiX_PmLASgUGw8voAYXZR6lBU0HuPy6PCkPsvwlVPhzesfTfTmz5BtNmHK67Q_0hGmG_rmRW12I0kMS3fccfUByXxIODmed0VpyiOC4pQl0W-1o-wyNpmcqY3QJ0hGACWW2SQkIAPePtCvHmuTZPTt0KTBnPolt_OYPDjwr1_RFlzoZEJdWK2D9aHzpho0Mn6-6ke3VsK6WSp_xutBvO9WS-jvfJeI4lC6Lpijb-va1xyq3T3AmK56Cz8auqhhHCqbqLxX092YVfpag9DO-aUmN_MX7JHeVuIWNJDQRe5OAICs7aJNyi4I6L-SK1ThMLtKpzXorrAu30OIimVj5od_OCv9Y9vhLnyJp3NrPp6ApfON93Vy_L9d6FUk92B6IgscBC_kWKD4LCZqePaBFOjLXyWdrVAFMTcby7hIdfwcLz4hu285oOCVCfXma1OjSET23-CGb4e-dova5sc-RhiZp4nzNDSACH5AZI0Un35Jyu8zCBk-g9G4kMQjeViiL6i7qXhOb7qPV587yye6j1VM_ZPG65lM2i_IY49sYIso59tRz25dp0Kn86CvgJVs0Hl0kFwukSTrs3lacCBx1mAhPMikqnpDaTW4MUZqPpms6EaphwMu9VEMZpDt8teT0RhQUfA1aW2dw1LliuUoxhXDX-bvs_x8JIonmOm5SWuvMUJIkrDYcRm1wMcLG_8OYV_Tlc3rGY9OChXvjwn9K5estEEPUKZK6i4yDdmz6QvnpPa-_0ZLuD4j_Jln-_IAB1XBRv_YLTctpLHIKSqRkdfCQNrUQsgdu6BuSCLhry5b415bvttFtyMeQIXD6TtHz_x5ULZF08Ke7hSs0AHbRBbI6IjqpWkgtDknIcrEDWRspj1siH_0znlNPIdrcz4NStt0coArqYSZeopx6WHcfIUnuwZvqLoxhWNlA3XkxkItRhhbMRiSAPS_ojTKOa2ng5fFcJ5WpK-MWcfVTJEHZ76AlQzKUe4fedgYRZCYIt3pFHjaPpRKhZjssIoSfkhV0sr-kWWz_I9dbwDC_d2cpVM_xALW6o296n-gpJto6C3K4\",\"summary\":[]},\"output_index\":0,\"sequence_number\":2}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"rs_0b86321b42b1ad5e016adf50481f5487d1816244e8f64a915c\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq31BINzpCqufGcEDBVQP8Fu3tSWIRjzs5PDUJjv1SqAMfzU3nYNbY_fPrw0V9_BBHyACHQhti4OEFUUoDCcuIs7yNUQmGaI5m8Xb31RTuxxKCe1kql8tXw5HVxfEv_a7RyvaWpa_20lh_KHLONOWRHh_do-n2SuBhma0wBeFTFmkdk4NSGoNgwmKP0r0QDHxbbVPwXfJpZQQXtbg4WuY2v1NCE2XgNCdxkioHp4Ye20aESCyLJnQHCrk2vAV68kJV9YRqe5sNiSXIvO-27Sn8_RJrXlozmXZ4_XWAQNpr26D_b5vmz6yJKVQ6zx4jErnaPacl5LzKG1qyCn94PpScB1HNTNnsMf6qqQjSgY6-auvIFaKUZBQEG29yh5NPP0030erc_CJdWrBO1SLdB5_u-Uz6dadsyOcz4NqzqMMFWs_j7OGsh2AjrDiKbbAUbDEen-CgBFHaev_aVaibzViP_F_TwL2Z1OeVlH1wpXqtn5MVhNe-RikxsDx4hZh3kza0nWtZUzwqnwhzEe_B3hcBSArbQOjfBwIHCq7gZpSPq5vBhaN8sIP98_YoS8E-QCZZJ14pmWWNxLahMhRoFE2zT55Ye83R-l0h3SjEjyGROzb__D-eUWC58ldwOLLoQMSlg1uk7PBl2NvWrpc_gVMPz4sOUOoeAyAViZFpUy_S6v3b0nIlobBrZWjBfloJwuqRz7aRzRC_8VMImdWKX7ESn1B8mIRmiaSZGcprh48ACX_-kI9jl-35Y3j8jcUlwtbmR7XfKDldZ7DWbXeIAqtX2p3WFzu_gFcHlDY0v-HUSJ1rSQm-L5zeEpcw6tHOwIJykuT_FirVMp9jW8MGAYHkjUjBg8DBrkyZWuGdDEDz7DsjkNK5i3vdO4eSvGVt2Mo9H9qs7KRN66tbIYRfFhI1Sc3oC7DZwwOAk9nEWZFfdvKnU83FkB_l2VB7iU5QHalYzKiifgDIQgK6qRCQ_SL2TEfBKecO2rKlGLKp5gNvzL7HP0oU7F7bc2AMJsAvxiKDypeXqFIUayP0Top5VLHeFTAraL_t34_KTiJGO1GZb3IBUyHqWLsZtGuWfSvnlicR64rN4leBNxKu-wqCZqMEVqAqdmydzaE0lq6caxSqxBmz68-5B5UIiGfarNWnF-KJU-Xm6oV1G0EBhF1mbUOxCcx2YUPCOYog0AkONfENVS-bCZV4nytMqI--XxfNGYAMBnR9ZyIO6Qc-nsPlsWc443FA8DPV1OYv3fYiWGg5DSJMEKDuFweJXCTBqzRPqFBoK5oxsvBlG0Q3VUFuVsWjEfTI7D2XmzOhh3mg9kE1H2XT879A_FzMmMG4IA8UwaEniKJ1HDzlaCKbuOSv0B0nLqjOsNhYMVcF65NZwlNg7pxs7cIDvkZL_AQiW4bDL7qJkRKKKY801Hw-JcMbTu-sAWTio35Gb9HzM6bynj_RiAECjbetVi7kPWJqk9Aq0JJb5Zz8QfnBKwSQego34a-SyteeB3vTPKhgFl237dcT2qPHIPhYhCHoZu1ZecHHUIc-tDGIquUzDZcDBo4oq4WfTaPwo0DHZHcV5oND-rdbUOLCJ8pmC7FQ5xwlsnSggbz33SxLMlFdMBdpAf9RRumEvZ38NncAMITJ7z4FfdlIYT9UDp0=\",\"summary\":[]},\"output_index\":0,\"sequence_number\":3}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_nfMLwtliP8Lmn5SMjdlCMUVh\",\"name\":\"bash\"},\"output_index\":1,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"obfuscation\":\"OkCA4IozQyJIvd\",\"output_index\":1,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"obfuscation\":\"SXAD8400i\",\"output_index\":1,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"obfuscation\":\"1YrzoXhwSvArk\",\"output_index\":1,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"printf\",\"item_id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"obfuscation\":\"61wpGCRCPK\",\"output_index\":1,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" pi\",\"item_id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"obfuscation\":\"2vA3Qx4muAon4\",\"output_index\":1,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_tool\",\"item_id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"obfuscation\":\"RLtMmbsddmd\",\"output_index\":1,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_ok\",\"item_id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"obfuscation\":\"6BDZ6RvOpWNDf\",\"output_index\":1,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"obfuscation\":\"h2SlIT7FrHqsN\",\"output_index\":1,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"obfuscation\":\"gi78AMYVz\",\"output_index\":1,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"obfuscation\":\"5jMn3I4oYZsVq4\",\"output_index\":1,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"120\",\"item_id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"obfuscation\":\"aQLc1hbJPoZPY\",\"output_index\":1,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"obfuscation\":\"vY9zGHQhB5IR4\",\"output_index\":1,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"obfuscation\":\"GwnErSdi4gwiUGu\",\"output_index\":1,\"sequence_number\":17}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"printf pi_tool_ok\\\",\\\"timeout\\\":120000}\",\"item_id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"output_index\":1,\"sequence_number\":18}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"printf pi_tool_ok\\\",\\\"timeout\\\":120000}\",\"call_id\":\"call_nfMLwtliP8Lmn5SMjdlCMUVh\",\"name\":\"bash\"},\"output_index\":1,\"sequence_number\":19}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0b86321b42b1ad5e016adf5047b89487d1aa5c6e63418ce6f2\",\"object\":\"response\",\"created_at\":1793019975,\"status\":\"completed\",\"access_programs\":null,\"background\":false,\"completed_at\":1793019976,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[{\"id\":\"rs_0b86321b42b1ad5e016adf50481f5487d1816244e8f64a915c\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq31BIUG469iwwdidwtn8QS6FDysrMIxGb0oGY2m98hbLfBZi-UhRU-4X_sfchJMxJH3Itq08C45iARupOSOx3RLQiRbXDeIldt8tqaGovbq1-ZkkUPROfatux_pVsEDyOja1FOdVfi0Cg2_VYqx22FvPYJrN8wpp_AEX173ulQ_c_kxXoy4yNQUsSpujNEQ9kcVlkr3L6j60pS8K0m_zlTgavsxxTq6l3NOl6yc9GMCnxkwmI-V3XSZtvM_hkmYd2Ls_xmLEfHTuQqtq0g3wjo2e7pY2VfzxNmfIlUK2BSFd9I2GhigMaPmxcP1mT9fwP319ycU5VBAjXFwNHUmMmuep5ODu4FGAtCP1fUoqb1J-BN4mOY4qTbDA1-66zbqdthW_yLeq48xIm1FYLA-TuU5LxismDPBqI92PsfNjmVvjgPBQPXv6Qjuz3SKNCaoWF7t1VfCU1tLINPue3r3rvNWuoguGfXt9btC_um6LJr0ze8UBaOH53VxmZRBpqxpxGM93B_AlCZXDLcNOwfTMkvig-B421YA8fA9PqAa4rocnVSiG1Z4cgp5ZOpzlewIB9QNXyyekaYERSk5rly1hj8B-prVhmGqWgh8yYdrCo-rA0U0leJisfc-ak-pVlYadBNos2WBKwyOTAESL6jfwtNFSn1b1dKeho6Zc79zRQ5tyB568rnkuU4JIH3Qcgc6d1KrcYaGk6d7s0YUIy6zYP68d3BbtjEvY3gAdyKOSS6MOwi5GdTQyeEXTt-ZeyEGrXFm2yjtVimyAy4kQ5npcM5yO5YMXSTOgPLnTWgAT2OzcSpRP2lnJO7QMRFnt6T6cdn_TnG64dudRERZ0SNBG3ibPs-dpq_tDma0LRqcpKakvg3MPGfNs0Ple3L8kpwbovtiofdH2cBe6FT_ikR82u5sIEQv3OzZ9o4y1fSYNXtgvqex8os9CKyyg4nGz3IKYF1xFTyD7fRGI4lrjoqllNCgSne3Lh2P77u0cw8MR-BfF_w9t-86MknxP63pSQ1G71Nvq9-V5LpFKQfgd-uNuCgVm8cIz12OOK0PPFdTvT5k8JHxyr9ApqsaH2VFzu0yREvMZtuoyTkER1Xs0n4-lElg9meI9mUyy6H3nVxwPJaFzdGCIuhspy04G4DgiHC9xVC44TRPnMhugzoeBhYLS4-K9SK5N6mWlJqqMYXn7yuqhwc2rPvHEM1QzrDhrAxCwNAOIa9AdfagiyzQ9Cfb9sbLkDXn2JxEYvUy5hIWUIE6W_mVqSXEmDBKTjCOD1lOLb-0bTKqIEdEY7CEn7AiayjxCbojl0e9l8xziNJSAFtyB0FHXEir9hMH6hPejDR6GkpgMS5-RJb_FvLNzZFewtLVUsEhdrfE4Uhr36Tbx-RaP7-O-tS7XG8LLO08jaDX-Nsc9LDPYURV9P8MmyKGDhH_zN3o0wiL71LT9_1JfJHHhFyU2rImynY3gzz06DoPf_QH_pfq5gfumqyMFhIgVKh3euKOFQacYYx-azi5prFhtbHxONgTxdyBofcF3PqNlCgls5HCL6XBH5abVuWU5BmBBSbFYcsGrVRjgqmuPawBesWIC5FaD8JVb-2L7Soswr-OIaXM1gdob91Zu85balTqKPqUrVCNLkUMrmSBqpmRZNB80=\",\"summary\":[]},{\"id\":\"fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"printf pi_tool_ok\\\",\\\"timeout\\\":120000}\",\"call_id\":\"call_nfMLwtliP8Lmn5SMjdlCMUVh\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17851-9740-712e-af1b-3238a14569b1\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":221,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":40,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":261},\"user\":null,\"metadata\":{}},\"sequence_number\":20}" + "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_05ec8b9b7b48b396016adff08600c487d198fd989c208153d7\",\"object\":\"response\",\"created_at\":1793060998,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17ac3-8a7d-75fe-8745-2b1913a10dce\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", + "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_05ec8b9b7b48b396016adff08600c487d198fd989c208153d7\",\"object\":\"response\",\"created_at\":1793060998,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17ac3-8a7d-75fe-8745-2b1913a10dce\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"rs_05ec8b9b7b48b396016adff0868d8887d1b9d77672e79da5c1\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq3_CGWNAjCDsCGS7pPm2oYnAaoJ6rfTwQAP6UXMfvF6_MDVzEiMb8Z2E20CmveDqGLh3o1UCko2kg6h_VOTvQbvQBbBFlATSLApP9oLAOl3gFE9rVNupCPFvQg8KEf0GH0_H1N1InaI2W-6Q-1jLTVk-4b3s7-uRf3KM4UVguURI-FZvbCLkdAzDH8Dn-mt6Yx6BHCHgrbYXCQSg9PYmUB9IHyTVEmvhM-Y9YBy0JhR59gVur9qFm0KqEdxQcDsXCPp8QVCa8SqDN9IjbfnkWwjxV_oE-ZfAf_wQ3C9i0J0WbUfkRXip674HAs3p4dwHHZOuZkxyfqvmYvBiKCcHqZVWDT-kbr4plKXPd0O97S7MlHM4P3o4mhq9NwMFyCqmqCIPMgAh3gTQD-YCR2WrPamqzv-sUEmIP-tq6v251x8R6QGJ_eZ6lzoM_fPmTEPOaUYXvejYdTARcliodX6YQetf2WetmJKdfz2V3S0Gi_izn24R-KP7cRU_ir4lYym5DEbW7UU1jHMc761Uj7yrFEK8dlyhx9OpPnUz9din3AE_JPXbutPbLu-3v5_3rusmg1VwbbPzGljiXa64Bee2FRjFF1-yszd1bpnehjYSeVPpMq2baXo8DUmpMETD6SfVDs5AiQphn1M8vqjj4PMDmHNBxXG3qoHR57T9n2FJ-yE6KfzwNG-zntFeYm_lDx-PcchSB1Mwc5js7pvJijYP3esQ4zJYm0iYXWe66kw3wIULlI6IL55YCewIE6NEerRaHzBgupCK9SvM7PToyfaHrjORJZpyx0Od9iL8xjg4C3sO8CaTL0I65FoOe-GJ2-QIiXwZee2O4HD7Tbwg4YlQS11uEp40q4iz1DuWYXPkfFUtX8igmarbfpTZr8b1Q9n1plL7raWOT2Z2ycvv69nAUVtx2L84uaxyrqH5WAr1Rc1xosBiPxomur8HGcZBcc2mWQxUIVohYnAnErqNkUkaDgjdQE3a805Cmfc1d4p1Ry0orSJYdIUTyG-qLvLbSvzyuQwfoZdr9Ka8Kj6Ihhpx1OYJ9Cn24sNLBWKfeRAENCHPFFHWHzY7G2wh-OfpFpInsebUap7TO8v9_VA-WTarQlWGAZ9BtTo80Ed78ghlYegY7MF0_Bf85MyM55SLJxMwjmZuWpuA9E4O5fJekZv0uwuYmXagT6h8yYLKbgy4T6_C_o9rlH2c992O8HcENG55UhoXT\",\"summary\":[]},\"output_index\":0,\"sequence_number\":2}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"rs_05ec8b9b7b48b396016adff0868d8887d1b9d77672e79da5c1\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq3_CGVNZcVNnLAs9Cqq_HSY78Bu6iPqw5A2pSzRVzQEIEAZ4c0Ge0C8E0JGlut07m4-lhWPTtWwlvhDQi-3av7Y17vAo-w2T3F_I54uSeAhNuU1FQF5iwh47aoWS1CdM8OGWexFk_axmW1CDmfO7-hwAahyOKbmOejFH556uFRI7TEbatYXPvruLvBHujifYWPi2XH1WI9N1K3r5Kpn9gmIqtrPOSDuMOj5PDma2_7vk7wPSSd1OiKmXc7rJUXngM8lQW49I-8lLrQGu8BbRr7X36vZajMnjZxjEYbnhRcvsh270pmp1lUJczQbpNq62kzEoL0j1e-e_S5lFl_ahPgGCOzfapFMCaeOdd5odRxq41VPKajN_qYK8fC9KdBmUaOsiXMq-rVwVaGq5OV8Sy7RAOrpdoatYoqUxjuMHeJ0OcTCN1x4eHmS1-ZpoAibIBlgpRgF_dxYZS24I2PZwVmee6HrutZlVc9hQ03wmkut4nNK3KIWjXyH1GH8S3sFYORz1xUTIj4KJ43MCSI5lPO_mzk5fGcOkfOd0RGVpot0_aAzxeU2AnV3k6TJtSvQ1aKRTDrAPFHt2Se1oxYaeYFzBycMR6lQ5H8MJsogJ126gyUSOkBT-vkN2mdUgt7L4LxrGfU4uKdTrsjoaDH4OKXKHjoNkOr_xU9LLor92fdtOQwYM0BL0Nkh1KQs2x_oqc7FGtlcekZuIxIvZxyjUVpg6aagaLi7d1Qrs9xCosqeCxJH8zDyBwbEvIJtT0C03DrVQUQUpbBciqeU72Gn6q3WtpnpPOTN9Hb_24t1V1YTCysPwVeS5WWCFrw7raezC1mTsqILOYAT31ADGMT5BD-sSU6x3YCcYys3J-xnVAHxT4vMiUWA_JoUnLLxubbmuVYGs3fqNLjNXXTpFpfCcXGaJJu4xWwTTt0VjuSxxTvxJdJmcZfFubqIaOPAQ9qURa1Lv48Q95pNarfMAaW1Vt_sak6kxNmdhM_R9TTJbZEZwZIhVdFUHXNUAPDoGHXnwlZoc4mYfqbiDY-uj2f9df987Tj3aA8NaHUU97LHrBZegrdmRu_B64LpsdGNKpOEHaqc7Mj_-D_9N2NocJcrUeX3zD4bXW0G77N0UCHOY8kI4UACz_vwBdj8dDABJOYwyTYBfYFuKfqsrwWp33figllKFMnOcizVjKPqe0Fo7eoDEVHotFwCjLuZ7geNZrhZUmKuxOT3RLnKC_7KC3IvZunigfBKT0UGtprEoAjxxyF5zOlbPhObiIyJf4DEnoODZnaqsLZs9hMfggBTgln1VA4B7sZIFfX7Qa_lkbkDTz2okxzmE_pJNOqxd7mpupah7f16dsXrYhQlM959WIQhWq8pLRB6rmmNp9blZzaL0CONPykwMyThhW0XQr6S5rmvje4QqExwv2k1VQzLPmpXeuWSNZJEglGsQ-tkOFu78m-4lxGib96rFHimQod9nzmM-SNJa22Md9_XSDMdpHs_0oUhxqIvJ-yQ48v6Oqf9yM0elOzjM4cBhVSk5-oZJXZqmIHqlstLvCupQDbt6lVqaLQylJQ1-e0asytZN7LztOQj-Vm9MxQYbmGi8gPvuIA-TB5_ipOImEPCzPj5L_TB9dLLekDfpsFfxB6eflSIojA4rns1k=\",\"summary\":[]},\"output_index\":0,\"sequence_number\":3}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_BxOAz9IH3YDU7L4rTQ1vwlT0\",\"name\":\"bash\"},\"output_index\":1,\"sequence_number\":4}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"obfuscation\":\"rkwzEKajHudcQH\",\"output_index\":1,\"sequence_number\":5}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"obfuscation\":\"gWoiiGYCD\",\"output_index\":1,\"sequence_number\":6}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"obfuscation\":\"bCth8FqPvV2lT\",\"output_index\":1,\"sequence_number\":7}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"printf\",\"item_id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"obfuscation\":\"EP5Yw86Rzy\",\"output_index\":1,\"sequence_number\":8}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" pi\",\"item_id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"obfuscation\":\"RwdeLTkbkEv4n\",\"output_index\":1,\"sequence_number\":9}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_tool\",\"item_id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"obfuscation\":\"t0l5TbyE6a8\",\"output_index\":1,\"sequence_number\":10}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_ok\",\"item_id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"obfuscation\":\"0IA2Q3HExXuiF\",\"output_index\":1,\"sequence_number\":11}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"obfuscation\":\"6sDDBWVsvFg40\",\"output_index\":1,\"sequence_number\":12}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"obfuscation\":\"LaOnqCdop\",\"output_index\":1,\"sequence_number\":13}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"obfuscation\":\"sltocUL0nBKBEk\",\"output_index\":1,\"sequence_number\":14}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"120\",\"item_id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"obfuscation\":\"XAPkNTZYw6BQ4\",\"output_index\":1,\"sequence_number\":15}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"obfuscation\":\"8UQqpaC9qbJvX\",\"output_index\":1,\"sequence_number\":16}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"obfuscation\":\"Qm9s5jiujMbE13O\",\"output_index\":1,\"sequence_number\":17}", + "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"printf pi_tool_ok\\\",\\\"timeout\\\":120000}\",\"item_id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"output_index\":1,\"sequence_number\":18}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"printf pi_tool_ok\\\",\\\"timeout\\\":120000}\",\"call_id\":\"call_BxOAz9IH3YDU7L4rTQ1vwlT0\",\"name\":\"bash\"},\"output_index\":1,\"sequence_number\":19}", + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_05ec8b9b7b48b396016adff08600c487d198fd989c208153d7\",\"object\":\"response\",\"created_at\":1793060998,\"status\":\"completed\",\"access_programs\":null,\"background\":false,\"completed_at\":1793060998,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[{\"id\":\"rs_05ec8b9b7b48b396016adff0868d8887d1b9d77672e79da5c1\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq3_CGKMeJP4vs3R13VmUdEFQbZHCzCWmx3ZK4xLUjYSak-Xsi04JJMLfM7Zb9KxaXibaqDVwqANgFK8Mich4RAuexi3If4oe00z5RSjU0F5vMII_J3nkHpNuI5FCYz2XAps01OPF7s8C7sAZEmsNqpkyEHXO5nrRRYnHtmhgsqMXaBFst-gR6O94GVy34NmnswC1LGkjcCFA74dp97LO6uX18edmubj1aQl5bqswkhbxNjDZYBa9b8TCpLvraUnjwJtN3HFVek_PKHmwD3D3n7WJeZT5IvGaVeTiLRna4nPbs2vUCONI_ChRCQO-l94AQNCwyM04hnhklCZwtwhVNrhm6ftJqMXwCVs0O2tO9xp3MHgoYRnQ_rOotBG2OLYAeDm0WMrHcNZr0WBiOEbVodgDdfq9_UYQhXvoc0biC7XETV9Dxl4fGa4DsCiv_0sK0PGZOaex--tXJZKYVPlZZcYDapjNtRlHEAzb1V7jPKpaTFAVgA79RUpkjGYzMo-zx5MHXJByna6-IH2P3sNcDTauW8-q7eVpQCplnyUP8RdK6TYfQHRPqGpIvBPuR-Iyw5XW_7mP89ohvUJzgcjdK-sRhJqwR4NcBzQVeb6VgR2RKWjmdKVcG94VG8ZMeUP3hcfnmq0daXEfIqejyM1neRqlg-3T-LIsrTb8yUWl3Rvi-9d3xNpxTiEQqIocyiAi5hNUpkmf3Arxl5PnWvDEFbzjk1M4izDSHndtYtRSGypiYKtuku3AMH6h4q8K73_7h08-JY_fnN3GujRitHOG3DJgk5p7nyJjpKr_TaMv3o-mPLObsmYqJOrpbs9WXamWy3Jjg73KGS-Jhwi-KULiu9wlffhivC27I7eIrUtucE6QLvL7U4nXqvRsGbAl9zyPLqVok6YfMHZcigi41iNQ5u-yrgLbx6xV9OZDhIVfXdYz-8L8OjJIQylvTl3NLTE7VIKTI7IYSTP7bn0wTE_x4yBbNrXPV3Ylb49QwSxyt1OwVOZWIB8qiWePzLAD1VmcJ6dqFlXaLhGjWrktsInLZmv5rTG-UPX_Sk90WWzoRtAf8_LYmrJgv8Se9atSk8JzOqT3uDDqpG4hETFKcqz_xXUw6XwDCCcPDBzrb4eEeci6bblnpg-OpoWyn0Q3dkx5B4nZa94pelCE7WT3JC98qWGZAvqZAfTFLBOoq55EU3tdoUlIwdAPBV3A8_RgYM4DPpdxSiok8u6oZdpfiDafBgSFYgms3FIev7Bfhwvlvy7NtiWL21CdlK7chdA8PROE6qfiA6OJU-yuqPqRGATDYVSpgQDkzzX_vRyz81RTgTWURpebbqnfCSlVggVjyi6V3uHUZSdUriV9-8aqzL3b6mID5uWiyJvE3fOPbAs5zn79a4lHwQKXHsw9vUi-Pg4c2lW9oQQV3mWpAXV2EqUNA6lW8QSzi26-yycFLJ5CJ6ECWTBp4lPs0w5FbTFqvi7quKmSGxbUQo2ph7cZZW7bb9_jfOhlx0NN50sQ939gMV6zKqzjAF2mdsBvqJQ-YA6rWv1oIlP8zCd7Q768OVYEK_sz5eCOm5mSUCWxK09-Xwr1mTad2VGwhmu9su4dLpkVv3jUlcBXkGDYunioDJvfyVkaZH4Ml7IyrvNdzAShGlvqeK7w=\",\"summary\":[]},{\"id\":\"fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"printf pi_tool_ok\\\",\\\"timeout\\\":120000}\",\"call_id\":\"call_BxOAz9IH3YDU7L4rTQ1vwlT0\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17ac3-8a7d-75fe-8745-2b1913a10dce\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":220,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":40,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":260},\"user\":null,\"metadata\":{}},\"sequence_number\":20}" ], "kind": "sse" }, @@ -101,11 +101,11 @@ "access-control-expose-headers": "X-Request-ID, Server-Timing-Total, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "a509ad5fbeb8051b-YYZ", + "cf-ray": "a50d96e4ea590c69-YYZ", "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 26 Oct 2026 13:06:15 GMT", + "date": "Tue, 27 Oct 2026 00:29:58 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "226", + "openai-processing-ms": "382", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -118,16 +118,16 @@ "x-ratelimit-remaining-tokens": "179999283", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_718e410526064111ad09440ad9f31a66" + "x-request-id": "req_201d31f562bb4c729ec7bf1f7e448fc6" }, "status": 200 } }, { "callIndex": 1, - "id": "bc3da232a7ed2a0a", + "id": "408e033402ac9952", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-10-26T13:06:17.626Z", + "recordedAt": "2026-10-27T00:29:59.490Z", "request": { "body": { "kind": "json", @@ -135,7 +135,7 @@ "include": ["reasoning.encrypted_content"], "input": [ { - "content": "You are a coding assistant. Use tools when requested.\n\n\n/Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/resolve-issue-red-green-73219c51/e2e/.bt-tmp/run-0w90vN/scenarios/pi-coding-agent-instrumentation\n", + "content": "You are a coding assistant. Use tools when requested.\n\n\n/Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/run-Y5pgi3/scenarios/pi-coding-agent-instrumentation\n", "role": "developer" }, { @@ -149,20 +149,20 @@ }, { "content": [], - "encrypted_content": "gAAAAABq31BINzpCqufGcEDBVQP8Fu3tSWIRjzs5PDUJjv1SqAMfzU3nYNbY_fPrw0V9_BBHyACHQhti4OEFUUoDCcuIs7yNUQmGaI5m8Xb31RTuxxKCe1kql8tXw5HVxfEv_a7RyvaWpa_20lh_KHLONOWRHh_do-n2SuBhma0wBeFTFmkdk4NSGoNgwmKP0r0QDHxbbVPwXfJpZQQXtbg4WuY2v1NCE2XgNCdxkioHp4Ye20aESCyLJnQHCrk2vAV68kJV9YRqe5sNiSXIvO-27Sn8_RJrXlozmXZ4_XWAQNpr26D_b5vmz6yJKVQ6zx4jErnaPacl5LzKG1qyCn94PpScB1HNTNnsMf6qqQjSgY6-auvIFaKUZBQEG29yh5NPP0030erc_CJdWrBO1SLdB5_u-Uz6dadsyOcz4NqzqMMFWs_j7OGsh2AjrDiKbbAUbDEen-CgBFHaev_aVaibzViP_F_TwL2Z1OeVlH1wpXqtn5MVhNe-RikxsDx4hZh3kza0nWtZUzwqnwhzEe_B3hcBSArbQOjfBwIHCq7gZpSPq5vBhaN8sIP98_YoS8E-QCZZJ14pmWWNxLahMhRoFE2zT55Ye83R-l0h3SjEjyGROzb__D-eUWC58ldwOLLoQMSlg1uk7PBl2NvWrpc_gVMPz4sOUOoeAyAViZFpUy_S6v3b0nIlobBrZWjBfloJwuqRz7aRzRC_8VMImdWKX7ESn1B8mIRmiaSZGcprh48ACX_-kI9jl-35Y3j8jcUlwtbmR7XfKDldZ7DWbXeIAqtX2p3WFzu_gFcHlDY0v-HUSJ1rSQm-L5zeEpcw6tHOwIJykuT_FirVMp9jW8MGAYHkjUjBg8DBrkyZWuGdDEDz7DsjkNK5i3vdO4eSvGVt2Mo9H9qs7KRN66tbIYRfFhI1Sc3oC7DZwwOAk9nEWZFfdvKnU83FkB_l2VB7iU5QHalYzKiifgDIQgK6qRCQ_SL2TEfBKecO2rKlGLKp5gNvzL7HP0oU7F7bc2AMJsAvxiKDypeXqFIUayP0Top5VLHeFTAraL_t34_KTiJGO1GZb3IBUyHqWLsZtGuWfSvnlicR64rN4leBNxKu-wqCZqMEVqAqdmydzaE0lq6caxSqxBmz68-5B5UIiGfarNWnF-KJU-Xm6oV1G0EBhF1mbUOxCcx2YUPCOYog0AkONfENVS-bCZV4nytMqI--XxfNGYAMBnR9ZyIO6Qc-nsPlsWc443FA8DPV1OYv3fYiWGg5DSJMEKDuFweJXCTBqzRPqFBoK5oxsvBlG0Q3VUFuVsWjEfTI7D2XmzOhh3mg9kE1H2XT879A_FzMmMG4IA8UwaEniKJ1HDzlaCKbuOSv0B0nLqjOsNhYMVcF65NZwlNg7pxs7cIDvkZL_AQiW4bDL7qJkRKKKY801Hw-JcMbTu-sAWTio35Gb9HzM6bynj_RiAECjbetVi7kPWJqk9Aq0JJb5Zz8QfnBKwSQego34a-SyteeB3vTPKhgFl237dcT2qPHIPhYhCHoZu1ZecHHUIc-tDGIquUzDZcDBo4oq4WfTaPwo0DHZHcV5oND-rdbUOLCJ8pmC7FQ5xwlsnSggbz33SxLMlFdMBdpAf9RRumEvZ38NncAMITJ7z4FfdlIYT9UDp0=", - "id": "rs_0b86321b42b1ad5e016adf50481f5487d1816244e8f64a915c", + "encrypted_content": "gAAAAABq3_CGVNZcVNnLAs9Cqq_HSY78Bu6iPqw5A2pSzRVzQEIEAZ4c0Ge0C8E0JGlut07m4-lhWPTtWwlvhDQi-3av7Y17vAo-w2T3F_I54uSeAhNuU1FQF5iwh47aoWS1CdM8OGWexFk_axmW1CDmfO7-hwAahyOKbmOejFH556uFRI7TEbatYXPvruLvBHujifYWPi2XH1WI9N1K3r5Kpn9gmIqtrPOSDuMOj5PDma2_7vk7wPSSd1OiKmXc7rJUXngM8lQW49I-8lLrQGu8BbRr7X36vZajMnjZxjEYbnhRcvsh270pmp1lUJczQbpNq62kzEoL0j1e-e_S5lFl_ahPgGCOzfapFMCaeOdd5odRxq41VPKajN_qYK8fC9KdBmUaOsiXMq-rVwVaGq5OV8Sy7RAOrpdoatYoqUxjuMHeJ0OcTCN1x4eHmS1-ZpoAibIBlgpRgF_dxYZS24I2PZwVmee6HrutZlVc9hQ03wmkut4nNK3KIWjXyH1GH8S3sFYORz1xUTIj4KJ43MCSI5lPO_mzk5fGcOkfOd0RGVpot0_aAzxeU2AnV3k6TJtSvQ1aKRTDrAPFHt2Se1oxYaeYFzBycMR6lQ5H8MJsogJ126gyUSOkBT-vkN2mdUgt7L4LxrGfU4uKdTrsjoaDH4OKXKHjoNkOr_xU9LLor92fdtOQwYM0BL0Nkh1KQs2x_oqc7FGtlcekZuIxIvZxyjUVpg6aagaLi7d1Qrs9xCosqeCxJH8zDyBwbEvIJtT0C03DrVQUQUpbBciqeU72Gn6q3WtpnpPOTN9Hb_24t1V1YTCysPwVeS5WWCFrw7raezC1mTsqILOYAT31ADGMT5BD-sSU6x3YCcYys3J-xnVAHxT4vMiUWA_JoUnLLxubbmuVYGs3fqNLjNXXTpFpfCcXGaJJu4xWwTTt0VjuSxxTvxJdJmcZfFubqIaOPAQ9qURa1Lv48Q95pNarfMAaW1Vt_sak6kxNmdhM_R9TTJbZEZwZIhVdFUHXNUAPDoGHXnwlZoc4mYfqbiDY-uj2f9df987Tj3aA8NaHUU97LHrBZegrdmRu_B64LpsdGNKpOEHaqc7Mj_-D_9N2NocJcrUeX3zD4bXW0G77N0UCHOY8kI4UACz_vwBdj8dDABJOYwyTYBfYFuKfqsrwWp33figllKFMnOcizVjKPqe0Fo7eoDEVHotFwCjLuZ7geNZrhZUmKuxOT3RLnKC_7KC3IvZunigfBKT0UGtprEoAjxxyF5zOlbPhObiIyJf4DEnoODZnaqsLZs9hMfggBTgln1VA4B7sZIFfX7Qa_lkbkDTz2okxzmE_pJNOqxd7mpupah7f16dsXrYhQlM959WIQhWq8pLRB6rmmNp9blZzaL0CONPykwMyThhW0XQr6S5rmvje4QqExwv2k1VQzLPmpXeuWSNZJEglGsQ-tkOFu78m-4lxGib96rFHimQod9nzmM-SNJa22Md9_XSDMdpHs_0oUhxqIvJ-yQ48v6Oqf9yM0elOzjM4cBhVSk5-oZJXZqmIHqlstLvCupQDbt6lVqaLQylJQ1-e0asytZN7LztOQj-Vm9MxQYbmGi8gPvuIA-TB5_ipOImEPCzPj5L_TB9dLLekDfpsFfxB6eflSIojA4rns1k=", + "id": "rs_05ec8b9b7b48b396016adff0868d8887d1b9d77672e79da5c1", "summary": [], "type": "reasoning" }, { "arguments": "{\"command\":\"printf pi_tool_ok\",\"timeout\":120000}", - "call_id": "call_nfMLwtliP8Lmn5SMjdlCMUVh", - "id": "fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7", + "call_id": "call_BxOAz9IH3YDU7L4rTQ1vwlT0", + "id": "fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca", "name": "bash", "type": "function_call" }, { - "call_id": "call_nfMLwtliP8Lmn5SMjdlCMUVh", + "call_id": "call_BxOAz9IH3YDU7L4rTQ1vwlT0", "output": "pi_tool_ok", "type": "function_call_output" } @@ -215,26 +215,26 @@ "response": { "body": { "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0b86321b42b1ad5e016adf50488dfc87d18fbbe77f9028820a\",\"object\":\"response\",\"created_at\":1793019976,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17851-9740-712e-af1b-3238a14569b1\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0b86321b42b1ad5e016adf50488dfc87d18fbbe77f9028820a\",\"object\":\"response\",\"created_at\":1793019976,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17851-9740-712e-af1b-3238a14569b1\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"rs_0b86321b42b1ad5e016adf5049369887d1b48119b94600893e\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq31BJoKK46j9aSrVQ1M4KuMloctdLmSnUwtQANv1bJ4k_Y4mn9ZslJkj6geZNhJjRrWU7loQ6SJrC5_NaytPcV553TDJx8cYektkgW8spHDueAwbw-HPRBi77p8-nIbW7Da7P68Wsfp5CyjmZlJml_THOrP8ASFeWxpdx9gw6HvzVoLp-ZM2ZTmuR6JQvWguITHVZVen5klvXK4pTqn0Oy1fAt5tvBN4sm6Pp4scRMrX_0xPtIRM-W57-0kJciICZySVObnsSWYhdyHlhntoRY0n2A83coAH46rPQbnyxJ97E-SFb8TAEQkx8YpdPea5ef3bxkfuE0jZ-x-dgLOjHr1WVZ7kAkPZNmPjD32mAhh6D2eQdbyYY181K75iuTH96-WXKkULnoyZudQw-gxsiHzmLu6qxNPDAQoFjoLBOoIdxJbh3SrVraMs476gbE9ATKdgum0MBdgzIT3NfpxCAXWAisj7JFz-PBWPRWDOFsgcSVu6LSE3N1M72Xt1r-mj6QNfitSIB_yWrg18Y312QH0aIjR1oQAUoLaCg40JgYNbjts1FvuJN3Zk76fh7zYJyREg6NaepKuW7TBsQi4AmerSe3hDGfXekeuYZ98aUAvlIltkxnxpr4aw0gXUlMVflkRk9WDdjaeAgEGS6NTsZHa8zOiB9SP5uPPwlp-_ZkJHfHBDW0vT-Xf1f2Przbbxoq3Ke-OjwWtHHU8_6YW6fCQy-E69mRKSyKjsTF9W96NDlzsowyYEI9FIRuI3Fhqm7Im1QuZU2WDD6x3L2UzCS_YIxz-Gl7MOn6yNhyPdNJsV_4MhDpQCNhKVzlQpgq2fcJZLbA1__A5D1tIt9f1o8O0IewUCS2HI7Qu-sd8IuOZWSmfkDleNvl2QpxIwx7ljaTfhkpdm04cQNNWa-NY4t-pRym_rE88E05ieL7yjuzEyzXDtwkXLwzP25j0vu-Y2QVY_llf-7_tnXWTxz63QtSAqWuZoqcp3IGBp3uPlj22-ReSUwpCud6ZaqzDIbAfEYcfTLQdd7pgXpA7Of34udjPgbFzBCV31ttsxlarKiQ8m3SCZZOZ_7tqKq5Z0hhxDVDM0_DXpnSU5_MptiB-ULY4gu9thfUPIv-F_vEuRGoGYiWNzB7okAeqqTntiiViCBgZ-uyHNn7qVx-xU3AV0hxpSehYnQJQBFeAxWGazDrEwVJgvjFhUGDXqK5YFldoPnpAnH\",\"summary\":[]},\"output_index\":0,\"sequence_number\":2}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"rs_0b86321b42b1ad5e016adf5049369887d1b48119b94600893e\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq31BJdF8AfASjmO2ndbenRB2c2Vj7MsZxvkI1YBi0_pCPqVTLIZtS4oT4fdhj7NLSFnSfdlSCc8kjjp43DfHaKPgDpHS_hWOHs10LcU_wc-mkG3SMD-m6xCrNlMKDAc6gHW8P8tfhwl2Ag3_zmrRvu9xDx1SVl1GpPR011TDzHtSPEpRZpvA8XdYScGWwnxRiUiyiZQcOv_JLO91K_CR71Rr40LmRPjLa-1DK3yRH74GvSYywAm5XMucLxpWLlipBgxbsxI1bssg57FRDPIvUUwX0F2uD2qd1TSt3kNI8bMJco6QmbzoHXTb8vUobqDAYWGsq5_QWbtBS_UBIjEdRxDZ-xpTQEPEqTU14T9cSGFpbg-G2lhmy1Ue9R_uszsr_QScak4rl94pOnmsPBG5iObksWVZ1iwQWgRfKd9_mWRC1xlCuPb_k_WtBjzT6QrwwKoG28ZwDg48JhzHMD7XZrcrFfVZPv3Y6jsC5rgAcQ8jBPTNFJbn5e4-CFUiZFRABV-YPCBLSGNu_urjaEP2uwwSnX_gmcwN7yclYUi0lWVmOcJ58T7RKkXGDSghZ7dCyslGCOSvHPYgzg1LzIOZa9MY4jtNY2gtI-3hSsq2Zlt8dGza6W0t0XHcV2pC6WgtmnLggoOULh8Wv6KKfAYDcGZaYewyGDZghrD4KIECiNynOCkfXNAi93LTYOA0UuRX72LrTCcc6i3E9LUnNj6o-P0v9RCgaECBIgtrD29pp1UNLh2Bb20cBl231nuOyH5HSJlzBAoHLJDKX7fg5HGGpcsrbFItwMRMxKK-nJMt4yYb52tThKVcn4zFKwpQKwDo6QjEe7kI14eqUP8akoMHh3X4Tykm3Qk5uPyZO9Q8xRlqEfuJ5CHTstwL3wjahc4FvW9BaTDWUgnfb54sYRoqbs1MccMCUc2ZnEeBN0zCBNyLOZMt2FwNxcfrUlccMNlBghnxSQtUfxztFVTMQZd3HbMOF7zvW4YUvi9uzjs1tAgAFeDo-NTx4utcDSc8B9CIloLe37xZBRNO9JZWUCD_ddRJ4VBDdl5uTVrZgZMOoHY3OHcVnNkdriakG8YOhyEcM7aJomwtT6pN5EbjasHNEfyp7_oA0LKylRW8EbDl51V5zl7zdWI1YQTC4IcKA_89vFNRPLvzPui8CDorei4jDS4j4RddIVTCnHU02yEp6x5coeC98lbMuPmluf2SXokOrA5saezfSL5MqhFok6oK2joBY1DYaWx12txiQ8c_QR2GDhzsTJkKthiM7oNVmXRc2g-Uv-17RhfXJkrmyrbFnPTSI2r_UYUB8bp5xm61F-XaJKr4ifGPfomB0KO_3YN4BKIFrSQqtONMuY3A9nWrJo9xejgQ2hlUiyTxGx9yssISGRRGdPixFaaZKzZwW9LRgDXIFkojSjrLMeMiLwcvIU_PYgtESnaTY5x4W5T3MdfQK0yhyxQ-yBQxrQvDvety5VnsXKJTsaUsfqoW3FMp3kCjvrWK_26IxE4tWKNzj0ZWXGcAYx2Npzh3OAKFvlYilGkGv2M-NxVpmf3lPDgDqLv6QSA3AhMxG3WuiwBsxqqX8cpgVJLzJZc3UDHvdM-Lzv6Kv4wNkQj4dmpl3z8FAV2-V4i3PmH81YAfuF01l0GSIjAk=\",\"summary\":[]},\"output_index\":0,\"sequence_number\":3}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":1,\"sequence_number\":4}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"PI\",\"item_id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"logprobs\":[],\"obfuscation\":\"KKYtqknL3Nzgpq\",\"output_index\":1,\"sequence_number\":6}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_C\",\"item_id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"logprobs\":[],\"obfuscation\":\"fCp1dbH69rPFBS\",\"output_index\":1,\"sequence_number\":7}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"OD\",\"item_id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"logprobs\":[],\"obfuscation\":\"2DFqlcDASqBgkS\",\"output_index\":1,\"sequence_number\":8}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ING\",\"item_id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"logprobs\":[],\"obfuscation\":\"64Nin47VBa4fh\",\"output_index\":1,\"sequence_number\":9}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_AGENT\",\"item_id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"logprobs\":[],\"obfuscation\":\"wpKnQbTJ8a\",\"output_index\":1,\"sequence_number\":10}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_OK\",\"item_id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"logprobs\":[],\"obfuscation\":\"4dlArS4ZYBesY\",\"output_index\":1,\"sequence_number\":11}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\n\",\"item_id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"logprobs\":[],\"obfuscation\":\"IO6hydX3q9Oa2Ak\",\"output_index\":1,\"sequence_number\":12}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"pi\",\"item_id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"logprobs\":[],\"obfuscation\":\"zCtUVMkB4dPa6R\",\"output_index\":1,\"sequence_number\":13}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_tool\",\"item_id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"logprobs\":[],\"obfuscation\":\"llSK6rgjMuA\",\"output_index\":1,\"sequence_number\":14}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_ok\",\"item_id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"logprobs\":[],\"obfuscation\":\"wnjBOCVvAAcCm\",\"output_index\":1,\"sequence_number\":15}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"logprobs\":[],\"output_index\":1,\"sequence_number\":16,\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"},\"sequence_number\":17}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"}],\"role\":\"assistant\"},\"output_index\":1,\"sequence_number\":18}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0b86321b42b1ad5e016adf50488dfc87d18fbbe77f9028820a\",\"object\":\"response\",\"created_at\":1793019976,\"status\":\"completed\",\"access_programs\":null,\"background\":false,\"completed_at\":1793019977,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[{\"id\":\"rs_0b86321b42b1ad5e016adf5049369887d1b48119b94600893e\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq31BJ88sFQIQ9oEdsMREM1qJE0yOfCQVWYylHg_xwz_TDxvF-sUxBfjpDQuy6uzr5TsfvpVAG5sQa74fpOHv7w6JWOpyIdoQAAuF5sXgsgTSIarbkVN55pVC11Xjwz1O_RcNehqcyMpk3pdTXGQykWvDAdcaaEbsVEG3cXMcJ2EEpgoE0if6riFnSVnT-cvTIkXAvV4ayTA4ZuPGA6ThsxSlY0P2J31d9jQt6tUdbzYUjaZyCOF2GuTkEkshuxSIxPUmjlyCVS0tJCqlPNveV53FLf8i-6ZtejiWLlCc-7M30PHFkjQ8gLQeV7MmacbKUTdNQz7ih3MKmkXllhBhbZ-5K020TrUG0KL_lto3YVGbeHmfOXPrpJedbNPBiW86Sm5peLVhjN4mBhZYIAtNrbNAQt86C9aOsSAo9N71JeQ73Gnn4ieFmNS5-m6akiNHFT7c73KDS4kw8w-U3D08Ih-Aur3C5-rupP0J1eMCvcgGV3-R8ibZh9bJxR4KH2To_UswipcWznd-4ybqsON5BYq0guC5PPGd5bFzm9Vi1jbwZbEaksSb6M2U8f6Lqm6mBNL9dP05bJwGwjgpwNOFObcNanHgV9x0rOxov6JaUSFSfv83FZGg4REPxgdhA7Y6zAOcLjQ2sok5Dq8Ob6tCvc5tHxbDfnTiui3tA11H4wNrjEi3y61tobXdnN7mKeM8-j9WhJAgU4Hl-rtKq3cOhPEHmbJkJo2nw5CcaViuaiO5O5egQBSxWOgfRYmVmBvP7ujkigC88qQXixw9JiRihE-0eZBFhCyO11JDfSSpjPw7fWaS6jMOaXoq_IVxpBuapiW0p-Ril6Z2b48TLUbV5DW55JFsfY3gFiAqKptHsMcvPMNxjSugMcRZCFe6ExUaqZ5UAJIGzlKlKKjITj1IVK9VLY5pzcOvXrM9dU4yhjz8JM2Q7GEOxL77ipzKh5CBkXfhM-KjY7Do1MvzWUWU-w3Bo0YMYsuxAVuCpTLkQ76BM3jQ87bJ9aJ53QOwO-bMo73RHx4JB2EY2QfJErXWuo0-j2AgrPlue5_wJbOb00Z_B7dKAz2Rg-L16UJ8OqboP5cPBsely22ruH3k6jZfOVYYjrCoWtAcoPw5Xda3tR5qja6CEcEnyPO6qvk-cZZxtwK5p1MsTAeUlGX8SHCc6mg4VHlmxpTE7_8B9JYUBDIVLnM8W_iMle5ZDHypQm4nar_4o7ae_193MxjXYqguXI2gvbsqtco8nARR0Qzdm_Z1WHjhbQdGJbH8Y323eEsuLLnIuzXQTiS-DbqSwizsYh5MFYxOzZcdeaVjJRTY-FpgZUCwIojEqt7J1gX2d_jFagqgA064XYuh3fYc6IBjvfKHwG8vVO1ojBq_GG2QJ7G2LSrMlBYM3HZPXdDVZIrYnPQzvG0IlECLWdQGsnbV3AgSyLXJ2J_XiLycaolGRZEHZVo1HjPh6IhN0FtQNU3wxkGRawKA1dVS7H4iA5Ezd8z35I5Z9pnI4z8QvVCF_OPmWm5sDHG0pU_jhMGD8xr5Znd84gRuXNjxouJKlUkhbQkmXxw3IDf_F2BoqDt2vVw78U2PQ9BMPFWPWrDlAUmFeGWYMNYKUS1FhNKIiwMtiyVZOfZfwnvzvdcooFScjbrKX0qU=\",\"summary\":[]},{\"id\":\"msg_0b86321b42b1ad5e016adf5049433887d1b1e3393a274b6862\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17851-9740-712e-af1b-3238a14569b1\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":272,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":28,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":300},\"user\":null,\"metadata\":{}},\"sequence_number\":19}" + "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_05ec8b9b7b48b396016adff0870a0087d1882ac4fee66c7225\",\"object\":\"response\",\"created_at\":1793060999,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17ac3-8a7d-75fe-8745-2b1913a10dce\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", + "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_05ec8b9b7b48b396016adff0870a0087d1882ac4fee66c7225\",\"object\":\"response\",\"created_at\":1793060999,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17ac3-8a7d-75fe-8745-2b1913a10dce\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"rs_05ec8b9b7b48b396016adff087613487d19c38798706903ada\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq3_CH0m09r96o5Q24vb4uj0CknRzDfNwweq1j3YgyyuO_FXlH4jH1g-SsJ3yx3LMg1SboSREgo1e2alTywM-PiUQhdNH4LNalvkkSk3rIyi1JiwI0jOZiyg30ak75OZnU_2-3rgGqIJfzNQ7NaCxGS0hgkc3BNR54NLi9aEX2AVSzGw4tD7s76WTVf5_8TOZUcfxatgDpbPwwVcDeO0JXNZy5fsq2aFpPnX689PZQfStQz-a1UVWEO9h_C5zMRn7rmN2E_AZ8I9b2RriqClEq3tyVAMdMqCVO8-LgZtJKfA0sHX3QnqKWFzMiVNAd4Qcsk3EIhNvWb3Y_huY9kzwtoGNXV3_6_OtqXrlCvPkESr9vyCc0WN0fT_Gs21SNUbwbEsa9iRek4bFreVFMTGd88aYkONUd5pF8jGXI5XbqkHtQbtL9WnOUKnHbs2FRAGfJlYbqYsP_7DTPYaV7YLgZFagOc_LUhb0LfzNquAaYF3t7yQrVEeBt8dxnRiNmkbwbuSWaLXgwiF_zFT4nPd6XR8L3p4Rj8leO-gV1gHK0QA0Kp5gGasdiI1jeteq3_KJ-DTGwZBkkWrb_yZxpdiZtBIuTTbUGXpCdqDC2RVYPk2W-uR3g4es3k91CloZsXJw7Yr1id-Uwm6-B2dqPmOGJotYEyTrqyBaYsAXbLH_0CTbHfKfcW9LaHXvSxbH-B75Iy_7Io8lxJ6JmufBgJlmLLsi351nGQ48H9sL1RnqjpW1iPX4pKPKKGwg4t-kq9kiC3c0-Xyq_C4TDhfuIaUKKSt9RsSzskJEDQWZ45nQkv5-g6cSJ1Qrn4GxtHZ2skmAFYjTW4vmkam_qycv5hIEumMi4YGllVmGbDkKN1Q5yDIAb0KBQZn2IbK905-dgnhTw5W_O8hPGXCggfbHKGHbz45Xy5KqHtaGvfy9b8sNofjTJJD6_FKxo48fizMlVZir94GDYoD7kP7OeXZeSaw6jR8pophO8FDq5OIX4HH_1IQKklrW5OAzx3ieCBpo8KGFKOdzPW78qLNH1KweyTlKwc-Lb3ENxjUPp5CMyca5heDoztFO7plCAoOAGDMpNGjzhDI5BAgoPUYWIVaVBbQwXt5h1cOnLZ237Lb5dZ_GmXZu1mxIRNKg5ZKVaHIhI2i-Ce8DdXyUwTO731WqZBSiE52i6m-CD84bu2YVFzDzu_LZse5HobQap0ahipNRb_1EadkAP\",\"summary\":[]},\"output_index\":0,\"sequence_number\":2}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"rs_05ec8b9b7b48b396016adff087613487d19c38798706903ada\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq3_CHNTFdHiDIBiHBvl7jziNajlfFu6fsu7u8AQR3UVs3zbX2Bdf9KJkqHnmKY-J0JTApHG11HWFTUN1A9y7_7yyu9bkw6_027HHP0yUBv3uhtl55zomk_lRW3Y02HM12KoS88XJ817elSB0X5olt2-etTgNxMPUiQgjC1NfEJbm1zlN1kem1_ZVcKbv0Vju-WdKqfIR7nR0WNx1Oajk3LGWSIULJjkRqEYqT98ZoilrP7rs5molprDi6cSOJ0IfkwJt55F8HgUG-qHKt3uchevwdaMElnq-01IVQATe_TpM28Xc1MICPnLDUulg5cV_OKyMe7NjANhIz6Y_pCnVLzr0fO2KiXLTVpjD5fWQnsaWcL_soCU5Wb1AjrdXCLWi94EfaQiOAwLgMCuz1InNSgHUKq7f-FQwd0krcaelJWEHFoSftRH2SIrtScU9sVlbo5WBFpa2EZrKxkDxg-UoBBfJFR3cWrTdLsRH7derwcsfjcdNSMJHxVNiVf22-uc8ydEY3zAVzvuE9vPC6vqiHIRH0h5UytPaJVZUlw8SuDu0kvnrSKjmCMx9tX4KT9zf--jTeFrqZzCHs5pHDAPPwwBwulFq6kBzhXx_mMYlq0bh4y6dV3mo0Ee7JcIZ9f4GVNJ7htUhF4x1ZfIYg21QrBX1IUhdnksmjrBuZDlIgvtzpAuSqxSZR3Zdsus3pL-U6-tmWQTbcYZVocgk8mK62cL6wk4lCQZ9l33TVODANKkTTt6EY--mDknb3_xeIdRqzCjtK1CbL_CqgPvQO7rk85egwpXDFyoyXmcjJXwUyF0EcPr4lKzln4kwwuySDNbZc2tOJILAhEAEF19hpzEe8254ioUzG2vMxdBjEv6vLKkU__7qxoyJyyd_52O57TPlGr4ZD20ZhwPqm61bfFjCOKAMEM8FUGpzG84jkbJYL6d1r-wMJV0vCPfHBLbjqsR3IzTjeX-gq3hpWTESYnLPMZLjVh2knW78ML8B4ODxq2G2zdjgQnBSglcWkZEU-kVOCEosm7xNtWgWEbRQKJjrpKOgXOPq2GaO81aLSgqfrv3b7AKNBh_pX_eAjj1ftBsa0zDTEgERoaqzlTs90SJs10sHwb33-DB6dPQb18xBlS0ZRDGdEP-sQ-QSOkyCRuo_vNU6s8FnYRyblKh4JA24IankvCyfaiqYmikxF1iuSZloGSBzyWoNv5S4ag9AesmBpCZ7qNQT1fMXC1iuoKFJwMbrH6KxVRLopmdOXi_vvpQwN6UpPig1ZWgb6zQjRxODRZK3le_gGeG01PyLa7RlpldlgJ1nfVc5RtyMQC9rMPwUWaf5tIeiryS4W4eLVnxw5VkJuAtQbkvCm2T3XCBDbJqMcKokD1DMI91ClFQTYKPhkQNFcrndYigDrdGtt_9dja7Nadfeo1l1FBicbjevTV2iloIanjpDKb_XDFjMzbAhpue8dJmERuU250fR9kBIK7Ha5vnC-0WMjXXllPvGOQdTaMVE-23N1VzTb5JgYBlArxCrR00qAchW6Qr-Cgi8cKQ8w4UzTbH1g9JPyuEyHs8WBPQpDxWwuOlmaygRHjCUufBQxpKo9_6lJptInMgQbYiVQcLgBN6_I_9Q8bONY1CM2EXqckckRtqhhj2BqwRQM50o=\",\"summary\":[]},\"output_index\":0,\"sequence_number\":3}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":1,\"sequence_number\":4}", + "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":5}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"PI\",\"item_id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"logprobs\":[],\"obfuscation\":\"a5Jprn6Ma0CyIO\",\"output_index\":1,\"sequence_number\":6}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_C\",\"item_id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"logprobs\":[],\"obfuscation\":\"M7XhceufPIorpc\",\"output_index\":1,\"sequence_number\":7}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"OD\",\"item_id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"logprobs\":[],\"obfuscation\":\"tEccF8EhsJMarS\",\"output_index\":1,\"sequence_number\":8}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ING\",\"item_id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"logprobs\":[],\"obfuscation\":\"VdytLbYYp37D4\",\"output_index\":1,\"sequence_number\":9}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_AGENT\",\"item_id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"logprobs\":[],\"obfuscation\":\"pio5185gPF\",\"output_index\":1,\"sequence_number\":10}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_OK\",\"item_id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"logprobs\":[],\"obfuscation\":\"f8l84Fx9QiqUz\",\"output_index\":1,\"sequence_number\":11}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\n\",\"item_id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"logprobs\":[],\"obfuscation\":\"FvwJ7gvNwedgRzo\",\"output_index\":1,\"sequence_number\":12}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"pi\",\"item_id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"logprobs\":[],\"obfuscation\":\"G7OI4vXdMhpUDR\",\"output_index\":1,\"sequence_number\":13}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_tool\",\"item_id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"logprobs\":[],\"obfuscation\":\"Kw3ucdYJa3E\",\"output_index\":1,\"sequence_number\":14}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_ok\",\"item_id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"logprobs\":[],\"obfuscation\":\"I842Ctdcb60oJ\",\"output_index\":1,\"sequence_number\":15}", + "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"logprobs\":[],\"output_index\":1,\"sequence_number\":16,\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"}", + "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"},\"sequence_number\":17}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"}],\"role\":\"assistant\"},\"output_index\":1,\"sequence_number\":18}", + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_05ec8b9b7b48b396016adff0870a0087d1882ac4fee66c7225\",\"object\":\"response\",\"created_at\":1793060999,\"status\":\"completed\",\"access_programs\":null,\"background\":false,\"completed_at\":1793060999,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[{\"id\":\"rs_05ec8b9b7b48b396016adff087613487d19c38798706903ada\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq3_CHJOGSLCvWBMI9lMvToB_u1kn2qzKS7DvpaGHN6xFVHZz19zHGGhtGueOIqLY62A7doAj9grjXir8z7xkWWV2JG_m52NE-yM4Dl33jXWoHfrn0wdiA7vdnrAc7wyaH7i9UTVPsNOc6qD-YbnXwsS_xuS-cOwM6yc3-ugt2cbZReISXssZI7CroP9b1zY07QaKFkXM8E_mY-0IifnQHTAIOuQmQ6kK5Fp6Ehx2aZtQ3dUIc4UzCOcxzFYrwtbcJGtiX2d-1RKfWn6fk65n0djNKuAYQMiLu5sNpPZQDc4kZhBtd3iNJLOB3gcPKS10mXOiKUx8N9q0QHM7Gr3O639IiSzkc9AYBXUhUfllJFRXOMqHxeLBE1eClAzYsI7fSdMsvHws_GzKPNtx7Xo59iuSTuWqXiKcJkWj9jl3FG0rnuB_n3kL6uTa93Z1ZZEKESnMM5ZSo7oubH5AjJqeDSSm7ySE5z5Sz-xhPBSd11CuHM5C0kM8ZJfg4g-BvaikB7bqbqpsbNZu6HjgmhHXzvmpqBhpelsLKtmSYKx4q3E5ry_kORRYq-4VOFt70RPgmKGwvSuy7pPEk_DMXQv6NLuVbqV4ban_ymnE1LKFjhfn4B8WctmmNG8G-XxNRkHNwJ-y7vnsZpORGUrd3-taoXebWMWN085ucdOfCm_Yie2uAQk4Wk-hSfYLO5p3J7VR0FAzEasEyFoJC7kwBhivqBqmvlNANphHIsEz2eEtCa_4c5bLpHs2huimLqyS6XHmrceyn19dKPFsvR050fbSaUMTV7r95c_tSVxi-JcbWu--CwVfmKUE6a2cYkzCWV7LjDMWU6ewAY1gO8eMI83DM_mhKQuJfKJWTk4yuzrZIQpZ2FMOcY-gVleyOcVtyx8Qk3L2hJ0e5WZQyb9XPdnB-IFDpMLP-jo8X-ouDMdjnbkUCL37OSL2YESnPKKUrcHzrb6PxRS9lVc4xe-CYVklje741Mdu65Kw5WgFM8EuZb9Ngb-WAT-SMhGroz07iulVEbSx_t_j7Jg30adqa54WwmyTjgF2_4YBDlgxPbPJM7-RKhtP3-A62XX5IkIRNXp9UfDJ2_oiKCWHOKArC84G_RbaRmKk6enEq1hkscN8Q3f8CCkWTtbtn2byi3UxXsS4gEDXSG2Z-oSUk7Ff1J3ur0AnQmT1IXeljNpKak0NDJx6jSBO6AfRc9YVvKWeA9fay_zHt2DaPqjwRZISqDhoh4ylAc3dvU3_8xAyImw9PNHYmtRL16nBxWgavB_QMIoApPvjeq-Nv6zekq8v4yQhwBL18sBQvuBmqOnFTBUqmkpe3TSkKJlhUFdZxbPqrtGGi5xBG6isqYabWhPtb2sRPgnkLE5picW1yn_c_lB28JL58qkhuGBPIwoTOx5fAMYU6U5CP0bC6Lu14DqIE2dYODUCYjMFgUezgphxmFQq2DrpPrPdo1oIpUl-PkUxwlkn-IKJ6X8_1BFmu4jT130rB1JKLpCZGIKLH204z_PmL8w7UhXoGe3SoX6DDj3EnvPw-V7gGGxnD2iMhZrYiAaghfwJ6K8XnvXmUuhJiaXPTo4g85b1wlLJUh8wrpU3R6hCgSXCEc_NJcPXDTkwfTrNAUYchouoe8x97qlCS8iQhuTH30uE=\",\"summary\":[]},{\"id\":\"msg_05ec8b9b7b48b396016adff087702887d1bb3b76bfddfe8ba3\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17ac3-8a7d-75fe-8745-2b1913a10dce\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":271,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":28,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":299},\"user\":null,\"metadata\":{}},\"sequence_number\":19}" ], "kind": "sse" }, @@ -242,11 +242,11 @@ "access-control-expose-headers": "X-Request-ID, Server-Timing-Total, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "a509ad64ee9a051b-YYZ", + "cf-ray": "a50d96eb8d2d0c69-YYZ", "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 26 Oct 2026 13:06:16 GMT", + "date": "Tue, 27 Oct 2026 00:29:59 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "391", + "openai-processing-ms": "189", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -259,7 +259,7 @@ "x-ratelimit-remaining-tokens": "179999232", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_12609f8134ea4096b1e480adeaa5d757" + "x-request-id": "req_2a3745f552614723a63c117386aae4e7" }, "status": 200 } diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v1.cassette.json b/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v1.cassette.json index 6dd102cf4..4cb9730ea 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v1.cassette.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v1.cassette.json @@ -2,9 +2,9 @@ "entries": [ { "callIndex": 0, - "id": "2df231c746c16f89", + "id": "732705731b290aa3", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-10-26T13:06:10.131Z", + "recordedAt": "2026-10-27T00:29:53.403Z", "request": { "body": { "kind": "json", @@ -12,7 +12,7 @@ "include": ["reasoning.encrypted_content"], "input": [ { - "content": "You are a coding assistant. Use tools when requested.\n\n\n/Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/resolve-issue-red-green-73219c51/e2e/.bt-tmp/run-0w90vN/scenarios/pi-coding-agent-instrumentation\n", + "content": "You are a coding assistant. Use tools when requested.\n\n\n/Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/run-Y5pgi3/scenarios/pi-coding-agent-instrumentation\n", "role": "developer" }, { @@ -73,27 +73,27 @@ "response": { "body": { "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_03733174ba77c7d2016adf50412f8087d187d3b3d750a16d1e\",\"object\":\"response\",\"created_at\":1793019969,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17851-7cd4-7300-9732-cf7a9f97b248\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_03733174ba77c7d2016adf50412f8087d187d3b3d750a16d1e\",\"object\":\"response\",\"created_at\":1793019969,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17851-7cd4-7300-9732-cf7a9f97b248\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"rs_03733174ba77c7d2016adf5041bcc087d19325f28f5cefd730\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq31BBM8nkglZW44QXhlDyhGJc5GOdLSk0fGEnY9xor1dya_QVjn1ni12lHw7csWGoEVeaPVVj34HFYZYiUXRuPPQSUv8v8H9ngqM3Mudi9c69ZuhfQWHnY7_8tYpjoPx_mRCJYI_2Gqf4gwot2_RrxO2uhFBEwgirSB_lJ51v5O9jYC_kTQwioHoJ7ZjnCuJdIZ1gmKpRqbkuKXrveh8Cd8LgxyVbi9ppDjP4mSzRs0roIr2HiquurYm-w_kq4sHktfPNE2WeXhLC6RFWoypsEQ0q6x5XUsY9IDtwmNA7Yi0aLdf3WvGsvPzcq-FUC0lFAhbMGobBm60MHFKUfVaJM5pGZYvIXAdRfNFrkeDvcCz2jfrUNj94RqYWezpprNTIjPEARhH3RhmWH_H-c5XHTV2C2DENLVkbsZ4R7BQ0nZFy-zlaOXNtIlWq___DP4skC5VojycMYWPJImndm2Y8H5qtJ8K4hPK-owVwLMankM1EpgyOXBqxH_c7ZHSlooA0Ob1hVpivFbOgx2FJ-tk8zMQnXlAjra5V1Bsf1ZM8B4EsW4YBBhJuPS5TBryVF4B9rlmKgU1MugxWdkcRivGem7LBvGIXzWj1WM_S5PZ_WDe5iGM3iOJr6Q6Qm1rCRYMm5K81Il3FWjplxq8mFCu7cPXwAfnux2mGDon5nJNnkk1iCZ4QL6Z-2OFtC9SHUzoAt-j6y_He80WD8VUVrZtbdj0WiI8-RTEtTI-hWMzy3ui9XzfUPP96F32LfU3FiMO70b_8w4cjAadA0ECSkmz6tEbz1RWbmXZBb41kU_S8O355q8LsTQINib5B7iMfibZz1l4y_U2DC2lUblXxzoSDqTYCK3Hgj1w8ULIvpLd_ML0qw31m41dsn3n3sb-PyYUzpUEd__yqIurwknIKjeVuMADJjwBow1ZkKIO_QCM3vIrh2TEtgqEHjE3Hn6SDBjiK9EhyeonRQ_CqWeXG0bQltJ6pvv5fJF1uL2iKOOVUbgDPgqXQke3htZTc9lKTCbHC0n6iv1uY2dlYgYIkgfq6O902urLOKJw3sbVHfup1v-8B4qb7c0tu1K3KR4Hou4df8AFf7iI7coWA0CCz5I9itIbEGu8uMn6t8psOt7HVYqHj-cYiENNjVMO6lVPhnQbs-yF00ESe7kWCDiCBdqC1OeVDu1HdPN7xRbq9OEYstIitWp9AJNtt9DPt_5ZQboVLeuhQ\",\"summary\":[]},\"output_index\":0,\"sequence_number\":2}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"rs_03733174ba77c7d2016adf5041bcc087d19325f28f5cefd730\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq31BB5qtMc6yF_wcnrDVUUn1Dtmrw5J3gjtxcovIUduIIHUgVk5SSHV4KhLlDjG6zt7ordNjOarnxFWPYHYvEPCGiFsMZCPLNyoiV6Mr5MxYhU6HWQZTmKTmyDgfJQlcKnPoTIg_4AFhKYf7vyS4UbMt6SesFGOYdM5tqmLXdf0hgv-O4dfiX-CiV1G1N6nJffcmY39VkIkZx7G90pUFuWsZkeZIYDoIFtKTEzfm10Nug2aH3soQhw4Y3pg6I1HDArNJJh0EDWxXJW8Ifm4uiE0CnGSdRjh6coguu9lo1sESF0b0rPPwYUXVU0eCD1chRa02J9yMWzN0uv038t9g4stdQRV8KboY6EvJ6_snA10k4vkdWC1huuqATr5XVg_RQu7qE9mMyUsK03_bPRvhsPeIvA8BfJ1qQIWyugzv-5Oe0Mh8GyMedWCUINlwjzlOg8g_zMeiQEM-VcNIy-uDr3G806GZRS_s7-OoXioCLV_0CzhkI-6Ly7P7yn-NhS4dI9heX3NCGFGgcSTneWwHAGWZB_b4ZyZG2G9142L3htC5VBwv3TIq5vE376T3MaOiv10Sq0l4Qu_oZ1N_66YxWqk0UfD-mDADNCUD4ERlobB8Z_YxEgHjGUiL1NAekgMh06h8ynhMPnsUBB7QDjvBZsqNg-kmANBy9m5-NWEi7N_QK96eiZq0xb59o46tJLW2fSwBn0PkbGKywQS1_dP3_On6DzNDzAzlcf50LJ3Ia9TNMcZyHbwIg_rPD_X29AQKuuTeEiYC_c8zFWUd_mEnmJrM061-4P6NZLyl43sXjIOWfpp0df3-V5-dSQ03G6LVYQ-L1PicDqaMfEoVdo_I6UZuxO-qmvAfpPRTchpb_xTAnNShqlyVGf-DJzqSYL3coT3vPQwtMEeITCP5bjoiZpuhiifATnQ5AXGoTYwR6BcP580oAHQS1JxyIw8lmRjYksQAXvaW7MxbGqLxnrrpih3cQd6uiQajVgYEne0Zkcnx5S_VTXX7Seq6LHBhWpg6ACGRpRC-j8XOVGhyp67E7SoWEjqs1HmdF-A7L1IICcYRCehdHeCmrEo3-yr-dbpvLwhu9vF5TsCeLmVTU_siuxYc6Gh6zAq4y453tCR2QyMQo5wrf2vLlGvkPKGr9OwfAAyEJAM3_3SnIsewiGOj0TzCraMep58Vcad3oyb31hdA8WDeTjiulKZ5YNTp_PwZ_CJPZR5e9bSjTS9HazPqwaa2OCtJBDEsC8JveQfm10mGbQyEdvD_OYjWYv-xYdJsHQeQeBbR4JpFvKO41tVSjJ8WLe3t34KSI3k6EZ5jQPg0ZaOOknqglRfXpGzHbfZY6tjFyUzxt5IqsG34NX4purO4zkAJ_nWbjAWRmAcWqVnTlT5a9ppy1aUbd_HJJPpTO8B0TweTWVKt8Y6U3kDhjvpwT-QOKhn_VAy-AbLvJBbZAOAh1XscW1wI9PJTBAZ3Wpyh_0fpcHCSCjsdKXCxMZMy-79tljxWvF8uN6_1SiH9TKdgmAZfJ29YYoU32VYKq8pgJdvGT5DG4Oc6gm0vUR48CcYu5HG6GyWg5vu7g796geR8n9PQ2Zw4pDDTIMJxqiF0hZks3W9hn5PryRooc536KhEc-avaZADdFA8ZLqdDCN6U=\",\"summary\":[]},\"output_index\":0,\"sequence_number\":3}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_3fCg1Ycahjqs70OPFddXVmKi\",\"name\":\"bash\"},\"output_index\":1,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"obfuscation\":\"XPZUo2ap6U5ulp\",\"output_index\":1,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"obfuscation\":\"nBTgLbgx2\",\"output_index\":1,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"obfuscation\":\"rlsNZ1gDt9R9R\",\"output_index\":1,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"printf\",\"item_id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"obfuscation\":\"GGmrJU4vJR\",\"output_index\":1,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" pi\",\"item_id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"obfuscation\":\"M65oWqi7iNWOi\",\"output_index\":1,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_tool\",\"item_id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"obfuscation\":\"btmzrXxlQLx\",\"output_index\":1,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_ok\",\"item_id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"obfuscation\":\"31PUOhrIc4dHT\",\"output_index\":1,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"obfuscation\":\"QeQvvPS8BdwG7\",\"output_index\":1,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"obfuscation\":\"CcI836Cah\",\"output_index\":1,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"obfuscation\":\"T595dfJDa66r7n\",\"output_index\":1,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"120\",\"item_id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"obfuscation\":\"ZtFESbFUY9oUy\",\"output_index\":1,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"obfuscation\":\"uaXM8XPyRqjyK\",\"output_index\":1,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"obfuscation\":\"CFXeG9EwxrxJWwp\",\"output_index\":1,\"sequence_number\":17}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"printf pi_tool_ok\\\",\\\"timeout\\\":120000}\",\"item_id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"output_index\":1,\"sequence_number\":18}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"printf pi_tool_ok\\\",\\\"timeout\\\":120000}\",\"call_id\":\"call_3fCg1Ycahjqs70OPFddXVmKi\",\"name\":\"bash\"},\"output_index\":1,\"sequence_number\":19}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_03733174ba77c7d2016adf50412f8087d187d3b3d750a16d1e\",\"object\":\"response\",\"created_at\":1793019969,\"status\":\"completed\",\"access_programs\":null,\"background\":false,\"completed_at\":1793019970,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[{\"id\":\"rs_03733174ba77c7d2016adf5041bcc087d19325f28f5cefd730\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq31BC5YlOY3W-i233Dtu0pIm4j69b4PYKudWpnklE9ef109txQe7GheKDZiIrDeGeicrrCNWAiYSnvfNDSG38lr_3X9X_y_j_drEXkeydHw1ohLexuLcMBlATmdru5TbJ4zB5FoqhjBlOgvMfFnS8GmyecTBHgg1CtZQ4rYqbkivCXZwcRB-bziock-VBl-AGmb0oKtDDwqTqof4yK33AXI3WOaTxng9Clnnl7-vAggCaS_wq188NB41rdYlOK6aeXwhiuEYAgpd1OPw8lRvIFbP5K3A8Oz1WNr1kXJYLMN-4dGGniJ7XYCXnGVkCwMRnoi0dGzkDoeaUUdW8bahoWmVP9T8uDC5nc3MfdwvY0bJ_saqXqoikllqB2ITkJAHIt1mNW7IyVCxjc8hPQvRbHTfV7Pc04U6wzUPCs9a33ZzUrDEmxfkOHeWRpOvlfNUqEH73IKvsuu31gXwO2_-zUSlExqoy5Pj4c3-Ud-0GhYGvTacFRLbHS90wqguT9_6IDsYZYfKj3kn3uHvQ7p1y1ER4mmbqvSYizvVQ3yhszMTor61NiL8Y03UECh9dsSJ3Ssa5PTcLWYfe2DDNcl-Kcmg1inuEFWNaf5ZAmmZTJYPUjtNqc5YBZRLpxKq6Rs87XLVDsjPcEyRWKZxQgXbdXmQCKlDqlZypHBmKGM5naLn_cjx_gxl99PxMcl-chWofLlUnTXZTG4FCaURIBkXWpnWMiEMpCkla_yUMBPUfNta7bM2bxL3nKF7HRge__68FesgzAXv7jRhu65Y9eT4DwrbVlBTK6GitDKhFdiGoBCLJ-kKjxTD5JN-tSQCu2tO_g5QPW5lTDk92hPPblxgAaG5jtqrQOFe68EtscF8ZUSZpqhRlTFQJj25DHA5OFoQTabERE64RKHOEV1cpHC5eJz3md51pTuQ8CIKAli3cgQadbOlbwtDWFWM0K7LjQ4yOtmc12lwv4XmbeHCBOImemhr1g4hyH9-rUKoD0-OR9q8OoIJZy17-qBW_NQ5UeA5D9SaD4TUV4bkgsJMfsGQhM-WvOJxmrjcC3LbldSb9tyzxqZzunAXjKZfq0-TkKLzUKjdqO2kh-0sxi-2nP7mOwK-e2oiIgjSDAHl88-AUlxLpJlNLSy_qS_QvNdNqzSjQpWpY6a4oAnXx2T65oAlVkng8AaQ_SMaidUjdQAoKPlohE1QUj9dGN33q1uFbiMd7yohmEptm3kWImpPcFWsoUFS6wReDrv0TYQ5_WNkU6g6fC346v7J00f_L4-JuMCjMvEAnSWtBvJH-PBNamS-V9F2xb9GlUqm4lik_8rpTy9m-EiZV6F_CLCcMcvhamTLtsghBN3Jt4-bLgXaWRGKp7PKcg5GEQK-wXghaG5bJZA45bHZ08BzqdrlPIzeJkNiLQLX_M9LV0UmyJeyRpSup7vN9WlR_csEr9GmAVU0V0dQAhtPfDferFUzZb7dBaWVndlaha2hUQeZ2UBmvrj1n3rJZki1R1da1ifpme6J0U7pftkAFKqwkwVyKFY_GhZuzGATwXOyktt83a9N5lFJfP1Su4nWe_eBBLsgoGKK1bzktsPte98y_UgqP0bQR8xSRRLsYPIaEpqCmCzJ3VWmN3e8XspXCqm9HVJ82bpdGGPCWfEc=\",\"summary\":[]},{\"id\":\"fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"printf pi_tool_ok\\\",\\\"timeout\\\":120000}\",\"call_id\":\"call_3fCg1Ycahjqs70OPFddXVmKi\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17851-7cd4-7300-9732-cf7a9f97b248\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":221,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":40,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":261},\"user\":null,\"metadata\":{}},\"sequence_number\":20}" + "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_05902bb47f22bf04016adff080cb8487d1a576194d17dca890\",\"object\":\"response\",\"created_at\":1793060992,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17ac3-7608-7510-b9e7-1f5d461d4b59\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", + "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_05902bb47f22bf04016adff080cb8487d1a576194d17dca890\",\"object\":\"response\",\"created_at\":1793060992,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17ac3-7608-7510-b9e7-1f5d461d4b59\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"rs_05902bb47f22bf04016adff0813aa887d1a883bf95d0ed47c4\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq3_CBVywuGg29uiv6N8ZWJCfk_XjEy8TL3O7FZEnX4uaG2kWgYf6-bQmh4xmTn_Nc-lSRWTQKHHfM0_NYf9v6WLb2ZK6ijtP95XCoghXPEVCGizMRcK-UHa9yHx6kwV2SOpBi5DQKOB9esGi3aElP9ccPdd7jA8XVIuPgp_xDrN9L9maNDKc9wqKmwj-d_uO2AzA0n-kHuY2s4AwXufhUSHokmE8NWLq9DRhplkA1UaRwkALvYqlWnJWs0JhD0awGTdF7ZJGvibDT1bcxKgTAkl4eZNnO0Z6dpstrqNeWyWhsqt9GADT9vtiuvwvcbYfLmaHuepWjXNHodXvOlOPuxhbDo1k4bsMd_nwf1yp2FjDnQT8WgN-ovRbi6rUFkSwp1mSQ--mBb_ubOqj9ANAFmganBa_0QwE-fEILYjD4t4R7x95k4tkMED__DJ9UUAOLGhYA0folhGhbiW4FkiXWJYaEi9FZb_hGUWivd_Vi47EB-Q0VSO_Rpkocj_-n2x337s2lraQEGt-BqaSb61e4XVfzx4tV1aHIvRAUcAuQuuk5EoygcDZ3aHTJUaypwoVCvY37zEixjwJBxq85IxIdkecB2tLWOj2LGJHDv9l_Gz9j9I9LOAASTajDyeYaKF5h_WSl9SJhXdRN-K-rJqNU7-h4jhk0d_xSlHIYsO_wZFPmZQDqumpMpmwJUoDzpxhUTf3gKoQTqbOREbHwpaWDH6ACM_oTRWcgLdnZShZ_7LmaZjsLTCnDH2CT0dPX81loC3mAljzFsWCrGntBVWrxB1Naz9h-8I0Eama8aonbvUhlHc-QOyFert4pM_w0oTaj3ErEaq8ztEQ31H5Wa4QlEg98A2Lxc11m3Nk1ZOc-BKFF0QES1_HRsM2jRmNxeHvyEczg3sr-C6dbKwZfOB4J_stoIsH0l1xJ2KSbEhlOXb_4m6D4mSeP4-44UuDp04w76N6YmKQgvQ9Me-sZcB4aCQSDnKiCztvkOKQ6NyaD2uPlEXZ9f9gsnCnVsmE7XQRU62x3C6rx5raMN_BjCSR17DjS1R6pjN8Vd9gXzr5MhaOxEcviN9T-9d_GCZoJc9Bt6588YvyZsXQFqtsg5PVPFK9Di4Hw48XSO0W-RWHSqjqr-PdDUF8pClMgBJaxrAODCnqHj1M7tpHv8SYl1KBb4l_OWOj0qVACaakk4ULWx5WLYd-wb2FnA2U5JLxTtVSW-T3D\",\"summary\":[]},\"output_index\":0,\"sequence_number\":2}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"rs_05902bb47f22bf04016adff0813aa887d1a883bf95d0ed47c4\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq3_CB-UcIigxrH5dWI3AEIvzJSjO2_bdFJEp5Bn47Y-ospdy2oDS-smXsDLBXBpGWnQC5odwYfT6JocrGiZty1Upq3QQhFS191Q3oXhkcmu3bgWnq4zWTWgaHNl5FdnNF4t26X4o3su8GI0p0GZCPYAzapb7pFaM0XKe3VpJjWus_j4ruQdGtKXdpOrdawZIjJbxXixv8a72MdMGPAV9fWVYxquOe4HMEfdYBwUaqMOjNwOFP2Sq4vf7m1KazIeU4U0wILRIOC4OwEMiIUtUNMsyq_1v16Z_0iPOz8KtHC6usgemuemdsofdBr7Smw9clEavZriyVsr3yggQMn6wwWj-BmQBvP5cLlE6J5j8hv-T__1zQgZ4UT4xMiXkxBEwXDzxFjlyR9JaBfMxUHMJg8CS0faLgaREGr79Hr8CEhG6Mp4L1Y2HsY6xspYSWyUXlnqB1a8L4SqfvJJJrr-cAH9fzMNUclZ47t62aPhJ9WldBf5RH-tpPMfBdS-rYhqOw7OJ7x5sS6GU8dvNUPQRQAFNrv0TF8wgPqBjLNaEvURpum2kk2VJV2-gcL15SDf-aMD50CbnUkMCkb6iBOa20RNPo3IG6kBwv_18f8LGzYRXqFMSOPUYR4eaOYkcga1U8XP-0bELHJkQVNUULfcsRTqwb8eVZ3yuaO1oas71n913oOLT6gX0Vc8RVYDpWLe2UmMGjSlehwPsVo2DRPVjUTtZLVHhDHimXeCxy1Ibq_8nP00ttzKDcL1S0JsEptx5KiOkOSDzAxsrL86EWl0xYh_hFADeQ5W-VZiIoGQ-VF-FqjKx31my7lJoNhfackEGFV1PUdiw5UKh6BKIfFMA8Xux8VP3PxlDgjVpVKMKVtpKVCTlbeoRrB02mhOdLRaS4cXo6tXBjH45fvUai_IEwHL2mCYGJ8LDN6Kuyhr6ubxqHtnEkh3pu9evT3HOWrt9j6a_9NFsSacnV-8z5VTD8p-Bniyz1rj5-tTjHyc2RHWJACHzeHoRJv7-dP-uOfJr4q-AS6wLiO5rtYENAQZEstrK22_rsCRmT1LzztP1X-F-ulw3vvG5acDKxpJtrIFylePHGnQJXVKuEbkmCBPO5-daAYuhP0yJmWhoBmsfsUnDANGhjcZysBRvk6svTgaJmwe-Dh57_bET_iZ1Hgq5BjcGynY2W3ow83f8eee9Egk3EovbTzHhOkHLbzePYCYiuuwwkLJn1cln6UdagvvQuCYpDYkFQUtDRUz23a4VAqmet1n5a3lgVJRiYXnZsyenprmI0O4rrU3AuojMXcQH0GWBJqe4l4nF2f9IYtkv6Lv6SD3Nh-vPNcfJha4Zjj8b4qivNycunu57OZzGrx8Q0l4SlqW9mL0tzjiCs-XF80MpHtTdg6Sp7ck1PiLYlhb1DD7zmPrZx2GkVw0w0bXUxEoW9-9UoZ2JdkOScHYlZsP095syBm25MNNxUW9UFWCRaUDUVqDMx3vB2krqUmuutnb78KmvCt5HWk9kk9YWTgTLtmT9ZsO23btQ1z3kMa61Usq23Kfbs1sXYZCw2ryWgY02UaV99FcGRm7Ym1IQgHjWUS2UIWkYDCZcyXxUzB7PYfas-D9GVjWzMT3TXrCXHSCWJvh8qSQu9k2jDc6kOqLrTgzA=\",\"summary\":[]},\"output_index\":0,\"sequence_number\":3}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_z1nocxjB1ozM66RNjCRMtY27\",\"name\":\"bash\"},\"output_index\":1,\"sequence_number\":4}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"obfuscation\":\"EA9NSpT6KrwmVK\",\"output_index\":1,\"sequence_number\":5}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"obfuscation\":\"aqz1QnTEQ\",\"output_index\":1,\"sequence_number\":6}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"obfuscation\":\"8tV9H9yX8W1mX\",\"output_index\":1,\"sequence_number\":7}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"printf\",\"item_id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"obfuscation\":\"wpJzgo7RP5\",\"output_index\":1,\"sequence_number\":8}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" pi\",\"item_id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"obfuscation\":\"ZuZxgviZSghyz\",\"output_index\":1,\"sequence_number\":9}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_tool\",\"item_id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"obfuscation\":\"wqjRlmFYT8m\",\"output_index\":1,\"sequence_number\":10}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_ok\",\"item_id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"obfuscation\":\"W9SbINGRAb2w3\",\"output_index\":1,\"sequence_number\":11}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"obfuscation\":\"u8bKG7YDEY9Gj\",\"output_index\":1,\"sequence_number\":12}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"obfuscation\":\"LuOnqdn8V\",\"output_index\":1,\"sequence_number\":13}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"obfuscation\":\"5rtAFoWrQ9IYwo\",\"output_index\":1,\"sequence_number\":14}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"120\",\"item_id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"obfuscation\":\"bFQpcDgpC3kDy\",\"output_index\":1,\"sequence_number\":15}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"obfuscation\":\"XJx1GEQrN2zjY\",\"output_index\":1,\"sequence_number\":16}", + "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"obfuscation\":\"OQXcsZXHbLAxKUc\",\"output_index\":1,\"sequence_number\":17}", + "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"printf pi_tool_ok\\\",\\\"timeout\\\":120000}\",\"item_id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"output_index\":1,\"sequence_number\":18}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"printf pi_tool_ok\\\",\\\"timeout\\\":120000}\",\"call_id\":\"call_z1nocxjB1ozM66RNjCRMtY27\",\"name\":\"bash\"},\"output_index\":1,\"sequence_number\":19}", + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_05902bb47f22bf04016adff080cb8487d1a576194d17dca890\",\"object\":\"response\",\"created_at\":1793060992,\"status\":\"completed\",\"access_programs\":null,\"background\":false,\"completed_at\":1793060993,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[{\"id\":\"rs_05902bb47f22bf04016adff0813aa887d1a883bf95d0ed47c4\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq3_CB30wf0roEn17sc6kGkjQOuGmFzMB2KbBLdhk10foZ5EaAblZRobIN0hZf_UKhalYw-RtP6IcxaAz5K71qErV_unq7Nob24fxhPi-t-PbwR8Ou2WZIEu-s56r3izQasGnvnJQtA-qxCTEz0BvJlw7KC4N-jiHnjYHosQMDhypg5Jkju1VchYww4apTTu2NI_7rHupKJtx9NGMa-3slj3wI7ttACejzmOT0r3q_XJR1XXH9YjgL1-FUpB_EDGguRBxvZUlw2HxDw9iadFEx-g6wH6SoZSeFQmYC2DJLOxvYuqmVxLZph4aVdHSAxS7IDczT42o3kUyFJfm9Ip1L32if600_QzX921yXkLiStTARJxSN7vMJedTdNsSprSlkcCjET2OYt3pR3TQQBkYiAiXZkwvvHICyk9mf0i4p1zTtIpyLlfYgZCKKYqJabZSy2leHpLVtI3WRtnXAQqcQm5tsv-xBKpRs02H4tkJHYVgLNPSDBSaiQB3vEEPC_sjZwRvi7YOhviDjR-9J1p_bWctQBlqO33DJPjSdyhObu1it33R7av84XVCmnLxr4Il96VCgfdndBwDbfwLDSOkEL6ya_i2u1B9aV7vkwV3-wxrR3ugHCEFL7vSVWYWowbTGjHxypjlYtrBSUVeXsifDLHP-JCbRX1uFjc5g89xEWgPX2q4PWbPmOSUKbBiW_JWEvMnLb3aOdED_cgtbriu30rADDmXEezpyIAD0E3wYwq8QbGOhsHHrVKHVmH0QMeNfzxCvZ1vmIKxk0U2DMBlfz8H_DitLsTqEhLlsu0Zf4f78M9g-EYh34GMNqjB0KIUHdcY6BjW3Nz2FFkc3y6CJkwK1mfCpT0gqju4sGpwbZFWauBRIrPlDNKJS6qiunflKSZ0qmnxbK6sGT51UUWDlMFj1pLmiOt1cdHkR49ae-ZwEg7wOlk_uXJmqixez_S8nZU27zzhNV4TuRUZk908waIDvdVBU-4hfH-OfeagRCjxxPNEZ6VCztYfG9-ozcBlucRaBGdX5V0D_6WwmJP17cpBLpXi7sFAPM0YiUNPrcHiNNBZRzOsrOzItdsd10R9zeiRJ1GcpI7smoDxOgF0aLB9Ux01sDX2ttEom6km_7YZqiNLNmDywP2l7_CSoOS542RsZ5tc5hKXFSYWmS2HINo0-dgbWQ5QeMuyb7rJR5gyezwO-WM0GP6wjFVm3nKHdj0An8I8V32_8iW8v-D4tGyHsipooIACzy1oeQeAoTCydhZzVK2wy5NNngczvdxlkOnfnEdk9_gpErZ4_ReEGrNHr9gohOYirIzcgeXwjHDEGueHNynRaXSJc1cMj93DhVq2-VdbhKD7PMgOMgOlXIhBReoquizsZJn5Q5OzLEmhBzdtldWNPFVkbloZsjsoMDrMeCE28bWtV6RgLKAX4yyBkf47JKZFrKH1Z73QEa4bV9ETesEkCV9ewALVyKiIXihA9Mef3cZocFSt5bvk_b5M6vdBBtGljNQ54EYHzpkSMDD5KBh6JhjmUD6qvKlvENWyxckhw2bbxtYyGCvyVA-KPm7BNCE21Y5ekRStemlNm4jtNpZ9Ro7_r-KpSijxbTY7fPVzsVqp_vPXdGtng6AwmGFmaAKPYGam407KmHDcYWJM=\",\"summary\":[]},{\"id\":\"fc_05902bb47f22bf04016adff081513487d19294263001cde5cb\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"printf pi_tool_ok\\\",\\\"timeout\\\":120000}\",\"call_id\":\"call_z1nocxjB1ozM66RNjCRMtY27\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17ac3-7608-7510-b9e7-1f5d461d4b59\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":220,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":40,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":260},\"user\":null,\"metadata\":{}},\"sequence_number\":20}" ], "kind": "sse" }, @@ -101,11 +101,11 @@ "access-control-expose-headers": "X-Request-ID, Server-Timing-Total, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "a509ad359ada051b-YYZ", + "cf-ray": "a50d96c43ffc0c69-YYZ", "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 26 Oct 2026 13:06:09 GMT", + "date": "Tue, 27 Oct 2026 00:29:53 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "582", + "openai-processing-ms": "380", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -118,16 +118,16 @@ "x-ratelimit-remaining-tokens": "179999283", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_cbb5026daf8345e1a9d41f691fee62d6" + "x-request-id": "req_7cab0e450e984b53affdf5016a389a38" }, "status": 200 } }, { "callIndex": 1, - "id": "741e0090b82f9531", + "id": "965612483da4119b", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-10-26T13:06:11.065Z", + "recordedAt": "2026-10-27T00:29:53.978Z", "request": { "body": { "kind": "json", @@ -135,7 +135,7 @@ "include": ["reasoning.encrypted_content"], "input": [ { - "content": "You are a coding assistant. Use tools when requested.\n\n\n/Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/resolve-issue-red-green-73219c51/e2e/.bt-tmp/run-0w90vN/scenarios/pi-coding-agent-instrumentation\n", + "content": "You are a coding assistant. Use tools when requested.\n\n\n/Users/abhijeetprasad/.superset/worktrees/braintrust-sdk-javascript/compare-instrumentation-pi-d012543b/e2e/.bt-tmp/run-Y5pgi3/scenarios/pi-coding-agent-instrumentation\n", "role": "developer" }, { @@ -149,20 +149,20 @@ }, { "content": [], - "encrypted_content": "gAAAAABq31BB5qtMc6yF_wcnrDVUUn1Dtmrw5J3gjtxcovIUduIIHUgVk5SSHV4KhLlDjG6zt7ordNjOarnxFWPYHYvEPCGiFsMZCPLNyoiV6Mr5MxYhU6HWQZTmKTmyDgfJQlcKnPoTIg_4AFhKYf7vyS4UbMt6SesFGOYdM5tqmLXdf0hgv-O4dfiX-CiV1G1N6nJffcmY39VkIkZx7G90pUFuWsZkeZIYDoIFtKTEzfm10Nug2aH3soQhw4Y3pg6I1HDArNJJh0EDWxXJW8Ifm4uiE0CnGSdRjh6coguu9lo1sESF0b0rPPwYUXVU0eCD1chRa02J9yMWzN0uv038t9g4stdQRV8KboY6EvJ6_snA10k4vkdWC1huuqATr5XVg_RQu7qE9mMyUsK03_bPRvhsPeIvA8BfJ1qQIWyugzv-5Oe0Mh8GyMedWCUINlwjzlOg8g_zMeiQEM-VcNIy-uDr3G806GZRS_s7-OoXioCLV_0CzhkI-6Ly7P7yn-NhS4dI9heX3NCGFGgcSTneWwHAGWZB_b4ZyZG2G9142L3htC5VBwv3TIq5vE376T3MaOiv10Sq0l4Qu_oZ1N_66YxWqk0UfD-mDADNCUD4ERlobB8Z_YxEgHjGUiL1NAekgMh06h8ynhMPnsUBB7QDjvBZsqNg-kmANBy9m5-NWEi7N_QK96eiZq0xb59o46tJLW2fSwBn0PkbGKywQS1_dP3_On6DzNDzAzlcf50LJ3Ia9TNMcZyHbwIg_rPD_X29AQKuuTeEiYC_c8zFWUd_mEnmJrM061-4P6NZLyl43sXjIOWfpp0df3-V5-dSQ03G6LVYQ-L1PicDqaMfEoVdo_I6UZuxO-qmvAfpPRTchpb_xTAnNShqlyVGf-DJzqSYL3coT3vPQwtMEeITCP5bjoiZpuhiifATnQ5AXGoTYwR6BcP580oAHQS1JxyIw8lmRjYksQAXvaW7MxbGqLxnrrpih3cQd6uiQajVgYEne0Zkcnx5S_VTXX7Seq6LHBhWpg6ACGRpRC-j8XOVGhyp67E7SoWEjqs1HmdF-A7L1IICcYRCehdHeCmrEo3-yr-dbpvLwhu9vF5TsCeLmVTU_siuxYc6Gh6zAq4y453tCR2QyMQo5wrf2vLlGvkPKGr9OwfAAyEJAM3_3SnIsewiGOj0TzCraMep58Vcad3oyb31hdA8WDeTjiulKZ5YNTp_PwZ_CJPZR5e9bSjTS9HazPqwaa2OCtJBDEsC8JveQfm10mGbQyEdvD_OYjWYv-xYdJsHQeQeBbR4JpFvKO41tVSjJ8WLe3t34KSI3k6EZ5jQPg0ZaOOknqglRfXpGzHbfZY6tjFyUzxt5IqsG34NX4purO4zkAJ_nWbjAWRmAcWqVnTlT5a9ppy1aUbd_HJJPpTO8B0TweTWVKt8Y6U3kDhjvpwT-QOKhn_VAy-AbLvJBbZAOAh1XscW1wI9PJTBAZ3Wpyh_0fpcHCSCjsdKXCxMZMy-79tljxWvF8uN6_1SiH9TKdgmAZfJ29YYoU32VYKq8pgJdvGT5DG4Oc6gm0vUR48CcYu5HG6GyWg5vu7g796geR8n9PQ2Zw4pDDTIMJxqiF0hZks3W9hn5PryRooc536KhEc-avaZADdFA8ZLqdDCN6U=", - "id": "rs_03733174ba77c7d2016adf5041bcc087d19325f28f5cefd730", + "encrypted_content": "gAAAAABq3_CB-UcIigxrH5dWI3AEIvzJSjO2_bdFJEp5Bn47Y-ospdy2oDS-smXsDLBXBpGWnQC5odwYfT6JocrGiZty1Upq3QQhFS191Q3oXhkcmu3bgWnq4zWTWgaHNl5FdnNF4t26X4o3su8GI0p0GZCPYAzapb7pFaM0XKe3VpJjWus_j4ruQdGtKXdpOrdawZIjJbxXixv8a72MdMGPAV9fWVYxquOe4HMEfdYBwUaqMOjNwOFP2Sq4vf7m1KazIeU4U0wILRIOC4OwEMiIUtUNMsyq_1v16Z_0iPOz8KtHC6usgemuemdsofdBr7Smw9clEavZriyVsr3yggQMn6wwWj-BmQBvP5cLlE6J5j8hv-T__1zQgZ4UT4xMiXkxBEwXDzxFjlyR9JaBfMxUHMJg8CS0faLgaREGr79Hr8CEhG6Mp4L1Y2HsY6xspYSWyUXlnqB1a8L4SqfvJJJrr-cAH9fzMNUclZ47t62aPhJ9WldBf5RH-tpPMfBdS-rYhqOw7OJ7x5sS6GU8dvNUPQRQAFNrv0TF8wgPqBjLNaEvURpum2kk2VJV2-gcL15SDf-aMD50CbnUkMCkb6iBOa20RNPo3IG6kBwv_18f8LGzYRXqFMSOPUYR4eaOYkcga1U8XP-0bELHJkQVNUULfcsRTqwb8eVZ3yuaO1oas71n913oOLT6gX0Vc8RVYDpWLe2UmMGjSlehwPsVo2DRPVjUTtZLVHhDHimXeCxy1Ibq_8nP00ttzKDcL1S0JsEptx5KiOkOSDzAxsrL86EWl0xYh_hFADeQ5W-VZiIoGQ-VF-FqjKx31my7lJoNhfackEGFV1PUdiw5UKh6BKIfFMA8Xux8VP3PxlDgjVpVKMKVtpKVCTlbeoRrB02mhOdLRaS4cXo6tXBjH45fvUai_IEwHL2mCYGJ8LDN6Kuyhr6ubxqHtnEkh3pu9evT3HOWrt9j6a_9NFsSacnV-8z5VTD8p-Bniyz1rj5-tTjHyc2RHWJACHzeHoRJv7-dP-uOfJr4q-AS6wLiO5rtYENAQZEstrK22_rsCRmT1LzztP1X-F-ulw3vvG5acDKxpJtrIFylePHGnQJXVKuEbkmCBPO5-daAYuhP0yJmWhoBmsfsUnDANGhjcZysBRvk6svTgaJmwe-Dh57_bET_iZ1Hgq5BjcGynY2W3ow83f8eee9Egk3EovbTzHhOkHLbzePYCYiuuwwkLJn1cln6UdagvvQuCYpDYkFQUtDRUz23a4VAqmet1n5a3lgVJRiYXnZsyenprmI0O4rrU3AuojMXcQH0GWBJqe4l4nF2f9IYtkv6Lv6SD3Nh-vPNcfJha4Zjj8b4qivNycunu57OZzGrx8Q0l4SlqW9mL0tzjiCs-XF80MpHtTdg6Sp7ck1PiLYlhb1DD7zmPrZx2GkVw0w0bXUxEoW9-9UoZ2JdkOScHYlZsP095syBm25MNNxUW9UFWCRaUDUVqDMx3vB2krqUmuutnb78KmvCt5HWk9kk9YWTgTLtmT9ZsO23btQ1z3kMa61Usq23Kfbs1sXYZCw2ryWgY02UaV99FcGRm7Ym1IQgHjWUS2UIWkYDCZcyXxUzB7PYfas-D9GVjWzMT3TXrCXHSCWJvh8qSQu9k2jDc6kOqLrTgzA=", + "id": "rs_05902bb47f22bf04016adff0813aa887d1a883bf95d0ed47c4", "summary": [], "type": "reasoning" }, { "arguments": "{\"command\":\"printf pi_tool_ok\",\"timeout\":120000}", - "call_id": "call_3fCg1Ycahjqs70OPFddXVmKi", - "id": "fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe", + "call_id": "call_z1nocxjB1ozM66RNjCRMtY27", + "id": "fc_05902bb47f22bf04016adff081513487d19294263001cde5cb", "name": "bash", "type": "function_call" }, { - "call_id": "call_3fCg1Ycahjqs70OPFddXVmKi", + "call_id": "call_z1nocxjB1ozM66RNjCRMtY27", "output": "pi_tool_ok", "type": "function_call_output" } @@ -215,26 +215,26 @@ "response": { "body": { "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_03733174ba77c7d2016adf504258d887d18268b9ec06df9f21\",\"object\":\"response\",\"created_at\":1793019970,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17851-7cd4-7300-9732-cf7a9f97b248\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_03733174ba77c7d2016adf504258d887d18268b9ec06df9f21\",\"object\":\"response\",\"created_at\":1793019970,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17851-7cd4-7300-9732-cf7a9f97b248\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"rs_03733174ba77c7d2016adf5042d16c87d1b3946ebd6bf8c547\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq31BClONKiMwPw18QYMRBNSVtUULK-yT83uu5FvgE8G6p56ECoowsWhZkA67WOfkEyZkuZ1QX60yvCGdergKRcRGvDBVLXM0Tt2JxzZt-KlNHW3uxMmm0AVJG_bmEWE7jWVPg1f--kdxzqldP9dZ2yIIhwR9WMqqQbbDSG4asqDa1qVD6_iDXZX6Qstbl_0uFLhctH-uNLbrEPEMB7lFLGVtovdSTpz0Bns_YzZhP8t2UKHz7EcQhqxaN9BwB1cDVaC9lwF406dk-qjmg0ol2_N1n01NvFXK1Z1werhVuLMyeb2hmpxbb2HdibkO25Mydiwk5gfoujr1jUKDAVvqJGYIS53YYogy9yz0sNPwvp6VdVGKjhf_stXDhKGnkmo0URof0JmxCJ3D4Ahz5M3ti44i3hkCnXuzIR-dMz0DUOUKTnCIB6AEnhxfDDNV43Y4rE4Hyg7--yQzIv3RRHCG0n5cRezlwOvJPUtot_vgfiN2o_tmwB1metwzFS9ckYeVARWY1HlT5m-46CG46DDu0zgirYCKZhbYrp4kVq-0APG6vPBGSrKmf0hl0nx_3SZuTJG0eY3wnBAXPdWpyRmsDixOU8glI0oJjpDNB7TRcofZDj_ZHlE9_gpbO1h6PlQVPXSoXQfFDqJbSTXNDJs4M2i6272x6EqdWMF2gjQhPcsbksNOT3EBLsAtvEbz19drVSLpxoazJldjUld7Y0fo1Cefoj9W4nZy11ERYU9Q9475iIPTXmnVp6Uio4Hipq26zOxoGV3PzsX2QVB26O7TJ5R-ggfpA7S3kIfaB8KGv4Wrd0WuaNdHezWasQ-NBE3T-dMEij1EUzytJe7_J5q5BEXu6gaLBs8f25x_l1cTUxlltQRJU3vwfsxMVYUdAN7ILYPNgidRPIYUHc0-UBwQACAKYCTcYiaLj5Mp-_FQhXyMcy4KKL1OOSoSYW8IKpkzdRN4qn5HYZSlY22yhHJYejyCGJV9ug6Afg7YxzDAUrb_ql-LGsBINQQn3Ng9M9xTgdK64-KHs-JnTvuMqeIsKkUXl8knenyt9ShIi6b8zxga4t8LChur9_7JguGdscfl2740AufqzBWQeWu8OaHkILGDsEzyQBuZNkZD3OVavwvcR3-bLhrDkD5Y-SqS0a2FvFU0xt6NKE3o6cWCPJmFVm_DEOrGpoCBsPrjm3NKM75_cKu70rKU-ePRoB_ZlbIkrOplu\",\"summary\":[]},\"output_index\":0,\"sequence_number\":2}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"rs_03733174ba77c7d2016adf5042d16c87d1b3946ebd6bf8c547\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq31BC8nl_sA-RlIhk40RLqsw-pt_aBM8QwRuVn20Wuq3FSkKyHOmyWWz0_Kjq3LeajLG8r3VX27io1BLlaW7O2I0tjIk11ECjLfYmjZGi0rWFfGdWUs_RclqxSIgg_k3VbHFWYGb8GPKWF2ALD8oyTfPAJ-ArzIOVd6hK_Sl-04M2sc_74v_gxuXsllNGCZ1_AHVwz39MlxlKtdMRYOqy0Z5M1l6m6oQu_gQDFwrQ0QiPK-Xq1M4ve7vWw1IVqSiPHOLKn2HSic9HK9S6V_NVt80y0A3xoj_tm6XvaAXX3g01BNl9rcbRlBwLUhOJ40GAa-PolmjVZrobhRVXMeC1wMgxWcZDNIWFJZALao-zaYiurMbUVpsjc5ihg6pe5Htk_c5c_TssT-Dps8uH3yN-WrPNFEwXvvyy2ZLMyPkDk2TrVj6rjTWkU0pgxU1UpaLcjya1ETTM1FiA7DXWcgT7k5ecgEltVb-v5WNvwAaEkejaEun9-WEhaXndT8Y5v7_4U-yqQimiu63ZBvZm38pGYtD2aBMpqCXc0W8xNRrDUYnnsMem1ng8OhknpACg_abSi9v5X6fzaYbuiMawvlxJHvkWy1drIawPmmqm-1iYRQejpZeaT39XiCoafqigFrhtUruUY0m2wncIjjx2LwvDQPhnkjfNFCaoCfuhcJWP34vOEUgd96Jy0lUje-EGnraOHrAXuE0_QqrW06I5IYaHggwbXXohhdUdv1iCzVm8lK28xX_0u8ZO4GYNQjNKuS2m7uKGZYqg29TxaAPvy8QJPJPH-vGqDifw9qrqznX0cAoxAqEeU_tnmNH5USRpijtTE6bEsz-zZ2fk3bkPO9pRQSlHxW9P1zjda3WgkY6oYykeufFFlt6C0cRoiHj5iSv9MLg55Mg30bU4p74iKxqWOqcM-eQG9XHvTFKR2Ak0X93ZcdpPTFgYi--qSUOH5RYKLwQZWDitbzqJlF4zp6TRwY1PsAzNReX31aPX3CPi3dJ3JGFmZm5oBrhaMzLq6_-s6oEshqPmeRgmDF8gEOUF9NwfUVOwP66d4aMDxUhDtmzRJawCP9YMkx4-IiL4VGCJhCt3RQn7481Iqre6E-mQPJVra8O67py5Bs-EsCWkMUwA-kmhyCT4oRK1EdmnSLnI3DLbV3_S8psMMYOjCUzrdW2I5Ujrgpy-5V0snDBbmR72FAbZn2f1ZbCUJPPOXRCg0WyAjVDJY-bee7Oq03gUsHQFLmGv3k62gR8TrvFnA0JGjQEcZ6CTBd3FROznN9PMAN66KtNFcrTfxjE0CRrnmRUKdKcd3mZatMxFi-_IuG7EbvfU_78knEqLuRGpRN1vDdSVQSCmevnJulfEzWELD-As5bF7VXdDRzrdAmGPdvaxkrUczLCZZSJl6pboW1GgM2nptqy4Fv0fcK7bPkC-U70ND1j9-x1RGx7E43AL0zraSmGjRfV7Co8x_V_4RNPkC-iBYdZ2_vwU1MA0YI6VXSB_ccgnQYBGb1PS4dYIvsfvYWOwwXBM2Yzkrr0h0weSgXY9s-EQCzfPiyh7a5wpGzlIhSjUs8c9t1-mGNCWX80iSAcUHgc6PgKS2lWEbivfLHLH_Sxus_UfR_tQh5K16hRs97qN1JBlI745CU-xkJhPI_E=\",\"summary\":[]},\"output_index\":0,\"sequence_number\":3}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":1,\"sequence_number\":4}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"PI\",\"item_id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"logprobs\":[],\"obfuscation\":\"CUAFxznIUIGmFY\",\"output_index\":1,\"sequence_number\":6}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_C\",\"item_id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"logprobs\":[],\"obfuscation\":\"lXNqJvexkJg6zy\",\"output_index\":1,\"sequence_number\":7}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"OD\",\"item_id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"logprobs\":[],\"obfuscation\":\"SmxPUBuHkkjlDy\",\"output_index\":1,\"sequence_number\":8}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ING\",\"item_id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"logprobs\":[],\"obfuscation\":\"ASIse2g9hRWcP\",\"output_index\":1,\"sequence_number\":9}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_AGENT\",\"item_id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"logprobs\":[],\"obfuscation\":\"2KD7OhVJ3j\",\"output_index\":1,\"sequence_number\":10}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_OK\",\"item_id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"logprobs\":[],\"obfuscation\":\"TOQ9CTb8IEsF9\",\"output_index\":1,\"sequence_number\":11}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\n\",\"item_id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"logprobs\":[],\"obfuscation\":\"UNw4zPAFJCVRAjM\",\"output_index\":1,\"sequence_number\":12}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"pi\",\"item_id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"logprobs\":[],\"obfuscation\":\"7Sy9gZc5xCPkSw\",\"output_index\":1,\"sequence_number\":13}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_tool\",\"item_id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"logprobs\":[],\"obfuscation\":\"r1F1gzfNtyg\",\"output_index\":1,\"sequence_number\":14}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_ok\",\"item_id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"logprobs\":[],\"obfuscation\":\"DNQZxiki0EgHx\",\"output_index\":1,\"sequence_number\":15}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"logprobs\":[],\"output_index\":1,\"sequence_number\":16,\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"},\"sequence_number\":17}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"}],\"role\":\"assistant\"},\"output_index\":1,\"sequence_number\":18}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_03733174ba77c7d2016adf504258d887d18268b9ec06df9f21\",\"object\":\"response\",\"created_at\":1793019970,\"status\":\"completed\",\"access_programs\":null,\"background\":false,\"completed_at\":1793019971,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[{\"id\":\"rs_03733174ba77c7d2016adf5042d16c87d1b3946ebd6bf8c547\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq31BDfTLh05wyrxOJbIORVvPdcuMrdtUiVXNswacVGdZA6WIqLWtEswjLVGzATUJSdCJNGcpGJX0YyPIZGProorL7xHRDVfGchN60jyL6QsBAo1u9q4W_lsqdY8QWX86NihhtJPGejTD7g7nYHsQV1N4km2uUlWHmiTfwmP9hV6wXhfsYCZZ5DI9n_tvtkRIgDYL4p7TDF5B3dLuUligQkaVcVXZF0qFJbMvl_f3bEjjsQQrpPAo6ZxepEQdGkCXUIC8BauqDci_MUEn-gqli8AhbzMp9HMz8VW7tFW0_LTOoW2sKOlyMl8RmYMkVfqyqACEwcTdNsRzpQF1AUUgSK1d6ckxQ_WZHOdHUs58HkDcCvBUST5ua6ZAnX2F0olJNGIEGnIqxZqkp7PPJEA9zIfW0Yj6_ftidunimnec8NbJh7M6ffyl52dg9VG23a0CRJHZiFAkIpP-Do7rUOE3ZLavKZ_AMA5Dtocd2P0gC22FTijHoEQdWvsoiEhOEcDgL_JqtC9oFi1AEs2ramwGNWdcO2J-qrAdSW7RzOsTtwaVbKzy5JYJinyMJcyiybBrhMPK5lmPpH9TKF0XICcgdHHtL7lM6MnSPul3R41sN4v8KOvAMoTGGdmrM2mmT2t3lcUoQmJbEHzGiY7t7KP9VmDSB4i8QDrSz3gqSZvRREnG6gr1lrSQJZbuO1oIJ42650K4-4shLt9815dHUYXka3cd48XkRZAXwkhjSVRrdzVaRklimo_5fSCea9F2LFfIgMzf3NniKt-2YNCsGuISbp4AwuxGUcbTpaPIJhzzjLTcgyz0h8y0CSFSxz7PKIJAWAtJ3n-FFboP3uMZYC0SalwXlVf0OPLvspfRYFZZ5bP4KDpvPUpjdYhEWfMWSC2-_3r7jJnuvUERAxEHMUdYNffDaPEKFOhVYJ_saaYrKS-SXL3hErSzrmyb_8Q91Q9IZkRAJthKWqrtyCz-8Snr4yJ60DWV848KMGLSccbRWkKNO2Z0x-_XjzSatWAvWgWraBMAjFeVK7XX8lKmIaO02y7rqsFJ7JGtPlp6Cn8eh8-w-ygilJ4uEzY-ohaHc5wiKNsI0j32jAVJ_rJQJhGzfBAVRow4wTP8FuJdpskVb_wayzSFVlg7uqNNyw9fAm_8t14b2H8AKc89ZuI9bWktFq5HzPdHXENOJyEdMZVWm2FFpFPSh0Z_-w4Yi3EGNe70VshApL8cpU6Jlo66GfAHLRn2qMdrUOn1LBjoJ-lDNTTEVAs1wNP1OsV7AVUHll58c90g5VFof_pwsD4xM_S3RQE9-D35DpEaEr5P77lgqO_mbBgJM_hzEzcbCrAcz7ocBGWDJMjMx-krmHYMR8mY0GIBkNosXCwZz9m0tY_qIeIcWzx3snIhvOg98ypAG176ONqE2KONhHygkkihqsgY46j9NMXkdjp4c-j9UQNWrGo-k5uPGHxqXW11l7napVd_S3w6RRnao0_zSAvfhORTVOG_j-JYZzk3qT7z-xJL1xs7nYVEuFnTmCt2iuY0-TIIMbn5ONmsFAz2Izss8i5Bwwhco5w_aKEvY5mWVtY8vSuU6R0GHXXddMoYDapMzjkF0H14gti3hKxwJrDntiBPF1eXZGU26_tE5k42xbvT7yd1_LP0=\",\"summary\":[]},{\"id\":\"msg_03733174ba77c7d2016adf5042e52487d1a25281a079706f0a\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17851-7cd4-7300-9732-cf7a9f97b248\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":272,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":28,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":300},\"user\":null,\"metadata\":{}},\"sequence_number\":19}" + "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_05902bb47f22bf04016adff08196e887d18a3be8b3398bc94d\",\"object\":\"response\",\"created_at\":1793060993,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17ac3-7608-7510-b9e7-1f5d461d4b59\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", + "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_05902bb47f22bf04016adff08196e887d18a3be8b3398bc94d\",\"object\":\"response\",\"created_at\":1793060993,\"status\":\"in_progress\",\"access_programs\":null,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17ac3-7608-7510-b9e7-1f5d461d4b59\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"rs_05902bb47f22bf04016adff081d43487d1b19dae8b6151655a\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq3_CBCp1IfIzi0N1gvhe-PXtKZtMlRynTGQ3Dc8_H85ZPy20D6gKuv1MGu8MR4PHERiOj2Ct7yb-FqT3kp8HKBdV_zkbEKyNgii6wzkeFea1MMTzBNu_flWb_OzBsiz1THi7bnNRD2VVPCjr0IeGB80mHTpDGAVkoTZwENHAxWxF7rf_5eRfD6cRIyIkT1sTqJg7SNeG9RCLUN8PWYlVCBWk0wdYT-c88HUo1zs5Jl1s_03KRSv7v1sUhPQLhcuhFiXsgaI70i2atSavD8sbuWFnbXNe-MRUcUVuo5uO0xgUVx5O_ICWS3vE_no03JyA3fV6T_JPmkGl7JkYZbBlrR0JfkoEPBj5R116zbRw4fnFmUn7E7RmDDn4fdArS3kn_8wxABFQiWUzF2XEDE-mdhnwvE2n5FtmnAaAgRdFxGHhY-7wBvHnsDSpQoRiuKWzuwjA1C2pw0R0MpR-WrcTxxD5Or25wUMQZOVPVVOO7jZ_n_sfOaCEf_etEnQpxr6BkS4k5LZSIhmwf15Xl-4TARvYn70pAtd6VH5MsriRFTh8SRZbobLgfh5z1VPD9m2AoJXd1wXIdZPaVzKVaf8uifJVFbr8SpsaM5vPD2NUx63GoqazvJMii9Jow-qs26zxFiLHM35zec8Tx2ylJaMbt_kGwPBtk6Y5GXNXddo3rvVe8d27UNtfk6ff6RQxhEu_R7tyx_l8eFWJjHxyzgKyiLzNLh6RQmt6gXPMBxKq7O9rKLKLJ9MD1E18Hnq8zBgLTKAy9P0H1xudiMlSyFoNI-F0cYxXIEXJQzR4Gq9xq0I0sKjTldEIV84J5LTCia9aTKCe_z9zjmYNw8uP_clTGGXm0znChNVzyoIHSryvCYSnlrU24UMUi9rpbhWeK7pC5KOWMpRNooZNIIo-tVPNuGhF_ZpfWyZLb5I7vnXN8t3u12LoagaGEmAQrwu8aU1LuPgvRZLyUumI9Yk6ZpxONBkeCFYMgciVvyEySNuTLRxT4JATJGYUnA7jryKBlIAFyoinTHgFF6bN9GFDkl2XQ3_QhL6u2trDOW2VvrZ_CQAFOExEJX8iZQNtlEKFBg8awVoBhIn76-bwEEnGKdl4vP8P4Ucx8D_TGZ8PNUE6e3geesfne25YkgFCKM4S5P9KOI1MVgKF4PYCj7GN8mVoUp0Fj--KPeMe8cBx2hbTnBa5hKqlJQo4v5s-QMJEY3sNKOg1u\",\"summary\":[]},\"output_index\":0,\"sequence_number\":2}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"rs_05902bb47f22bf04016adff081d43487d1b19dae8b6151655a\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq3_CBz24bDLRyvqQyYI3oEnCKIt6INnesk4n_kQ_zm-WimGS6MN_6MkaMOqehjXU7wOnSjToQUcxQ6ZBCsS6Z6UTuX_JsamZRQ66G1dCxeE9Lq1J99zjhU7ceP-L61VW2VMNjvnVIAs4R4_7owTTsQTO_0h2XD4z-CQuEqf2KvUPkM39TahRTgpGAbA_k0DZrARTjHVTWC-owjtBIpgU0f9N97bVK-lBPS93Nzyg95ZIUXX1_AVIuRw6ORy8zQWPyDm4lmbwA7ouKsmdcUkEQ4V24RCCBWxIkJigrXWnv6mARRkYTrzWifRP__MbsDf0Ub6bahR5VTHjif0jxRoSRYIrN5EmJqHM80obUz4VcAOEJEN45H72UycUg2duYjudhIOYasop0vkL2O_1_mgjJJt_jxcy62bOfqrINV1a0mmdKOQdtJNsqcwRv8oIIuomHdRPP7CssxuuP5aTs9lXu80txd9C-UJaSIvnQKUmv4dChzro4jSplBmfO0dZFlasOvboioeSgIDTqUwq1f8TN3N9VhI7-V9y-KaXzeTo5fSKSvTgoCTkeeE3Z2K8ep3G5y1SjsGLt4_BDmzoWs5F3PgK_h16pgEQCIGrgNXaqlADWtaaoqN64UW5pYYP8HVVA4dpXc1835GfsB2Tr2ZX2QKXGx1-ZzzIiYGzUI9uPGtffxTwq5srSiZw5EZ1wzVftIAKEd2KZAT-y735lUm483z3CAKlsv0-XdV21ArLhFDTAJMn-osZc6s1-3Kx-rpoamMBRgQq3b9UTXyd5abxzYZYeZcb62chJxDwr6_NADCyMxLUtZiGpIpE4wSEOr5IbTFjC2jgo4MukmP7xb2tuqGOxxjKVXFgoxhccBDLiiuLuWzpE9myvkJTA_1nKRX7RGuzcw8DSdeJTD7ou2ANA7LsdW70NXUZQVCDQTM_FgZHUdPB7T8HdYXnMyfbV3g2g_92uAy1mxePDzFw2Ko9o1K-zuKJlPreo7Z5bxTJVJYKT6OJtPoactUvG-0iPwZOVPeAGywHoUN1RC7V7m-2AVjSo82iWXqXaG2lKJim7IOPhz5zgxdjuCHURKwk3hWapJeYFqaNdcefJ_kg8t1Ov3NyPcdlI1lm9adK_907NG1ws2nhPIqq9JyYsI1AeiGOnc2hjNKI3j6nHxpRkZ3R5SNax6V1oXfqLRX_h6nwMNQVTxEdtYc9uLj-jylC7_93Qk6Vbd7NkgC0yJ04Vhxh9Ut8hOX60LmkQpqNGqPR6Qrmv0oNsOhfKrG6otWdCGsrDKm3oxdho__I8_YNhtYxrHgWUTrk0yD4I0pq-fbGKK_knzjt4jM3wc7meozjRyhRIjubmJWaetK737EGK1SgiC6vNoig2r0Ms-MryF6aUY7eLzas0PC-VTJdG_XDIDpA00t4bh3p1L2XgL6K9VMx8ed2kI8TW6uORsPXrFMjBsr99qApSyq3Yo4V3vZpzaxUuwLms-bhcEMZSXhtzoeDIll5uWHL4Kb5_yXh10oVQELoM3JuMmmhLsgfiwhX0em_ny3zHHopdwmsSjVBRMcA6eQWS3wP4Af-E0AwJjFqvR6pIbRJe1vPou92JZuDuBbRSVY6TXm9MhYDbt1MjQ7wWBVgW1LHXFVAipwQ8ZuMrPpSXphs=\",\"summary\":[]},\"output_index\":0,\"sequence_number\":3}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":1,\"sequence_number\":4}", + "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":5}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"PI\",\"item_id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"logprobs\":[],\"obfuscation\":\"MLV0SGypM0AWih\",\"output_index\":1,\"sequence_number\":6}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_C\",\"item_id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"logprobs\":[],\"obfuscation\":\"2ZTirH075sKrIP\",\"output_index\":1,\"sequence_number\":7}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"OD\",\"item_id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"logprobs\":[],\"obfuscation\":\"efr4g8ajPfN1RH\",\"output_index\":1,\"sequence_number\":8}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ING\",\"item_id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"logprobs\":[],\"obfuscation\":\"rYgnjgdjlwnm7\",\"output_index\":1,\"sequence_number\":9}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_AGENT\",\"item_id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"logprobs\":[],\"obfuscation\":\"dV8FSn0Nwo\",\"output_index\":1,\"sequence_number\":10}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_OK\",\"item_id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"logprobs\":[],\"obfuscation\":\"UGsxySNuIUFAu\",\"output_index\":1,\"sequence_number\":11}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\n\",\"item_id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"logprobs\":[],\"obfuscation\":\"Tac6N6LjlnOvkh0\",\"output_index\":1,\"sequence_number\":12}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"pi\",\"item_id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"logprobs\":[],\"obfuscation\":\"zN7ntJUGJ0Vg0H\",\"output_index\":1,\"sequence_number\":13}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_tool\",\"item_id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"logprobs\":[],\"obfuscation\":\"TGtEotvDSjz\",\"output_index\":1,\"sequence_number\":14}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_ok\",\"item_id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"logprobs\":[],\"obfuscation\":\"GNknRkmRD0ow7\",\"output_index\":1,\"sequence_number\":15}", + "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"logprobs\":[],\"output_index\":1,\"sequence_number\":16,\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"}", + "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"},\"sequence_number\":17}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"}],\"role\":\"assistant\"},\"output_index\":1,\"sequence_number\":18}", + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_05902bb47f22bf04016adff08196e887d18a3be8b3398bc94d\",\"object\":\"response\",\"created_at\":1793060993,\"status\":\"completed\",\"access_programs\":null,\"background\":false,\"completed_at\":1793060993,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":128000,\"max_tool_calls\":null,\"model\":\"gpt-5-nano-2025-08-07\",\"moderation\":null,\"output\":[{\"id\":\"rs_05902bb47f22bf04016adff081d43487d1b19dae8b6151655a\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABq3_CBAxBul0kHw4_ykOiHpboPR83LoDDxHVlT_uMyl8ezMaf1dWQy-eu9DmrT5D6_e6BkToFiKFFeCnVrFbwCwZOQfvMcECeVyK-7bK0k4Q_kQgz5Y_4xutWksIsg6Ggg1vThk0-4PVokkRHsS_v45ZW6X8p94eFeuEo46ucb6-SQ0OnK4FVoIiTSUgoSRVwemJ-DDs3J_7YJIpvB3tGMyKMon0yKfREsjj3BW5HSJKYuCN0PgaTbKkX27vqSzxGNXBM2spYXroHK5l9DoP4l4bYOsxSr0wrT7OAP2nQ1g4N3IXKgh9dvigP8sxPJ0iJebXcCE8Lc2dEn44srT_CjxPOvRrvxnlJ48QX1oIuMU3El-IzdbO0SxNTnuX2gGYtRdzt67BZB9t5wvXeO_F6vc6n1BFTUbKxACW0lBqWVzAdaHz8By5Q3aY6P1jHDQwJ51xLUOiEmzIXopSEwTVMHbjGPa8dWAMfyOx17d2f_dSK8DhK_YIAH_rCDkzN59-q7nSQmUqKeJSXPpODW_tSBhhIqqXKPu151sWehQmYYygNGcSnxU9YBC_ofqsViyY_Av1_Pj0BBCydovcAxbxOpNpLNkZYjit25MFmciyZ5ZwzDSr2l1bWzshZ8aSvZ1DBc0pOqTgLUYZxeAsbBOJcsrzBSDfOTzLxA2mkFwMwO1wM_bLCmCzycRrx60qgRxmqPOQ-A7IC1uf7QURBuMM-yA284eskuDe1ctL3-8G858Gx9NKYunWW1meX03zZdcL1gr-ya2rJCamzQuBWOCmquVOWsdTj5RXDLU8QLxmnE1RywcJnNUW6USeUoNTRff_CmDOx5-7RDf5m3vg0As4uIGjWe6RGf76OUN6gzSHVGso3XnhFSdXVpXR5HIvIFUH_PfV8rznkF97rLyMqiY7gZYFh0a-nixIN-szezEigo5rdKhhErocJDlnhh3HbAWdmUQPMMZyqzdKchluffuOoIuk-TCBnp-Y9YVFObPyVzaFtEXXBDhx0VywUoGYSOMQkpWfhWqYq-8bjoIFugVLzIaGDkOMpqHes3u73oCpzxqHGPLCgpWMh5yl1olxr6rOMfwSrG4jJvEMk2_RfrXIOWzutbapttON31qAl9BNsMyXkAdIQ7PCqMaHdHv8ePjkIopyv1M57mbIliLNjb8VvsPAn0EalaqZemGJw65CikL2MlY84ZLQ_OnMire8WCCMvuz9Iab5E5ScLvZi5RaNZO-oddUb3-DtPWnjke-a5Tk1SX6TY7I0h5M7Lt4sXlgcbRlJCYXNzVPheF0YA99i7sdKvKomhhvOoAyw-qTqKVHwkc2t7T95T1blOgRKwuRNzbn9hxnKtdm1NKbGA0uh1TCuTJ6XT_wG7KykLXfsLT5qsigEFTPikZYCMC7nLNa1RHQDH0T6vB4zDH2Ei4b0kpATnr_YVOdcQI5gZ4gBfMzEnga0f2MUkyqJWJjDzyEle1dB2d9TT9hnGNM9Nw2TofPWDdVjPZPI_11KKTfGfgeMmZxyuV3NGwYZryOWkNB4IqqnM3ASDaCgj4Rj9R16aY7yt6Y7yYenoGi52TXfK1M_A6Efr6q3q5fXfkHN2KZ4jbDmJA4zG8GHfTFjoZsTocRMagzeCSsBKMTV_M9m8WOKtNRrw=\",\"summary\":[]},{\"id\":\"msg_05902bb47f22bf04016adff081e63887d1b53f3c0e7c5c2ad1\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PI_CODING_AGENT_OK\\npi_tool_ok\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"01a17ac3-7608-7510-b9e7-1f5d461d4b59\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"minimal\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\",\"timeout\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Shell command to execute\"},\"timeout\":{\"anyOf\":[{\"type\":\"number\",\"description\":\"Timeout in seconds (optional, no default timeout)\"},{\"type\":\"null\"}]}},\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":271,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":28,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":299},\"user\":null,\"metadata\":{}},\"sequence_number\":19}" ], "kind": "sse" }, @@ -242,11 +242,11 @@ "access-control-expose-headers": "X-Request-ID, Server-Timing-Total, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "a509ad3dfa48051b-YYZ", + "cf-ray": "a50d96c98c6f0c69-YYZ", "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 26 Oct 2026 13:06:10 GMT", + "date": "Tue, 27 Oct 2026 00:29:53 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "304", + "openai-processing-ms": "182", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -259,7 +259,7 @@ "x-ratelimit-remaining-tokens": "179999232", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_23bb6b46b3c048b39a999f987768a5e3" + "x-request-id": "req_58fec088144645d4b4bc5c4fc79da8c4" }, "status": 200 } diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json index f0764dee2..b69e26ea6 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json @@ -11,6 +11,95 @@ "name": "AgentSession.prompt", "type": "task", "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\"}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.max_retry_delay_ms": 0, + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + "pi_coding_agent.operation": "agent.streamFn", + "pi_coding_agent.response_id": "msg_011CgRoA9MLPbh7ZBCTQXMgD", + "pi_coding_agent.session_id": "", + "pi_coding_agent.stop_reason": "toolUse", + "pi_coding_agent.transport": "auto", + "provider": "anthropic", + "tools": [ + { + "function": { + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "name": "bash", + "parameters": { + "properties": { + "command": { + "description": "Bash command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": [ + "command" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 57, + "estimated_cost": 0.001786, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1501, + "time_to_first_token": 0, + "tokens": 1558 + } + }, { "name": "anthropic.messages.create", "type": "llm", @@ -67,9 +156,14 @@ "metadata": { "model": "claude-haiku-4-5", "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.max_retry_delay_ms": 0, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "agent.streamFn", + "pi_coding_agent.response_id": "msg_011CgRoACeXDKGoyf2QJB3rN", + "pi_coding_agent.session_id": "", "pi_coding_agent.stop_reason": "stop", + "pi_coding_agent.transport": "auto", "provider": "anthropic", "tools": [ { @@ -99,12 +193,14 @@ }, "metrics": { "completion_tokens": 23, - "duration": 0, + "estimated_cost": 0.00169, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 2335, + "prompt_tokens": 1575, "time_to_first_token": 0, - "tokens": 2358 + "tokens": 1598 } }, { @@ -130,7 +226,9 @@ "metadata": { "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } } ], @@ -147,24 +245,31 @@ ], "metadata": { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", + "pi_coding_agent.version": "0.79.1", "provider": "anthropic" - }, - "metrics": { - "duration": 0 } } ], "metadata": { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } } ], "metadata": { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } } ] diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt index 28df14572..907404ef3 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt @@ -1,11 +1,13 @@ span_tree: ā””ā”€ā”€ pi-coding-agent-root [task] metadata: { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } ā””ā”€ā”€ pi-coding-agent-prompt-operation metadata: { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } ā””ā”€ā”€ AgentSession.prompt [task] input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." @@ -21,15 +23,105 @@ span_tree: ] metadata: { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", + "pi_coding_agent.version": "0.79.1", "provider": "anthropic" } - metrics: { - "duration": 0 - } + ā”œā”€ā”€ anthropic.messages.create [llm] + ā”‚ input: [ + ā”‚ { + ā”‚ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + ā”‚ "role": "system" + ā”‚ }, + ā”‚ { + ā”‚ "content": [ + ā”‚ { + ā”‚ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + ā”‚ "type": "text" + ā”‚ } + ā”‚ ], + ā”‚ "role": "user" + ā”‚ } + ā”‚ ] + ā”‚ output: [ + ā”‚ { + ā”‚ "finish_reason": "tool_calls", + ā”‚ "index": 0, + ā”‚ "message": { + ā”‚ "content": null, + ā”‚ "role": "assistant", + ā”‚ "tool_calls": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "arguments": "{\"command\":\"printf pi_tool_ok\"}", + ā”‚ "name": "bash" + ā”‚ }, + ā”‚ "id": "", + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ } + ā”‚ ] + ā”‚ metadata: { + ā”‚ "model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.api": "anthropic-messages", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, + ā”‚ "pi_coding_agent.model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + ā”‚ "pi_coding_agent.operation": "agent.streamFn", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoA9MLPbh7ZBCTQXMgD", + ā”‚ "pi_coding_agent.session_id": "", + ā”‚ "pi_coding_agent.stop_reason": "toolUse", + ā”‚ "pi_coding_agent.transport": "auto", + ā”‚ "provider": "anthropic", + ā”‚ "tools": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + ā”‚ "name": "bash", + ā”‚ "parameters": { + ā”‚ "properties": { + ā”‚ "command": { + ā”‚ "description": "Bash command to execute", + ā”‚ "type": "string" + ā”‚ }, + ā”‚ "timeout": { + ā”‚ "description": "Timeout in seconds (optional, no default timeout)", + ā”‚ "type": "number" + ā”‚ } + ā”‚ }, + ā”‚ "required": [ + ā”‚ "command" + ā”‚ ], + ā”‚ "type": "object" + ā”‚ } + ā”‚ }, + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ metrics: { + ā”‚ "completion_tokens": 57, + ā”‚ "estimated_cost": 0.001786, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, + ā”‚ "prompt_cache_creation_tokens": 0, + ā”‚ "prompt_cached_tokens": 0, + ā”‚ "prompt_tokens": 1501, + ā”‚ "time_to_first_token": 0, + ā”‚ "tokens": 1558 + ā”‚ } ā”œā”€ā”€ anthropic.messages.create [llm] ā”‚ input: [ ā”‚ { @@ -83,9 +175,14 @@ span_tree: ā”‚ metadata: { ā”‚ "model": "claude-haiku-4-5", ā”‚ "pi_coding_agent.api": "anthropic-messages", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, ā”‚ "pi_coding_agent.model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", ā”‚ "pi_coding_agent.operation": "agent.streamFn", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoACeXDKGoyf2QJB3rN", + ā”‚ "pi_coding_agent.session_id": "", ā”‚ "pi_coding_agent.stop_reason": "stop", + ā”‚ "pi_coding_agent.transport": "auto", ā”‚ "provider": "anthropic", ā”‚ "tools": [ ā”‚ { @@ -115,12 +212,14 @@ span_tree: ā”‚ } ā”‚ metrics: { ā”‚ "completion_tokens": 23, - ā”‚ "duration": 0, + ā”‚ "estimated_cost": 0.00169, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, ā”‚ "prompt_cache_creation_tokens": 0, ā”‚ "prompt_cached_tokens": 0, - ā”‚ "prompt_tokens": 2335, + ā”‚ "prompt_tokens": 1575, ā”‚ "time_to_first_token": 0, - ā”‚ "tokens": 2358 + ā”‚ "tokens": 1598 ā”‚ } ā””ā”€ā”€ bash [tool] input: { @@ -137,6 +236,8 @@ span_tree: metadata: { "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } ā””ā”€ā”€ pi-tool-child diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json index f64cb51bc..541232ffc 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json @@ -11,6 +11,96 @@ "name": "AgentSession.prompt", "type": "task", "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- You can inspect PI_* environment variables for current model and session details.\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\"}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.max_retry_delay_ms": 0, + "pi_coding_agent.model": "claude-haiku-4-5-20251001", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.response_id": "msg_011CgRoAfADfwQ2T1NQ6BDoJ", + "pi_coding_agent.session_id": "", + "pi_coding_agent.stop_reason": "toolUse", + "pi_coding_agent.transport": "auto", + "provider": "anthropic", + "selected_model": "anthropic/claude-haiku-4-5", + "tools": [ + { + "function": { + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "name": "bash", + "parameters": { + "properties": { + "command": { + "description": "Shell command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": [ + "command" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 57, + "estimated_cost": 0.001802, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1517, + "time_to_first_token": 0, + "tokens": 1574 + } + }, { "name": "anthropic.messages.create", "type": "llm", @@ -51,7 +141,7 @@ } ], "role": "tool", - "tool_call_id": "tool-call-1" + "tool_call_id": "" } ], "output": [ @@ -59,18 +149,24 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } ], "metadata": { - "model": "claude-haiku-4-5", + "model": "claude-haiku-4-5-20251001", "pi_coding_agent.api": "anthropic-messages", - "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.max_retry_delay_ms": 0, + "pi_coding_agent.model": "claude-haiku-4-5-20251001", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.response_id": "msg_011CgRoAiZc2juDBqHM7TR7G", + "pi_coding_agent.session_id": "", "pi_coding_agent.stop_reason": "stop", + "pi_coding_agent.transport": "auto", "provider": "anthropic", + "selected_model": "anthropic/claude-haiku-4-5", "tools": [ { "function": { @@ -98,13 +194,15 @@ ] }, "metrics": { - "completion_tokens": 3, - "duration": 0, + "completion_tokens": 23, + "estimated_cost": 0.001706, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 5, + "prompt_tokens": 1591, "time_to_first_token": 0, - "tokens": 8 + "tokens": 1614 } }, { @@ -128,9 +226,11 @@ ] }, "metadata": { - "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } } ], @@ -140,31 +240,38 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } ], "metadata": { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", + "pi_coding_agent.version": "0.84.3", "provider": "anthropic" - }, - "metrics": { - "duration": 0 } } ], "metadata": { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } } ], "metadata": { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } } ] diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt index 4f3fe0ca2..a83c26925 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt @@ -1,11 +1,13 @@ span_tree: ā””ā”€ā”€ pi-coding-agent-root [task] metadata: { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } ā””ā”€ā”€ pi-coding-agent-prompt-operation metadata: { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } ā””ā”€ā”€ AgentSession.prompt [task] input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." @@ -14,22 +16,113 @@ span_tree: "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } ] metadata: { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", + "pi_coding_agent.version": "0.84.3", "provider": "anthropic" } - metrics: { - "duration": 0 - } + ā”œā”€ā”€ anthropic.messages.create [llm] + ā”‚ input: [ + ā”‚ { + ā”‚ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- You can inspect PI_* environment variables for current model and session details.\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + ā”‚ "role": "system" + ā”‚ }, + ā”‚ { + ā”‚ "content": [ + ā”‚ { + ā”‚ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + ā”‚ "type": "text" + ā”‚ } + ā”‚ ], + ā”‚ "role": "user" + ā”‚ } + ā”‚ ] + ā”‚ output: [ + ā”‚ { + ā”‚ "finish_reason": "tool_calls", + ā”‚ "index": 0, + ā”‚ "message": { + ā”‚ "content": null, + ā”‚ "role": "assistant", + ā”‚ "tool_calls": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "arguments": "{\"command\":\"printf pi_tool_ok\"}", + ā”‚ "name": "bash" + ā”‚ }, + ā”‚ "id": "", + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ } + ā”‚ ] + ā”‚ metadata: { + ā”‚ "model": "claude-haiku-4-5-20251001", + ā”‚ "pi_coding_agent.api": "anthropic-messages", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, + ā”‚ "pi_coding_agent.model": "claude-haiku-4-5-20251001", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoAfADfwQ2T1NQ6BDoJ", + ā”‚ "pi_coding_agent.session_id": "", + ā”‚ "pi_coding_agent.stop_reason": "toolUse", + ā”‚ "pi_coding_agent.transport": "auto", + ā”‚ "provider": "anthropic", + ā”‚ "selected_model": "anthropic/claude-haiku-4-5", + ā”‚ "tools": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + ā”‚ "name": "bash", + ā”‚ "parameters": { + ā”‚ "properties": { + ā”‚ "command": { + ā”‚ "description": "Shell command to execute", + ā”‚ "type": "string" + ā”‚ }, + ā”‚ "timeout": { + ā”‚ "description": "Timeout in seconds (optional, no default timeout)", + ā”‚ "type": "number" + ā”‚ } + ā”‚ }, + ā”‚ "required": [ + ā”‚ "command" + ā”‚ ], + ā”‚ "type": "object" + ā”‚ } + ā”‚ }, + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ metrics: { + ā”‚ "completion_tokens": 57, + ā”‚ "estimated_cost": 0.001802, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, + ā”‚ "prompt_cache_creation_tokens": 0, + ā”‚ "prompt_cached_tokens": 0, + ā”‚ "prompt_tokens": 1517, + ā”‚ "time_to_first_token": 0, + ā”‚ "tokens": 1574 + ā”‚ } ā”œā”€ā”€ anthropic.messages.create [llm] ā”‚ input: [ ā”‚ { @@ -67,7 +160,7 @@ span_tree: ā”‚ } ā”‚ ], ā”‚ "role": "tool", - ā”‚ "tool_call_id": "tool-call-1" + ā”‚ "tool_call_id": "" ā”‚ } ā”‚ ] ā”‚ output: [ @@ -75,18 +168,24 @@ span_tree: ā”‚ "finish_reason": "stop", ā”‚ "index": 0, ā”‚ "message": { - ā”‚ "content": "PI_CODING_AGENT_OK pi_tool_ok", + ā”‚ "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", ā”‚ "role": "assistant" ā”‚ } ā”‚ } ā”‚ ] ā”‚ metadata: { - ā”‚ "model": "claude-haiku-4-5", + ā”‚ "model": "claude-haiku-4-5-20251001", ā”‚ "pi_coding_agent.api": "anthropic-messages", - ā”‚ "pi_coding_agent.model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, + ā”‚ "pi_coding_agent.model": "claude-haiku-4-5-20251001", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoAiZc2juDBqHM7TR7G", + ā”‚ "pi_coding_agent.session_id": "", ā”‚ "pi_coding_agent.stop_reason": "stop", + ā”‚ "pi_coding_agent.transport": "auto", ā”‚ "provider": "anthropic", + ā”‚ "selected_model": "anthropic/claude-haiku-4-5", ā”‚ "tools": [ ā”‚ { ā”‚ "function": { @@ -114,13 +213,15 @@ span_tree: ā”‚ ] ā”‚ } ā”‚ metrics: { - ā”‚ "completion_tokens": 3, - ā”‚ "duration": 0, + ā”‚ "completion_tokens": 23, + ā”‚ "estimated_cost": 0.001706, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, ā”‚ "prompt_cache_creation_tokens": 0, ā”‚ "prompt_cached_tokens": 0, - ā”‚ "prompt_tokens": 5, + ā”‚ "prompt_tokens": 1591, ā”‚ "time_to_first_token": 0, - ā”‚ "tokens": 8 + ā”‚ "tokens": 1614 ā”‚ } ā””ā”€ā”€ bash [tool] input: { @@ -135,8 +236,10 @@ span_tree: ] } metadata: { - "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } ā””ā”€ā”€ pi-tool-child diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json index f64cb51bc..df6823120 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json @@ -11,6 +11,96 @@ "name": "AgentSession.prompt", "type": "task", "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- You can inspect PI_* environment variables for current model and session details.\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\"}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.max_retry_delay_ms": 0, + "pi_coding_agent.model": "claude-haiku-4-5-20251001", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.response_id": "msg_011CgRoAfADfwQ2T1NQ6BDoJ", + "pi_coding_agent.session_id": "", + "pi_coding_agent.stop_reason": "toolUse", + "pi_coding_agent.transport": "auto", + "provider": "anthropic", + "selected_model": "anthropic/claude-haiku-4-5", + "tools": [ + { + "function": { + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "name": "bash", + "parameters": { + "properties": { + "command": { + "description": "Shell command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": [ + "command" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 57, + "estimated_cost": 0.001802, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1517, + "time_to_first_token": 0, + "tokens": 1574 + } + }, { "name": "anthropic.messages.create", "type": "llm", @@ -51,7 +141,7 @@ } ], "role": "tool", - "tool_call_id": "tool-call-1" + "tool_call_id": "" } ], "output": [ @@ -59,18 +149,24 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } ], "metadata": { - "model": "claude-haiku-4-5", + "model": "claude-haiku-4-5-20251001", "pi_coding_agent.api": "anthropic-messages", - "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.max_retry_delay_ms": 0, + "pi_coding_agent.model": "claude-haiku-4-5-20251001", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.response_id": "msg_011CgRoAiZc2juDBqHM7TR7G", + "pi_coding_agent.session_id": "", "pi_coding_agent.stop_reason": "stop", + "pi_coding_agent.transport": "auto", "provider": "anthropic", + "selected_model": "anthropic/claude-haiku-4-5", "tools": [ { "function": { @@ -98,13 +194,15 @@ ] }, "metrics": { - "completion_tokens": 3, - "duration": 0, + "completion_tokens": 23, + "estimated_cost": 0.001706, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 5, + "prompt_tokens": 1591, "time_to_first_token": 0, - "tokens": 8 + "tokens": 1614 } }, { @@ -128,9 +226,11 @@ ] }, "metadata": { - "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } } ], @@ -140,31 +240,37 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } ], "metadata": { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", "provider": "anthropic" - }, - "metrics": { - "duration": 0 } } ], "metadata": { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } } ], "metadata": { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } } ] diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt index 4f3fe0ca2..eb7c7cbdf 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt @@ -1,11 +1,13 @@ span_tree: ā””ā”€ā”€ pi-coding-agent-root [task] metadata: { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } ā””ā”€ā”€ pi-coding-agent-prompt-operation metadata: { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } ā””ā”€ā”€ AgentSession.prompt [task] input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." @@ -14,22 +16,112 @@ span_tree: "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } ] metadata: { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", "provider": "anthropic" } - metrics: { - "duration": 0 - } + ā”œā”€ā”€ anthropic.messages.create [llm] + ā”‚ input: [ + ā”‚ { + ā”‚ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- You can inspect PI_* environment variables for current model and session details.\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + ā”‚ "role": "system" + ā”‚ }, + ā”‚ { + ā”‚ "content": [ + ā”‚ { + ā”‚ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + ā”‚ "type": "text" + ā”‚ } + ā”‚ ], + ā”‚ "role": "user" + ā”‚ } + ā”‚ ] + ā”‚ output: [ + ā”‚ { + ā”‚ "finish_reason": "tool_calls", + ā”‚ "index": 0, + ā”‚ "message": { + ā”‚ "content": null, + ā”‚ "role": "assistant", + ā”‚ "tool_calls": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "arguments": "{\"command\":\"printf pi_tool_ok\"}", + ā”‚ "name": "bash" + ā”‚ }, + ā”‚ "id": "", + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ } + ā”‚ ] + ā”‚ metadata: { + ā”‚ "model": "claude-haiku-4-5-20251001", + ā”‚ "pi_coding_agent.api": "anthropic-messages", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, + ā”‚ "pi_coding_agent.model": "claude-haiku-4-5-20251001", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoAfADfwQ2T1NQ6BDoJ", + ā”‚ "pi_coding_agent.session_id": "", + ā”‚ "pi_coding_agent.stop_reason": "toolUse", + ā”‚ "pi_coding_agent.transport": "auto", + ā”‚ "provider": "anthropic", + ā”‚ "selected_model": "anthropic/claude-haiku-4-5", + ā”‚ "tools": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + ā”‚ "name": "bash", + ā”‚ "parameters": { + ā”‚ "properties": { + ā”‚ "command": { + ā”‚ "description": "Shell command to execute", + ā”‚ "type": "string" + ā”‚ }, + ā”‚ "timeout": { + ā”‚ "description": "Timeout in seconds (optional, no default timeout)", + ā”‚ "type": "number" + ā”‚ } + ā”‚ }, + ā”‚ "required": [ + ā”‚ "command" + ā”‚ ], + ā”‚ "type": "object" + ā”‚ } + ā”‚ }, + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ metrics: { + ā”‚ "completion_tokens": 57, + ā”‚ "estimated_cost": 0.001802, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, + ā”‚ "prompt_cache_creation_tokens": 0, + ā”‚ "prompt_cached_tokens": 0, + ā”‚ "prompt_tokens": 1517, + ā”‚ "time_to_first_token": 0, + ā”‚ "tokens": 1574 + ā”‚ } ā”œā”€ā”€ anthropic.messages.create [llm] ā”‚ input: [ ā”‚ { @@ -67,7 +159,7 @@ span_tree: ā”‚ } ā”‚ ], ā”‚ "role": "tool", - ā”‚ "tool_call_id": "tool-call-1" + ā”‚ "tool_call_id": "" ā”‚ } ā”‚ ] ā”‚ output: [ @@ -75,18 +167,24 @@ span_tree: ā”‚ "finish_reason": "stop", ā”‚ "index": 0, ā”‚ "message": { - ā”‚ "content": "PI_CODING_AGENT_OK pi_tool_ok", + ā”‚ "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", ā”‚ "role": "assistant" ā”‚ } ā”‚ } ā”‚ ] ā”‚ metadata: { - ā”‚ "model": "claude-haiku-4-5", + ā”‚ "model": "claude-haiku-4-5-20251001", ā”‚ "pi_coding_agent.api": "anthropic-messages", - ā”‚ "pi_coding_agent.model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, + ā”‚ "pi_coding_agent.model": "claude-haiku-4-5-20251001", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoAiZc2juDBqHM7TR7G", + ā”‚ "pi_coding_agent.session_id": "", ā”‚ "pi_coding_agent.stop_reason": "stop", + ā”‚ "pi_coding_agent.transport": "auto", ā”‚ "provider": "anthropic", + ā”‚ "selected_model": "anthropic/claude-haiku-4-5", ā”‚ "tools": [ ā”‚ { ā”‚ "function": { @@ -114,13 +212,15 @@ span_tree: ā”‚ ] ā”‚ } ā”‚ metrics: { - ā”‚ "completion_tokens": 3, - ā”‚ "duration": 0, + ā”‚ "completion_tokens": 23, + ā”‚ "estimated_cost": 0.001706, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, ā”‚ "prompt_cache_creation_tokens": 0, ā”‚ "prompt_cached_tokens": 0, - ā”‚ "prompt_tokens": 5, + ā”‚ "prompt_tokens": 1591, ā”‚ "time_to_first_token": 0, - ā”‚ "tokens": 8 + ā”‚ "tokens": 1614 ā”‚ } ā””ā”€ā”€ bash [tool] input: { @@ -135,8 +235,10 @@ span_tree: ] } metadata: { - "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } ā””ā”€ā”€ pi-tool-child diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json index f0764dee2..6a97e2f37 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json @@ -11,6 +11,95 @@ "name": "AgentSession.prompt", "type": "task", "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\"}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.max_retry_delay_ms": 0, + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + "pi_coding_agent.operation": "agent.streamFn", + "pi_coding_agent.response_id": "msg_011CgRoA9MLPbh7ZBCTQXMgD", + "pi_coding_agent.session_id": "", + "pi_coding_agent.stop_reason": "toolUse", + "pi_coding_agent.transport": "auto", + "provider": "anthropic", + "tools": [ + { + "function": { + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "name": "bash", + "parameters": { + "properties": { + "command": { + "description": "Bash command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": [ + "command" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 57, + "estimated_cost": 0.001786, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1501, + "time_to_first_token": 0, + "tokens": 1558 + } + }, { "name": "anthropic.messages.create", "type": "llm", @@ -67,9 +156,14 @@ "metadata": { "model": "claude-haiku-4-5", "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.max_retry_delay_ms": 0, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "agent.streamFn", + "pi_coding_agent.response_id": "msg_011CgRoACeXDKGoyf2QJB3rN", + "pi_coding_agent.session_id": "", "pi_coding_agent.stop_reason": "stop", + "pi_coding_agent.transport": "auto", "provider": "anthropic", "tools": [ { @@ -99,12 +193,14 @@ }, "metrics": { "completion_tokens": 23, - "duration": 0, + "estimated_cost": 0.00169, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 2335, + "prompt_tokens": 1575, "time_to_first_token": 0, - "tokens": 2358 + "tokens": 1598 } }, { @@ -130,7 +226,9 @@ "metadata": { "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } } ], @@ -147,24 +245,30 @@ ], "metadata": { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", "provider": "anthropic" - }, - "metrics": { - "duration": 0 } } ], "metadata": { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } } ], "metadata": { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } } ] diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt index 28df14572..de43ee80b 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt @@ -1,11 +1,13 @@ span_tree: ā””ā”€ā”€ pi-coding-agent-root [task] metadata: { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } ā””ā”€ā”€ pi-coding-agent-prompt-operation metadata: { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } ā””ā”€ā”€ AgentSession.prompt [task] input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." @@ -21,15 +23,104 @@ span_tree: ] metadata: { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", "provider": "anthropic" } - metrics: { - "duration": 0 - } + ā”œā”€ā”€ anthropic.messages.create [llm] + ā”‚ input: [ + ā”‚ { + ā”‚ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + ā”‚ "role": "system" + ā”‚ }, + ā”‚ { + ā”‚ "content": [ + ā”‚ { + ā”‚ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + ā”‚ "type": "text" + ā”‚ } + ā”‚ ], + ā”‚ "role": "user" + ā”‚ } + ā”‚ ] + ā”‚ output: [ + ā”‚ { + ā”‚ "finish_reason": "tool_calls", + ā”‚ "index": 0, + ā”‚ "message": { + ā”‚ "content": null, + ā”‚ "role": "assistant", + ā”‚ "tool_calls": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "arguments": "{\"command\":\"printf pi_tool_ok\"}", + ā”‚ "name": "bash" + ā”‚ }, + ā”‚ "id": "", + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ } + ā”‚ ] + ā”‚ metadata: { + ā”‚ "model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.api": "anthropic-messages", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, + ā”‚ "pi_coding_agent.model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + ā”‚ "pi_coding_agent.operation": "agent.streamFn", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoA9MLPbh7ZBCTQXMgD", + ā”‚ "pi_coding_agent.session_id": "", + ā”‚ "pi_coding_agent.stop_reason": "toolUse", + ā”‚ "pi_coding_agent.transport": "auto", + ā”‚ "provider": "anthropic", + ā”‚ "tools": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + ā”‚ "name": "bash", + ā”‚ "parameters": { + ā”‚ "properties": { + ā”‚ "command": { + ā”‚ "description": "Bash command to execute", + ā”‚ "type": "string" + ā”‚ }, + ā”‚ "timeout": { + ā”‚ "description": "Timeout in seconds (optional, no default timeout)", + ā”‚ "type": "number" + ā”‚ } + ā”‚ }, + ā”‚ "required": [ + ā”‚ "command" + ā”‚ ], + ā”‚ "type": "object" + ā”‚ } + ā”‚ }, + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ metrics: { + ā”‚ "completion_tokens": 57, + ā”‚ "estimated_cost": 0.001786, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, + ā”‚ "prompt_cache_creation_tokens": 0, + ā”‚ "prompt_cached_tokens": 0, + ā”‚ "prompt_tokens": 1501, + ā”‚ "time_to_first_token": 0, + ā”‚ "tokens": 1558 + ā”‚ } ā”œā”€ā”€ anthropic.messages.create [llm] ā”‚ input: [ ā”‚ { @@ -83,9 +174,14 @@ span_tree: ā”‚ metadata: { ā”‚ "model": "claude-haiku-4-5", ā”‚ "pi_coding_agent.api": "anthropic-messages", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, ā”‚ "pi_coding_agent.model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", ā”‚ "pi_coding_agent.operation": "agent.streamFn", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoACeXDKGoyf2QJB3rN", + ā”‚ "pi_coding_agent.session_id": "", ā”‚ "pi_coding_agent.stop_reason": "stop", + ā”‚ "pi_coding_agent.transport": "auto", ā”‚ "provider": "anthropic", ā”‚ "tools": [ ā”‚ { @@ -115,12 +211,14 @@ span_tree: ā”‚ } ā”‚ metrics: { ā”‚ "completion_tokens": 23, - ā”‚ "duration": 0, + ā”‚ "estimated_cost": 0.00169, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, ā”‚ "prompt_cache_creation_tokens": 0, ā”‚ "prompt_cached_tokens": 0, - ā”‚ "prompt_tokens": 2335, + ā”‚ "prompt_tokens": 1575, ā”‚ "time_to_first_token": 0, - ā”‚ "tokens": 2358 + ā”‚ "tokens": 1598 ā”‚ } ā””ā”€ā”€ bash [tool] input: { @@ -137,6 +235,8 @@ span_tree: metadata: { "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } ā””ā”€ā”€ pi-tool-child diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-auto-hook.span-tree.json index 081624d6b..6bac49cd3 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-auto-hook.span-tree.json @@ -11,6 +11,95 @@ "name": "AgentSession.prompt", "type": "task", "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\"}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.max_retry_delay_ms": 0, + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.response_id": "msg_011CgRoB5EKhUC27bPHCWqda", + "pi_coding_agent.session_id": "", + "pi_coding_agent.stop_reason": "toolUse", + "pi_coding_agent.transport": "auto", + "provider": "anthropic", + "tools": [ + { + "function": { + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "name": "bash", + "parameters": { + "properties": { + "command": { + "description": "Bash command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": [ + "command" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 57, + "estimated_cost": 0.001775, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1490, + "time_to_first_token": 0, + "tokens": 1547 + } + }, { "name": "anthropic.messages.create", "type": "llm", @@ -51,7 +140,7 @@ } ], "role": "tool", - "tool_call_id": "tool-call-1" + "tool_call_id": "" } ], "output": [ @@ -59,7 +148,7 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } @@ -67,9 +156,14 @@ "metadata": { "model": "claude-haiku-4-5", "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.max_retry_delay_ms": 0, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.response_id": "msg_011CgRoB8p7tu3WfibUMqmjY", + "pi_coding_agent.session_id": "", "pi_coding_agent.stop_reason": "stop", + "pi_coding_agent.transport": "auto", "provider": "anthropic", "tools": [ { @@ -98,13 +192,15 @@ ] }, "metrics": { - "completion_tokens": 3, - "duration": 0, + "completion_tokens": 23, + "estimated_cost": 0.001679, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 5, + "prompt_tokens": 1564, "time_to_first_token": 0, - "tokens": 8 + "tokens": 1587 } }, { @@ -128,9 +224,11 @@ ] }, "metadata": { - "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } } ], @@ -140,31 +238,38 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } ], "metadata": { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", + "pi_coding_agent.version": "0.81.0", "provider": "anthropic" - }, - "metrics": { - "duration": 0 } } ], "metadata": { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } } ], "metadata": { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } } ] diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-auto-hook.span-tree.txt index 95db54189..7a6a94017 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-auto-hook.span-tree.txt @@ -1,11 +1,13 @@ span_tree: ā””ā”€ā”€ pi-coding-agent-root [task] metadata: { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } ā””ā”€ā”€ pi-coding-agent-prompt-operation metadata: { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } ā””ā”€ā”€ AgentSession.prompt [task] input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." @@ -14,22 +16,112 @@ span_tree: "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } ] metadata: { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", + "pi_coding_agent.version": "0.81.0", "provider": "anthropic" } - metrics: { - "duration": 0 - } + ā”œā”€ā”€ anthropic.messages.create [llm] + ā”‚ input: [ + ā”‚ { + ā”‚ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + ā”‚ "role": "system" + ā”‚ }, + ā”‚ { + ā”‚ "content": [ + ā”‚ { + ā”‚ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + ā”‚ "type": "text" + ā”‚ } + ā”‚ ], + ā”‚ "role": "user" + ā”‚ } + ā”‚ ] + ā”‚ output: [ + ā”‚ { + ā”‚ "finish_reason": "tool_calls", + ā”‚ "index": 0, + ā”‚ "message": { + ā”‚ "content": null, + ā”‚ "role": "assistant", + ā”‚ "tool_calls": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "arguments": "{\"command\":\"printf pi_tool_ok\"}", + ā”‚ "name": "bash" + ā”‚ }, + ā”‚ "id": "", + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ } + ā”‚ ] + ā”‚ metadata: { + ā”‚ "model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.api": "anthropic-messages", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, + ā”‚ "pi_coding_agent.model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoB5EKhUC27bPHCWqda", + ā”‚ "pi_coding_agent.session_id": "", + ā”‚ "pi_coding_agent.stop_reason": "toolUse", + ā”‚ "pi_coding_agent.transport": "auto", + ā”‚ "provider": "anthropic", + ā”‚ "tools": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + ā”‚ "name": "bash", + ā”‚ "parameters": { + ā”‚ "properties": { + ā”‚ "command": { + ā”‚ "description": "Bash command to execute", + ā”‚ "type": "string" + ā”‚ }, + ā”‚ "timeout": { + ā”‚ "description": "Timeout in seconds (optional, no default timeout)", + ā”‚ "type": "number" + ā”‚ } + ā”‚ }, + ā”‚ "required": [ + ā”‚ "command" + ā”‚ ], + ā”‚ "type": "object" + ā”‚ } + ā”‚ }, + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ metrics: { + ā”‚ "completion_tokens": 57, + ā”‚ "estimated_cost": 0.001775, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, + ā”‚ "prompt_cache_creation_tokens": 0, + ā”‚ "prompt_cached_tokens": 0, + ā”‚ "prompt_tokens": 1490, + ā”‚ "time_to_first_token": 0, + ā”‚ "tokens": 1547 + ā”‚ } ā”œā”€ā”€ anthropic.messages.create [llm] ā”‚ input: [ ā”‚ { @@ -67,7 +159,7 @@ span_tree: ā”‚ } ā”‚ ], ā”‚ "role": "tool", - ā”‚ "tool_call_id": "tool-call-1" + ā”‚ "tool_call_id": "" ā”‚ } ā”‚ ] ā”‚ output: [ @@ -75,7 +167,7 @@ span_tree: ā”‚ "finish_reason": "stop", ā”‚ "index": 0, ā”‚ "message": { - ā”‚ "content": "PI_CODING_AGENT_OK pi_tool_ok", + ā”‚ "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", ā”‚ "role": "assistant" ā”‚ } ā”‚ } @@ -83,9 +175,14 @@ span_tree: ā”‚ metadata: { ā”‚ "model": "claude-haiku-4-5", ā”‚ "pi_coding_agent.api": "anthropic-messages", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, ā”‚ "pi_coding_agent.model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoB8p7tu3WfibUMqmjY", + ā”‚ "pi_coding_agent.session_id": "", ā”‚ "pi_coding_agent.stop_reason": "stop", + ā”‚ "pi_coding_agent.transport": "auto", ā”‚ "provider": "anthropic", ā”‚ "tools": [ ā”‚ { @@ -114,13 +211,15 @@ span_tree: ā”‚ ] ā”‚ } ā”‚ metrics: { - ā”‚ "completion_tokens": 3, - ā”‚ "duration": 0, + ā”‚ "completion_tokens": 23, + ā”‚ "estimated_cost": 0.001679, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, ā”‚ "prompt_cache_creation_tokens": 0, ā”‚ "prompt_cached_tokens": 0, - ā”‚ "prompt_tokens": 5, + ā”‚ "prompt_tokens": 1564, ā”‚ "time_to_first_token": 0, - ā”‚ "tokens": 8 + ā”‚ "tokens": 1587 ā”‚ } ā””ā”€ā”€ bash [tool] input: { @@ -135,8 +234,10 @@ span_tree: ] } metadata: { - "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } ā””ā”€ā”€ pi-tool-child diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-auto-hook.span-tree.json index 88e5e6ffd..52fac2539 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-auto-hook.span-tree.json @@ -11,6 +11,95 @@ "name": "AgentSession.prompt", "type": "task", "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\"}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.max_retry_delay_ms": 0, + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.response_id": "msg_011CgRoBcprVdjxx3eTrUun6", + "pi_coding_agent.session_id": "", + "pi_coding_agent.stop_reason": "toolUse", + "pi_coding_agent.transport": "auto", + "provider": "anthropic", + "tools": [ + { + "function": { + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "name": "bash", + "parameters": { + "properties": { + "command": { + "description": "Bash command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": [ + "command" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 57, + "estimated_cost": 0.001775, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1490, + "time_to_first_token": 0, + "tokens": 1547 + } + }, { "name": "anthropic.messages.create", "type": "llm", @@ -51,7 +140,7 @@ } ], "role": "tool", - "tool_call_id": "tool-call-1" + "tool_call_id": "" } ], "output": [ @@ -59,7 +148,7 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } @@ -67,9 +156,14 @@ "metadata": { "model": "claude-haiku-4-5", "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.max_retry_delay_ms": 0, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.response_id": "msg_011CgRoBgNR4wNZe9UdQehi2", + "pi_coding_agent.session_id": "", "pi_coding_agent.stop_reason": "stop", + "pi_coding_agent.transport": "auto", "provider": "anthropic", "tools": [ { @@ -98,13 +192,15 @@ ] }, "metrics": { - "completion_tokens": 3, - "duration": 0, + "completion_tokens": 23, + "estimated_cost": 0.001679, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 5, + "prompt_tokens": 1564, "time_to_first_token": 0, - "tokens": 8 + "tokens": 1587 } }, { @@ -128,9 +224,11 @@ ] }, "metadata": { - "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } } ], @@ -140,31 +238,38 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } ], "metadata": { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", + "pi_coding_agent.version": "0.81.1", "provider": "anthropic" - }, - "metrics": { - "duration": 0 } } ], "metadata": { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } } ], "metadata": { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } } ] diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-auto-hook.span-tree.txt index d905b3d8d..0771a021b 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-auto-hook.span-tree.txt @@ -1,11 +1,13 @@ span_tree: ā””ā”€ā”€ pi-coding-agent-root [task] metadata: { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } ā””ā”€ā”€ pi-coding-agent-prompt-operation metadata: { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } ā””ā”€ā”€ AgentSession.prompt [task] input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." @@ -14,22 +16,112 @@ span_tree: "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } ] metadata: { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", + "pi_coding_agent.version": "0.81.1", "provider": "anthropic" } - metrics: { - "duration": 0 - } + ā”œā”€ā”€ anthropic.messages.create [llm] + ā”‚ input: [ + ā”‚ { + ā”‚ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + ā”‚ "role": "system" + ā”‚ }, + ā”‚ { + ā”‚ "content": [ + ā”‚ { + ā”‚ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + ā”‚ "type": "text" + ā”‚ } + ā”‚ ], + ā”‚ "role": "user" + ā”‚ } + ā”‚ ] + ā”‚ output: [ + ā”‚ { + ā”‚ "finish_reason": "tool_calls", + ā”‚ "index": 0, + ā”‚ "message": { + ā”‚ "content": null, + ā”‚ "role": "assistant", + ā”‚ "tool_calls": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "arguments": "{\"command\":\"printf pi_tool_ok\"}", + ā”‚ "name": "bash" + ā”‚ }, + ā”‚ "id": "", + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ } + ā”‚ ] + ā”‚ metadata: { + ā”‚ "model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.api": "anthropic-messages", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, + ā”‚ "pi_coding_agent.model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoBcprVdjxx3eTrUun6", + ā”‚ "pi_coding_agent.session_id": "", + ā”‚ "pi_coding_agent.stop_reason": "toolUse", + ā”‚ "pi_coding_agent.transport": "auto", + ā”‚ "provider": "anthropic", + ā”‚ "tools": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + ā”‚ "name": "bash", + ā”‚ "parameters": { + ā”‚ "properties": { + ā”‚ "command": { + ā”‚ "description": "Bash command to execute", + ā”‚ "type": "string" + ā”‚ }, + ā”‚ "timeout": { + ā”‚ "description": "Timeout in seconds (optional, no default timeout)", + ā”‚ "type": "number" + ā”‚ } + ā”‚ }, + ā”‚ "required": [ + ā”‚ "command" + ā”‚ ], + ā”‚ "type": "object" + ā”‚ } + ā”‚ }, + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ metrics: { + ā”‚ "completion_tokens": 57, + ā”‚ "estimated_cost": 0.001775, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, + ā”‚ "prompt_cache_creation_tokens": 0, + ā”‚ "prompt_cached_tokens": 0, + ā”‚ "prompt_tokens": 1490, + ā”‚ "time_to_first_token": 0, + ā”‚ "tokens": 1547 + ā”‚ } ā”œā”€ā”€ anthropic.messages.create [llm] ā”‚ input: [ ā”‚ { @@ -67,7 +159,7 @@ span_tree: ā”‚ } ā”‚ ], ā”‚ "role": "tool", - ā”‚ "tool_call_id": "tool-call-1" + ā”‚ "tool_call_id": "" ā”‚ } ā”‚ ] ā”‚ output: [ @@ -75,7 +167,7 @@ span_tree: ā”‚ "finish_reason": "stop", ā”‚ "index": 0, ā”‚ "message": { - ā”‚ "content": "PI_CODING_AGENT_OK pi_tool_ok", + ā”‚ "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", ā”‚ "role": "assistant" ā”‚ } ā”‚ } @@ -83,9 +175,14 @@ span_tree: ā”‚ metadata: { ā”‚ "model": "claude-haiku-4-5", ā”‚ "pi_coding_agent.api": "anthropic-messages", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, ā”‚ "pi_coding_agent.model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoBgNR4wNZe9UdQehi2", + ā”‚ "pi_coding_agent.session_id": "", ā”‚ "pi_coding_agent.stop_reason": "stop", + ā”‚ "pi_coding_agent.transport": "auto", ā”‚ "provider": "anthropic", ā”‚ "tools": [ ā”‚ { @@ -114,13 +211,15 @@ span_tree: ā”‚ ] ā”‚ } ā”‚ metrics: { - ā”‚ "completion_tokens": 3, - ā”‚ "duration": 0, + ā”‚ "completion_tokens": 23, + ā”‚ "estimated_cost": 0.001679, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, ā”‚ "prompt_cache_creation_tokens": 0, ā”‚ "prompt_cached_tokens": 0, - ā”‚ "prompt_tokens": 5, + ā”‚ "prompt_tokens": 1564, ā”‚ "time_to_first_token": 0, - ā”‚ "tokens": 8 + ā”‚ "tokens": 1587 ā”‚ } ā””ā”€ā”€ bash [tool] input: { @@ -135,8 +234,10 @@ span_tree: ] } metadata: { - "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } ā””ā”€ā”€ pi-tool-child diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-wrapped.span-tree.json index 88e5e6ffd..e2e8d7593 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-wrapped.span-tree.json @@ -11,6 +11,95 @@ "name": "AgentSession.prompt", "type": "task", "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\"}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.max_retry_delay_ms": 0, + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.response_id": "msg_011CgRoBcprVdjxx3eTrUun6", + "pi_coding_agent.session_id": "", + "pi_coding_agent.stop_reason": "toolUse", + "pi_coding_agent.transport": "auto", + "provider": "anthropic", + "tools": [ + { + "function": { + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "name": "bash", + "parameters": { + "properties": { + "command": { + "description": "Bash command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": [ + "command" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 57, + "estimated_cost": 0.001775, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1490, + "time_to_first_token": 0, + "tokens": 1547 + } + }, { "name": "anthropic.messages.create", "type": "llm", @@ -51,7 +140,7 @@ } ], "role": "tool", - "tool_call_id": "tool-call-1" + "tool_call_id": "" } ], "output": [ @@ -59,7 +148,7 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } @@ -67,9 +156,14 @@ "metadata": { "model": "claude-haiku-4-5", "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.max_retry_delay_ms": 0, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.response_id": "msg_011CgRoBgNR4wNZe9UdQehi2", + "pi_coding_agent.session_id": "", "pi_coding_agent.stop_reason": "stop", + "pi_coding_agent.transport": "auto", "provider": "anthropic", "tools": [ { @@ -98,13 +192,15 @@ ] }, "metrics": { - "completion_tokens": 3, - "duration": 0, + "completion_tokens": 23, + "estimated_cost": 0.001679, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 5, + "prompt_tokens": 1564, "time_to_first_token": 0, - "tokens": 8 + "tokens": 1587 } }, { @@ -128,9 +224,11 @@ ] }, "metadata": { - "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } } ], @@ -140,31 +238,37 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } ], "metadata": { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", "provider": "anthropic" - }, - "metrics": { - "duration": 0 } } ], "metadata": { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } } ], "metadata": { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } } ] diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-wrapped.span-tree.txt index d905b3d8d..d3c88c3b8 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-wrapped.span-tree.txt @@ -1,11 +1,13 @@ span_tree: ā””ā”€ā”€ pi-coding-agent-root [task] metadata: { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } ā””ā”€ā”€ pi-coding-agent-prompt-operation metadata: { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } ā””ā”€ā”€ AgentSession.prompt [task] input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." @@ -14,22 +16,111 @@ span_tree: "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } ] metadata: { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", "provider": "anthropic" } - metrics: { - "duration": 0 - } + ā”œā”€ā”€ anthropic.messages.create [llm] + ā”‚ input: [ + ā”‚ { + ā”‚ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + ā”‚ "role": "system" + ā”‚ }, + ā”‚ { + ā”‚ "content": [ + ā”‚ { + ā”‚ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + ā”‚ "type": "text" + ā”‚ } + ā”‚ ], + ā”‚ "role": "user" + ā”‚ } + ā”‚ ] + ā”‚ output: [ + ā”‚ { + ā”‚ "finish_reason": "tool_calls", + ā”‚ "index": 0, + ā”‚ "message": { + ā”‚ "content": null, + ā”‚ "role": "assistant", + ā”‚ "tool_calls": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "arguments": "{\"command\":\"printf pi_tool_ok\"}", + ā”‚ "name": "bash" + ā”‚ }, + ā”‚ "id": "", + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ } + ā”‚ ] + ā”‚ metadata: { + ā”‚ "model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.api": "anthropic-messages", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, + ā”‚ "pi_coding_agent.model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoBcprVdjxx3eTrUun6", + ā”‚ "pi_coding_agent.session_id": "", + ā”‚ "pi_coding_agent.stop_reason": "toolUse", + ā”‚ "pi_coding_agent.transport": "auto", + ā”‚ "provider": "anthropic", + ā”‚ "tools": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + ā”‚ "name": "bash", + ā”‚ "parameters": { + ā”‚ "properties": { + ā”‚ "command": { + ā”‚ "description": "Bash command to execute", + ā”‚ "type": "string" + ā”‚ }, + ā”‚ "timeout": { + ā”‚ "description": "Timeout in seconds (optional, no default timeout)", + ā”‚ "type": "number" + ā”‚ } + ā”‚ }, + ā”‚ "required": [ + ā”‚ "command" + ā”‚ ], + ā”‚ "type": "object" + ā”‚ } + ā”‚ }, + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ metrics: { + ā”‚ "completion_tokens": 57, + ā”‚ "estimated_cost": 0.001775, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, + ā”‚ "prompt_cache_creation_tokens": 0, + ā”‚ "prompt_cached_tokens": 0, + ā”‚ "prompt_tokens": 1490, + ā”‚ "time_to_first_token": 0, + ā”‚ "tokens": 1547 + ā”‚ } ā”œā”€ā”€ anthropic.messages.create [llm] ā”‚ input: [ ā”‚ { @@ -67,7 +158,7 @@ span_tree: ā”‚ } ā”‚ ], ā”‚ "role": "tool", - ā”‚ "tool_call_id": "tool-call-1" + ā”‚ "tool_call_id": "" ā”‚ } ā”‚ ] ā”‚ output: [ @@ -75,7 +166,7 @@ span_tree: ā”‚ "finish_reason": "stop", ā”‚ "index": 0, ā”‚ "message": { - ā”‚ "content": "PI_CODING_AGENT_OK pi_tool_ok", + ā”‚ "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", ā”‚ "role": "assistant" ā”‚ } ā”‚ } @@ -83,9 +174,14 @@ span_tree: ā”‚ metadata: { ā”‚ "model": "claude-haiku-4-5", ā”‚ "pi_coding_agent.api": "anthropic-messages", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, ā”‚ "pi_coding_agent.model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoBgNR4wNZe9UdQehi2", + ā”‚ "pi_coding_agent.session_id": "", ā”‚ "pi_coding_agent.stop_reason": "stop", + ā”‚ "pi_coding_agent.transport": "auto", ā”‚ "provider": "anthropic", ā”‚ "tools": [ ā”‚ { @@ -114,13 +210,15 @@ span_tree: ā”‚ ] ā”‚ } ā”‚ metrics: { - ā”‚ "completion_tokens": 3, - ā”‚ "duration": 0, + ā”‚ "completion_tokens": 23, + ā”‚ "estimated_cost": 0.001679, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, ā”‚ "prompt_cache_creation_tokens": 0, ā”‚ "prompt_cached_tokens": 0, - ā”‚ "prompt_tokens": 5, + ā”‚ "prompt_tokens": 1564, ā”‚ "time_to_first_token": 0, - ā”‚ "tokens": 8 + ā”‚ "tokens": 1587 ā”‚ } ā””ā”€ā”€ bash [tool] input: { @@ -135,8 +233,10 @@ span_tree: ] } metadata: { - "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } ā””ā”€ā”€ pi-tool-child diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-wrapped.span-tree.json index 081624d6b..5c8aabc9d 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-wrapped.span-tree.json @@ -11,6 +11,95 @@ "name": "AgentSession.prompt", "type": "task", "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\"}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.max_retry_delay_ms": 0, + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.response_id": "msg_011CgRoB5EKhUC27bPHCWqda", + "pi_coding_agent.session_id": "", + "pi_coding_agent.stop_reason": "toolUse", + "pi_coding_agent.transport": "auto", + "provider": "anthropic", + "tools": [ + { + "function": { + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "name": "bash", + "parameters": { + "properties": { + "command": { + "description": "Bash command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": [ + "command" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 57, + "estimated_cost": 0.001775, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1490, + "time_to_first_token": 0, + "tokens": 1547 + } + }, { "name": "anthropic.messages.create", "type": "llm", @@ -51,7 +140,7 @@ } ], "role": "tool", - "tool_call_id": "tool-call-1" + "tool_call_id": "" } ], "output": [ @@ -59,7 +148,7 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } @@ -67,9 +156,14 @@ "metadata": { "model": "claude-haiku-4-5", "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.max_retry_delay_ms": 0, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.response_id": "msg_011CgRoB8p7tu3WfibUMqmjY", + "pi_coding_agent.session_id": "", "pi_coding_agent.stop_reason": "stop", + "pi_coding_agent.transport": "auto", "provider": "anthropic", "tools": [ { @@ -98,13 +192,15 @@ ] }, "metrics": { - "completion_tokens": 3, - "duration": 0, + "completion_tokens": 23, + "estimated_cost": 0.001679, + "prompt_cache_creation_1h_tokens": 0, + "prompt_cache_creation_5m_tokens": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 5, + "prompt_tokens": 1564, "time_to_first_token": 0, - "tokens": 8 + "tokens": 1587 } }, { @@ -128,9 +224,11 @@ ] }, "metadata": { - "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } } ], @@ -140,31 +238,37 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } ], "metadata": { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", "provider": "anthropic" - }, - "metrics": { - "duration": 0 } } ], "metadata": { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } } ], "metadata": { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } } ] diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-wrapped.span-tree.txt index 95db54189..abbeb3a18 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-wrapped.span-tree.txt @@ -1,11 +1,13 @@ span_tree: ā””ā”€ā”€ pi-coding-agent-root [task] metadata: { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } ā””ā”€ā”€ pi-coding-agent-prompt-operation metadata: { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } ā””ā”€ā”€ AgentSession.prompt [task] input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." @@ -14,22 +16,111 @@ span_tree: "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK pi_tool_ok", + "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", "role": "assistant" } } ] metadata: { "model": "claude-haiku-4-5", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "off", "provider": "anthropic" } - metrics: { - "duration": 0 - } + ā”œā”€ā”€ anthropic.messages.create [llm] + ā”‚ input: [ + ā”‚ { + ā”‚ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + ā”‚ "role": "system" + ā”‚ }, + ā”‚ { + ā”‚ "content": [ + ā”‚ { + ā”‚ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + ā”‚ "type": "text" + ā”‚ } + ā”‚ ], + ā”‚ "role": "user" + ā”‚ } + ā”‚ ] + ā”‚ output: [ + ā”‚ { + ā”‚ "finish_reason": "tool_calls", + ā”‚ "index": 0, + ā”‚ "message": { + ā”‚ "content": null, + ā”‚ "role": "assistant", + ā”‚ "tool_calls": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "arguments": "{\"command\":\"printf pi_tool_ok\"}", + ā”‚ "name": "bash" + ā”‚ }, + ā”‚ "id": "", + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ } + ā”‚ ] + ā”‚ metadata: { + ā”‚ "model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.api": "anthropic-messages", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, + ā”‚ "pi_coding_agent.model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", + ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoB5EKhUC27bPHCWqda", + ā”‚ "pi_coding_agent.session_id": "", + ā”‚ "pi_coding_agent.stop_reason": "toolUse", + ā”‚ "pi_coding_agent.transport": "auto", + ā”‚ "provider": "anthropic", + ā”‚ "tools": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + ā”‚ "name": "bash", + ā”‚ "parameters": { + ā”‚ "properties": { + ā”‚ "command": { + ā”‚ "description": "Bash command to execute", + ā”‚ "type": "string" + ā”‚ }, + ā”‚ "timeout": { + ā”‚ "description": "Timeout in seconds (optional, no default timeout)", + ā”‚ "type": "number" + ā”‚ } + ā”‚ }, + ā”‚ "required": [ + ā”‚ "command" + ā”‚ ], + ā”‚ "type": "object" + ā”‚ } + ā”‚ }, + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ metrics: { + ā”‚ "completion_tokens": 57, + ā”‚ "estimated_cost": 0.001775, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, + ā”‚ "prompt_cache_creation_tokens": 0, + ā”‚ "prompt_cached_tokens": 0, + ā”‚ "prompt_tokens": 1490, + ā”‚ "time_to_first_token": 0, + ā”‚ "tokens": 1547 + ā”‚ } ā”œā”€ā”€ anthropic.messages.create [llm] ā”‚ input: [ ā”‚ { @@ -67,7 +158,7 @@ span_tree: ā”‚ } ā”‚ ], ā”‚ "role": "tool", - ā”‚ "tool_call_id": "tool-call-1" + ā”‚ "tool_call_id": "" ā”‚ } ā”‚ ] ā”‚ output: [ @@ -75,7 +166,7 @@ span_tree: ā”‚ "finish_reason": "stop", ā”‚ "index": 0, ā”‚ "message": { - ā”‚ "content": "PI_CODING_AGENT_OK pi_tool_ok", + ā”‚ "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", ā”‚ "role": "assistant" ā”‚ } ā”‚ } @@ -83,9 +174,14 @@ span_tree: ā”‚ metadata: { ā”‚ "model": "claude-haiku-4-5", ā”‚ "pi_coding_agent.api": "anthropic-messages", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, ā”‚ "pi_coding_agent.model": "claude-haiku-4-5", + ā”‚ "pi_coding_agent.model_name": "Claude Haiku 4.5 (latest)", ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.response_id": "msg_011CgRoB8p7tu3WfibUMqmjY", + ā”‚ "pi_coding_agent.session_id": "", ā”‚ "pi_coding_agent.stop_reason": "stop", + ā”‚ "pi_coding_agent.transport": "auto", ā”‚ "provider": "anthropic", ā”‚ "tools": [ ā”‚ { @@ -114,13 +210,15 @@ span_tree: ā”‚ ] ā”‚ } ā”‚ metrics: { - ā”‚ "completion_tokens": 3, - ā”‚ "duration": 0, + ā”‚ "completion_tokens": 23, + ā”‚ "estimated_cost": 0.001679, + ā”‚ "prompt_cache_creation_1h_tokens": 0, + ā”‚ "prompt_cache_creation_5m_tokens": 0, ā”‚ "prompt_cache_creation_tokens": 0, ā”‚ "prompt_cached_tokens": 0, - ā”‚ "prompt_tokens": 5, + ā”‚ "prompt_tokens": 1564, ā”‚ "time_to_first_token": 0, - ā”‚ "tokens": 8 + ā”‚ "tokens": 1587 ā”‚ } ā””ā”€ā”€ bash [tool] input: { @@ -135,8 +233,10 @@ span_tree: ] } metadata: { - "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.call.id": "", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } ā””ā”€ā”€ pi-tool-child diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-auto-hook.span-tree.json index 0fc9c5a1e..f62e9739a 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-auto-hook.span-tree.json @@ -11,6 +11,95 @@ "name": "AgentSession.prompt", "type": "task", "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are a coding assistant. Use tools when requested.\n\n\n/e2e/scenarios/pi-coding-agent-instrumentation/\n", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\",\"timeout\":120000}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-5-nano", + "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.max_retry_delay_ms": 0, + "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", + "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.reasoning": "minimal", + "pi_coding_agent.response_id": "resp_05902bb47f22bf04016adff080cb8487d1a576194d17dca890", + "pi_coding_agent.session_id": "", + "pi_coding_agent.stop_reason": "toolUse", + "pi_coding_agent.transport": "auto", + "provider": "openai", + "tools": [ + { + "function": { + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "name": "bash", + "parameters": { + "properties": { + "command": { + "description": "Shell command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": [ + "command" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 40, + "estimated_cost": 0.000027000000000000006, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 220, + "time_to_first_token": 0, + "tokens": 260 + } + }, { "name": "openai.responses.create", "type": "llm", @@ -51,7 +140,7 @@ } ], "role": "tool", - "tool_call_id": "call_3fCg1Ycahjqs70OPFddXVmKi|fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe" + "tool_call_id": "call_z1nocxjB1ozM66RNjCRMtY27|fc_05902bb47f22bf04016adff081513487d19294263001cde5cb" } ], "output": [ @@ -67,9 +156,15 @@ "metadata": { "model": "gpt-5-nano", "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.max_retry_delay_ms": 0, "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.reasoning": "minimal", + "pi_coding_agent.response_id": "resp_05902bb47f22bf04016adff08196e887d18a3be8b3398bc94d", + "pi_coding_agent.session_id": "", "pi_coding_agent.stop_reason": "stop", + "pi_coding_agent.transport": "auto", "provider": "openai", "tools": [ { @@ -98,13 +193,14 @@ ] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 28, - "duration": 0, + "estimated_cost": 0.000024750000000000002, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 272, + "prompt_tokens": 271, "time_to_first_token": 0, - "tokens": 300 + "tokens": 299 } }, { @@ -131,13 +227,15 @@ "exit_code": 0, "output": "pi_tool_ok", "truncated": false, - "wall_time_seconds": 0 + "wall_time_seconds": "" } }, "metadata": { - "gen_ai.tool.call.id": "call_3fCg1Ycahjqs70OPFddXVmKi|fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe", + "gen_ai.tool.call.id": "call_z1nocxjB1ozM66RNjCRMtY27|fc_05902bb47f22bf04016adff081513487d19294263001cde5cb", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } } ], @@ -154,24 +252,31 @@ ], "metadata": { "model": "gpt-5-nano", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "minimal", + "pi_coding_agent.version": "1.4.0", "provider": "openai" - }, - "metrics": { - "duration": 0 } } ], "metadata": { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } } ], "metadata": { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } } ] diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-auto-hook.span-tree.txt index 26ba3c641..612cb9fc9 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-auto-hook.span-tree.txt @@ -1,11 +1,13 @@ span_tree: ā””ā”€ā”€ pi-coding-agent-root [task] metadata: { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } ā””ā”€ā”€ pi-coding-agent-prompt-operation metadata: { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } ā””ā”€ā”€ AgentSession.prompt [task] input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." @@ -21,15 +23,105 @@ span_tree: ] metadata: { "model": "gpt-5-nano", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "minimal", + "pi_coding_agent.version": "1.4.0", "provider": "openai" } - metrics: { - "duration": 0 - } + ā”œā”€ā”€ openai.responses.create [llm] + ā”‚ input: [ + ā”‚ { + ā”‚ "content": "You are a coding assistant. Use tools when requested.\n\n\n/e2e/scenarios/pi-coding-agent-instrumentation/\n", + ā”‚ "role": "system" + ā”‚ }, + ā”‚ { + ā”‚ "content": [ + ā”‚ { + ā”‚ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + ā”‚ "type": "text" + ā”‚ } + ā”‚ ], + ā”‚ "role": "user" + ā”‚ } + ā”‚ ] + ā”‚ output: [ + ā”‚ { + ā”‚ "finish_reason": "tool_calls", + ā”‚ "index": 0, + ā”‚ "message": { + ā”‚ "content": null, + ā”‚ "role": "assistant", + ā”‚ "tool_calls": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "arguments": "{\"command\":\"printf pi_tool_ok\",\"timeout\":120000}", + ā”‚ "name": "bash" + ā”‚ }, + ā”‚ "id": "", + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ } + ā”‚ ] + ā”‚ metadata: { + ā”‚ "model": "gpt-5-nano", + ā”‚ "pi_coding_agent.api": "openai-responses", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, + ā”‚ "pi_coding_agent.model": "gpt-5-nano", + ā”‚ "pi_coding_agent.model_name": "GPT-5 Nano", + ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.reasoning": "minimal", + ā”‚ "pi_coding_agent.response_id": "resp_05902bb47f22bf04016adff080cb8487d1a576194d17dca890", + ā”‚ "pi_coding_agent.session_id": "", + ā”‚ "pi_coding_agent.stop_reason": "toolUse", + ā”‚ "pi_coding_agent.transport": "auto", + ā”‚ "provider": "openai", + ā”‚ "tools": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + ā”‚ "name": "bash", + ā”‚ "parameters": { + ā”‚ "properties": { + ā”‚ "command": { + ā”‚ "description": "Shell command to execute", + ā”‚ "type": "string" + ā”‚ }, + ā”‚ "timeout": { + ā”‚ "description": "Timeout in seconds (optional, no default timeout)", + ā”‚ "type": "number" + ā”‚ } + ā”‚ }, + ā”‚ "required": [ + ā”‚ "command" + ā”‚ ], + ā”‚ "type": "object" + ā”‚ } + ā”‚ }, + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ metrics: { + ā”‚ "completion_reasoning_tokens": 0, + ā”‚ "completion_tokens": 40, + ā”‚ "estimated_cost": 0.000027000000000000006, + ā”‚ "prompt_cache_creation_tokens": 0, + ā”‚ "prompt_cached_tokens": 0, + ā”‚ "prompt_tokens": 220, + ā”‚ "time_to_first_token": 0, + ā”‚ "tokens": 260 + ā”‚ } ā”œā”€ā”€ openai.responses.create [llm] ā”‚ input: [ ā”‚ { @@ -67,7 +159,7 @@ span_tree: ā”‚ } ā”‚ ], ā”‚ "role": "tool", - ā”‚ "tool_call_id": "call_3fCg1Ycahjqs70OPFddXVmKi|fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe" + ā”‚ "tool_call_id": "call_z1nocxjB1ozM66RNjCRMtY27|fc_05902bb47f22bf04016adff081513487d19294263001cde5cb" ā”‚ } ā”‚ ] ā”‚ output: [ @@ -83,9 +175,15 @@ span_tree: ā”‚ metadata: { ā”‚ "model": "gpt-5-nano", ā”‚ "pi_coding_agent.api": "openai-responses", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, ā”‚ "pi_coding_agent.model": "gpt-5-nano", + ā”‚ "pi_coding_agent.model_name": "GPT-5 Nano", ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.reasoning": "minimal", + ā”‚ "pi_coding_agent.response_id": "resp_05902bb47f22bf04016adff08196e887d18a3be8b3398bc94d", + ā”‚ "pi_coding_agent.session_id": "", ā”‚ "pi_coding_agent.stop_reason": "stop", + ā”‚ "pi_coding_agent.transport": "auto", ā”‚ "provider": "openai", ā”‚ "tools": [ ā”‚ { @@ -114,13 +212,14 @@ span_tree: ā”‚ ] ā”‚ } ā”‚ metrics: { + ā”‚ "completion_reasoning_tokens": 0, ā”‚ "completion_tokens": 28, - ā”‚ "duration": 0, + ā”‚ "estimated_cost": 0.000024750000000000002, ā”‚ "prompt_cache_creation_tokens": 0, ā”‚ "prompt_cached_tokens": 0, - ā”‚ "prompt_tokens": 272, + ā”‚ "prompt_tokens": 271, ā”‚ "time_to_first_token": 0, - ā”‚ "tokens": 300 + ā”‚ "tokens": 299 ā”‚ } ā””ā”€ā”€ bash [tool] input: { @@ -138,12 +237,14 @@ span_tree: "exit_code": 0, "output": "pi_tool_ok", "truncated": false, - "wall_time_seconds": 0 + "wall_time_seconds": "" } } metadata: { - "gen_ai.tool.call.id": "call_3fCg1Ycahjqs70OPFddXVmKi|fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe", + "gen_ai.tool.call.id": "call_z1nocxjB1ozM66RNjCRMtY27|fc_05902bb47f22bf04016adff081513487d19294263001cde5cb", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } ā””ā”€ā”€ pi-tool-child diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-auto-hook.span-tree.json index 01a93937c..1d549fcb6 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-auto-hook.span-tree.json @@ -11,6 +11,95 @@ "name": "AgentSession.prompt", "type": "task", "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are a coding assistant. Use tools when requested.\n\n\n/e2e/scenarios/pi-coding-agent-instrumentation/\n", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\",\"timeout\":120000}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-5-nano", + "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.max_retry_delay_ms": 0, + "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", + "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.reasoning": "minimal", + "pi_coding_agent.response_id": "resp_05ec8b9b7b48b396016adff08600c487d198fd989c208153d7", + "pi_coding_agent.session_id": "", + "pi_coding_agent.stop_reason": "toolUse", + "pi_coding_agent.transport": "auto", + "provider": "openai", + "tools": [ + { + "function": { + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "name": "bash", + "parameters": { + "properties": { + "command": { + "description": "Shell command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": [ + "command" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 40, + "estimated_cost": 0.000027000000000000006, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 220, + "time_to_first_token": 0, + "tokens": 260 + } + }, { "name": "openai.responses.create", "type": "llm", @@ -51,7 +140,7 @@ } ], "role": "tool", - "tool_call_id": "call_nfMLwtliP8Lmn5SMjdlCMUVh|fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7" + "tool_call_id": "call_BxOAz9IH3YDU7L4rTQ1vwlT0|fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca" } ], "output": [ @@ -67,9 +156,15 @@ "metadata": { "model": "gpt-5-nano", "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.max_retry_delay_ms": 0, "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.reasoning": "minimal", + "pi_coding_agent.response_id": "resp_05ec8b9b7b48b396016adff0870a0087d1882ac4fee66c7225", + "pi_coding_agent.session_id": "", "pi_coding_agent.stop_reason": "stop", + "pi_coding_agent.transport": "auto", "provider": "openai", "tools": [ { @@ -98,13 +193,14 @@ ] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 28, - "duration": 0, + "estimated_cost": 0.000024750000000000002, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 272, + "prompt_tokens": 271, "time_to_first_token": 0, - "tokens": 300 + "tokens": 299 } }, { @@ -131,13 +227,15 @@ "exit_code": 0, "output": "pi_tool_ok", "truncated": false, - "wall_time_seconds": 0 + "wall_time_seconds": "" } }, "metadata": { - "gen_ai.tool.call.id": "call_nfMLwtliP8Lmn5SMjdlCMUVh|fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7", + "gen_ai.tool.call.id": "call_BxOAz9IH3YDU7L4rTQ1vwlT0|fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } } ], @@ -154,24 +252,31 @@ ], "metadata": { "model": "gpt-5-nano", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "minimal", + "pi_coding_agent.version": "1.4.0", "provider": "openai" - }, - "metrics": { - "duration": 0 } } ], "metadata": { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } } ], "metadata": { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } } ] diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-auto-hook.span-tree.txt index 1bbf50e0d..e0940eab5 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-auto-hook.span-tree.txt @@ -1,11 +1,13 @@ span_tree: ā””ā”€ā”€ pi-coding-agent-root [task] metadata: { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } ā””ā”€ā”€ pi-coding-agent-prompt-operation metadata: { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } ā””ā”€ā”€ AgentSession.prompt [task] input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." @@ -21,15 +23,105 @@ span_tree: ] metadata: { "model": "gpt-5-nano", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "minimal", + "pi_coding_agent.version": "1.4.0", "provider": "openai" } - metrics: { - "duration": 0 - } + ā”œā”€ā”€ openai.responses.create [llm] + ā”‚ input: [ + ā”‚ { + ā”‚ "content": "You are a coding assistant. Use tools when requested.\n\n\n/e2e/scenarios/pi-coding-agent-instrumentation/\n", + ā”‚ "role": "system" + ā”‚ }, + ā”‚ { + ā”‚ "content": [ + ā”‚ { + ā”‚ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + ā”‚ "type": "text" + ā”‚ } + ā”‚ ], + ā”‚ "role": "user" + ā”‚ } + ā”‚ ] + ā”‚ output: [ + ā”‚ { + ā”‚ "finish_reason": "tool_calls", + ā”‚ "index": 0, + ā”‚ "message": { + ā”‚ "content": null, + ā”‚ "role": "assistant", + ā”‚ "tool_calls": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "arguments": "{\"command\":\"printf pi_tool_ok\",\"timeout\":120000}", + ā”‚ "name": "bash" + ā”‚ }, + ā”‚ "id": "", + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ } + ā”‚ ] + ā”‚ metadata: { + ā”‚ "model": "gpt-5-nano", + ā”‚ "pi_coding_agent.api": "openai-responses", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, + ā”‚ "pi_coding_agent.model": "gpt-5-nano", + ā”‚ "pi_coding_agent.model_name": "GPT-5 Nano", + ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.reasoning": "minimal", + ā”‚ "pi_coding_agent.response_id": "resp_05ec8b9b7b48b396016adff08600c487d198fd989c208153d7", + ā”‚ "pi_coding_agent.session_id": "", + ā”‚ "pi_coding_agent.stop_reason": "toolUse", + ā”‚ "pi_coding_agent.transport": "auto", + ā”‚ "provider": "openai", + ā”‚ "tools": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + ā”‚ "name": "bash", + ā”‚ "parameters": { + ā”‚ "properties": { + ā”‚ "command": { + ā”‚ "description": "Shell command to execute", + ā”‚ "type": "string" + ā”‚ }, + ā”‚ "timeout": { + ā”‚ "description": "Timeout in seconds (optional, no default timeout)", + ā”‚ "type": "number" + ā”‚ } + ā”‚ }, + ā”‚ "required": [ + ā”‚ "command" + ā”‚ ], + ā”‚ "type": "object" + ā”‚ } + ā”‚ }, + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ metrics: { + ā”‚ "completion_reasoning_tokens": 0, + ā”‚ "completion_tokens": 40, + ā”‚ "estimated_cost": 0.000027000000000000006, + ā”‚ "prompt_cache_creation_tokens": 0, + ā”‚ "prompt_cached_tokens": 0, + ā”‚ "prompt_tokens": 220, + ā”‚ "time_to_first_token": 0, + ā”‚ "tokens": 260 + ā”‚ } ā”œā”€ā”€ openai.responses.create [llm] ā”‚ input: [ ā”‚ { @@ -67,7 +159,7 @@ span_tree: ā”‚ } ā”‚ ], ā”‚ "role": "tool", - ā”‚ "tool_call_id": "call_nfMLwtliP8Lmn5SMjdlCMUVh|fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7" + ā”‚ "tool_call_id": "call_BxOAz9IH3YDU7L4rTQ1vwlT0|fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca" ā”‚ } ā”‚ ] ā”‚ output: [ @@ -83,9 +175,15 @@ span_tree: ā”‚ metadata: { ā”‚ "model": "gpt-5-nano", ā”‚ "pi_coding_agent.api": "openai-responses", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, ā”‚ "pi_coding_agent.model": "gpt-5-nano", + ā”‚ "pi_coding_agent.model_name": "GPT-5 Nano", ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.reasoning": "minimal", + ā”‚ "pi_coding_agent.response_id": "resp_05ec8b9b7b48b396016adff0870a0087d1882ac4fee66c7225", + ā”‚ "pi_coding_agent.session_id": "", ā”‚ "pi_coding_agent.stop_reason": "stop", + ā”‚ "pi_coding_agent.transport": "auto", ā”‚ "provider": "openai", ā”‚ "tools": [ ā”‚ { @@ -114,13 +212,14 @@ span_tree: ā”‚ ] ā”‚ } ā”‚ metrics: { + ā”‚ "completion_reasoning_tokens": 0, ā”‚ "completion_tokens": 28, - ā”‚ "duration": 0, + ā”‚ "estimated_cost": 0.000024750000000000002, ā”‚ "prompt_cache_creation_tokens": 0, ā”‚ "prompt_cached_tokens": 0, - ā”‚ "prompt_tokens": 272, + ā”‚ "prompt_tokens": 271, ā”‚ "time_to_first_token": 0, - ā”‚ "tokens": 300 + ā”‚ "tokens": 299 ā”‚ } ā””ā”€ā”€ bash [tool] input: { @@ -138,12 +237,14 @@ span_tree: "exit_code": 0, "output": "pi_tool_ok", "truncated": false, - "wall_time_seconds": 0 + "wall_time_seconds": "" } } metadata: { - "gen_ai.tool.call.id": "call_nfMLwtliP8Lmn5SMjdlCMUVh|fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7", + "gen_ai.tool.call.id": "call_BxOAz9IH3YDU7L4rTQ1vwlT0|fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } ā””ā”€ā”€ pi-tool-child diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-wrapped.span-tree.json index 01a93937c..5e54472f7 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-wrapped.span-tree.json @@ -11,6 +11,95 @@ "name": "AgentSession.prompt", "type": "task", "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are a coding assistant. Use tools when requested.\n\n\n/e2e/scenarios/pi-coding-agent-instrumentation/\n", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\",\"timeout\":120000}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-5-nano", + "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.max_retry_delay_ms": 0, + "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", + "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.reasoning": "minimal", + "pi_coding_agent.response_id": "resp_05ec8b9b7b48b396016adff08600c487d198fd989c208153d7", + "pi_coding_agent.session_id": "", + "pi_coding_agent.stop_reason": "toolUse", + "pi_coding_agent.transport": "auto", + "provider": "openai", + "tools": [ + { + "function": { + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "name": "bash", + "parameters": { + "properties": { + "command": { + "description": "Shell command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": [ + "command" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 40, + "estimated_cost": 0.000027000000000000006, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 220, + "time_to_first_token": 0, + "tokens": 260 + } + }, { "name": "openai.responses.create", "type": "llm", @@ -51,7 +140,7 @@ } ], "role": "tool", - "tool_call_id": "call_nfMLwtliP8Lmn5SMjdlCMUVh|fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7" + "tool_call_id": "call_BxOAz9IH3YDU7L4rTQ1vwlT0|fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca" } ], "output": [ @@ -67,9 +156,15 @@ "metadata": { "model": "gpt-5-nano", "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.max_retry_delay_ms": 0, "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.reasoning": "minimal", + "pi_coding_agent.response_id": "resp_05ec8b9b7b48b396016adff0870a0087d1882ac4fee66c7225", + "pi_coding_agent.session_id": "", "pi_coding_agent.stop_reason": "stop", + "pi_coding_agent.transport": "auto", "provider": "openai", "tools": [ { @@ -98,13 +193,14 @@ ] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 28, - "duration": 0, + "estimated_cost": 0.000024750000000000002, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 272, + "prompt_tokens": 271, "time_to_first_token": 0, - "tokens": 300 + "tokens": 299 } }, { @@ -131,13 +227,15 @@ "exit_code": 0, "output": "pi_tool_ok", "truncated": false, - "wall_time_seconds": 0 + "wall_time_seconds": "" } }, "metadata": { - "gen_ai.tool.call.id": "call_nfMLwtliP8Lmn5SMjdlCMUVh|fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7", + "gen_ai.tool.call.id": "call_BxOAz9IH3YDU7L4rTQ1vwlT0|fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } } ], @@ -154,24 +252,30 @@ ], "metadata": { "model": "gpt-5-nano", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "minimal", "provider": "openai" - }, - "metrics": { - "duration": 0 } } ], "metadata": { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } } ], "metadata": { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } } ] diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-wrapped.span-tree.txt index 1bbf50e0d..d9934c13a 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-latest-wrapped.span-tree.txt @@ -1,11 +1,13 @@ span_tree: ā””ā”€ā”€ pi-coding-agent-root [task] metadata: { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } ā””ā”€ā”€ pi-coding-agent-prompt-operation metadata: { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } ā””ā”€ā”€ AgentSession.prompt [task] input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." @@ -21,15 +23,104 @@ span_tree: ] metadata: { "model": "gpt-5-nano", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "minimal", "provider": "openai" } - metrics: { - "duration": 0 - } + ā”œā”€ā”€ openai.responses.create [llm] + ā”‚ input: [ + ā”‚ { + ā”‚ "content": "You are a coding assistant. Use tools when requested.\n\n\n/e2e/scenarios/pi-coding-agent-instrumentation/\n", + ā”‚ "role": "system" + ā”‚ }, + ā”‚ { + ā”‚ "content": [ + ā”‚ { + ā”‚ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + ā”‚ "type": "text" + ā”‚ } + ā”‚ ], + ā”‚ "role": "user" + ā”‚ } + ā”‚ ] + ā”‚ output: [ + ā”‚ { + ā”‚ "finish_reason": "tool_calls", + ā”‚ "index": 0, + ā”‚ "message": { + ā”‚ "content": null, + ā”‚ "role": "assistant", + ā”‚ "tool_calls": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "arguments": "{\"command\":\"printf pi_tool_ok\",\"timeout\":120000}", + ā”‚ "name": "bash" + ā”‚ }, + ā”‚ "id": "", + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ } + ā”‚ ] + ā”‚ metadata: { + ā”‚ "model": "gpt-5-nano", + ā”‚ "pi_coding_agent.api": "openai-responses", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, + ā”‚ "pi_coding_agent.model": "gpt-5-nano", + ā”‚ "pi_coding_agent.model_name": "GPT-5 Nano", + ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.reasoning": "minimal", + ā”‚ "pi_coding_agent.response_id": "resp_05ec8b9b7b48b396016adff08600c487d198fd989c208153d7", + ā”‚ "pi_coding_agent.session_id": "", + ā”‚ "pi_coding_agent.stop_reason": "toolUse", + ā”‚ "pi_coding_agent.transport": "auto", + ā”‚ "provider": "openai", + ā”‚ "tools": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + ā”‚ "name": "bash", + ā”‚ "parameters": { + ā”‚ "properties": { + ā”‚ "command": { + ā”‚ "description": "Shell command to execute", + ā”‚ "type": "string" + ā”‚ }, + ā”‚ "timeout": { + ā”‚ "description": "Timeout in seconds (optional, no default timeout)", + ā”‚ "type": "number" + ā”‚ } + ā”‚ }, + ā”‚ "required": [ + ā”‚ "command" + ā”‚ ], + ā”‚ "type": "object" + ā”‚ } + ā”‚ }, + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ metrics: { + ā”‚ "completion_reasoning_tokens": 0, + ā”‚ "completion_tokens": 40, + ā”‚ "estimated_cost": 0.000027000000000000006, + ā”‚ "prompt_cache_creation_tokens": 0, + ā”‚ "prompt_cached_tokens": 0, + ā”‚ "prompt_tokens": 220, + ā”‚ "time_to_first_token": 0, + ā”‚ "tokens": 260 + ā”‚ } ā”œā”€ā”€ openai.responses.create [llm] ā”‚ input: [ ā”‚ { @@ -67,7 +158,7 @@ span_tree: ā”‚ } ā”‚ ], ā”‚ "role": "tool", - ā”‚ "tool_call_id": "call_nfMLwtliP8Lmn5SMjdlCMUVh|fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7" + ā”‚ "tool_call_id": "call_BxOAz9IH3YDU7L4rTQ1vwlT0|fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca" ā”‚ } ā”‚ ] ā”‚ output: [ @@ -83,9 +174,15 @@ span_tree: ā”‚ metadata: { ā”‚ "model": "gpt-5-nano", ā”‚ "pi_coding_agent.api": "openai-responses", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, ā”‚ "pi_coding_agent.model": "gpt-5-nano", + ā”‚ "pi_coding_agent.model_name": "GPT-5 Nano", ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.reasoning": "minimal", + ā”‚ "pi_coding_agent.response_id": "resp_05ec8b9b7b48b396016adff0870a0087d1882ac4fee66c7225", + ā”‚ "pi_coding_agent.session_id": "", ā”‚ "pi_coding_agent.stop_reason": "stop", + ā”‚ "pi_coding_agent.transport": "auto", ā”‚ "provider": "openai", ā”‚ "tools": [ ā”‚ { @@ -114,13 +211,14 @@ span_tree: ā”‚ ] ā”‚ } ā”‚ metrics: { + ā”‚ "completion_reasoning_tokens": 0, ā”‚ "completion_tokens": 28, - ā”‚ "duration": 0, + ā”‚ "estimated_cost": 0.000024750000000000002, ā”‚ "prompt_cache_creation_tokens": 0, ā”‚ "prompt_cached_tokens": 0, - ā”‚ "prompt_tokens": 272, + ā”‚ "prompt_tokens": 271, ā”‚ "time_to_first_token": 0, - ā”‚ "tokens": 300 + ā”‚ "tokens": 299 ā”‚ } ā””ā”€ā”€ bash [tool] input: { @@ -138,12 +236,14 @@ span_tree: "exit_code": 0, "output": "pi_tool_ok", "truncated": false, - "wall_time_seconds": 0 + "wall_time_seconds": "" } } metadata: { - "gen_ai.tool.call.id": "call_nfMLwtliP8Lmn5SMjdlCMUVh|fc_0b86321b42b1ad5e016adf50483ed487d180778d474ab895e7", + "gen_ai.tool.call.id": "call_BxOAz9IH3YDU7L4rTQ1vwlT0|fc_05ec8b9b7b48b396016adff086b14087d18dff476353a325ca", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } ā””ā”€ā”€ pi-tool-child diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-wrapped.span-tree.json index 0fc9c5a1e..20c4836da 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-wrapped.span-tree.json @@ -11,6 +11,95 @@ "name": "AgentSession.prompt", "type": "task", "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are a coding assistant. Use tools when requested.\n\n\n/e2e/scenarios/pi-coding-agent-instrumentation/\n", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\",\"timeout\":120000}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-5-nano", + "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.max_retry_delay_ms": 0, + "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", + "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.reasoning": "minimal", + "pi_coding_agent.response_id": "resp_05902bb47f22bf04016adff080cb8487d1a576194d17dca890", + "pi_coding_agent.session_id": "", + "pi_coding_agent.stop_reason": "toolUse", + "pi_coding_agent.transport": "auto", + "provider": "openai", + "tools": [ + { + "function": { + "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + "name": "bash", + "parameters": { + "properties": { + "command": { + "description": "Shell command to execute", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds (optional, no default timeout)", + "type": "number" + } + }, + "required": [ + "command" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 40, + "estimated_cost": 0.000027000000000000006, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 220, + "time_to_first_token": 0, + "tokens": 260 + } + }, { "name": "openai.responses.create", "type": "llm", @@ -51,7 +140,7 @@ } ], "role": "tool", - "tool_call_id": "call_3fCg1Ycahjqs70OPFddXVmKi|fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe" + "tool_call_id": "call_z1nocxjB1ozM66RNjCRMtY27|fc_05902bb47f22bf04016adff081513487d19294263001cde5cb" } ], "output": [ @@ -67,9 +156,15 @@ "metadata": { "model": "gpt-5-nano", "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.max_retry_delay_ms": 0, "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.reasoning": "minimal", + "pi_coding_agent.response_id": "resp_05902bb47f22bf04016adff08196e887d18a3be8b3398bc94d", + "pi_coding_agent.session_id": "", "pi_coding_agent.stop_reason": "stop", + "pi_coding_agent.transport": "auto", "provider": "openai", "tools": [ { @@ -98,13 +193,14 @@ ] }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 28, - "duration": 0, + "estimated_cost": 0.000024750000000000002, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 272, + "prompt_tokens": 271, "time_to_first_token": 0, - "tokens": 300 + "tokens": 299 } }, { @@ -131,13 +227,15 @@ "exit_code": 0, "output": "pi_tool_ok", "truncated": false, - "wall_time_seconds": 0 + "wall_time_seconds": "" } }, "metadata": { - "gen_ai.tool.call.id": "call_3fCg1Ycahjqs70OPFddXVmKi|fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe", + "gen_ai.tool.call.id": "call_z1nocxjB1ozM66RNjCRMtY27|fc_05902bb47f22bf04016adff081513487d19294263001cde5cb", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } } ], @@ -154,24 +252,30 @@ ], "metadata": { "model": "gpt-5-nano", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "minimal", "provider": "openai" - }, - "metrics": { - "duration": 0 } } ], "metadata": { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } } ], "metadata": { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } } ] diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-wrapped.span-tree.txt index 26ba3c641..d4da5b67f 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v1-wrapped.span-tree.txt @@ -1,11 +1,13 @@ span_tree: ā””ā”€ā”€ pi-coding-agent-root [task] metadata: { - "scenario": "pi-coding-agent-instrumentation" + "scenario": "pi-coding-agent-instrumentation", + "testRunId": "" } ā””ā”€ā”€ pi-coding-agent-prompt-operation metadata: { - "operation": "prompt" + "operation": "prompt", + "testRunId": "" } ā””ā”€ā”€ AgentSession.prompt [task] input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." @@ -21,15 +23,104 @@ span_tree: ] metadata: { "model": "gpt-5-nano", + "pi_coding_agent.active_tools": [ + "bash" + ], "pi_coding_agent.api": "openai-responses", + "pi_coding_agent.expand_prompt_templates": false, "pi_coding_agent.model": "gpt-5-nano", + "pi_coding_agent.model_name": "GPT-5 Nano", "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.session_id": "", "pi_coding_agent.source": "rpc", + "pi_coding_agent.thinking_level": "minimal", "provider": "openai" } - metrics: { - "duration": 0 - } + ā”œā”€ā”€ openai.responses.create [llm] + ā”‚ input: [ + ā”‚ { + ā”‚ "content": "You are a coding assistant. Use tools when requested.\n\n\n/e2e/scenarios/pi-coding-agent-instrumentation/\n", + ā”‚ "role": "system" + ā”‚ }, + ā”‚ { + ā”‚ "content": [ + ā”‚ { + ā”‚ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + ā”‚ "type": "text" + ā”‚ } + ā”‚ ], + ā”‚ "role": "user" + ā”‚ } + ā”‚ ] + ā”‚ output: [ + ā”‚ { + ā”‚ "finish_reason": "tool_calls", + ā”‚ "index": 0, + ā”‚ "message": { + ā”‚ "content": null, + ā”‚ "role": "assistant", + ā”‚ "tool_calls": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "arguments": "{\"command\":\"printf pi_tool_ok\",\"timeout\":120000}", + ā”‚ "name": "bash" + ā”‚ }, + ā”‚ "id": "", + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ } + ā”‚ ] + ā”‚ metadata: { + ā”‚ "model": "gpt-5-nano", + ā”‚ "pi_coding_agent.api": "openai-responses", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, + ā”‚ "pi_coding_agent.model": "gpt-5-nano", + ā”‚ "pi_coding_agent.model_name": "GPT-5 Nano", + ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.reasoning": "minimal", + ā”‚ "pi_coding_agent.response_id": "resp_05902bb47f22bf04016adff080cb8487d1a576194d17dca890", + ā”‚ "pi_coding_agent.session_id": "", + ā”‚ "pi_coding_agent.stop_reason": "toolUse", + ā”‚ "pi_coding_agent.transport": "auto", + ā”‚ "provider": "openai", + ā”‚ "tools": [ + ā”‚ { + ā”‚ "function": { + ā”‚ "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", + ā”‚ "name": "bash", + ā”‚ "parameters": { + ā”‚ "properties": { + ā”‚ "command": { + ā”‚ "description": "Shell command to execute", + ā”‚ "type": "string" + ā”‚ }, + ā”‚ "timeout": { + ā”‚ "description": "Timeout in seconds (optional, no default timeout)", + ā”‚ "type": "number" + ā”‚ } + ā”‚ }, + ā”‚ "required": [ + ā”‚ "command" + ā”‚ ], + ā”‚ "type": "object" + ā”‚ } + ā”‚ }, + ā”‚ "type": "function" + ā”‚ } + ā”‚ ] + ā”‚ } + ā”‚ metrics: { + ā”‚ "completion_reasoning_tokens": 0, + ā”‚ "completion_tokens": 40, + ā”‚ "estimated_cost": 0.000027000000000000006, + ā”‚ "prompt_cache_creation_tokens": 0, + ā”‚ "prompt_cached_tokens": 0, + ā”‚ "prompt_tokens": 220, + ā”‚ "time_to_first_token": 0, + ā”‚ "tokens": 260 + ā”‚ } ā”œā”€ā”€ openai.responses.create [llm] ā”‚ input: [ ā”‚ { @@ -67,7 +158,7 @@ span_tree: ā”‚ } ā”‚ ], ā”‚ "role": "tool", - ā”‚ "tool_call_id": "call_3fCg1Ycahjqs70OPFddXVmKi|fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe" + ā”‚ "tool_call_id": "call_z1nocxjB1ozM66RNjCRMtY27|fc_05902bb47f22bf04016adff081513487d19294263001cde5cb" ā”‚ } ā”‚ ] ā”‚ output: [ @@ -83,9 +174,15 @@ span_tree: ā”‚ metadata: { ā”‚ "model": "gpt-5-nano", ā”‚ "pi_coding_agent.api": "openai-responses", + ā”‚ "pi_coding_agent.max_retry_delay_ms": 0, ā”‚ "pi_coding_agent.model": "gpt-5-nano", + ā”‚ "pi_coding_agent.model_name": "GPT-5 Nano", ā”‚ "pi_coding_agent.operation": "agent.streamFunction", + ā”‚ "pi_coding_agent.reasoning": "minimal", + ā”‚ "pi_coding_agent.response_id": "resp_05902bb47f22bf04016adff08196e887d18a3be8b3398bc94d", + ā”‚ "pi_coding_agent.session_id": "", ā”‚ "pi_coding_agent.stop_reason": "stop", + ā”‚ "pi_coding_agent.transport": "auto", ā”‚ "provider": "openai", ā”‚ "tools": [ ā”‚ { @@ -114,13 +211,14 @@ span_tree: ā”‚ ] ā”‚ } ā”‚ metrics: { + ā”‚ "completion_reasoning_tokens": 0, ā”‚ "completion_tokens": 28, - ā”‚ "duration": 0, + ā”‚ "estimated_cost": 0.000024750000000000002, ā”‚ "prompt_cache_creation_tokens": 0, ā”‚ "prompt_cached_tokens": 0, - ā”‚ "prompt_tokens": 272, + ā”‚ "prompt_tokens": 271, ā”‚ "time_to_first_token": 0, - ā”‚ "tokens": 300 + ā”‚ "tokens": 299 ā”‚ } ā””ā”€ā”€ bash [tool] input: { @@ -138,12 +236,14 @@ span_tree: "exit_code": 0, "output": "pi_tool_ok", "truncated": false, - "wall_time_seconds": 0 + "wall_time_seconds": "" } } metadata: { - "gen_ai.tool.call.id": "call_3fCg1Ycahjqs70OPFddXVmKi|fc_03733174ba77c7d2016adf5041e83c87d19bf04f8ab269fbbe", + "gen_ai.tool.call.id": "call_z1nocxjB1ozM66RNjCRMtY27|fc_05902bb47f22bf04016adff081513487d19294263001cde5cb", "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" + "pi_coding_agent.tool.is_error": false, + "pi_coding_agent.tool.name": "bash", + "tool_approval": "approved" } ā””ā”€ā”€ pi-tool-child diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/assertions.ts b/e2e/scenarios/pi-coding-agent-instrumentation/assertions.ts index 03e1f0990..3673f80c0 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/assertions.ts +++ b/e2e/scenarios/pi-coding-agent-instrumentation/assertions.ts @@ -1,10 +1,10 @@ -import { beforeAll, describe, expect, test } from "vitest"; +import { describe, expect, test } from "vitest"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { effectiveScenarioTimeoutMs, withScenarioHarness, - type ScenarioRunContext, + type runNodeScenarioDir, } from "../../helpers/scenario-harness"; import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; import { @@ -16,51 +16,27 @@ import { import { ROOT_NAME, TOOL_CHILD_SPAN_NAME } from "./scenario.impl.mjs"; type RunPiCodingAgentScenario = (harness: { - runNodeScenarioDir: (options: { - entry: string; - nodeArgs: string[]; - runContext?: ScenarioRunContext; - scenarioDir: string; - timeoutMs: number; - }) => Promise; - runScenarioDir: (options: { - entry: string; - runContext?: ScenarioRunContext; - scenarioDir: string; - timeoutMs: number; - }) => Promise; + runNodeScenarioDir: typeof runNodeScenarioDir; }) => Promise; -const METADATA_KEYS = [ - "provider", - "model", - "operation", - "scenario", - "tools", - "gen_ai.tool.call.id", - "gen_ai.tool.name", - "pi_coding_agent.api", - "pi_coding_agent.model", - "pi_coding_agent.operation", - "pi_coding_agent.source", - "pi_coding_agent.stop_reason", - "pi_coding_agent.tool.name", -] as const; - -function normalizeToolCallIds(value: unknown): unknown { +function normalizePiSnapshotValue(value: unknown): unknown { if (typeof value === "string" && value.startsWith("toolu_")) { return ""; } if (Array.isArray(value)) { - return value.map((entry) => normalizeToolCallIds(entry)); + return value.map((entry) => normalizePiSnapshotValue(entry)); } if (value && typeof value === "object") { return Object.fromEntries( Object.entries(value).map(([key, entry]) => [ key, - normalizeToolCallIds(entry), + key === "pi_coding_agent.session_id" + ? "" + : key === "wall_time_seconds" && typeof entry === "number" + ? "" + : normalizePiSnapshotValue(entry), ]), ); } @@ -70,22 +46,11 @@ function normalizeToolCallIds(value: unknown): unknown { function snapshotFields(event: CapturedLogEvent): SpanTreeFields { const fields = spanTreeFields(event); - const metadata = - fields.metadata && - typeof fields.metadata === "object" && - !Array.isArray(fields.metadata) - ? Object.fromEntries( - Object.entries(fields.metadata).filter(([key]) => - METADATA_KEYS.includes(key as (typeof METADATA_KEYS)[number]), - ), - ) - : undefined; - return { ...fields, - input: normalizeToolCallIds(fields.input), - output: normalizeToolCallIds(fields.output), - metadata: normalizeToolCallIds(metadata), + input: normalizePiSnapshotValue(fields.input), + output: normalizePiSnapshotValue(fields.output), + metadata: normalizePiSnapshotValue(fields.metadata), }; } @@ -102,7 +67,7 @@ function summarize( ): SpanTreeEntry[] { const operation = findLatestSpan(events, "pi-coding-agent-prompt-operation"); const task = findPiTask(events); - const llm = findChildSpans(events, llmSpanName, task?.span.id).at(-1); + const llmSpans = findChildSpans(events, llmSpanName, task?.span.id); const tool = findChildSpans(events, "bash", task?.span.id).at(-1); const toolChild = findChildSpans( events, @@ -114,7 +79,7 @@ function summarize( findLatestSpan(events, ROOT_NAME), operation, task, - llm, + ...llmSpans, tool, toolChild, ].flatMap((event) => @@ -142,127 +107,48 @@ export function definePiCodingAgentInstrumentationAssertions(options: { provider === "openai" ? "openai.responses.create" : "anthropic.messages.create"; - const api = provider === "openai" ? "openai-responses" : "anthropic-messages"; - const modelId = provider === "openai" ? "gpt-5-nano" : "claude-haiku-4-5"; const snapshotPath = resolveFileSnapshotPath( options.testFileUrl, `${options.snapshotName}.span-tree.json`, ); - const timeoutMs = effectiveScenarioTimeoutMs(options.timeoutMs); - const testConfig = { timeout: timeoutMs }; - describe(options.name, () => { - let events: CapturedLogEvent[] = []; - let setupError: string | undefined; - - beforeAll(async () => { - try { + test( + "captures the prompt, LLM, and tool span tree", + { timeout: effectiveScenarioTimeoutMs(options.timeoutMs) }, + async () => { + let events: CapturedLogEvent[] = []; await withScenarioHarness(async (harness) => { await options.runScenario(harness); events = harness.events(); }); - } catch (error) { - setupError = error instanceof Error ? error.message : String(error); - } - }, timeoutMs); - - test( - "captures prompt task with nested LLM and tool spans", - testConfig, - () => { - expect(setupError).toBeUndefined(); const operation = findLatestSpan( events, "pi-coding-agent-prompt-operation", ); + expect( + findChildSpans(events, "AgentSession.prompt", operation?.span.id), + ).toHaveLength(1); const task = findPiTask(events); - const llmSpans = findChildSpans(events, llmSpanName, task?.span.id); - const llm = llmSpans.at(-1); - const tool = findChildSpans(events, "bash", task?.span.id).at(-1); - const toolChild = findChildSpans( - events, - TOOL_CHILD_SPAN_NAME, - tool?.span.id, - ).at(-1); - - expect(operation).toBeDefined(); - expect(task).toBeDefined(); - expect(task?.span.parentIds).toEqual([operation?.span.id ?? ""]); - expect(task?.span.type).toBe("task"); - expect(task?.row.metadata).toMatchObject({ - "pi_coding_agent.operation": "AgentSession.prompt", - provider, - }); - for (const metric of [ - "prompt_tokens", - "completion_tokens", - "tokens", - "prompt_cached_tokens", - "prompt_cache_creation_tokens", - "estimated_cost", - ]) { - expect(task?.metrics).not.toHaveProperty(metric); - } - expect(task?.metrics).toMatchObject({ duration: expect.any(Number) }); - - expect(llm).toBeDefined(); - expect(llmSpans).toHaveLength(2); - expect(llm?.span.type).toBe("llm"); - expect(llm?.row.metadata).toMatchObject({ - "pi_coding_agent.api": api, - provider, - }); - expect(String(llm?.row.metadata?.model)).toContain(modelId); - for (const llmSpan of llmSpans) { - expect(llmSpan.input).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - role: "system", - content: expect.stringContaining("coding assistant"), - }), - ]), - ); - expect(llmSpan.row.metadata?.tools).toEqual([ - { - type: "function", - function: expect.objectContaining({ - name: "bash", - description: expect.any(String), - parameters: expect.any(Object), - }), - }, - ]); - } - expect(llm?.output).toBeDefined(); - for (const llmSpan of llmSpans) { - expect(llmSpan.metrics).toMatchObject({ - completion_tokens: expect.any(Number), - prompt_tokens: expect.any(Number), - tokens: expect.any(Number), + const llms = findChildSpans(events, llmSpanName, task?.span.id); + const tools = findChildSpans(events, "bash", task?.span.id); + expect(llms).toHaveLength(2); + expect(tools).toHaveLength(1); + expect( + findChildSpans(events, TOOL_CHILD_SPAN_NAME, tools[0]?.span.id), + ).toHaveLength(1); + for (const span of [task, ...llms, ...tools]) { + expect(span?.metrics).toMatchObject({ + start: expect.any(Number), + end: expect.any(Number), }); } - - expect(tool).toBeDefined(); - expect(tool?.span.type).toBe("tool"); - expect(tool?.input).toMatchObject({ - command: expect.stringContaining("printf pi_tool_ok"), - }); - expect(tool?.row.metadata).toMatchObject({ - "gen_ai.tool.name": "bash", - }); - expect(tool?.row.metadata?.["gen_ai.tool.call.id"]).toEqual( - expect.any(String), + // Snapshots assert the full payloads and hierarchy; counts above also + // catch duplicate spans that summarize() would otherwise select past. + await matchSpanTreeSnapshot( + summarize(events, llmSpanName), + snapshotPath, ); - expect(JSON.stringify(tool?.output)).toContain("pi_tool_ok"); - - expect(toolChild).toBeDefined(); - expect(toolChild?.span.parentIds).toEqual([tool?.span.id ?? ""]); }, ); - - test("matches the shared span tree snapshot", testConfig, async () => { - expect(setupError).toBeUndefined(); - await matchSpanTreeSnapshot(summarize(events, llmSpanName), snapshotPath); - }); }); } diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs b/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs index 406b8c928..497dcd52d 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs @@ -10,7 +10,6 @@ export const TOOL_CHILD_SPAN_NAME = "pi-tool-child"; async function runPiCodingAgentScenario({ decorateSDK, sdk }) { const instrumentedSDK = decorateSDK ? decorateSDK(sdk) : sdk; - const useFakeStream = process.env.PI_CODING_AGENT_FAKE_STREAM === "true"; const provider = process.env.PI_CODING_AGENT_PROVIDER ?? "anthropic"; const modelId = provider === "openai" ? "gpt-5-nano" : "claude-haiku-4-5"; const apiKey = @@ -35,13 +34,8 @@ async function runPiCodingAgentScenario({ decorateSDK, sdk }) { let modelRegistry; if (ModelRuntime) { modelRuntime = await ModelRuntime.create({ modelsPath: null }); - if (!useFakeStream) { - modelRuntime.registerProvider(provider, { baseUrl }); - } - await modelRuntime.setRuntimeApiKey( - provider, - useFakeStream ? "pi-coding-agent-e2e-placeholder" : apiKey, - ); + modelRuntime.registerProvider(provider, { baseUrl }); + await modelRuntime.setRuntimeApiKey(provider, apiKey); modelRegistry = new ModelRegistry(modelRuntime); } else { authStorage = AuthStorage.inMemory(); @@ -104,51 +98,6 @@ async function runPiCodingAgentScenario({ decorateSDK, sdk }) { childSpan.end(); } }; - if (useFakeStream) { - let streamCallCount = 0; - session.agent.streamFunction = async (streamModel) => { - streamCallCount += 1; - const message = { - api: "anthropic-messages", - content: - streamCallCount === 1 - ? [ - { - arguments: { command: "printf pi_tool_ok" }, - id: "tool-call-1", - name: "bash", - type: "toolCall", - }, - ] - : [ - { - text: "PI_CODING_AGENT_OK pi_tool_ok", - type: "text", - }, - ], - model: streamModel.id, - provider: streamModel.provider, - role: "assistant", - stopReason: streamCallCount === 1 ? "toolUse" : "stop", - timestamp: Date.now(), - usage: { - cacheRead: 0, - cacheWrite: 0, - input: 5, - output: 3, - totalTokens: 8, - }, - }; - return { - async *[Symbol.asyncIterator]() { - yield { message, type: "done" }; - }, - async result() { - return message; - }, - }; - }; - } await session.prompt( "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", { expandPromptTemplates: false, source: "rpc" }, diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/scenario.test.ts b/e2e/scenarios/pi-coding-agent-instrumentation/scenario.test.ts index 40bdc526e..57d437390 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/scenario.test.ts +++ b/e2e/scenarios/pi-coding-agent-instrumentation/scenario.test.ts @@ -13,58 +13,20 @@ const scenarioDir = await prepareScenarioDir({ const TIMEOUT_MS = 240_000; const piCodingAgentScenarios = await Promise.all( [ + { dependencyName: "pi-coding-agent-v0", provider: "anthropic" as const }, { - autoEntry: "scenario.pi-coding-agent-v079.mjs", - autoSnapshotName: "pi-coding-agent-v0-auto-hook", - dependencyName: "pi-coding-agent-v0", - variantKey: "pi-coding-agent-v0", - wrapperEntry: "scenario.pi-coding-agent-v079-wrapped.mjs", - wrapperSnapshotName: "pi-coding-agent-v0-wrapped", - }, - { - autoEntry: "scenario.pi-coding-agent-v079.mjs", - autoSnapshotName: "pi-coding-agent-v0-latest-auto-hook", dependencyName: "pi-coding-agent-v0-latest", - fakeStream: true, - variantKey: "pi-coding-agent-v0-latest", - wrapperEntry: "scenario.pi-coding-agent-v079-wrapped.mjs", - wrapperSnapshotName: "pi-coding-agent-v0-latest-wrapped", - }, - { - autoEntry: "scenario.pi-coding-agent-v079.mjs", - autoSnapshotName: "pi-coding-agent-v081-auto-hook", - dependencyName: "pi-coding-agent-v081", - fakeStream: true, - variantKey: "pi-coding-agent-v081", - wrapperEntry: "scenario.pi-coding-agent-v079-wrapped.mjs", - wrapperSnapshotName: "pi-coding-agent-v081-wrapped", + provider: "anthropic" as const, }, + { dependencyName: "pi-coding-agent-v081", provider: "anthropic" as const }, { - autoEntry: "scenario.pi-coding-agent-v079.mjs", - autoSnapshotName: "pi-coding-agent-v081-latest-auto-hook", dependencyName: "pi-coding-agent-v081-latest", - fakeStream: true, - variantKey: "pi-coding-agent-v081-latest", - wrapperEntry: "scenario.pi-coding-agent-v079-wrapped.mjs", - wrapperSnapshotName: "pi-coding-agent-v081-latest-wrapped", + provider: "anthropic" as const, }, + { dependencyName: "pi-coding-agent-v1", provider: "openai" as const }, { - autoEntry: "scenario.pi-coding-agent-v079.mjs", - autoSnapshotName: "pi-coding-agent-v1-auto-hook", - dependencyName: "pi-coding-agent-v1", - provider: "openai" as const, - variantKey: "pi-coding-agent-v1", - wrapperEntry: "scenario.pi-coding-agent-v079-wrapped.mjs", - wrapperSnapshotName: "pi-coding-agent-v1-wrapped", - }, - { - autoEntry: "scenario.pi-coding-agent-v079.mjs", - autoSnapshotName: "pi-coding-agent-v1-latest-auto-hook", dependencyName: "pi-coding-agent-v1-latest", provider: "openai" as const, - variantKey: "pi-coding-agent-v1-latest", - wrapperEntry: "scenario.pi-coding-agent-v079-wrapped.mjs", - wrapperSnapshotName: "pi-coding-agent-v1-latest-wrapped", }, ].map(async (scenario) => ({ ...scenario, @@ -78,72 +40,47 @@ const piCodingAgentScenarios = await Promise.all( describe.concurrent("variants", () => { for (const piCodingAgentScenario of piCodingAgentScenarios) { describe.sequential( - `pi coding agent ${piCodingAgentScenario.version}`, + `${piCodingAgentScenario.dependencyName} (${piCodingAgentScenario.version})`, () => { - definePiCodingAgentInstrumentationAssertions({ - name: "wrapped instrumentation", - provider: piCodingAgentScenario.provider, - runScenario: async ({ runNodeScenarioDir }) => { - await runNodeScenarioDir({ - entry: piCodingAgentScenario.wrapperEntry, - env: { - PI_CODING_AGENT_PACKAGE_NAME: - piCodingAgentScenario.dependencyName, - ...(piCodingAgentScenario.provider - ? { PI_CODING_AGENT_PROVIDER: piCodingAgentScenario.provider } - : {}), - ...(piCodingAgentScenario.fakeStream - ? { PI_CODING_AGENT_FAKE_STREAM: "true" } - : {}), - }, - runContext: { - ...(piCodingAgentScenario.fakeStream - ? { cassette: false as const } - : {}), - variantKey: piCodingAgentScenario.variantKey, - originalScenarioDir, - }, - scenarioDir, - timeoutMs: TIMEOUT_MS, - }); + for (const mode of [ + { + name: "wrapped instrumentation", + entry: "scenario.pi-coding-agent-v079-wrapped.mjs", + suffix: "wrapped", + nodeArgs: [], }, - snapshotName: piCodingAgentScenario.wrapperSnapshotName, - testFileUrl: import.meta.url, - timeoutMs: TIMEOUT_MS, - }); - - definePiCodingAgentInstrumentationAssertions({ - name: "auto-hook instrumentation", - provider: piCodingAgentScenario.provider, - runScenario: async ({ runNodeScenarioDir }) => { - await runNodeScenarioDir({ - entry: piCodingAgentScenario.autoEntry, - env: { - PI_CODING_AGENT_PACKAGE_NAME: - piCodingAgentScenario.dependencyName, - ...(piCodingAgentScenario.provider - ? { PI_CODING_AGENT_PROVIDER: piCodingAgentScenario.provider } - : {}), - ...(piCodingAgentScenario.fakeStream - ? { PI_CODING_AGENT_FAKE_STREAM: "true" } - : {}), - }, - nodeArgs: ["--import", "braintrust/hook.mjs"], - runContext: { - ...(piCodingAgentScenario.fakeStream - ? { cassette: false as const } - : {}), - variantKey: piCodingAgentScenario.variantKey, - originalScenarioDir, - }, - scenarioDir, - timeoutMs: TIMEOUT_MS, - }); + { + name: "auto-hook instrumentation", + entry: "scenario.pi-coding-agent-v079.mjs", + suffix: "auto-hook", + nodeArgs: ["--import", "braintrust/hook.mjs"], }, - snapshotName: piCodingAgentScenario.autoSnapshotName, - testFileUrl: import.meta.url, - timeoutMs: TIMEOUT_MS, - }); + ]) { + definePiCodingAgentInstrumentationAssertions({ + name: mode.name, + provider: piCodingAgentScenario.provider, + runScenario: async ({ runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: mode.entry, + nodeArgs: mode.nodeArgs, + env: { + PI_CODING_AGENT_PACKAGE_NAME: + piCodingAgentScenario.dependencyName, + PI_CODING_AGENT_PROVIDER: piCodingAgentScenario.provider, + }, + runContext: { + variantKey: piCodingAgentScenario.dependencyName, + originalScenarioDir, + }, + scenarioDir, + timeoutMs: TIMEOUT_MS, + }); + }, + snapshotName: `${piCodingAgentScenario.dependencyName}-${mode.suffix}`, + testFileUrl: import.meta.url, + timeoutMs: TIMEOUT_MS, + }); + } }, ); } diff --git a/js/src/auto-instrumentations/configs/pi-coding-agent.test.ts b/js/src/auto-instrumentations/configs/pi-coding-agent.test.ts index 4d61ea950..e5b42c518 100644 --- a/js/src/auto-instrumentations/configs/pi-coding-agent.test.ts +++ b/js/src/auto-instrumentations/configs/pi-coding-agent.test.ts @@ -1,28 +1,10 @@ import { describe, expect, it } from "vitest"; import { readDisabledInstrumentationEnvConfig } from "../../instrumentation/config"; import { getDefaultInstrumentationConfigs } from "./all"; -import { piCodingAgentConfigs } from "./pi-coding-agent"; const piCodingAgentChannelName = "AgentSession.prompt"; describe("piCodingAgentConfigs", () => { - it("targets the Pi Coding Agent library AgentSession.prompt entrypoint", () => { - expect(piCodingAgentConfigs).toHaveLength(1); - expect(piCodingAgentConfigs[0]).toMatchObject({ - channelName: piCodingAgentChannelName, - module: { - name: "@earendil-works/pi-coding-agent", - versionRange: ">=0.79.0 <2.0.0", - filePath: "dist/core/agent-session.js", - }, - functionQuery: { - className: "AgentSession", - methodName: "prompt", - }, - }); - expect(piCodingAgentConfigs[0].module.filePath).not.toContain("cli"); - }); - it("is included by default and disabled by Pi Coding Agent env aliases", () => { expect( getDefaultInstrumentationConfigs().some( diff --git a/js/src/instrumentation/plugins/pi-coding-agent-context.test.ts b/js/src/instrumentation/plugins/pi-coding-agent-context.test.ts index a9eec3480..e50d43794 100644 --- a/js/src/instrumentation/plugins/pi-coding-agent-context.test.ts +++ b/js/src/instrumentation/plugins/pi-coding-agent-context.test.ts @@ -8,7 +8,6 @@ import { import { configureNode } from "../../node/config"; import type { PiAgentEvent, - PiAgentEventListener, PiAgentSession, PiStreamFunction, PiTool, @@ -142,6 +141,54 @@ describe("Pi tool execution context", () => { }, ); + it("parents native nested tools and their application spans under the calling tool", async () => { + const observed = new Map(); + const tool: PiTool = { + name: "lookup", + execute: async (id) => { + observed.set(String(id), currentSpan()); + if (id === "parent") { + fixture.start("nested", "parent"); + await tool.execute!("nested"); + await fixture.end("nested"); + expect(currentSpan()).toBe(observed.get("parent")); + } else { + await nextTurn(); + startSpan({ name: "nested-child" }).end(); + } + }, + }; + const fixture = makeFixture(tool); + await experiment.traced(async () => { + await fixture.prompt(async () => { + fixture.start("parent"); + await tool.execute!("parent"); + await fixture.end("parent"); + // An unavailable native parent must not lose the tool span. + await fixture.start("orphan", "unknown"); + await fixture.end("orphan"); + }); + }); + const rows = (await background.drain()).flatMap((row) => + "span_id" in row ? [row] : [], + ); + const nested = rows.find( + (row) => row.span_id === observed.get("nested")?.spanId, + ); + expect(nested?.span_parents).toEqual([observed.get("parent")?.spanId]); + expect( + rows.find((row) => row.span_attributes?.name === "nested-child") + ?.span_parents, + ).toEqual([nested?.span_id]); + const prompt = rows.find( + (row) => row.span_attributes?.name === "AgentSession.prompt", + ); + const orphan = rows.find( + (row) => row.metadata?.["gen_ai.tool.call.id"] === "orphan", + ); + expect(orphan?.span_parents).toEqual([prompt?.span_id]); + }); + it.each(["same prompt", "overlapping prompts"] as const)( "isolates concurrent tools in %s", async (mode) => { @@ -331,23 +378,26 @@ function makeFixture( throw new Error("This context test must not call a model"); }, ) { - const listeners = new Set(); const agent: PiAgentSession["agent"] & { state: { tools: PiTool[] } | undefined; } = { state: { tools: [tool] }, [property]: streamFunction, - subscribe(listener: PiAgentEventListener) { - listeners.add(listener); + }; + const sessionListeners = new Set<(event: PiAgentEvent) => void>(); + const session: PiAgentSession = { + agent, + prompt: async () => {}, + subscribe(listener) { + sessionListeners.add(listener); return () => { - listeners.delete(listener); + sessionListeners.delete(listener); }; }, }; - const session: PiAgentSession = { agent, prompt: async () => {} }; - const emit = async (event: PiAgentEvent) => { - for (const listener of listeners) - await listener(event, new AbortController().signal); + const emit = (event: PiAgentEvent) => { + // AgentSession emits to subscribers synchronously without awaiting them. + for (const listener of sessionListeners) listener(event); }; return { agent, @@ -358,10 +408,11 @@ function makeFixture( ["test", undefined], { session }, ), - start: (toolCallId: string) => + start: (toolCallId: string, parentToolCallId?: string) => emit({ type: "tool_execution_start", toolCallId, + parentToolCallId, toolName: tool.name, args: {}, }), diff --git a/js/src/instrumentation/plugins/pi-coding-agent-interceptors.ts b/js/src/instrumentation/plugins/pi-coding-agent-interceptors.ts index a154bcd66..5362cba5a 100644 --- a/js/src/instrumentation/plugins/pi-coding-agent-interceptors.ts +++ b/js/src/instrumentation/plugins/pi-coding-agent-interceptors.ts @@ -15,5 +15,8 @@ export const piCodingAgentChannels = defineInterceptor( >({ channelName: "AgentSession.prompt", }), + toolExecute: channel({ + channelName: "tool.execute", + }), }, ); diff --git a/js/src/instrumentation/plugins/pi-coding-agent-plugin.test.ts b/js/src/instrumentation/plugins/pi-coding-agent-plugin.test.ts index deb9ff787..eabfed2a8 100644 --- a/js/src/instrumentation/plugins/pi-coding-agent-plugin.test.ts +++ b/js/src/instrumentation/plugins/pi-coding-agent-plugin.test.ts @@ -1,16 +1,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -const { - interceptorsByName, - mockNewTracingChannel, - mockRemoveInterceptor, - mockStartSpan, -} = vi.hoisted(() => ({ - interceptorsByName: new Map(), - mockNewTracingChannel: vi.fn(), - mockRemoveInterceptor: vi.fn(), - mockStartSpan: vi.fn(), -})); +const { interceptorsByName, mockNewTracingChannel, mockStartSpan } = vi.hoisted( + () => ({ + interceptorsByName: new Map(), + mockNewTracingChannel: vi.fn(), + mockStartSpan: vi.fn(), + }), +); vi.mock("../../isomorph", async (importOriginal) => { const actual = await importOriginal(); @@ -19,6 +15,7 @@ vi.mock("../../isomorph", async (importOriginal) => { ...actual, default: { ...actual.default, + getEnv: (name: string) => process.env[name], newAsyncLocalStorage: () => new AsyncLocalStorage(), }, }; @@ -31,15 +28,18 @@ vi.mock("../../global-instrumentation-hooks", async (importOriginal) => ({ newGlobalInvocationHook: mockNewTracingChannel, })); -vi.mock("../../logger", () => ({ +vi.mock("../../logger", async (importOriginal) => ({ + Attachment: (await importOriginal()) + .Attachment, startSpan: (...args: unknown[]) => mockStartSpan(...args), withCurrent: (_span: unknown, callback: () => unknown) => callback(), })); -import { isAutoInstrumentationSuppressed } from "../auto-instrumentation-suppression"; import type { + PiAssistantMessage, PiContext, PiModel, + PiSimpleStreamOptions, } from "../../vendor-sdk-types/pi-coding-agent"; import { PiCodingAgentPlugin } from "./pi-coding-agent-plugin"; @@ -63,10 +63,23 @@ describe("PiCodingAgentPlugin", () => { spans = []; interceptorsByName.clear(); mockNewTracingChannel.mockImplementation((name: string) => ({ - intercept: vi.fn((interceptor) => { + intercept(interceptor: unknown) { interceptorsByName.set(name, interceptor); - return mockRemoveInterceptor; - }), + return () => { + interceptorsByName.delete(name); + }; + }, + invoke( + target: (...args: unknown[]) => unknown, + receiver: unknown, + args: unknown[], + additional: object, + ) { + const interceptor = interceptorsByName.get(name); + return interceptor + ? interceptor(target, receiver, args, additional) + : Reflect.apply(target, receiver, args); + }, })); mockStartSpan.mockImplementation((args: any) => { const span = { @@ -89,139 +102,10 @@ describe("PiCodingAgentPlugin", () => { plugin.disable(); } vi.clearAllMocks(); + vi.useRealTimers(); + vi.unstubAllEnvs(); }); - it("registers and removes the prompt interceptor", () => { - const plugin = enablePlugin(plugins); - - expect(interceptorsByName.has(PROMPT_CHANNEL)).toBe(true); - expect(mockRemoveInterceptor).not.toHaveBeenCalled(); - - plugin.disable(); - - expect(mockRemoveInterceptor).toHaveBeenCalledTimes(1); - }); - - it.each([ - { property: "streamFn" as const, version: "0.79.1" }, - { property: "streamFunction" as const, version: "0.81.0" }, - ])( - "uses interceptor ALS for prompt, LLM, and tool spans with $property", - async ({ property, version }) => { - const interceptor = promptInterceptor(enablePlugin(plugins)); - const finalMessage = makeAssistantMessage("done"); - const originalStreamFn = vi.fn(async () => { - expect(isAutoInstrumentationSuppressed()).toBe(true); - return makeStream(finalMessage); - }); - const agent = makeAgent(originalStreamFn, property); - const tool = bashTool(); - agent.state.tools = [tool]; - const session = makeSession(agent); - const context = { - systemPrompt: "system", - messages: [ - { - role: "user", - content: [{ type: "text", text: "hello" }], - timestamp: 1, - }, - ], - tools: [tool], - }; - - await interceptor( - async function (this: typeof session) { - expect(isAutoInstrumentationSuppressed()).toBe(true); - const stream = await this.agent[property](anthropicModel(), context, { - apiKey: "secret", - headers: { authorization: "secret" }, - reasoning: "low", - }); - await this.agent.emit({ - args: { command: "printf pi_tool_ok" }, - toolCallId: "tool-1", - toolName: "bash", - type: "tool_execution_start", - }); - await tool.execute?.("tool-1", { - command: "printf pi_tool_ok", - }); - expect(isAutoInstrumentationSuppressed()).toBe(true); - await this.agent.emit({ - isError: false, - result: { stdout: "pi_tool_ok" }, - toolCallId: "tool-1", - toolName: "bash", - type: "tool_execution_end", - }); - await stream.result(); - await this.agent.emit({ - message: finalMessage, - toolResults: [], - turnIndex: 0, - type: "turn_end", - }); - }, - session, - ["hello", undefined], - { moduleVersion: version }, - ); - - expect(isAutoInstrumentationSuppressed()).toBe(false); - expect(agent[property]).not.toBe(originalStreamFn); - expect(originalStreamFn).toHaveBeenCalledWith( - anthropicModel(), - context, - expect.objectContaining({ apiKey: "secret" }), - ); - - const taskSpan = findSpan(spans, "AgentSession.prompt"); - const llmSpan = findSpan(spans, "anthropic.messages.create"); - const toolSpan = findSpan(spans, "bash"); - expect(llmSpan?.args.event.input).toEqual([ - { role: "system", content: "system" }, - { role: "user", content: [{ type: "text", text: "hello" }] }, - ]); - expect(llmSpan?.args.event.metadata).toMatchObject({ - model: "claude-haiku-4-5", - "pi_coding_agent.operation": `agent.${property}`, - provider: "anthropic", - tools: [ - { - type: "function", - function: expect.objectContaining({ name: "bash" }), - }, - ], - }); - expect(llmSpan?.args.event.metadata).not.toHaveProperty("apiKey"); - expect(llmSpan?.args.event.metadata).not.toHaveProperty("headers"); - expect(llmSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ - metrics: expect.objectContaining({ - completion_tokens: 3, - prompt_tokens: 5, - tokens: 8, - }), - output: expect.any(Array), - }), - ); - expect(toolSpan?.args.event.input).toEqual({ - command: "printf pi_tool_ok", - }); - expect(toolSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ output: { stdout: "pi_tool_ok" } }), - ); - const finalTaskLog = taskSpan?.log.mock.calls.at(-1)?.[0]; - expect(finalTaskLog?.metrics).toEqual({ - duration: expect.any(Number), - end: expect.any(Number), - start: expect.any(Number), - }); - expect(taskSpan?.end).toHaveBeenCalledTimes(1); - }, - ); - it("preserves Pi 1.4 prompt deltas and resolves tool updates without mutation", async () => { const original = { name: "lookup", @@ -337,168 +221,662 @@ describe("PiCodingAgentPlugin", () => { expect(originalStream.mock.calls[0]?.[1]).toBe(context); } - it("counts cached usage once across multiple LLM turns", async () => { + it("captures per-call usage and cost without double-counting parent rollups", async () => { const interceptor = promptInterceptor(enablePlugin(plugins)); - // Pi's `input` excludes cached tokens (its OpenAI client subtracts them from - // `input_tokens`; Anthropic reports them separately), so 5 + 20 + 4 + 3 = 32. - const message = { - ...makeAssistantMessage("done"), - usage: { - cacheRead: 20, - cacheWrite: 4, + // Pi input excludes cache usage; reasoning is already included in output. + const usages: NonNullable[] = [ + { input: 5, - output: 3, - totalTokens: 32, + output: 10, + reasoning: 6, + cacheRead: 20, + cacheWrite: 8, + cacheWrite1h: 3, + cost: { total: 0.002 }, }, - }; - const agent = makeAgent(vi.fn(async () => makeStream(message))); - const session = makeSession(agent); - + { input: 5, output: 3, cacheRead: 20, cacheWrite: 4, totalTokens: 32 }, + ]; + const expected: Record[] = [ + { + prompt_tokens: 33, + completion_tokens: 10, + tokens: 43, + completion_reasoning_tokens: 6, + prompt_cached_tokens: 20, + prompt_cache_creation_tokens: 8, + prompt_cache_creation_5m_tokens: 5, + prompt_cache_creation_1h_tokens: 3, + estimated_cost: 0.002, + }, + { + prompt_tokens: 29, + completion_tokens: 3, + tokens: 32, + prompt_cached_tokens: 20, + prompt_cache_creation_tokens: 4, + }, + ]; + const originalStream = vi.fn(); + const agent = makeAgent(originalStream); await interceptor( - async function (this: typeof session) { - for (let turnIndex = 0; turnIndex < 2; turnIndex++) { - const stream = await this.agent.streamFn(anthropicModel(), { - messages: [{ role: "user", content: "hello" }], + async () => { + for (const usage of usages) { + const message = { ...makeAssistantMessage("done"), usage }; + originalStream.mockReturnValueOnce(makeStream(message)); + const stream = await agent.streamFn(anthropicModel(), { + messages: [], }); await stream.result(); - await this.agent.emit({ - type: "turn_end", - message, - toolResults: [], - turnIndex, - }); + await agent.emit({ type: "turn_end", message, toolResults: [] }); } }, - session, + makeSession(agent), ["hello", undefined], {}, ); - - const expected = { - completion_tokens: 3, - prompt_cache_creation_tokens: 4, - prompt_cached_tokens: 20, - prompt_tokens: 29, - tokens: 32, - }; - const llmSpans = spans.filter( + const llms = spans.filter( (span) => span.args.spanAttributes.type === "llm", ); - expect(llmSpans).toHaveLength(2); - for (const span of llmSpans) { - expect(span.log).toHaveBeenCalledWith( - expect.objectContaining({ metrics: expect.objectContaining(expected) }), - ); - } - for (const [metric, value] of Object.entries(expected)) { + expect(llms).toHaveLength(usages.length); + llms.forEach((span, index) => + expect(span.log.mock.calls.at(-1)?.[0].metrics).toEqual({ + ...expected[index], + start: expect.any(Number), + end: expect.any(Number), + }), + ); + for (const metric of Object.keys(expected[0])) { const total = spans.reduce( (sum, span) => sum + (span.log.mock.calls.at(-1)?.[0].metrics?.[metric] ?? 0), 0, ); - expect(total).toBe(value * 2); + expect(total).toBe( + expected.reduce((sum, usage) => sum + (usage[metric] ?? 0), 0), + ); } }); - it("isolates overlapping prompts on the same agent without prompt matching", async () => { + async function captureMessage( + message: PiAssistantMessage, + options?: PiSimpleStreamOptions, + model: PiModel = anthropicModel(), + ) { const interceptor = promptInterceptor(enablePlugin(plugins)); - const originalStreamFn = vi.fn(async () => - makeStream(makeAssistantMessage("done")), - ); - const agent = makeAgent(originalStreamFn); - const session = makeSession(agent); - const firstGate = deferred(); - const secondGate = deferred(); - - const first = interceptor( - async function (this: typeof session) { - await firstGate.promise; - const stream = await this.agent.streamFn(anthropicModel(), { - messages: [{ role: "user", content: "same prompt" }], - }); - await stream.result(); - }, - session, - ["same prompt", undefined], - {}, - ); - const sharedStreamFn = agent.streamFn; - const second = interceptor( - async function (this: typeof session) { - await secondGate.promise; - const stream = await this.agent.streamFn(anthropicModel(), { - messages: [{ role: "user", content: "same prompt" }], - }); - await stream.result(); + const agent = makeAgent(vi.fn(async () => makeStream(message))); + await interceptor( + async () => { + const stream = await agent.streamFn(model, { messages: [] }, options); + expect(await stream.result()).toBe(message); + await agent.emit({ type: "message_end", message }); + await agent.emit({ type: "turn_end", message, toolResults: [] }); }, - session, - ["same prompt", undefined], + makeSession(agent), + ["hello", undefined], {}, ); + return spans.find((span) => span.args.spanAttributes.type === "llm")!; + } - expect(agent.streamFn).toBe(sharedStreamFn); - secondGate.resolve(); - await second; - firstGate.resolve(); - await first; + it.each([ + { + name: "missing usage", + usage: { input: 2 }, + expected: { prompt_tokens: 2 }, + }, + { + name: "invalid counts and cost", + usage: { + input: 2, + output: 3.5, + cacheRead: -1, + reasoning: Infinity, + totalTokens: NaN, + cost: { total: Infinity }, + }, + expected: {}, + }, + { + name: "reported totals, explicit zeros, and unpriced models", + usage: { + input: 2, + output: 3, + totalTokens: 9, + reasoning: 0, + cacheWrite: 0, + cacheWrite1h: 0, + cost: { total: 0 }, + }, + expected: { + prompt_tokens: 2, + completion_tokens: 3, + tokens: 9, + completion_reasoning_tokens: 0, + prompt_cache_creation_tokens: 0, + prompt_cache_creation_1h_tokens: 0, + prompt_cache_creation_5m_tokens: 0, + }, + }, + ])("preserves metric completeness for $name", async ({ usage, expected }) => { + const llm = await captureMessage({ + ...makeAssistantMessage("done"), + usage, + }); + expect(llm.log.mock.calls.at(-1)?.[0].metrics).toEqual({ + ...expected, + start: expect.any(Number), + end: expect.any(Number), + }); + }); - const llmSpans = spans.filter( - (span) => span.name === "anthropic.messages.create", + it("attributes physical model dispatch while preserving the selected virtual model", async () => { + const llm = await captureMessage( + { + ...makeAssistantMessage("done"), + responseModel: "physical-model", + thinkingLevel: "high", + providerThinkingLevel: "medium", + }, + undefined, + { ...anthropicModel(), id: "virtual-model" }, ); - expect(llmSpans).toHaveLength(2); - expect( - spans.filter((span) => span.name === "AgentSession.prompt"), - ).toHaveLength(2); + expect(llm.log.mock.calls.at(-1)?.[0].metadata).toMatchObject({ + model: "physical-model", + provider: "anthropic", + selected_model: "anthropic/virtual-model", + "pi_coding_agent.thinking_level": "high", + "pi_coding_agent.provider_thinking_level": "medium", + }); + expect(llm.name).toBe("anthropic.messages.create"); + }); - await agent.streamFn(anthropicModel(), { - messages: [{ role: "user", content: "outside prompt" }], + it("allowlists request controls without capturing arbitrary provider metadata", async () => { + const options = { + temperature: 0.2, + maxTokens: 20, + toolChoice: "none" as const, + samplingParams: { + top_p: 0.9, + temperature: 0.1, + tool_choice: "auto", + parallel_tool_calls: false, + presence_penalty: undefined, + secret: "secret", + }, + metadata: { user_id: "private-user" }, + apiKey: "secret", + headers: { authorization: "secret" }, + }; + const before = structuredClone(options); + const llm = await captureMessage(makeAssistantMessage("done"), options, { + id: "model", + provider: "openai", + api: "openai-responses", + samplingParams: { + frequency_penalty: 0.5, + top_p: 0.5, + presence_penalty: 0.7, + secret: "secret", + }, }); - expect( - spans.filter((span) => span.name === "anthropic.messages.create"), - ).toHaveLength(2); + const metadata = llm.args.event.metadata; + expect(metadata).toMatchObject({ + temperature: 0.1, + max_tokens: 20, + top_p: 0.9, + frequency_penalty: 0.5, + tool_choice: "auto", + parallel_tool_calls: false, + }); + expect(metadata).not.toHaveProperty("presence_penalty"); + expect(JSON.stringify(metadata)).not.toMatch(/secret|private-user/); + expect(options).toEqual(before); + }); + + it("does not capture OpenAI-only sampling overrides for other providers", async () => { + const llm = await captureMessage(makeAssistantMessage("done"), { + temperature: 0.2, + toolChoice: "none", + samplingParams: { temperature: 0.1, top_p: 0.9 }, + }); + expect(llm.args.event.metadata).toMatchObject({ + temperature: 0.2, + tool_choice: "none", + }); + expect(llm.args.event.metadata).not.toHaveProperty("top_p"); + }); + + it("omits unserializable sampling fields while retaining valid request metadata", async () => { + const callback = vi.fn(); + const cycle: Record = {}; + cycle.self = cycle; + const llm = await captureMessage( + makeAssistantMessage("done"), + { + samplingParams: { + top_p: callback, + stop: cycle, + temperature: 0.2, + frequency_penalty: Infinity, + }, + }, + { ...anthropicModel(), api: "openai-responses" }, + ); + expect(llm.args.event.metadata.temperature).toBe(0.2); + for (const field of ["top_p", "stop", "frequency_penalty"]) + expect(llm.args.event.metadata).not.toHaveProperty(field); + expect(callback).not.toHaveBeenCalled(); }); - it("keeps deferred follow-up prompts open for their ALS-owned turn", async () => { + it.each(["success", "failure", "blocked"] as const)( + "records tool approval only when the executor ran: %s", + async (outcome) => { + const interceptor = promptInterceptor(enablePlugin(plugins)); + const agent = makeAgent( + vi.fn(() => makeStream(makeAssistantMessage("done"))), + ); + const failure = new Error("tool failed"); + const tool = { + name: "lookup", + execute: vi.fn((_toolCallId: string) => { + if (outcome === "failure") throw failure; + return "result"; + }), + }; + agent.state.tools = [tool]; + await interceptor( + async () => { + await agent.emit({ + type: "tool_execution_start", + toolCallId: "call", + toolName: "lookup", + args: {}, + }); + if (outcome === "failure") + expect(() => tool.execute("call")).toThrow(failure); + if (outcome === "success") + expect(tool.execute("call")).toBe("result"); + await agent.emit({ + type: "tool_execution_end", + toolCallId: "call", + toolName: "lookup", + result: + outcome === "blocked" + ? "Tool execution was blocked" + : outcome === "failure" + ? failure + : "result", + isError: outcome !== "success", + }); + }, + makeSession(agent), + ["hello", undefined], + {}, + ); + const log = findSpan(spans, "lookup")?.log.mock.calls.at(-1)?.[0]; + if (outcome === "blocked") + expect(log.metadata).not.toHaveProperty("tool_approval"); + else expect(log.metadata.tool_approval).toBe("approved"); + if (outcome === "failure") expect(log.error).toBe(failure); + if (outcome === "blocked") expect(log.error).toContain("blocked"); + }, + ); + + it.each(["error", "aborted"])( + "marks %s responses as failures without returning stale prompt output", + async (stopReason) => { + const message = { + ...makeAssistantMessage("partial"), + stopReason, + errorMessage: "request failed", + }; + const llm = await captureMessage(message); + expect(llm.log.mock.calls.at(-1)?.[0]).toMatchObject({ + error: "request failed", + output: expect.any(Array), + }); + const task = findSpan(spans, "AgentSession.prompt"); + expect(task?.log.mock.calls.at(-1)?.[0]).toMatchObject({ + error: "request failed", + }); + expect(task?.log.mock.calls.at(-1)?.[0].output).toBeUndefined(); + }, + ); + + it.each(["text_delta", "thinking_delta", "toolcall_delta"])( + "measures TTFT from the first nonempty %s", + async (type) => { + vi.useFakeTimers(); + vi.setSystemTime(1_000); + const message = makeAssistantMessage("done"); + const events = [ + { type: "start", partial: message }, + { type: "text_start", partial: message }, + { type, delta: "", partial: message }, + { type, delta: "first", partial: message }, + { type, delta: "second", partial: message }, + { type: "done", message }, + ]; + const { stream } = makeIteratorBackedStream(events); + const agent = makeAgent(vi.fn(async () => stream)); + const interceptor = promptInterceptor(enablePlugin(plugins)); + await interceptor( + async () => { + const observed = await agent.streamFn(anthropicModel(), { + messages: [], + }); + const iterator = observed[Symbol.asyncIterator](); + for (let i = 0; i < events.length; i++) { + vi.setSystemTime(2_000 + i * 1_000); + expect((await iterator.next()).value).toBe(events[i]); + } + }, + makeSession(agent), + ["hello", undefined], + {}, + ); + expect( + findSpan(spans, "anthropic.messages.create")?.log.mock.calls.at(-1)?.[0] + .metrics.time_to_first_token, + ).toBe(4); + }, + ); + + it.each(["steer", "followUp"] as const)( + "does not trace queued %s prompts or correlate their later events", + async (streamingBehavior) => { + const interceptor = promptInterceptor(enablePlugin(plugins)); + const agent = makeAgent( + vi.fn(() => makeStream(makeAssistantMessage("done"))), + ); + const session = { ...makeSession(agent), isStreaming: true }; + let later: (() => Promise) | undefined; + const result = Promise.resolve(); + const returned = interceptor( + () => { + later = async () => { + const stream = await agent.streamFn(anthropicModel(), { + messages: [], + }); + await stream.result(); + await agent.emit({ + type: "turn_end", + message: makeAssistantMessage("queued"), + toolResults: [], + }); + }; + return result; + }, + session, + ["queued", { streamingBehavior }], + {}, + ); + expect(returned).toBe(result); + await returned; + await later!(); + expect(spans).toHaveLength(0); + expect(agent.subscribe).not.toHaveBeenCalled(); + }, + ); + + it("finishes a direct idle prompt even when it specifies followUp behavior", async () => { const interceptor = promptInterceptor(enablePlugin(plugins)); const agent = makeAgent( - vi.fn(async () => makeStream(makeAssistantMessage("done"))), + vi.fn(() => makeStream(makeAssistantMessage("done"))), ); - const session = makeSession(agent); - const turnGate = deferred(); - let backgroundTurn: Promise | undefined; + const session = { ...makeSession(agent), isStreaming: false }; + const result = Promise.resolve(); + expect( + interceptor( + () => result, + session, + ["direct", { streamingBehavior: "followUp" }], + {}, + ), + ).toBe(result); + await result; + expect(findSpan(spans, "AgentSession.prompt")?.end).toHaveBeenCalledTimes( + 1, + ); + }); + it.each([false, true])( + "preserves stream and promise identities with async=%s", + async (asyncStream) => { + const interceptor = promptInterceptor(enablePlugin(plugins)); + const stream = makeStream(makeAssistantMessage("done")); + const completion = stream.result(); + stream.result = vi.fn(() => completion); + class StreamPromise extends Promise { + helper() { + return "kept"; + } + } + const streamResult = asyncStream + ? new StreamPromise((resolve) => resolve(stream)) + : stream; + const agent = makeAgent(vi.fn(() => streamResult)); + const session = makeSession(agent); + let promptResult: Promise; + const returned = interceptor( + () => { + promptResult = (async () => { + const observed = agent.streamFn(anthropicModel(), { messages: [] }); + expect(observed).toBe(streamResult); + if (observed instanceof StreamPromise) + expect(observed.helper()).toBe("kept"); + expect(await observed).toBe(stream); + const result = stream.result(); + expect(result).toBe(completion); + expect(await result).toMatchObject({ role: "assistant" }); + })(); + return promptResult; + }, + session, + ["hello", undefined], + {}, + ); + expect(returned).toBe(promptResult!); + await returned; + }, + ); + + it("does not let input extraction failures replace or retry the provider call", async () => { + const interceptor = promptInterceptor(enablePlugin(plugins)); + const stream = makeStream(makeAssistantMessage("done")); + const target = vi.fn(() => stream); + const agent = makeAgent(target); await interceptor( - async function (this: typeof session) { - backgroundTurn = (async () => { - await turnGate.promise; - await this.agent.emit({ - message: makeAssistantMessage("follow-up done"), - toolResults: [], - turnIndex: 1, - type: "turn_end", - }); - })(); + async () => { + const context = { + get messages(): never { + throw new Error("invalid context"); + }, + }; + expect(await agent.streamFn(anthropicModel(), context)).toBe(stream); }, - session, - ["follow up", { streamingBehavior: "followUp" }], + makeSession(agent), + ["hello", undefined], {}, ); + expect(target).toHaveBeenCalledTimes(1); + }); - const taskSpan = findSpan(spans, "AgentSession.prompt"); - expect(taskSpan?.end).not.toHaveBeenCalled(); + it.each(["false", "true"])( + "applies attachment consent %s to prompt, conversation, and tool output", + async (consent) => { + vi.stubEnv("BRAINTRUST_CAPTURE_ATTACHMENTS", consent); + const enabled = consent === "true" || consent === "1"; + const png = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aWQAAAABJRU5ErkJggg=="; + const image = { + type: "image" as const, + mimeType: "image/png", + data: png, + }; + const context: PiContext = { + messages: [ + { role: "user", content: [image] }, + { + role: "toolResult", + toolCallId: "previous", + toolName: "image", + content: [image], + }, + ], + }; + const before = structuredClone(context); + const interceptor = promptInterceptor(enablePlugin(plugins)); + const message = makeAssistantMessage("done"); + const agent = makeAgent(vi.fn(() => makeStream(message))); + const result = { + content: [{ type: "text", text: "image result" }, image], + details: { useful: true }, + }; + await interceptor( + async () => { + const stream = await agent.streamFn(anthropicModel(), context); + await stream.result(); + await agent.emit({ + type: "tool_execution_start", + toolCallId: "image", + toolName: "image", + args: {}, + }); + await agent.emit({ + type: "tool_execution_end", + toolCallId: "image", + toolName: "image", + result, + isError: true, + }); + }, + makeSession(agent), + ["describe", { images: [image] }], + {}, + ); + const prompt = findSpan(spans, "AgentSession.prompt")!; + const llm = findSpan(spans, "anthropic.messages.create")!; + const tool = findSpan(spans, "image")!; + const loggedImages = [ + prompt.args.event.input[0].content[1].image_url.url, + llm.args.event.input[0].content[0].image_url.url, + llm.args.event.input[1].content[0].image_url.url, + tool.log.mock.calls.at(-1)?.[0].output.content[1].image_url.url, + ]; + for (const logged of loggedImages) { + if (enabled) expect(logged.constructor.name).toBe("Attachment"); + else expect(logged).toBe(""); + } + expect( + JSON.stringify(spans.flatMap((span) => span.log.mock.calls)), + ).not.toContain(png); + expect(context).toEqual(before); + expect(result.content[1]).toBe(image); + for (const span of [prompt, llm]) { + expect(span.log.mock.calls.at(-1)?.[0].metrics).not.toHaveProperty( + "duration", + ); + } + }, + ); - turnGate.resolve(); - await backgroundTurn; + it("does not read inline bytes when attachment capture is disabled", async () => { + vi.stubEnv("BRAINTRUST_CAPTURE_ATTACHMENTS", "false"); + const data = vi.fn(() => { + throw new Error("must not read bytes"); + }); + const image = { + type: "image" as const, + mimeType: "image/png", + get data(): string { + return data(); + }, + }; + const interceptor = promptInterceptor(enablePlugin(plugins)); + const agent = makeAgent( + vi.fn(() => makeStream(makeAssistantMessage("done"))), + ); + await interceptor( + async () => { + const stream = await agent.streamFn(anthropicModel(), { + messages: [{ role: "user", content: [image] }], + }); + await stream.result(); + }, + makeSession(agent), + ["describe", { images: [image] }], + {}, + ); + expect(data).not.toHaveBeenCalled(); + expect(findSpan(spans, "anthropic.messages.create")).toBeDefined(); + }); - expect(taskSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ output: expect.any(Array) }), + it("preserves visible reasoning and tool-call correlation in subsequent context", async () => { + const message: PiAssistantMessage = { + ...makeAssistantMessage("answer"), + content: [ + { + type: "thinking", + thinking: "reason", + signature: "private-signature", + }, + { type: "thinking", thinking: "hidden", redacted: true }, + { type: "text", text: "answer" }, + { + type: "toolCall", + id: "call-1", + name: "lookup", + arguments: { query: "x" }, + }, + ], + }; + const interceptor = promptInterceptor(enablePlugin(plugins)); + const agent = makeAgent(vi.fn(() => makeStream(message))); + await interceptor( + async () => { + const stream = await agent.streamFn(anthropicModel(), { + messages: [ + message, + { + role: "toolResult", + toolCallId: "call-1", + toolName: "lookup", + content: [{ type: "text", text: "found" }], + }, + ], + }); + await stream.result(); + }, + makeSession(agent), + ["hello", undefined], + {}, + ); + const llm = findSpan(spans, "anthropic.messages.create")!; + const expected = { + role: "assistant", + content: "answer", + reasoning_content: "reason", + tool_calls: [ + { + id: "call-1", + type: "function", + function: { name: "lookup", arguments: '{"query":"x"}' }, + }, + ], + }; + expect(llm.args.event.input).toEqual([ + expected, + { + role: "tool", + tool_call_id: "call-1", + content: [{ type: "text", text: "found" }], + }, + ]); + expect(llm.log.mock.calls.at(-1)?.[0].output[0].message).toEqual(expected); + expect(JSON.stringify(llm.log.mock.calls)).not.toMatch( + /private-signature|hidden/, ); - expect(taskSpan?.end).toHaveBeenCalledTimes(1); - expect(taskSpan?.log.mock.calls.at(-1)?.[0].metrics).toEqual({ - duration: expect.any(Number), - end: expect.any(Number), - start: expect.any(Number), - }); }); it("preserves full-iterator stream behavior", async () => { @@ -534,12 +912,16 @@ describe("PiCodingAgentPlugin", () => { }), ); expect(llmSpan?.end).toHaveBeenCalledTimes(1); + expect(llmSpan?.log.mock.calls.at(-1)?.[0].metrics).not.toHaveProperty( + "time_to_first_token", + ); }); it("forwards iterator failures and closes the underlying iterator", async () => { const interceptor = promptInterceptor(enablePlugin(plugins)); const { iterator, stream } = makeIteratorBackedStream([]); - iterator.next.mockRejectedValueOnce(new Error("stream next failed")); + const error = new Error("stream next failed"); + iterator.next.mockRejectedValueOnce(error); const agent = makeAgent(vi.fn(async () => stream)); const session = makeSession(agent); @@ -561,11 +943,11 @@ describe("PiCodingAgentPlugin", () => { const taskSpan = findSpan(spans, "AgentSession.prompt"); expect(iterator.return).toHaveBeenCalledTimes(1); expect(llmSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ error: "stream next failed" }), + expect.objectContaining({ error }), ); expect(llmSpan?.end).toHaveBeenCalledTimes(1); expect(taskSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ error: "stream next failed" }), + expect.objectContaining({ error }), ); }); @@ -575,6 +957,7 @@ describe("PiCodingAgentPlugin", () => { vi.fn(async () => makeStream(makeAssistantMessage("unused"))), ); const session = makeSession(agent); + const error = new Error("boom"); await expect( interceptor( @@ -582,7 +965,7 @@ describe("PiCodingAgentPlugin", () => { await this.agent.streamFn(anthropicModel(), { messages: [{ role: "user", content: "hello" }], }); - throw new Error("boom"); + throw error; }, session, ["hello", undefined], @@ -593,11 +976,11 @@ describe("PiCodingAgentPlugin", () => { const llmSpan = findSpan(spans, "anthropic.messages.create"); const taskSpan = findSpan(spans, "AgentSession.prompt"); expect(llmSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ error: "boom" }), + expect.objectContaining({ error }), ); expect(llmSpan?.end).toHaveBeenCalledTimes(1); expect(taskSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ error: "boom" }), + expect.objectContaining({ error }), ); expect(taskSpan?.end).toHaveBeenCalledTimes(1); }); @@ -633,6 +1016,57 @@ describe("PiCodingAgentPlugin", () => { await result; expect(taskSpan?.end).toHaveBeenCalledTimes(1); }); + + it("keeps installed tool wrappers working across SDK copies", async () => { + const first = enablePlugin(plugins); + const tool = { name: "lookup", execute: vi.fn((_id: string) => "result") }; + const agent = makeAgent( + vi.fn(() => makeStream(makeAssistantMessage("done"))), + ); + agent.state.tools = [tool]; + const session = makeSession(agent); + await promptInterceptor(first)( + async () => {}, + session, + ["first", undefined], + {}, + ); + const installedWrapper = tool.execute; + first.disable(); + + vi.resetModules(); + const { PiCodingAgentPlugin: OtherCopy } = + await import("./pi-coding-agent-plugin"); + const second = new OtherCopy(); + plugins.push(second); + second.enable(); + await promptInterceptor(second)( + async () => { + await agent.emit({ + type: "tool_execution_start", + toolCallId: "call", + toolName: "lookup", + args: {}, + }); + expect(tool.execute("call")).toBe("result"); + await agent.emit({ + type: "tool_execution_end", + toolCallId: "call", + toolName: "lookup", + result: "result", + isError: false, + }); + }, + session, + ["second", undefined], + {}, + ); + expect(tool.execute).toBe(installedWrapper); + expect( + findSpan(spans, "lookup")?.log.mock.calls.at(-1)?.[0].metadata + .tool_approval, + ).toBe("approved"); + }); }); function enablePlugin(plugins: PiCodingAgentPlugin[]): PiCodingAgentPlugin { @@ -661,26 +1095,7 @@ function anthropicModel() { }; } -function bashTool() { - return { - description: "Run a shell command.", - name: "bash", - execute: vi.fn(async (..._args: unknown[]) => { - expect(isAutoInstrumentationSuppressed()).toBe(false); - await Promise.resolve(); - expect(isAutoInstrumentationSuppressed()).toBe(false); - return { stdout: "pi_tool_ok" }; - }), - parameters: { - type: "object", - properties: { - command: { type: "string" }, - }, - }, - }; -} - -function makeAssistantMessage(text: string) { +function makeAssistantMessage(text: string): PiAssistantMessage { return { api: "anthropic-messages", content: [{ type: "text", text }], @@ -698,7 +1113,7 @@ function makeAssistantMessage(text: string) { }; } -function makeStream(message: ReturnType) { +function makeStream(message: PiAssistantMessage) { return { async *[Symbol.asyncIterator]() { yield { partial: message, type: "start" }; @@ -756,6 +1171,7 @@ function makeAgent( function makeSession(agent: ReturnType) { return { agent, + subscribe: agent.subscribe.bind(agent), getActiveToolNames: () => ["bash"], model: anthropicModel(), prompt: vi.fn(), diff --git a/js/src/instrumentation/plugins/pi-coding-agent-plugin.ts b/js/src/instrumentation/plugins/pi-coding-agent-plugin.ts index 9ecfe50b5..7e3e2ff1c 100644 --- a/js/src/instrumentation/plugins/pi-coding-agent-plugin.ts +++ b/js/src/instrumentation/plugins/pi-coding-agent-plugin.ts @@ -30,27 +30,29 @@ import type { PiToolCall, PiToolResultMessage, } from "../../vendor-sdk-types/pi-coding-agent"; -import { processInputAttachments } from "../../wrappers/attachment-utils"; +import { + isAutoCaptureAttachmentsEnabled, + processInputAttachments, +} from "../../wrappers/attachment-utils"; import { runWithAutoInstrumentationAllowed, runWithAutoInstrumentationSuppressed, } from "../auto-instrumentation-suppression"; import type { ChannelMessage } from "../core/tracing-types"; +import { observeResult, runInstrumentation } from "../core/observe-result"; import { piCodingAgentChannels } from "./pi-coding-agent-interceptors"; type PiPromptState = { activeLlmSpans: Set; activeToolSpans: Map; agent: PiAgent; - deferCompletionUntilTurnEnd: boolean; finalized: boolean; onFinalize?: (state: PiPromptState) => void; metadata: Record; output?: unknown; - promptCallEnded: boolean; + error?: string; span: Span; startTime: number; - turnEnded: boolean; }; type PiLlmSpanState = { @@ -63,6 +65,7 @@ type PiLlmSpanState = { type PiToolSpanState = { span: Span; + executed: boolean; }; type PiAgentPatchState = { @@ -72,7 +75,7 @@ type PiAgentPatchState = { }; const piAgentPatchStates = new WeakMap(); -const piAgentEventSubscriptions = new WeakSet(); +const piEventSubscriptions = new WeakSet(); const PI_TOOL_EXECUTE_WRAPPED = Symbol.for( "braintrust.pi_coding_agent.tool_execute_wrapped", ); @@ -83,6 +86,14 @@ export class PiCodingAgentPlugin extends BasePlugin { protected onEnable(): void { this.interceptPrompt(); + this.unsubscribers.push( + piCodingAgentChannels.toolExecute.intercept((target, receiver, args) => + tracePiToolExecution( + () => Reflect.apply(target, receiver, args), + args[0], + ), + ), + ); } protected onDisable(): void { @@ -118,7 +129,9 @@ export class PiCodingAgentPlugin extends BasePlugin { } if (!state) { - return runWithAutoInstrumentationSuppressed(invokeTarget); + return promptContextStore().run(undefined, () => + runWithAutoInstrumentationSuppressed(invokeTarget), + ); } this.activePromptStates.add(state); @@ -132,14 +145,11 @@ export class PiCodingAgentPlugin extends BasePlugin { throw error; } - return Promise.resolve(result).then( - (value) => { - finishPiPromptCall(state); - return value; - }, + return observeResult( + result, + () => finishPiPromptRun(state), (error) => { finishPiPromptRun(state, error); - throw error; }, ); }), @@ -150,14 +160,6 @@ export class PiCodingAgentPlugin extends BasePlugin { } } -function finishPiPromptCall(state: PiPromptState): void { - state.promptCallEnded = true; - if (state.deferCompletionUntilTurnEnd && !state.turnEnded) { - return; - } - finishPiPromptRun(state); -} - function startPiPromptRun( event: ChannelMessage, onFinalize?: (state: PiPromptState) => void, @@ -167,14 +169,21 @@ function startPiPromptRun( if (!session || !isPiAgent(agent)) { return undefined; } - installPiAgentInstrumentation(agent); + // A prompt submitted during streaming only enqueues work. Its result arrives + // through a separate lifecycle, which is excluded from automatic tracing. + if (event.arguments[1]?.streamingBehavior && session.isStreaming !== false) { + return undefined; + } + installPiAgentInstrumentation(agent, session); wrapPiToolExecutors(agent.state?.tools); const metadata = { ...extractSessionMetadata(session), ...extractPromptOptionsMetadata(event.arguments[1]), "pi_coding_agent.operation": "AgentSession.prompt", - provider: session.model?.provider ?? agent.state?.model?.provider ?? "pi", + ...(session.model?.provider || agent.state?.model?.provider + ? { provider: session.model?.provider ?? agent.state?.model?.provider } + : {}), ...(session.model?.id || agent.state?.model?.id ? { model: session.model?.id ?? agent.state?.model?.id } : {}), @@ -195,22 +204,15 @@ function startPiPromptRun( INSTRUMENTATION_NAMES.PI_CODING_AGENT, ), ); - const options = event.arguments[1]; - const state: PiPromptState = { activeLlmSpans: new Set(), activeToolSpans: new Map(), agent, - deferCompletionUntilTurnEnd: - options?.streamingBehavior === "followUp" || - options?.streamingBehavior === "steer", finalized: false, metadata, onFinalize, - promptCallEnded: false, span, startTime: getCurrentUnixTimestamp(), - turnEnded: false, }; return state; @@ -220,7 +222,9 @@ function extractSession( event: ChannelMessage, ): PiAgentSession | undefined { const candidate = event.session ?? event.self; - return isObject(candidate) && typeof candidate.prompt === "function" + return isObject(candidate) && + typeof candidate.prompt === "function" && + typeof candidate.subscribe === "function" ? (candidate as PiAgentSession) : undefined; } @@ -229,8 +233,7 @@ function isPiAgent(value: unknown): value is PiAgent { return ( isObject(value) && (typeof value.streamFunction === "function" || - typeof value.streamFn === "function") && - typeof value.subscribe === "function" + typeof value.streamFn === "function") ); } @@ -243,7 +246,12 @@ function currentPiPromptState(): PiPromptState | undefined { return promptContextStore().getStore(); } -function installPiAgentInstrumentation(agent: PiAgent): void { +function installPiAgentInstrumentation( + agent: PiAgent, + session: PiAgentSession, +): void { + // Pi's replaceable provider callback may come from application code outside + // transformed modules. Patch its live value to cover those custom providers. const property = typeof agent.streamFunction === "function" ? "streamFunction" : "streamFn"; const streamFunction = agent[property]; @@ -268,24 +276,26 @@ function installPiAgentInstrumentation(agent: PiAgent): void { piAgentPatchStates.set(agent, patchState); } - if (piAgentEventSubscriptions.has(agent)) { + // Session events include calls made through ctx.executeTool(), which the + // underlying agent does not emit. + if (piEventSubscriptions.has(session)) { return; } try { - agent.subscribe(async (event) => { + session.subscribe((event) => { const state = currentPiPromptState(); if (!state || state.agent !== agent || state.finalized) { return; } try { - await runWithAutoInstrumentationSuppressed(() => + runWithAutoInstrumentationSuppressed(() => handlePiAgentEvent(state, event), ); } catch (error) { logInstrumentationError("Pi Coding Agent event", error); } }); - piAgentEventSubscriptions.add(agent); + piEventSubscriptions.add(session); } catch (error) { logInstrumentationError("Pi Coding Agent event subscription", error); } @@ -296,7 +306,7 @@ function makeInstrumentedStreamFunction( originalStreamFunction: PiStreamFunction, property: PiAgentPatchState["property"], ): PiStreamFunction { - return async function instrumentedPiStreamFunction( + return function instrumentedPiStreamFunction( this: unknown, model: PiModel, context: PiContext, @@ -309,25 +319,31 @@ function makeInstrumentedStreamFunction( return invokeOriginal(); } - wrapPiToolExecutors(agent.state?.tools); - wrapPiToolExecutors(context.tools); - const llmState = await startPiLlmSpan( - state, - model, - context, - property, - options, + runInstrumentation(() => { + wrapPiToolExecutors(agent.state?.tools); + wrapPiToolExecutors(context.tools); + }); + const llmState = runInstrumentation(() => + startPiLlmSpan(state, model, context, property, options), ); try { - const stream = await runWithAutoInstrumentationSuppressed(invokeOriginal); - return patchAssistantMessageStream(stream, state, llmState); + const result = runWithAutoInstrumentationSuppressed(invokeOriginal); + if (!llmState) return result; + return observeResult( + result, + (stream) => { + patchAssistantMessageStream(stream, state, llmState); + }, + (error) => finishPiLlmSpan(state, llmState, undefined, error), + ); } catch (error) { - finishPiLlmSpan(state, llmState, undefined, error); + if (llmState) finishPiLlmSpan(state, llmState, undefined, error); throw error; } }; } +// Tools are runtime-provided callbacks that the module transformer cannot target. function wrapPiToolExecutors(tools: PiTool[] | undefined): void { if (!tools) { return; @@ -346,20 +362,10 @@ function wrapPiToolExecutors(tools: PiTool[] | undefined): void { } const wrappedExecute = function (this: unknown, ...args: unknown[]) { - const invokeOriginal = () => - runWithAutoInstrumentationAllowed(() => - Reflect.apply(execute, this, args), - ); - const state = currentPiPromptState(); - const toolCallId = args[0]; - const toolState = - !state?.finalized && typeof toolCallId === "string" - ? state?.activeToolSpans.get(toolCallId) - : undefined; - - return toolState - ? withCurrent(toolState.span, invokeOriginal) - : invokeOriginal(); + // Prompts may still be running after their plugin is disabled. + return runWithAutoInstrumentationAllowed(() => + piCodingAgentChannels.toolExecute.invoke(execute, this, args, {}), + ); }; Object.defineProperty(wrappedExecute, PI_TOOL_EXECUTE_WRAPPED, { configurable: false, @@ -374,31 +380,49 @@ function wrapPiToolExecutors(tools: PiTool[] | undefined): void { } } -async function startPiLlmSpan( +function tracePiToolExecution(call: () => T, toolCallId: unknown): T { + return runWithAutoInstrumentationAllowed(() => { + const state = currentPiPromptState(); + const toolState = + !state?.finalized && typeof toolCallId === "string" + ? state?.activeToolSpans.get(toolCallId) + : undefined; + if (!toolState) return call(); + toolState.executed = true; + return withCurrent(toolState.span, call); + }); +} + +function startPiLlmSpan( state: PiPromptState, model: PiModel, context: PiContext, property: PiAgentPatchState["property"], options?: PiSimpleStreamOptions, -): Promise { +): PiLlmSpanState { const metadata = { ...extractModelMetadata(model), ...extractStreamOptionsMetadata(options), + ...extractSamplingMetadata(model, options), ...extractToolMetadata(resolvePiContextTools(context)), "pi_coding_agent.operation": `agent.${property}`, }; - const span = startBaseSpan( - withSpanInstrumentationName( - { - event: { - input: processInputAttachments(normalizePiContextInput(context)), - metadata, + const span = withCurrent(state.span, () => + startBaseSpan( + withSpanInstrumentationName( + { + event: { + input: processInputAttachments( + normalizePiContextInput(context), + isAutoCaptureAttachmentsEnabled(), + ), + metadata, + }, + name: getLlmSpanName(model), + spanAttributes: { type: SpanTypeAttribute.LLM }, }, - name: getLlmSpanName(model), - parent: await state.span.export(), - spanAttributes: { type: SpanTypeAttribute.LLM }, - }, - INSTRUMENTATION_NAMES.PI_CODING_AGENT, + INSTRUMENTATION_NAMES.PI_CODING_AGENT, + ), ), ); const llmState = { @@ -428,14 +452,13 @@ function patchAssistantMessageStream( streamRecord.result = function patchedPiResult( this: PiAssistantMessageEventStream, ) { - return Promise.resolve(Reflect.apply(originalResult, this, [])).then( + return observeResult( + Reflect.apply(originalResult, this, []), (message) => { finishPiLlmSpan(promptState, llmState, message); - return message; }, (error) => { finishPiLlmSpan(promptState, llmState, undefined, error); - throw error; }, ); }; @@ -462,7 +485,13 @@ function patchAssistantMessageStream( return result; } - recordPiAssistantMessageEvent(promptState, llmState, result.value); + runInstrumentation(() => + recordPiAssistantMessageEvent( + promptState, + llmState, + result.value, + ), + ); return result; } catch (error) { finishPiLlmSpan(promptState, llmState, undefined, error); @@ -539,7 +568,12 @@ function recordFirstTokenMetric( ): void { if ( state.metrics.time_to_first_token !== undefined || - event.type === "start" + !["text_delta", "thinking_delta", "toolcall_delta"].includes( + event.type ?? "", + ) || + !("delta" in event) || + typeof event.delta !== "string" || + event.delta.length === 0 ) { return; } @@ -548,10 +582,7 @@ function recordFirstTokenMetric( getCurrentUnixTimestamp() - state.startTime; } -async function handlePiAgentEvent( - state: PiPromptState, - event: PiAgentEvent, -): Promise { +function handlePiAgentEvent(state: PiPromptState, event: PiAgentEvent): void { if (state.finalized) { return; } @@ -559,20 +590,16 @@ async function handlePiAgentEvent( switch (event.type) { case "message_end": if (isPiAssistantMessage(event.message)) { - state.output = extractAssistantOutput(event.message); + recordPiPromptOutput(state, event.message); } return; case "turn_end": - state.turnEnded = true; if (isPiAssistantMessage(event.message)) { - state.output = extractAssistantOutput(event.message); - } - if (state.promptCallEnded && state.deferCompletionUntilTurnEnd) { - finishPiPromptRun(state); + recordPiPromptOutput(state, event.message); } return; case "tool_execution_start": - await startPiToolSpan(state, event); + startPiToolSpan(state, event); return; case "tool_execution_end": finishPiToolSpan(state, event); @@ -582,10 +609,25 @@ async function handlePiAgentEvent( } } -async function startPiToolSpan( +function piMessageError(message: PiAssistantMessage): string | undefined { + if (message.stopReason === "error" || message.stopReason === "aborted") { + return message.errorMessage || `Pi response ${message.stopReason}`; + } + return undefined; +} + +function recordPiPromptOutput( + state: PiPromptState, + message: PiAssistantMessage, +): void { + state.error = piMessageError(message); + state.output = state.error ? undefined : extractAssistantOutput(message); +} + +function startPiToolSpan( state: PiPromptState, event: Extract, -): Promise { +): void { if (!event.toolCallId || state.activeToolSpans.has(event.toolCallId)) { return; } @@ -595,21 +637,28 @@ async function startPiToolSpan( "gen_ai.tool.name": event.toolName, "pi_coding_agent.tool.name": event.toolName, }; - const span = startBaseSpan( - withSpanInstrumentationName( - { - event: { - input: event.args, - metadata, + const parent = + (event.parentToolCallId && + state.activeToolSpans.get(event.parentToolCallId)?.span) || + state.span; + // Session subscribers run synchronously; install the span before Pi invokes + // the executor rather than yielding while exporting its parent context. + const span = withCurrent(parent, () => + startBaseSpan( + withSpanInstrumentationName( + { + event: { + input: event.args, + metadata, + }, + name: event.toolName || "tool", + spanAttributes: { type: SpanTypeAttribute.TOOL }, }, - name: event.toolName || "tool", - parent: await state.span.export(), - spanAttributes: { type: SpanTypeAttribute.TOOL }, - }, - INSTRUMENTATION_NAMES.PI_CODING_AGENT, + INSTRUMENTATION_NAMES.PI_CODING_AGENT, + ), ), ); - state.activeToolSpans.set(event.toolCallId, { span }); + state.activeToolSpans.set(event.toolCallId, { span, executed: false }); } function finishPiToolSpan( @@ -627,12 +676,23 @@ function finishPiToolSpan( "gen_ai.tool.name": event.toolName, "pi_coding_agent.tool.name": event.toolName, "pi_coding_agent.tool.is_error": event.isError, + ...(toolState.executed ? { tool_approval: "approved" } : {}), }; try { - safeLog(toolState.span, { - ...(event.isError ? { error: toLoggedError(event.result) } : {}), + const output = runInstrumentation(() => + normalizePiToolOutput(event.result), + ); + toolState.span.log({ + ...(event.isError + ? { + error: + event.result instanceof Error + ? event.result + : toLoggedError(output), + } + : {}), metadata, - output: event.result, + output, }); } finally { toolState.span.end(); @@ -650,22 +710,20 @@ function finishPiPromptRun(state: PiPromptState, error?: unknown): void { const metadata = { ...state.metadata, - ...extractModelMetadata(state.agent.state?.model), + ...runInstrumentation(() => extractModelMetadata(state.agent.state?.model)), }; try { - safeLog(state.span, { - ...(error ? { error: toLoggedError(error) } : {}), + state.span.log({ + ...(error !== undefined || state.error + ? { error: error ?? state.error } + : {}), metadata, // Usage belongs to LLM spans; repeating it here doubles trace rollups and cost. - metrics: buildDurationMetrics(state.startTime), + metrics: buildTimingMetrics(state.startTime), output: state.output, }); } finally { - try { - state.span.end(); - } catch (endError) { - logInstrumentationError("Pi Coding Agent prompt end", endError); - } + state.span.end(); } } @@ -687,34 +745,32 @@ function finishPiLlmSpan( llmState.finalized = true; promptState.activeLlmSpans.delete(llmState); - const messageError = message?.stopReason === "error" && message.errorMessage; - const metrics = { - ...extractUsageMetrics(message?.usage), - ...cleanMetrics(llmState.metrics), - ...buildDurationMetrics(llmState.startTime), - }; - - try { - safeLog(llmState.span, { - ...(error || messageError - ? { error: toLoggedError(error ?? messageError) } - : {}), + const data = runInstrumentation(() => { + const messageError = message && piMessageError(message); + return { + ...(error || messageError ? { error: error ?? messageError } : {}), metadata: { ...llmState.metadata, - ...(message ? extractAssistantMetadata(message) : {}), + ...(message + ? extractAssistantMetadata(message, llmState.metadata) + : {}), + }, + metrics: { + ...extractUsageMetrics(message?.usage), + ...cleanMetrics(llmState.metrics), + ...buildTimingMetrics(llmState.startTime), }, - metrics, ...(message ? { output: extractAssistantOutput(message) } : {}), - }); - } finally { - llmState.span.end(); - } + }; + }); + if (data) llmState.span.log(data); + llmState.span.end(); } function finishOpenToolSpans(state: PiPromptState, error?: unknown): void { for (const [, toolState] of state.activeToolSpans) { - safeLog(toolState.span, { - error: error ? toLoggedError(error) : "Pi tool did not complete", + toolState.span.log({ + error: error ?? "Pi tool did not complete", }); toolState.span.end(); } @@ -819,7 +875,7 @@ function normalizeAssistantMessage(message: PiAssistantMessage): unknown { return { role: "assistant", content: text || (toolCalls.length > 0 ? null : ""), - ...(thinking ? { reasoning: thinking } : {}), + ...(thinking ? { reasoning_content: thinking } : {}), ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}), }; } @@ -847,7 +903,9 @@ function normalizeUserContent( return { type: "image_url", image_url: { - url: `data:${part.mimeType};base64,${part.data}`, + url: isAutoCaptureAttachmentsEnabled() + ? `data:${part.mimeType};base64,${part.data}` + : "", }, }; } @@ -855,6 +913,19 @@ function normalizeUserContent( }); } +function normalizePiToolOutput(result: unknown): unknown { + const output = + isObject(result) && Array.isArray(result.content) + ? { + ...result, + content: normalizeUserContent( + result.content as Array, + ), + } + : result; + return processInputAttachments(output, isAutoCaptureAttachmentsEnabled()); +} + function normalizeToolCall(toolCall: PiToolCall): unknown { return { id: toolCall.id, @@ -867,13 +938,16 @@ function normalizeToolCall(toolCall: PiToolCall): unknown { } function extractAssistantOutput(message: PiAssistantMessage): unknown { - return processInputAttachments([ - { - finish_reason: normalizeStopReason(message.stopReason), - index: 0, - message: normalizeAssistantMessage(message), - }, - ]); + return processInputAttachments( + [ + { + finish_reason: normalizeStopReason(message.stopReason), + index: 0, + message: normalizeAssistantMessage(message), + }, + ], + isAutoCaptureAttachmentsEnabled(), + ); } function isPiSystemMessage(message: PiMessage): message is PiSystemMessage { @@ -926,20 +1000,18 @@ function extractPromptInput( if (!images || images.length === 0) { return text; } - return processInputAttachments([ - { - role: "user", - content: [ - { type: "text", text: text ?? "" }, - ...images.map((image) => ({ - type: "image_url", - image_url: { - url: `data:${image.mimeType};base64,${image.data}`, - }, - })), - ], - }, - ]); + return processInputAttachments( + [ + { + role: "user", + content: normalizeUserContent([ + { type: "text", text: text ?? "" }, + ...images, + ]), + }, + ], + isAutoCaptureAttachmentsEnabled(), + ); } function extractToolMetadata( @@ -978,7 +1050,15 @@ function extractModelMetadata( function extractAssistantMetadata( message: PiAssistantMessage, + requestMetadata: Record, ): Record { + const model = message.responseModel ?? message.model; + const provider = message.provider ?? requestMetadata.provider; + const selectionChanged = + model && + requestMetadata.model && + requestMetadata.provider && + (model !== requestMetadata.model || provider !== requestMetadata.provider); return { ...(message.provider ? { provider: message.provider } : {}), ...(message.responseModel || message.model @@ -992,6 +1072,20 @@ function extractAssistantMetadata( ...(message.responseId ? { "pi_coding_agent.response_id": message.responseId } : {}), + ...(selectionChanged + ? { + selected_model: `${requestMetadata.provider}/${requestMetadata.model}`, + } + : {}), + ...(message.thinkingLevel + ? { "pi_coding_agent.thinking_level": message.thinkingLevel } + : {}), + ...(message.providerThinkingLevel + ? { + "pi_coding_agent.provider_thinking_level": + message.providerThinkingLevel, + } + : {}), ...(message.stopReason ? { "pi_coding_agent.stop_reason": message.stopReason } : {}), @@ -1053,6 +1147,9 @@ function extractStreamOptionsMetadata( ...(options.maxTokens !== undefined ? { max_tokens: options.maxTokens } : {}), + ...(options.toolChoice === "auto" || options.toolChoice === "none" + ? { tool_choice: options.toolChoice } + : {}), ...(options.reasoning ? { "pi_coding_agent.reasoning": options.reasoning } : {}), @@ -1074,12 +1171,50 @@ function extractStreamOptionsMetadata( ...(options.maxRetryDelayMs !== undefined ? { "pi_coding_agent.max_retry_delay_ms": options.maxRetryDelayMs } : {}), - ...(options.metadata - ? { "pi_coding_agent.metadata": options.metadata } - : {}), }; } +function extractSamplingMetadata( + model: PiModel, + options?: PiSimpleStreamOptions, +): Record { + // Pi merges samplingParams over named fields only for OpenAI-compatible APIs. + if ( + ![ + "openai-completions", + "openai-responses", + "azure-openai-responses", + ].includes(model.api ?? "") + ) { + return {}; + } + const metadata: Record = {}; + for (const field of [ + "temperature", + "top_p", + "max_tokens", + "frequency_penalty", + "presence_penalty", + "stop", + "response_format", + "tool_choice", + "parallel_tool_calls", + "max_tool_calls", + ] as const) { + const request = options?.samplingParams; + const value = + request && Object.prototype.hasOwnProperty.call(request, field) + ? request[field] + : model.samplingParams?.[field]; + // Request metadata must be JSON data, never executable or cyclic values. + // Snapshot it independently so one invalid field does not lose the call. + if (typeof value === "number" && !Number.isFinite(value)) continue; + const serialized = runInstrumentation(() => JSON.stringify(value)); + if (serialized !== undefined) metadata[field] = JSON.parse(serialized); + } + return metadata; +} + function getLlmSpanName(model: PiModel): string { switch (model.api) { case "anthropic-messages": @@ -1109,23 +1244,64 @@ function extractUsageMetrics( return {}; } - return cleanMetrics({ - completion_tokens: usage.output, - prompt_cache_creation_tokens: usage.cacheWrite, - prompt_cached_tokens: usage.cacheRead, - // Pi reports cache reads and writes outside `input`; Braintrust's `prompt_tokens` includes them. - prompt_tokens: - typeof usage.input === "number" - ? usage.input + (usage.cacheRead ?? 0) + (usage.cacheWrite ?? 0) - : undefined, - tokens: usage.totalTokens ?? usage.tokens, - }); + const metrics = cleanMetrics( + { + completion_tokens: usage.output, + completion_reasoning_tokens: usage.reasoning, + prompt_cache_creation_tokens: usage.cacheWrite, + prompt_cache_creation_1h_tokens: usage.cacheWrite1h, + prompt_cached_tokens: usage.cacheRead, + prompt_tokens: usage.input, + tokens: usage.totalTokens ?? usage.tokens, + }, + Number.isSafeInteger, + ); + // Pi reports cache reads and writes outside `input`. Detail fields remain + // subsets of these inclusive totals and must not be added a second time. + if (metrics.prompt_tokens !== undefined) { + if ( + (usage.cacheRead !== undefined && + metrics.prompt_cached_tokens === undefined) || + (usage.cacheWrite !== undefined && + metrics.prompt_cache_creation_tokens === undefined) + ) { + delete metrics.prompt_tokens; + } else { + metrics.prompt_tokens += + (metrics.prompt_cached_tokens ?? 0) + + (metrics.prompt_cache_creation_tokens ?? 0); + } + } + if ( + metrics.tokens === undefined && + metrics.prompt_tokens !== undefined && + metrics.completion_tokens !== undefined + ) { + metrics.tokens = metrics.prompt_tokens + metrics.completion_tokens; + } + if ( + metrics.prompt_cache_creation_tokens !== undefined && + metrics.prompt_cache_creation_1h_tokens !== undefined && + metrics.prompt_cache_creation_tokens >= + metrics.prompt_cache_creation_1h_tokens + ) { + metrics.prompt_cache_creation_5m_tokens = + metrics.prompt_cache_creation_tokens - + metrics.prompt_cache_creation_1h_tokens; + } + // Pi uses zero for unpriced models. Leave those to backend registry pricing. + const cost = usage.cost?.total; + return { + ...cleanMetrics(metrics, Number.isSafeInteger), + ...(typeof cost === "number" && Number.isFinite(cost) && cost > 0 + ? { estimated_cost: cost } + : {}), + }; } -function buildDurationMetrics(startTime: number): Record { +function buildTimingMetrics(startTime: number): Record { const end = getCurrentUnixTimestamp(); return { - duration: end - startTime, end, start: startTime, }; @@ -1133,10 +1309,11 @@ function buildDurationMetrics(startTime: number): Record { function cleanMetrics( metrics: Record, + isValidNumber: (value: number) => boolean = Number.isFinite, ): Record { const cleaned: Record = {}; for (const [key, value] of Object.entries(metrics)) { - if (typeof value === "number" && Number.isFinite(value) && value >= 0) { + if (typeof value === "number" && value >= 0 && isValidNumber(value)) { cleaned[key] = value; } } @@ -1168,14 +1345,6 @@ function stringifyUnknown(value: unknown): string { } } -function safeLog(span: Span, event: Parameters[0]): void { - try { - span.log(event); - } catch (error) { - logInstrumentationError("Pi Coding Agent span log", error); - } -} - function logInstrumentationError(context: string, error: unknown): void { debugLogger.debug(`${context}:`, error); } diff --git a/js/src/vendor-sdk-types/pi-coding-agent.ts b/js/src/vendor-sdk-types/pi-coding-agent.ts index 9adbed91c..0e8193d28 100644 --- a/js/src/vendor-sdk-types/pi-coding-agent.ts +++ b/js/src/vendor-sdk-types/pi-coding-agent.ts @@ -23,7 +23,9 @@ export interface PiAgentSession { readonly sessionId?: string; readonly sessionName?: string; readonly thinkingLevel?: string; + readonly isStreaming?: boolean; prompt(text: string, options?: PiPromptOptions): Promise; + subscribe(listener: (event: PiAgentEvent) => void): () => void; getActiveToolNames?: () => string[]; dispose?: () => void; [key: string | symbol]: unknown; @@ -42,7 +44,6 @@ export interface PiAgent { /** @deprecated Renamed to streamFunction in pi-agent-core 0.81.0. */ streamFn?: PiStreamFunction; streamFunction?: PiStreamFunction; - subscribe(listener: PiAgentEventListener): () => void; readonly state?: { model?: PiModel; systemPrompt?: string; @@ -53,11 +54,6 @@ export interface PiAgent { [key: string | symbol]: unknown; } -export type PiAgentEventListener = ( - event: PiAgentEvent, - signal: AbortSignal, -) => Promise | void; - export type PiAgentEvent = | { type: "agent_start" } | { type: "agent_end"; messages: PiMessage[] } @@ -80,6 +76,7 @@ export type PiAgentEvent = toolCallId: string; toolName: string; args: unknown; + parentToolCallId?: string; } | { type: "tool_execution_update"; @@ -94,6 +91,7 @@ export type PiAgentEvent = toolName: string; result: unknown; isError: boolean; + parentToolCallId?: string; }; export type PiStreamFunction = ( @@ -105,6 +103,8 @@ export type PiStreamFunction = ( export interface PiSimpleStreamOptions { temperature?: number; maxTokens?: number; + toolChoice?: "auto" | "none"; + samplingParams?: Record; signal?: AbortSignal; apiKey?: string; headers?: Record; @@ -129,6 +129,7 @@ export interface PiModel { input?: string[]; contextWindow?: number; maxTokens?: number; + samplingParams?: Record; [key: string]: unknown; } @@ -203,8 +204,11 @@ export interface PiToolCall { export interface PiUsage { input?: number; output?: number; + reasoning?: number; cacheRead?: number; cacheWrite?: number; + cacheWrite1h?: number; + cost?: { total?: number }; totalTokens?: number; tokens?: number; [key: string]: unknown; @@ -218,6 +222,8 @@ export interface PiAssistantMessage { model?: string; responseModel?: string; responseId?: string; + thinkingLevel?: string; + providerThinkingLevel?: string; usage?: PiUsage; stopReason?: string; errorMessage?: string; diff --git a/js/src/wrappers/pi-coding-agent.test.ts b/js/src/wrappers/pi-coding-agent.test.ts index 678b6481c..391bfb00c 100644 --- a/js/src/wrappers/pi-coding-agent.test.ts +++ b/js/src/wrappers/pi-coding-agent.test.ts @@ -1,4 +1,3 @@ -import { debugLogger } from "../debug-logger"; import { afterEach, describe, expect, it, vi } from "vitest"; const { invoke } = vi.hoisted(() => ({ @@ -23,21 +22,7 @@ describe("wrapPiCodingAgentSDK", () => { vi.clearAllMocks(); }); - it("returns invalid modules unchanged", () => { - const warnSpy = vi - .spyOn(debugLogger.notice, "warn") - .mockImplementation(() => true); - const sdk = { Session: class {} }; - - expect(wrapPiCodingAgentSDK(sdk)).toBe(sdk); - expect(warnSpy).toHaveBeenCalledWith( - "Unsupported Pi Coding Agent SDK. Not wrapping.", - ); - - warnSpy.mockRestore(); - }); - - it("wraps AgentSession.prompt", async () => { + it("wraps AgentSession.prompt once while preserving receiver, arguments, and result", async () => { class AgentSession { async prompt(text: string) { return text; @@ -46,6 +31,7 @@ describe("wrapPiCodingAgentSDK", () => { const sdk = { AgentSession }; wrapPiCodingAgentSDK(sdk); + wrapPiCodingAgentSDK(sdk); const session = new sdk.AgentSession(); await expect(session.prompt("hello")).resolves.toBe("hello"); @@ -59,21 +45,6 @@ describe("wrapPiCodingAgentSDK", () => { ]); }); - it("patches AgentSession.prompt only once", async () => { - class AgentSession { - async prompt(text: string) { - return text; - } - } - - const sdk = { AgentSession }; - wrapPiCodingAgentSDK(sdk); - wrapPiCodingAgentSDK(sdk); - - await new sdk.AgentSession().prompt("hello"); - expect(invoke).toHaveBeenCalledTimes(1); - }); - it("preserves prompt rejections", async () => { class AgentSession { async prompt() { From 4a930f12988092234f00d2eba2fd68dc0fef7b8e Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 27 Oct 2026 11:19:06 -0400 Subject: [PATCH 2/2] chore(pi): add instrumentation changeset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > šŸ¤– **This text was AI-generated.** Add a patch release note describing the Pi instrumentation fixes. --- .changeset/fix-pi-instrumentation.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .changeset/fix-pi-instrumentation.md diff --git a/.changeset/fix-pi-instrumentation.md b/.changeset/fix-pi-instrumentation.md new file mode 100644 index 000000000..fb8e00631 --- /dev/null +++ b/.changeset/fix-pi-instrumentation.md @@ -0,0 +1,10 @@ +--- +"braintrust": patch +--- + +fix(pi): align agent instrumentation with tracing spec + +Capture nested session tool calls and their application spans, record tool +approval only when execution is observed, and preserve tracing context across +SDK copies. Align Pi errors, attachments, usage metrics, and metadata with the +instrumentation specification.