diff --git a/package-lock.json b/package-lock.json index 16a3c3bca..173c49fc9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -72,6 +72,7 @@ "next-themes": "^0.4.6", "papaparse": "^5.5.3", "pngjs": "^7.0.0", + "prefix-safe-json": "0.2.0", "qrcode": "^1.5.4", "radix-ui": "^1.4.3", "react": "19.2.3", @@ -3123,34 +3124,6 @@ "balanced-match": "^1.0.0" } }, - "node_modules/@electron/universal/node_modules/fs-extra": { - "version": "11.3.4", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.4.tgz", - "integrity": "sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^6.0.1", - "universalify": "^2.0.0" - }, - "engines": { - "node": ">=14.14" - } - }, - "node_modules/@electron/universal/node_modules/jsonfile": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", - "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", - "dev": true, - "license": "MIT", - "dependencies": { - "universalify": "^2.0.0" - }, - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, "node_modules/@electron/universal/node_modules/minimatch": { "version": "9.0.9", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", @@ -3167,14 +3140,26 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/@electron/universal/node_modules/universalify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", - "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "node_modules/@electron/windows-sign": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@electron/windows-sign/-/windows-sign-1.2.2.tgz", + "integrity": "sha512-dfZeox66AvdPtb2lD8OsIIQh12Tp0GNCRUDfBHIKGpbmopZto2/A8nSpYYLoedPIHpqkeblZ/k8OV0Gy7PYuyQ==", "dev": true, - "license": "MIT", + "license": "BSD-2-Clause", + "optional": true, + "peer": true, + "dependencies": { + "cross-dirname": "^0.1.0", + "debug": "^4.3.4", + "fs-extra": "^11.1.1", + "minimist": "^1.2.8", + "postject": "^1.0.0-alpha.6" + }, + "bin": { + "electron-windows-sign": "bin/electron-windows-sign.js" + }, "engines": { - "node": ">= 10.0.0" + "node": ">=14.14" } }, "node_modules/@emnapi/core": { @@ -13144,6 +13129,15 @@ "integrity": "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g==", "license": "MIT" }, + "node_modules/cross-dirname": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/cross-dirname/-/cross-dirname-0.1.0.tgz", + "integrity": "sha512-+R08/oI0nl3vfPcqftZRpytksBXDzOUveBq/NBVx0sUp1axwzPQrKinNx5yd5sxPu8j1wIy8AfnVQ+5eFdha6Q==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -16311,6 +16305,44 @@ "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", "license": "MIT" }, + "node_modules/fs-extra": { + "version": "11.4.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.4.0.tgz", + "integrity": "sha512-EQsFzMUJkCKGr1ePqlYADkIUmHW1s3ZXr5Yqy6wbGrfUCphpl2maM/kyOIRA2HpP3AaFQTZXD4ldjek+nccddA==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/fs-extra/node_modules/jsonfile": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz", + "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/fs-extra/node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, "node_modules/fs-minipass": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-3.0.3.tgz", @@ -16335,7 +16367,6 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -22796,6 +22827,36 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/postject": { + "version": "1.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/postject/-/postject-1.0.0-alpha.6.tgz", + "integrity": "sha512-b9Eb8h2eVqNE8edvKdwqkrY6O7kAwmI8kcnBv1NScolYJbo59XUF0noFq+lxbC1yN20bmC0WBEbDC5H/7ASb0A==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "commander": "^9.4.0" + }, + "bin": { + "postject": "dist/cli.js" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/postject/node_modules/commander": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", + "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "engines": { + "node": "^12.20.0 || >=14" + } + }, "node_modules/powershell-utils": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/powershell-utils/-/powershell-utils-0.1.0.tgz", @@ -22834,6 +22895,40 @@ "node": ">=10" } }, + "node_modules/prefix-safe-json": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/prefix-safe-json/-/prefix-safe-json-0.2.0.tgz", + "integrity": "sha512-J10P2w9U58lspbmP+Mz1TeZKCZfbyAk9gEZMlTOt1YyouchzRvi/8BaNbjyf0fG3uqe1YC/ss/wPApqWBfb8Cg==", + "license": "MIT OR Apache-2.0", + "dependencies": { + "ajv": "^8.20.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/prefix-safe-json/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/prefix-safe-json/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", diff --git a/package.json b/package.json index 6c014752e..f74848093 100644 --- a/package.json +++ b/package.json @@ -110,6 +110,7 @@ "next-themes": "^0.4.6", "papaparse": "^5.5.3", "pngjs": "^7.0.0", + "prefix-safe-json": "0.2.0", "qrcode": "^1.5.4", "radix-ui": "^1.4.3", "react": "19.2.3", diff --git a/src/__tests__/unit/execution-guard.test.ts b/src/__tests__/unit/execution-guard.test.ts new file mode 100644 index 000000000..0c7737cbe --- /dev/null +++ b/src/__tests__/unit/execution-guard.test.ts @@ -0,0 +1,457 @@ +/** + * execution-guard.ts regression coverage. + * + * Context: `codepilot_cli_tools_install`'s `execute()` used to call + * `execAsync(command, ...)` directly and unconditionally the moment the AI + * SDK finished parsing that one tool call's arguments — with zero awareness + * of whether the *step* containing it (which may include other tool calls + * or trailing text) ever reached a safe terminal state. A tool call can be + * syntactically complete and still belong to a generation that was cut off + * by a token limit, a provider error, or a content filter immediately + * after. Structural JSON validity was being treated as proof of intent, + * which it is not. + * + * `execution-guard.ts` fixes this by deferring the real side effect until + * `agent-loop.ts` — which already knows the step's `finishReason` once its + * `fullStream` is fully consumed — resolves it against a `prefix-safe-json` + * execution guard fed the same stream events. + * + * Deferred registrations are additionally isolated per `executionScopeId` + * (a fresh id per `runAgentLoop` invocation): `toolCallId` alone is + * provider-generated per call and not guaranteed unique across concurrent + * turns, so two overlapping turns must never be able to resolve or discard + * each other's pending commands even if their toolCallIds collide. + */ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import type { ModelMessage } from 'ai'; +import { + createStepGuard, + registerDeferredExecution, + resolvePendingExecutions, + discardPendingExecutions, + applyResolvedExecutionsToMessages, +} from '@/lib/execution-guard'; + +/** Minimal fullStream part shapes this guard actually reads (AI SDK v6/v7). */ +function toolInputStart(id: string, toolName: string) { + return { type: 'tool-input-start' as const, id, toolName }; +} +function toolInputDelta(id: string, delta: string) { + return { type: 'tool-input-delta' as const, id, delta }; +} +function toolInputEnd(id: string) { + return { type: 'tool-input-end' as const, id }; +} +function finish(finishReason: string) { + return { type: 'finish' as const, finishReason, totalUsage: {} }; +} + +const COMMAND_ARGS = JSON.stringify({ command: 'brew install ffmpeg' }); + +describe('execution-guard — deferred tool execution requires a positively confirmed safe step', () => { + it('SAFE: complete arguments + a safe finish reason ("tool-calls") — the real command runs exactly once', async () => { + const scopeId = 'scope-safe'; + const toolCallId = 'call_safe'; + let runCount = 0; + const fakeInstall = async () => { + runCount++; + return 'Successfully installed "ffmpeg".'; + }; + + const guard = await createStepGuard(); + const queuedMessage = registerDeferredExecution(scopeId, toolCallId, fakeInstall); + assert.match(queuedMessage, /queued/i, 'execute() must return an immediate, non-blocking placeholder'); + + guard.push(toolInputStart(toolCallId, 'codepilot_cli_tools_install')); + guard.push(toolInputDelta(toolCallId, COMMAND_ARGS)); + guard.push(toolInputEnd(toolCallId)); + guard.push(finish('tool-calls')); + + const resolved = await resolvePendingExecutions(scopeId, guard, { providerReason: 'tool-calls' }); + + assert.equal(runCount, 1, 'the real install must run exactly once for a safely-completed step'); + assert.equal(resolved.length, 1); + assert.equal(resolved[0].executed, true); + assert.equal(resolved[0].outcomeText, 'Successfully installed "ffmpeg".'); + }); + + it('UNSAFE: complete-looking arguments + finishReason "length" — the real command never runs', async () => { + const scopeId = 'scope-length'; + const toolCallId = 'call_length'; + let runCount = 0; + const fakeInstall = async () => { + runCount++; + return 'Successfully installed "ffmpeg".'; + }; + + const guard = await createStepGuard(); + registerDeferredExecution(scopeId, toolCallId, fakeInstall); + + // This specific tool call's own JSON is fully formed — the truncation + // happened to something else later in the same step. + guard.push(toolInputStart(toolCallId, 'codepilot_cli_tools_install')); + guard.push(toolInputDelta(toolCallId, COMMAND_ARGS)); + guard.push(toolInputEnd(toolCallId)); + guard.push(finish('length')); + + const resolved = await resolvePendingExecutions(scopeId, guard, { providerReason: 'length' }); + + assert.equal(runCount, 0, 'a step that ended with finishReason "length" must never execute'); + assert.equal(resolved.length, 1); + assert.equal(resolved[0].executed, false); + }); + + it('UNSAFE: the tool call\'s own arguments are truncated mid-stream — the real command never runs', async () => { + const scopeId = 'scope-truncated'; + const toolCallId = 'call_truncated'; + let runCount = 0; + const fakeInstall = async () => { + runCount++; + return 'Successfully installed "ffmpeg".'; + }; + + const guard = await createStepGuard(); + registerDeferredExecution(scopeId, toolCallId, fakeInstall); + + // Stream cuts off mid-argument — tool-input-end never arrives. + guard.push(toolInputStart(toolCallId, 'codepilot_cli_tools_install')); + guard.push(toolInputDelta(toolCallId, '{"command":"brew inst')); + guard.push(finish('length')); + + const resolved = await resolvePendingExecutions(scopeId, guard, { providerReason: 'length' }); + + assert.equal(runCount, 0, 'truncated arguments must never reach real execution'); + assert.equal(resolved.length, 1); + assert.equal(resolved[0].executed, false); + }); + + it('UNSAFE: provider error ends the step — the real command never runs', async () => { + const scopeId = 'scope-error'; + const toolCallId = 'call_error'; + let runCount = 0; + const fakeInstall = async () => { + runCount++; + return 'Successfully installed "ffmpeg".'; + }; + + const guard = await createStepGuard(); + registerDeferredExecution(scopeId, toolCallId, fakeInstall); + + guard.push(toolInputStart(toolCallId, 'codepilot_cli_tools_install')); + guard.push(toolInputDelta(toolCallId, COMMAND_ARGS)); + guard.push(toolInputEnd(toolCallId)); + guard.push({ type: 'error' as const, error: new Error('upstream 500') }); + + const resolved = await resolvePendingExecutions(scopeId, guard, { providerReason: 'error' }); + + assert.equal(runCount, 0, 'a step that errored must never execute a deferred command'); + assert.equal(resolved.length, 1); + assert.equal(resolved[0].executed, false); + }); + + it('UNSAFE: unrecognized/unknown terminal state — the real command never runs', async () => { + const scopeId = 'scope-unknown'; + const toolCallId = 'call_unknown'; + let runCount = 0; + const fakeInstall = async () => { + runCount++; + return 'Successfully installed "ffmpeg".'; + }; + + const guard = await createStepGuard(); + registerDeferredExecution(scopeId, toolCallId, fakeInstall); + + guard.push(toolInputStart(toolCallId, 'codepilot_cli_tools_install')); + guard.push(toolInputDelta(toolCallId, COMMAND_ARGS)); + guard.push(toolInputEnd(toolCallId)); + // No finish/error/abort part at all — the stream just stops. finish()'s + // own meta fallback is the only source of a terminal reason here, and it + // is deliberately NOT "complete". + const resolved = await resolvePendingExecutions(scopeId, guard, { providerReason: 'unknown' }); + + assert.equal(runCount, 0, 'an unconfirmed terminal state must never execute a deferred command'); + assert.equal(resolved.length, 1); + assert.equal(resolved[0].executed, false); + }); + + it('a tool call nobody deferred (no matching registration) produces no resolution and is silently ignored', async () => { + const scopeId = 'scope-unrelated'; + const guard = await createStepGuard(); + guard.push(toolInputStart('call_unrelated', 'some_other_tool')); + guard.push(toolInputDelta('call_unrelated', '{}')); + guard.push(toolInputEnd('call_unrelated')); + guard.push(finish('tool-calls')); + + const resolved = await resolvePendingExecutions(scopeId, guard, { providerReason: 'tool-calls' }); + assert.equal(resolved.length, 0); + }); +}); + +describe('execution-guard — concurrent turns are isolated by executionScopeId', () => { + it('two scopes registering the SAME toolCallId ("call_1") never cross-resolve, even in adversarial order', async () => { + const scopeA = 'scope-A'; + const scopeB = 'scope-B'; + const toolCallId = 'call_1'; + + let aRunCount = 0; + let bRunCount = 0; + registerDeferredExecution(scopeA, toolCallId, async () => { + aRunCount++; + return 'A: installed successfully'; + }); + registerDeferredExecution(scopeB, toolCallId, async () => { + bRunCount++; + return 'B: installed successfully'; + }); + + const guardA = await createStepGuard(); + guardA.push(toolInputStart(toolCallId, 'codepilot_cli_tools_install')); + guardA.push(toolInputDelta(toolCallId, COMMAND_ARGS)); + guardA.push(toolInputEnd(toolCallId)); + guardA.push(finish('tool-calls')); // scope A: safe + + const guardB = await createStepGuard(); + guardB.push(toolInputStart(toolCallId, 'codepilot_cli_tools_install')); + guardB.push(toolInputDelta(toolCallId, COMMAND_ARGS)); + guardB.push(toolInputEnd(toolCallId)); + guardB.push(finish('length')); // scope B: unsafe + + // Adversarial order: resolve the UNSAFE scope first, so if scope + // isolation were broken (e.g. a shared toolCallId-only map), B's + // resolution could delete/consume A's still-pending registration + // before A ever gets a chance to resolve it. + const resolvedB = await resolvePendingExecutions(scopeB, guardB, { providerReason: 'length' }); + const resolvedA = await resolvePendingExecutions(scopeA, guardA, { providerReason: 'tool-calls' }); + + assert.equal(bRunCount, 0, 'B must never execute — its step ended unsafely'); + assert.equal(resolvedB.length, 1); + assert.equal(resolvedB[0].executed, false); + + assert.equal(aRunCount, 1, 'A must execute exactly once — its own step ended safely'); + assert.equal(resolvedA.length, 1); + assert.equal(resolvedA[0].executed, true); + assert.equal(resolvedA[0].outcomeText, 'A: installed successfully'); + + // Neither scope's outcome carries the other scope's result text — + // proves B's resolution could not have consumed A's closure or vice + // versa (an accidental cross-resolve would show the wrong string here). + assert.notEqual(resolvedA[0].outcomeText, 'B: installed successfully'); + }); + + it('resolving scope A twice in a row (A can never consume its own registration a second time, let alone B\'s) is a no-op the second time', async () => { + const scopeA = 'scope-A2'; + const toolCallId = 'call_1'; + let runCount = 0; + registerDeferredExecution(scopeA, toolCallId, async () => { + runCount++; + return 'ok'; + }); + + const guard = await createStepGuard(); + guard.push(toolInputStart(toolCallId, 'codepilot_cli_tools_install')); + guard.push(toolInputDelta(toolCallId, COMMAND_ARGS)); + guard.push(toolInputEnd(toolCallId)); + guard.push(finish('tool-calls')); + + const first = await resolvePendingExecutions(scopeA, guard, { providerReason: 'tool-calls' }); + assert.equal(first.length, 1); + assert.equal(runCount, 1); + + // resolvePendingExecutions clears the scope's map after resolving it + // once; a second call for the same (now-empty) scope must find nothing. + const secondGuard = await createStepGuard(); + secondGuard.push(toolInputStart(toolCallId, 'codepilot_cli_tools_install')); + secondGuard.push(toolInputDelta(toolCallId, COMMAND_ARGS)); + secondGuard.push(toolInputEnd(toolCallId)); + secondGuard.push(finish('tool-calls')); + const second = await resolvePendingExecutions(scopeA, secondGuard, { providerReason: 'tool-calls' }); + assert.equal(second.length, 0, 'nothing was registered for this scope the second time'); + assert.equal(runCount, 1, 'the original closure must not run again'); + }); +}); + +describe('execution-guard — fail-closed cleanup', () => { + it('discardPendingExecutions removes a scope\'s registration; a LATER, unrelated scope reusing the same toolCallId never runs the old closure', async () => { + const discardedScope = 'scope-discarded'; + const laterScope = 'scope-later'; + const toolCallId = 'call_1'; // deliberately identical to prove no leakage + + let oldRunCount = 0; + registerDeferredExecution(discardedScope, toolCallId, async () => { + oldRunCount++; + return 'OLD closure ran — this must never happen'; + }); + + // Simulates agent-loop.ts's teardown `finally` firing (abort/timeout/ + // thrown error) before this scope's step ever resolved. + discardPendingExecutions(discardedScope); + // Safe to call again / on an already-empty scope. + discardPendingExecutions(discardedScope); + + let newRunCount = 0; + registerDeferredExecution(laterScope, toolCallId, async () => { + newRunCount++; + return 'NEW closure ran — correct'; + }); + + const guard = await createStepGuard(); + guard.push(toolInputStart(toolCallId, 'codepilot_cli_tools_install')); + guard.push(toolInputDelta(toolCallId, COMMAND_ARGS)); + guard.push(toolInputEnd(toolCallId)); + guard.push(finish('tool-calls')); + + const resolved = await resolvePendingExecutions(laterScope, guard, { providerReason: 'tool-calls' }); + + assert.equal(oldRunCount, 0, 'the discarded scope\'s closure must never run'); + assert.equal(newRunCount, 1, 'the later scope\'s own closure must run exactly once'); + assert.equal(resolved.length, 1); + assert.equal(resolved[0].outcomeText, 'NEW closure ran — correct'); + + // Also prove the discarded scope produces nothing if "resolved" itself + // (rather than relying only on the run-count above). + const discardedGuard = await createStepGuard(); + discardedGuard.push(toolInputStart(toolCallId, 'codepilot_cli_tools_install')); + discardedGuard.push(toolInputDelta(toolCallId, COMMAND_ARGS)); + discardedGuard.push(toolInputEnd(toolCallId)); + discardedGuard.push(finish('tool-calls')); + const discardedResolution = await resolvePendingExecutions(discardedScope, discardedGuard, { + providerReason: 'tool-calls', + }); + assert.equal(discardedResolution.length, 0); + }); + + it('a registration left unmatched by any guard decision is discarded fail-closed, not carried forward', async () => { + const scopeId = 'scope-unmatched'; + const toolCallId = 'call_never_seen'; + let runCount = 0; + registerDeferredExecution(scopeId, toolCallId, async () => { + runCount++; + return 'should never run'; + }); + + // A guard that never saw any event at all for this toolCallId. + const guard = await createStepGuard(); + const resolved = await resolvePendingExecutions(scopeId, guard, { providerReason: 'tool-calls' }); + + assert.equal(runCount, 0); + assert.equal(resolved.length, 1); + assert.equal(resolved[0].executed, false); + + // Confirms it was actually removed, not left pending for a future call. + const guard2 = await createStepGuard(); + const resolvedAgain = await resolvePendingExecutions(scopeId, guard2, { providerReason: 'tool-calls' }); + assert.equal(resolvedAgain.length, 0); + }); +}); + +describe('execution-guard — contrast: the pre-fix pattern (direct, unconditional execute) this replaces', () => { + it('demonstrates why the old pattern was unsafe: with no guard at all, the same "length"-terminated step would have run the command', async () => { + // This reproduces the pre-patch shape of codepilot_cli_tools_install's + // execute(): `execute: async ({ command }) => { await execAsync(command, ...) }` + // — no `finishReason`, no stream-completion check of any kind. The + // fullStream carrying finishReason: "length" for this exact scenario + // is identical to the SAFE test above; only the presence of the guard + // changes the outcome. + let runCount = 0; + const unguardedExecute = async (command: string) => { + void command; + runCount++; // this is the pre-fix behavior: runs immediately, no gate + return 'Successfully installed "ffmpeg".'; + }; + + await unguardedExecute('brew install ffmpeg'); + + assert.equal( + runCount, + 1, + 'the unguarded pattern executes unconditionally, even for a step that (per the SAFE/UNSAFE tests above) the guard would have rejected', + ); + }); +}); + +describe('applyResolvedExecutionsToMessages — the model transcript must show the real outcome, never "Queued"', () => { + function toolResultMessage(toolCallId: string, toolName: string, value: string): ModelMessage { + return { + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId, + toolName, + output: { type: 'text', value }, + }, + ], + } as ModelMessage; + } + + it('SAFE: the queued placeholder is replaced with the real success text; "Queued" never appears', () => { + const messages: ModelMessage[] = [ + toolResultMessage('call_1', 'codepilot_cli_tools_install', 'Queued — this command will run once the response that requested it is confirmed complete.'), + ]; + const resolvedExecutions = [ + { toolCallId: 'call_1', executed: true, outcomeText: 'Successfully installed ffmpeg' }, + ]; + + const fixed = applyResolvedExecutionsToMessages(messages, resolvedExecutions); + + const part = (fixed[0].content as Array<{ output: { value: string } }>)[0]; + assert.equal(part.output.value, 'Successfully installed ffmpeg'); + assert.doesNotMatch(part.output.value, /queued/i); + }); + + it('UNSAFE: the queued placeholder is replaced with an explicit skipped result; "Queued" never appears and success is never claimed', () => { + const messages: ModelMessage[] = [ + toolResultMessage('call_1', 'codepilot_cli_tools_install', 'Queued — this command will run once the response that requested it is confirmed complete.'), + ]; + const resolvedExecutions = [ + { toolCallId: 'call_1', executed: false, outcomeText: 'Execution skipped: the surrounding model response was not safely completed (reason: length).', reason: 'length' }, + ]; + + const fixed = applyResolvedExecutionsToMessages(messages, resolvedExecutions); + + const part = (fixed[0].content as Array<{ output: { value: string } }>)[0]; + assert.doesNotMatch(part.output.value, /queued/i); + assert.doesNotMatch(part.output.value, /success/i); + assert.match(part.output.value, /skip/i); + }); + + it('an unrelated tool-result message (different toolCallId) is returned structurally unchanged', () => { + const unrelated = toolResultMessage('call_other', 'codepilot_cli_tools_list', 'unaffected output'); + const messages: ModelMessage[] = [unrelated]; + const resolvedExecutions = [ + { toolCallId: 'call_1', executed: true, outcomeText: 'Successfully installed ffmpeg' }, + ]; + + const fixed = applyResolvedExecutionsToMessages(messages, resolvedExecutions); + + assert.equal(fixed[0], unrelated, 'an untouched message must be the same reference, not a clone'); + }); + + it('multiple tool-result parts in the same message: only the matching deferred toolCallId is replaced', () => { + const messages: ModelMessage[] = [ + { + role: 'tool', + content: [ + { type: 'tool-result', toolCallId: 'call_deferred', toolName: 'codepilot_cli_tools_install', output: { type: 'text', value: 'Queued — this command will run once the response that requested it is confirmed complete.' } }, + { type: 'tool-result', toolCallId: 'call_untouched', toolName: 'codepilot_cli_tools_list', output: { type: 'text', value: 'original list output' } }, + ], + } as ModelMessage, + ]; + const resolvedExecutions = [ + { toolCallId: 'call_deferred', executed: true, outcomeText: 'Successfully installed ffmpeg' }, + ]; + + const fixed = applyResolvedExecutionsToMessages(messages, resolvedExecutions); + const parts = fixed[0].content as Array<{ toolCallId: string; output: { value: string } }>; + + assert.equal(parts[0].output.value, 'Successfully installed ffmpeg'); + assert.equal(parts[1].output.value, 'original list output', 'the unrelated part in the same message must be untouched'); + }); + + it('no resolved executions: messages are returned unchanged', () => { + const messages: ModelMessage[] = [toolResultMessage('call_1', 'codepilot_cli_tools_list', 'output')]; + const fixed = applyResolvedExecutionsToMessages(messages, []); + assert.equal(fixed, messages, 'the exact same array reference is returned when there is nothing to fix'); + }); +}); diff --git a/src/lib/agent-loop.ts b/src/lib/agent-loop.ts index f47c24550..48738172a 100644 --- a/src/lib/agent-loop.ts +++ b/src/lib/agent-loop.ts @@ -10,6 +10,7 @@ * compatible with the existing frontend contract (useSSEStream.ts). */ +import { randomUUID } from 'node:crypto'; import { streamText, type LanguageModel, type ToolSet, type ModelMessage } from 'ai'; import type { SSEEvent, TokenUsage, MediaBlock, ExternalSource } from '@/types'; import { subscribeBuiltinEvents } from './harness/builtin-event-bus'; @@ -34,6 +35,12 @@ import { wrapController } from './safe-stream'; import { buildNativeErrorEventData } from './agent-loop-error-event'; import { buildToolErrorResultData } from './agent-loop-tool-error'; import { repairIncompleteToolHistory } from './tool-history-integrity'; +import { + createStepGuard, + resolvePendingExecutions, + discardPendingExecutions, + applyResolvedExecutionsToMessages, +} from './execution-guard'; import { createNativeTimeoutController, resolveNativeTimeoutConfig, @@ -178,6 +185,15 @@ export function runAgentLoop(options: AgentLoopOptions): ReadableStream const controller = wrapController(controllerRaw, (kind) => { console.warn(`[agent-loop] late ${kind} after stream close — silently dropped`); }); + // Isolates codepilot_cli_tools_install's deferred execution state + // (execution-guard.ts) to THIS runAgentLoop invocation. toolCallId + // alone is not a safe isolation key: it's provider-generated per + // call and not guaranteed unique across concurrent turns (see + // runtime/native-turn-registry.ts for the same concern already + // handled for abort controllers). Discarded unconditionally in the + // teardown `finally` below so an abort/timeout/thrown error can + // never leave a queued shell command for a later turn to resolve. + const executionScopeId = randomUUID(); const keepAliveTimer = setInterval(() => { controller.enqueue(formatSSE({ type: 'keep_alive', data: '' })); }, KEEPALIVE_INTERVAL_MS); @@ -265,6 +281,7 @@ export function runAgentLoop(options: AgentLoopOptions): ReadableStream prompt, mode: permissionMode, sessionId, + executionScopeId, emitSSE: (event) => { controller.enqueue(formatSSE(event as SSEEvent)); }, @@ -387,6 +404,13 @@ export function runAgentLoop(options: AgentLoopOptions): ReadableStream while (step < maxSteps) { step++; providerStreamTelemetry.resetStep(); + // Fed every fullStream event below; resolved once this step's + // stream is fully consumed and finishReason is known. See + // execution-guard.ts for why deferred execution (not a check + // inside execute()) is the only deadlock-free way to gate a + // tool whose real side effect must wait for step-level proof + // the generation that requested it actually completed. + const stepGuard = await createStepGuard(); // Build provider options (Anthropic-specific). // Shared sanitizer applies Opus 4.7 migration guards (manual @@ -698,6 +722,7 @@ export function runAgentLoop(options: AgentLoopOptions): ReadableStream // Phase 4 ① — timeout observer: clears connect on start-step, // first-token on the first output part; tracks per-tool timers. timeoutCtl.onStreamPart(event as { type: string; toolCallId?: string }); + stepGuard.push(event); switch (event.type) { case 'text-delta': hasContent = true; @@ -833,12 +858,45 @@ export function runAgentLoop(options: AgentLoopOptions): ReadableStream // Step's stream fully consumed — clear step-scoped timeout budgets. timeoutCtl.onStepEnd(); + // Resolve any tool call that deferred its real side effect (see + // execution-guard.ts) against this step's now-known terminal + // state. Only calls the guard confirms as `execute` actually run; + // everything else is discarded, never having taken effect. + const stepFinishReason = await result.finishReason; + const resolvedExecutions = await resolvePendingExecutions(executionScopeId, stepGuard, { + providerReason: String(stepFinishReason), + }); + // outcomeText is the single source of truth for what happened — + // used verbatim here (UI) and below via applyResolvedExecutionsToMessages + // (model history), so the two can never disagree about the result. + for (const resolved of resolvedExecutions) { + controller.enqueue(formatSSE({ + type: 'tool_result', + data: JSON.stringify({ + tool_use_id: resolved.toolCallId, + content: resolved.outcomeText, + is_error: !resolved.executed, + }), + })); + } + // AI SDK's response metadata is the Runtime/Provider fact for the // model that actually answered. Keep the last step's value and // expose it in the terminal SSE result so managed Native Sub Agents // can fail closed on an upstream fallback instead of echoing the // requested catalog route as though it were effective. const responseData = await result.response; + // Deferred tool calls (see execution-guard.ts) reported an + // immediate "Queued…" placeholder as their tool-result — the AI + // SDK already baked that placeholder into responseData.messages. + // Replace it with the now-known real outcome before this becomes + // part of the model's own history; otherwise the model would + // permanently believe "Queued…" was the final result, or a later + // step could act as though an unexecuted command had succeeded. + const stepResponseMessages = applyResolvedExecutionsToMessages( + responseData.messages, + resolvedExecutions, + ); runtimeReportedModel = responseData.modelId?.trim() || runtimeReportedModel; // An in-band error part can finish with all AI SDK promises @@ -893,8 +951,10 @@ export function runAgentLoop(options: AgentLoopOptions): ReadableStream // Update messages for next iteration. // streamText returns the full message list including our input + model response. - // Use response.messages which contains properly typed ModelMessage[]. - messages = [...messages, ...responseData.messages] as ModelMessage[]; + // Use response.messages (with deferred-execution placeholders already + // replaced by their real outcome above) which contains properly typed + // ModelMessage[]. + messages = [...messages, ...stepResponseMessages] as ModelMessage[]; } // 6a. Emit skill-nudge if the run was complex enough to warrant saving as a Skill. @@ -1017,6 +1077,14 @@ export function runAgentLoop(options: AgentLoopOptions): ReadableStream // session id gets reused (see contract note in // harness/builtin-event-bus.ts about cross-turn leakage). unsubscribeMediaSideChannel(); + // Fail-closed: every step already resolves its own deferred + // executions in this scope, so this is normally a no-op. It exists + // for the abort/timeout/thrown-error paths, which skip straight to + // this `finally` without ever reaching a step's own resolution — + // without it, a registered-but-never-resolved command would sit in + // execution-guard.ts's process-wide map indefinitely. Safe to call + // even when the scope is already empty. + discardPendingExecutions(executionScopeId); controller.enqueue(formatSSE({ type: 'done', data: '' })); controller.close(); } diff --git a/src/lib/agent-tools.ts b/src/lib/agent-tools.ts index e435d97cb..928fe93d9 100644 --- a/src/lib/agent-tools.ts +++ b/src/lib/agent-tools.ts @@ -101,6 +101,14 @@ export interface AssembleToolsOptions { agentRunId?: string; childSessionId?: string; }; + /** + * Per-runAgentLoop-invocation isolation id, threaded through to + * getBuiltinTools -> codepilot_cli_tools_install so its deferred + * execution registration (execution-guard.ts) can never cross-resolve + * with a concurrent turn. See runtime/native-turn-registry.ts for the + * same identity-isolation concern already handled for abort controllers. + */ + executionScopeId?: string; } export interface AssembleToolsResult { @@ -147,6 +155,7 @@ export function assembleTools(options: AssembleToolsOptions = {}): AssembleTools providerId: options.providerId || options.sessionProviderId, grokVideoAvailable: options.grokVideoAvailable, safeReadOnly: true, + executionScopeId: options.executionScopeId, }); const safeBuiltin = Object.fromEntries( Object.entries(builtinTools).filter(([name]) => PERMISSION_SAFE_TOOLS.has(name)), @@ -166,6 +175,7 @@ export function assembleTools(options: AssembleToolsOptions = {}): AssembleTools sessionId: options.sessionId ?? options.permissionContext?.sessionId, providerId: options.providerId || options.sessionProviderId, grokVideoAvailable: options.grokVideoAvailable, + executionScopeId: options.executionScopeId, }); // External MCP tools from connected servers diff --git a/src/lib/builtin-tools/cli-tools.ts b/src/lib/builtin-tools/cli-tools.ts index 8045ef969..498913377 100644 --- a/src/lib/builtin-tools/cli-tools.ts +++ b/src/lib/builtin-tools/cli-tools.ts @@ -31,6 +31,7 @@ import { import { detectAllCliTools, invalidateDetectCache } from '@/lib/cli-tools-detect'; import { CLI_TOOLS_CATALOG, EXTRA_WELL_KNOWN_BINS } from '@/lib/cli-tools-catalog'; import { getExpandedPath } from '@/lib/platform'; +import { registerDeferredExecution } from '@/lib/execution-guard'; const execFileAsync = promisify(execFile); const execAsync = promisify(exec); @@ -117,8 +118,15 @@ When listing tools with format="json", each tool includes: agentFriendly (design /** * Create CLI tools as Vercel AI SDK ToolSet. * Can be used by both Native Runtime and as reference for SDK Runtime. + * + * `executionScopeId` isolates codepilot_cli_tools_install's deferred + * execution registration to the specific runAgentLoop invocation that + * created this tool set — see execution-guard.ts. Native Runtime's only + * caller (builtin-tools/index.ts) always supplies one; the SDK Runtime + * path (cli-tools-mcp.ts) has its own independent implementation and + * never calls this factory at all. */ -export function createCliToolsTools() { +export function createCliToolsTools(executionScopeId?: string) { return { // ── LIST ───────────────────────────────────────────────────── codepilot_cli_tools_list: tool({ @@ -256,7 +264,7 @@ export function createCliToolsTools() { command: z.string().describe('The install command to execute, e.g. "brew install ffmpeg"'), name: z.string().optional().describe('Display name for the tool. If omitted, extracted from the command.'), }), - execute: async ({ command, name }) => { + execute: async ({ command, name }, { toolCallId }) => registerDeferredExecution(executionScopeId, toolCallId, async () => { try { const expandedPath = getExpandedPath(); const installMethod = extractInstallMethod(command); @@ -401,7 +409,7 @@ export function createCliToolsTools() { const msg = error instanceof Error ? error.message : 'Command execution failed'; return `Installation failed: ${msg}`; } - }, + }), }), // ── ADD ────────────────────────────────────────────────────── diff --git a/src/lib/builtin-tools/index.ts b/src/lib/builtin-tools/index.ts index 6f8944792..f4e39298e 100644 --- a/src/lib/builtin-tools/index.ts +++ b/src/lib/builtin-tools/index.ts @@ -58,6 +58,12 @@ export interface GetBuiltinToolsOptions { * notify / media import). See assembleTools('plan'). */ safeReadOnly?: boolean; + /** + * Per-runAgentLoop-invocation isolation id, threaded through to + * codepilot_cli_tools_install so its deferred execution registration + * (execution-guard.ts) can never cross-resolve with a concurrent turn. + */ + executionScopeId?: string; } /** @@ -384,7 +390,7 @@ function getToolGroups(options: GetBuiltinToolsOptions): BuiltinToolGroup[] { name: 'codepilot-cli-tools', systemPrompt: CLI_TOOLS_SYSTEM_PROMPT, condition: 'always', - tools: createCliToolsTools(), + tools: createCliToolsTools(options.executionScopeId), }); } catch (error) { reportLoadFailure('codepilot-cli-tools', error); } diff --git a/src/lib/execution-guard.ts b/src/lib/execution-guard.ts new file mode 100644 index 000000000..06efb9787 --- /dev/null +++ b/src/lib/execution-guard.ts @@ -0,0 +1,217 @@ +/** + * execution-guard.ts — defers a tool's real side effect until the surrounding + * AI SDK step has positively confirmed it finished safely. + * + * Why this exists: `streamText()`'s own `execute()` callback on a tool runs + * autonomously, mid-stream, as soon as that one call's arguments parse — it + * has no way to know whether the *step* it belongs to (which may contain + * other tool calls, or trailing text) later ends in `finishReason: "length"`, + * a provider error, or a content filter. A tool call can look completely + * valid on its own and still be part of a generation that was never + * confirmed complete. Structural JSON validity is not proof the model's + * intent for this turn was fully expressed. + * + * `execute()` cannot simply await that proof itself: the step's `fullStream` + * can't reach its own `finish` part until every in-flight `execute()` call + * for that step has already resolved (that's how the AI SDK's step + * lifecycle works), so waiting inside `execute()` would deadlock. Instead, + * `execute()` registers the real side effect here and returns an immediate + * "queued" result; `agent-loop.ts` — which already iterates the step's + * `fullStream` and already learns `finishReason` once the step ends — feeds + * every event into a `prefix-safe-json` execution guard and, once the step + * is over, resolves each pending registration against the guard's decision + * for that exact `toolCallId`. Only a call whose surrounding step positively + * confirmed completion ever actually runs. + * + * Scoping: pending registrations are keyed by `executionScopeId + toolCallId`, + * not `toolCallId` alone. `toolCallId` is provider-generated per call and is + * not guaranteed unique across concurrent turns/sessions (two overlapping + * turns can both produce e.g. "call_1") — see runtime/native-turn-registry.ts + * for the same identity-isolation concern already handled for abort + * controllers, for the same underlying reason (module-local state shared + * across concurrent Native turns). `executionScopeId` is a fresh random id + * created once per `runAgentLoop()` invocation and threaded through tool + * assembly, so two turns can never resolve or discard each other's pending + * commands even if their `toolCallId`s collide. + */ +import type { AiSdkExecutionGuard } from 'prefix-safe-json'; +import type { ModelMessage } from 'ai'; + +interface PendingExecution { + run: () => Promise; +} + +const pendingByScope = new Map>(); + +/** + * Called by a tool's `execute()`. Registers the real side effect and + * returns immediately — never runs `run` itself. Fails closed (refuses to + * register, returns an explicit error) if no scope is available, since a + * registration with no scope could never be resolved or safely discarded. + */ +export function registerDeferredExecution( + executionScopeId: string | undefined, + toolCallId: string, + run: () => Promise, +): string { + if (!executionScopeId) { + return 'Installation blocked: no execution scope was available to verify this command completes safely.'; + } + let scoped = pendingByScope.get(executionScopeId); + if (!scoped) { + scoped = new Map(); + pendingByScope.set(executionScopeId, scoped); + } + scoped.set(toolCallId, { run }); + return ( + 'Queued — this command will run once the response that requested it is confirmed complete. ' + + 'You will see the real result as a follow-up.' + ); +} + +export async function createStepGuard(): Promise { + // Dynamic import: `prefix-safe-json` is ESM-only (no "require" export + // condition), and this project's "moduleResolution": "bundler" + tsx test + // harness combination cannot statically resolve an ESM-only package from + // a .ts source file. A dynamic import resolves correctly in every runtime + // (Next.js/webpack, plain Node, and the node:test harness alike) and costs + // nothing here — this runs once per agent step, not per token. + const { createAiSdkExecutionGuard } = await import('prefix-safe-json'); + return createAiSdkExecutionGuard(); +} + +export interface ResolvedExecution { + toolCallId: string; + executed: boolean; + /** + * The exact text to surface to both the UI (follow-up SSE tool_result) + * and the model's own transcript (via applyResolvedExecutionsToMessages) + * for this call. Computed once here so the two can never drift out of + * agreement about what actually happened. + */ + outcomeText: string; + /** Present when `executed` is false: the guard's reason code. */ + reason?: string; +} + +function skippedText(reason: string | undefined): string { + return ( + 'Execution skipped: the surrounding model response was not safely completed' + + (reason ? ` (reason: ${reason}).` : '.') + ); +} + +/** + * Called by `agent-loop.ts` once a step's `fullStream` is fully consumed. + * Resolves every pending registration in this scope that the step's events + * touched, against the guard's terminal decision, running the real side + * effect only for calls the guard confirms as `action: "execute"`. + * + * Fail-closed: any registration in this scope left unmatched by a guard + * decision (e.g. registered but never observed reaching a terminal state) + * is discarded here too, never carried into a later step or scope. + */ +export async function resolvePendingExecutions( + executionScopeId: string, + guard: AiSdkExecutionGuard, + finishMeta?: { reason?: string; providerReason?: string }, +): Promise { + const { decisions } = guard.finish(finishMeta as Parameters[0]); + const resolved: ResolvedExecution[] = []; + const scoped = pendingByScope.get(executionScopeId); + if (!scoped) return resolved; + + for (const decision of decisions) { + // toolCallId is optional on ExecutionDecision; a decision without one + // can never correspond to a registration (registerDeferredExecution + // always requires a real toolCallId), so it can't be resolved here. + const toolCallId = decision.toolCallId; + if (!toolCallId) continue; + const entry = scoped.get(toolCallId); + if (!entry) continue; + scoped.delete(toolCallId); + + if (decision.action === 'execute') { + try { + const result = await entry.run(); + resolved.push({ toolCallId, executed: true, outcomeText: result }); + } catch (error) { + const msg = error instanceof Error ? error.message : 'Command execution failed'; + resolved.push({ toolCallId, executed: true, outcomeText: `Installation failed: ${msg}` }); + } + } else { + resolved.push({ toolCallId, executed: false, outcomeText: skippedText(decision.reason), reason: decision.reason }); + } + } + + // Anything still left in this scope never matched a decision this step + // (e.g. registered but the guard never observed a terminal state for it). + // Discard it unconfirmed rather than let it linger into a future step. + for (const toolCallId of scoped.keys()) { + resolved.push({ toolCallId, executed: false, outcomeText: skippedText('unconfirmed') }); + } + pendingByScope.delete(executionScopeId); + + return resolved; +} + +/** + * Discards every pending deferred execution for a scope without running + * any of them. Called from agent-loop.ts's teardown (finally block) so an + * abort, timeout, or thrown error can never leave a queued shell command + * available for a later, unrelated turn to accidentally resolve. Safe to + * call when the scope is already empty or was never created. + */ +export function discardPendingExecutions(executionScopeId: string): void { + pendingByScope.delete(executionScopeId); +} + +type UnknownToolResultPart = { + type?: unknown; + toolCallId?: unknown; + [key: string]: unknown; +}; + +function partsOf(message: ModelMessage): UnknownToolResultPart[] | null { + return Array.isArray(message.content) ? (message.content as UnknownToolResultPart[]) : null; +} + +/** + * Replaces the temporary "Queued…" tool-result text for each resolved + * deferred execution with its real outcome, before these messages are + * appended to the conversation the model sees on the next step. + * + * Without this, the model's own history would permanently show "Queued…" + * as the tool's result and never learn whether the command actually ran — + * or worse, a later step could act as though a command it never actually + * executed had already succeeded. + * + * Only touches `role: "tool"` message parts whose `toolCallId` exactly + * matches a resolved execution from this step; every other message and + * part is returned unchanged (same reference, not cloned), matching the + * existing immutable-transform style used by tool-history-integrity.ts. + */ +export function applyResolvedExecutionsToMessages( + messages: readonly ModelMessage[], + resolvedExecutions: readonly ResolvedExecution[], +): ModelMessage[] { + if (resolvedExecutions.length === 0) return messages as ModelMessage[]; + const byToolCallId = new Map(resolvedExecutions.map((r) => [r.toolCallId, r])); + + return messages.map((message) => { + if (message.role !== 'tool') return message; + const parts = partsOf(message); + if (!parts) return message; + + let changed = false; + const nextParts = parts.map((part) => { + if (part.type !== 'tool-result' || typeof part.toolCallId !== 'string') return part; + const resolvedEntry = byToolCallId.get(part.toolCallId); + if (!resolvedEntry) return part; + changed = true; + return { ...part, output: { type: 'text', value: resolvedEntry.outcomeText } }; + }); + + return changed ? ({ ...message, content: nextParts } as ModelMessage) : message; + }) as ModelMessage[]; +}