From 8f785c9151f18d3c40ce8b1d490fd52244fb0be2 Mon Sep 17 00:00:00 2001 From: yannickmonney Date: Thu, 8 Oct 2026 12:44:14 +0200 Subject: [PATCH 1/8] fix(platform): end Claude turns on stdin EOF instead of a reap A Claude Code turn holds its stdin open as the steering channel, so the CLI never exits after its result. The drain window waited out a 1.5 s grace after every turn-ended and then cancelled the exec (a group SIGTERM): every reply paid 1.5 s, the exit code was lost, and the CLI was killed while it could still be writing its transcript. Once the turn has ended and its background-task ledger is empty, the window now closes the held stdin (the runnerd EOF write the harness contract already described but nobody sent). The CLI finishes on its own and the drain reports its real exit; in the stand-in transport the turn ends ~20 ms after the result instead of 1.5 s. The grace and the reap stay as the fallback when the CLI ignores the EOF or the write fails. Stdin stays open while a background task runs, and close-stdin harnesses are never sent an EOF. A steer that lands after the reply now meets STDIN_CLOSED and takes the existing settled-run fallback (a fresh run carrying the comment) instead of being written into a process about to be SIGTERMed. --- .../agent_host.empty_answer.test.ts | 2 + .../agent_host.retry_resume.test.ts | 2 + .../automations/agent_host.shutdown.test.ts | 2 + .../core/chat/external_turn_shared.test.ts | 143 +++++++++++++++--- .../backend/core/chat/external_turn_shared.ts | 65 ++++++-- .../agent_run_host.budget_exceeded.test.ts | 2 + .../agent_run_host.credential_rotated.test.ts | 2 + .../tasks/agent_run_host.empty_answer.test.ts | 2 + 8 files changed, 193 insertions(+), 27 deletions(-) diff --git a/services/platform/backend/core/automations/agent_host.empty_answer.test.ts b/services/platform/backend/core/automations/agent_host.empty_answer.test.ts index afd9823181..5430babed8 100644 --- a/services/platform/backend/core/automations/agent_host.empty_answer.test.ts +++ b/services/platform/backend/core/automations/agent_host.empty_answer.test.ts @@ -32,6 +32,8 @@ vi.mock('../node_only/sandbox/helpers/session_client', async (importActual) => { >(); return { ...actual, + // The window closes a Claude turn's held stdin; no spawner answers here. + sessionWriteExecStdin: async () => ({ ok: true }), sessionGetExecCheckpoint: async () => null, sessionPutExecCheckpoint: async () => undefined, drainSessionExecResilient: async ( diff --git a/services/platform/backend/core/automations/agent_host.retry_resume.test.ts b/services/platform/backend/core/automations/agent_host.retry_resume.test.ts index dbfd28a9eb..54ebab5dd2 100644 --- a/services/platform/backend/core/automations/agent_host.retry_resume.test.ts +++ b/services/platform/backend/core/automations/agent_host.retry_resume.test.ts @@ -23,6 +23,8 @@ vi.mock('../node_only/sandbox/helpers/session_client', async (importActual) => { >(); return { ...actual, + // The window closes a Claude turn's held stdin; no spawner answers here. + sessionWriteExecStdin: async () => ({ ok: true }), sessionGetExecCheckpoint: async () => null, sessionPutExecCheckpoint: async () => undefined, drainSessionExecResilient: async ( diff --git a/services/platform/backend/core/automations/agent_host.shutdown.test.ts b/services/platform/backend/core/automations/agent_host.shutdown.test.ts index 4aeb96f668..d0bafba852 100644 --- a/services/platform/backend/core/automations/agent_host.shutdown.test.ts +++ b/services/platform/backend/core/automations/agent_host.shutdown.test.ts @@ -18,6 +18,8 @@ vi.mock('../node_only/sandbox/helpers/session_client', async (importActual) => { >(); return { ...actual, + // The window closes a Claude turn's held stdin; no spawner answers here. + sessionWriteExecStdin: async () => ({ ok: true }), sessionGetExecCheckpoint: async () => null, sessionPutExecCheckpoint: async () => undefined, sessionCancelExec: async (_sessionId: string, execId: string) => { diff --git a/services/platform/backend/core/chat/external_turn_shared.test.ts b/services/platform/backend/core/chat/external_turn_shared.test.ts index 4cd7d3ca53..81b673ebaf 100644 --- a/services/platform/backend/core/chat/external_turn_shared.test.ts +++ b/services/platform/backend/core/chat/external_turn_shared.test.ts @@ -8,8 +8,10 @@ * - a `turn-ended` whose background-task ledger is still open is a lingering * turn, not a finished one — the process stays alive until the ledger * settles (the `types.ts` `task-started`/`task-settled` contract); - * - a real exit still finalizes a held result, and a plain hold-stdin turn - * with no background work is still cut and reaped after the grace. + * - a real exit still finalizes a held result; + * - a hold-stdin turn with no background work gets its stdin closed and + * ends on the CLI's own exit, and is cut and reaped after the grace only + * when it does not exit. * * And how a terminal window classifies: a turn that ended cleanly with * nothing at all from the model (live: a serving cluster that failed @@ -55,6 +57,11 @@ const transport = vi.hoisted(() => ({ } }, resumedAt: [] as number[], + stdinWrites: [] as Array<{ execId: string; eof?: boolean }>, + /** How the live exec answers a stdin EOF: by exiting, by ignoring it, or + * the write itself failing. */ + onEof: 'exit' as 'exit' | 'ignore' | 'fail', + exitOnEof: undefined as (() => void) | undefined, })); vi.mock('../node_only/sandbox/helpers/session_client', () => ({ @@ -86,6 +93,20 @@ vi.mock('../node_only/sandbox/helpers/session_client', () => ({ transport.cancelled.push(execId); return true; }, + sessionWriteExecStdin: async ( + _sessionId: string, + execId: string, + write: { dataBase64?: string; eof?: boolean }, + ) => { + transport.stdinWrites.push({ execId, eof: write.eof }); + if (transport.onEof === 'fail') throw new Error('spawner unreachable'); + // Only the exec live at the write exits; a write after the drain ended + // must not end the next test's exec. + const exit = transport.exitOnEof; + if (write.eof === true && transport.onEof === 'exit' && exit !== undefined) + setTimeout(exit, 20); + return { ok: true }; + }, drainSessionExecResilient: async ( _sessionId: string, _body: unknown, @@ -133,13 +154,19 @@ vi.mock('../node_only/sandbox/helpers/session_client', () => ({ } if (transport.exitAfterStdout) return { exitCode: transport.exitCode, errorCode: transport.errorCode }; - // A live exec: the drain only ends when the window (or the cut) aborts. - await new Promise((_resolve, reject) => { - signal.addEventListener('abort', () => reject(signal.reason), { - once: true, + // A live exec: the drain only ends when the window (or the cut) aborts, + // or when the process exits on a stdin EOF. + try { + await new Promise((resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { + once: true, + }); + transport.exitOnEof = resolve; }); - }); - throw new Error('unreachable'); + } finally { + transport.exitOnEof = undefined; + } + return { exitCode: 0 }; }, })); @@ -220,6 +247,9 @@ describe('drainHarnessWindow end-of-turn rules', () => { transport.exitCode = 0; transport.errorCode = undefined; transport.protocolFailure = false; + transport.stdinWrites = []; + transport.onEof = 'exit'; + transport.exitOnEof = undefined; vi.spyOn(console, 'warn').mockImplementation(() => {}); }); @@ -263,7 +293,11 @@ describe('drainHarnessWindow end-of-turn rules', () => { transport.stdout = ndjson([CLAUDE_TASK_SETTLED]); const terminal = await drainHarnessWindow({ ...args, windowMs: 2000 }); expect(terminal.kind).toBe('terminal'); - expect(transport.cancelled).toEqual(['checkpoint-bg']); + // Stdin closes only once the restored ledger settles; the CLI exits. + expect(transport.stdinWrites).toEqual([ + { execId: 'checkpoint-bg', eof: true }, + ]); + expect(transport.cancelled).toEqual([]); }); it.each(['missing', 'failed'] as const)( @@ -963,7 +997,10 @@ describe('drainHarnessWindow end-of-turn rules', () => { windowMs: 10000, }); expect(terminal.kind).toBe('terminal'); - expect(transport.cancelled).toEqual(['replay-background']); + expect(transport.stdinWrites).toEqual([ + { execId: 'replay-background', eof: true }, + ]); + expect(transport.cancelled).toEqual([]); } finally { await replay.dispose(); await rm(directory, { recursive: true, force: true }); @@ -977,6 +1014,7 @@ describe('drainHarnessWindow end-of-turn rules', () => { CLAUDE_RESULT, CLAUDE_TASK_SETTLED, ]); + transport.onEof = 'ignore'; const result = await drainHarnessWindow({ sessionId: 'sandbox', @@ -992,12 +1030,15 @@ describe('drainHarnessWindow end-of-turn rules', () => { expect(result.ended?.status).toBe('completed'); expect(classifyHarnessEnd(result).errored).toBe(false); } - // The lingering hold-stdin process is reaped exactly once. + // Stdin closes only once the ledger is empty, and the lingering + // hold-stdin process is reaped exactly once. + expect(transport.stdinWrites).toEqual([{ execId: 'bg-claude', eof: true }]); expect(transport.cancelled).toEqual(['bg-claude']); }); - it('cuts and reaps a plain hold-stdin turn with no background work', async () => { + it('closes the stdin of a plain hold-stdin turn and ends on its own exit, without the grace or a reap', async () => { transport.stdout = ndjson([CLAUDE_INIT, CLAUDE_RESULT]); + const started = Date.now(); const result = await drainHarnessWindow({ sessionId: 'sandbox', @@ -1008,12 +1049,74 @@ describe('drainHarnessWindow end-of-turn rules', () => { expect(result.kind).toBe('terminal'); if (result.kind === 'terminal') { - expect(result.exited).toBe(false); + expect(result.exited).toBe(true); + expect(result.execResult?.exitCode).toBe(0); expect(result.ended?.finalText).toBe( 'The report is being generated in the background.', ); + expect(classifyHarnessEnd(result).errored).toBe(false); } - expect(transport.cancelled).toEqual(['plain-claude']); + expect(transport.stdinWrites).toEqual([ + { execId: 'plain-claude', eof: true }, + ]); + expect(transport.cancelled).toEqual([]); + // Well inside the 1.5 s grace the turn used to sit out on every reply. + expect(Date.now() - started).toBeLessThan(1_000); + }); + + it.each(['ignore', 'fail'] as const)( + 'cuts and reaps a hold-stdin turn after the grace when its stdin EOF is %sed', + async (onEof) => { + transport.stdout = ndjson([CLAUDE_INIT, CLAUDE_RESULT]); + transport.onEof = onEof; + + const result = await drainHarnessWindow({ + sessionId: 'sandbox', + execId: 'plain-claude', + harness: 'claude-code', + windowMs: 10_000, + }); + + expect(result.kind).toBe('terminal'); + if (result.kind === 'terminal') { + expect(result.exited).toBe(false); + expect(result.ended?.finalText).toBe( + 'The report is being generated in the background.', + ); + } + expect(transport.stdinWrites).toEqual([ + { execId: 'plain-claude', eof: true }, + ]); + expect(transport.cancelled).toEqual(['plain-claude']); + }, + ); + + it('never closes stdin while a background task is open, nor on a close-stdin harness', async () => { + transport.stdout = ndjson([ + CLAUDE_INIT, + CLAUDE_TASK_STARTED, + CLAUDE_RESULT, + ]); + const lingering = await drainHarnessWindow({ + sessionId: 'sandbox', + execId: 'bg-claude', + harness: 'claude-code', + windowMs: 50, + }); + expect(lingering.kind).toBe('running'); + + // Another exec: the stand-in keeps one checkpoint for all of them. + transport.checkpoint = null; + transport.stdout = readFixture('pi', 'shell-turn'); + transport.exitAfterStdout = true; + const closed = await drainHarnessWindow({ + sessionId: 'sandbox', + execId: 'pi-turn', + harness: 'pi', + }); + expect(closed.kind).toBe('terminal'); + + expect(transport.stdinWrites).toEqual([]); }); }); @@ -1070,6 +1173,9 @@ describe('classifyHarnessEnd', () => { transport.exitCode = 0; transport.errorCode = undefined; transport.protocolFailure = false; + transport.stdinWrites = []; + transport.onEof = 'exit'; + transport.exitOnEof = undefined; vi.spyOn(console, 'warn').mockImplementation(() => {}); }); @@ -1143,18 +1249,19 @@ describe('classifyHarnessEnd', () => { }); it('fails a turn whose model answered nothing (the live empty 200)', async () => { - // As it ran: the held-stdin CLI lingers after its result, so the turn - // ends on the grace cut, not on an exit. + // The held-stdin CLI lingers after its result until the window closes + // its stdin; the turn then ends on the CLI's own exit, which is clean. const result = await drainCapture('empty-answer-turn', { exits: false }); - expect(result.exited).toBe(false); + expect(result.exited).toBe(true); + expect(result.execResult?.exitCode).toBe(0); expect(result.ended?.isError).toBe(false); expect(classifyHarnessEnd(result)).toEqual({ errored: true, reason: EMPTY_ANSWER, emptyAnswer: true, }); - expect(transport.cancelled).toEqual(['exec-empty-answer-turn']); + expect(transport.cancelled).toEqual([]); }); it('keeps a reasoning-only turn: the result counts its tokens', async () => { diff --git a/services/platform/backend/core/chat/external_turn_shared.ts b/services/platform/backend/core/chat/external_turn_shared.ts index 2e404d6802..b4ad22d363 100644 --- a/services/platform/backend/core/chat/external_turn_shared.ts +++ b/services/platform/backend/core/chat/external_turn_shared.ts @@ -48,6 +48,7 @@ import { sessionCancelExec, sessionGetExecCheckpoint, sessionPutExecCheckpoint, + sessionWriteExecStdin, type ExecCursor, type SessionExecCheckpoint, sessionStageFiles, @@ -65,10 +66,11 @@ import { export const SKILLS_DIR = 'workspace/.tale/skills'; /** One drain window; well under the Convex action execution ceiling. */ const DRAIN_WINDOW_MS = 90_000; -/** After the parser sees `turn-ended`, how long to keep draining for the - * exec's natural exit (which carries a close-stdin harness's exit code) - * before cutting the window. A hold-stdin harness (claude-code) never exits - * on its own — without the cut, every reply would sit out the full window. */ +/** After the parser sees `turn-ended` with no background task open, how long + * to keep draining for the exec's natural exit (which carries its exit code) + * before cutting the window and reaping the exec. A hold-stdin harness + * (claude-code) exits once its stdin is closed, which the window does at that + * moment; the cut is the fallback for a process that does not. */ const TURN_ENDED_EXIT_GRACE_MS = 1_500; /** Floor between two mid-window notifications of the accumulating output — * the cadence of the `onText`/`onTimeline` progress sinks, so a host's @@ -116,6 +118,15 @@ export function isManagedHarness(harness: string): boolean { return def?.credentialPolicy.managed === true; } +/** Whether a harness holds its stdin open as a steering channel (the + * `ndjson-user-message` stdin mode): such a CLI answers turn after turn and + * exits only on stdin EOF. */ +function harnessHoldsStdin(harness: string): boolean { + if (!isHarnessSlug(harness)) return false; + const def = loadHarnesses().find((h) => h.slug === harness); + return def?.exec.stdin.mode === 'ndjson-user-message'; +} + /** Whether a harness mounts MCP servers — and so the platform bridge every * workspace tool rides. A harness whose YAML declares `capabilities.mcp: * false` (Pi, Hermes, Cursor) can call none of them, so a tool that exists @@ -457,12 +468,46 @@ export async function drainHarnessWindow(args: { // A hold-stdin harness (claude-code) lingers after its reply waiting for // more input, so its process exit can be a whole window away from the - // `turn-ended` event that actually ends the turn. Cut the drain shortly - // after the parser sees `turn-ended`; the grace lets a harness that DOES - // exit deliver its terminal result (and exit code) first. + // `turn-ended` event that actually ends the turn. Once the turn has ended + // with no background task open, the window closes that stdin — the CLI's + // own way to finish: it writes its transcript and exits with its real exit + // code, which the drain then reports. The grace bounds the wait for that + // exit (and for a close-stdin harness's own); when it elapses the drain is + // cut and the exec reaped, so a process that ignores the EOF cannot hold + // the turn open. let replayComplete = args.start !== undefined; const turnEndedCut = new AbortController(); let turnEndedGrace: ReturnType | undefined; + const holdsStdin = harnessHoldsStdin(args.harness); + let stdinClosed = false; + const closeHeldStdin = () => { + if (!holdsStdin || stdinClosed) return; + stdinClosed = true; + void Promise.resolve() + .then(() => + sessionWriteExecStdin(args.sessionId, args.execId, { eof: true }), + ) + .then( + (wrote) => { + // STDIN_CLOSED: an earlier window already sent it; NOT_FOUND: the + // exec is gone. Neither leaves anything for the grace cut to miss. + if ( + !wrote.ok && + wrote.reason !== 'STDIN_CLOSED' && + wrote.reason !== 'NOT_FOUND' + ) { + console.warn( + `[harness-window] ${args.execId}: stdin EOF refused (${wrote.reason ?? 'unknown'}); the grace cut reaps the exec`, + ); + } + }, + (err: unknown) => + console.warn( + `[harness-window] ${args.execId}: stdin EOF failed; the grace cut reaps the exec:`, + err, + ), + ); + }; // The background-task ledger (`types.ts` contract): a harness that // launched background work reports `task-started`/`task-settled` pairs, @@ -477,6 +522,7 @@ export async function drainHarnessWindow(args: { () => turnEndedCut.abort(), TURN_ENDED_EXIT_GRACE_MS, ); + closeHeldStdin(); }; const disarmTurnEndedCut = () => { if (turnEndedGrace === undefined) return; @@ -830,8 +876,9 @@ export async function drainHarnessWindow(args: { }; } - // A harness that lingers after its turn (held-open stdin) has ended the turn - // but not the process — reap it so it can't hold the session. + // A harness that lingers after its turn (held-open stdin it did not exit + // on) has ended the turn but not the process — reap it so it can't hold + // the session. if (!exited && ended !== undefined) { await sessionCancelExec(args.sessionId, args.execId).catch((err) => console.warn('[harness-window] linger reap failed:', err), diff --git a/services/platform/backend/core/tasks/agent_run_host.budget_exceeded.test.ts b/services/platform/backend/core/tasks/agent_run_host.budget_exceeded.test.ts index d07d8c97a5..8f1617bee8 100644 --- a/services/platform/backend/core/tasks/agent_run_host.budget_exceeded.test.ts +++ b/services/platform/backend/core/tasks/agent_run_host.budget_exceeded.test.ts @@ -39,6 +39,8 @@ vi.mock('../node_only/sandbox/helpers/session_client', async (importActual) => { >(); return { ...actual, + // The window closes a Claude turn's held stdin; no spawner answers here. + sessionWriteExecStdin: async () => ({ ok: true }), sessionGetExecCheckpoint: async () => null, sessionPutExecCheckpoint: async () => undefined, drainSessionExecResilient: async ( diff --git a/services/platform/backend/core/tasks/agent_run_host.credential_rotated.test.ts b/services/platform/backend/core/tasks/agent_run_host.credential_rotated.test.ts index 93d3fba23d..a82f538540 100644 --- a/services/platform/backend/core/tasks/agent_run_host.credential_rotated.test.ts +++ b/services/platform/backend/core/tasks/agent_run_host.credential_rotated.test.ts @@ -40,6 +40,8 @@ vi.mock('../node_only/sandbox/helpers/session_client', async (importActual) => { >(); return { ...actual, + // The window closes a Claude turn's held stdin; no spawner answers here. + sessionWriteExecStdin: async () => ({ ok: true }), sessionGetExecCheckpoint: async () => io.checkpoint, sessionPutExecCheckpoint: async ( _sessionId: string, diff --git a/services/platform/backend/core/tasks/agent_run_host.empty_answer.test.ts b/services/platform/backend/core/tasks/agent_run_host.empty_answer.test.ts index 766a6828dd..1a45d30a7e 100644 --- a/services/platform/backend/core/tasks/agent_run_host.empty_answer.test.ts +++ b/services/platform/backend/core/tasks/agent_run_host.empty_answer.test.ts @@ -42,6 +42,8 @@ vi.mock('../node_only/sandbox/helpers/session_client', async (importActual) => { >(); return { ...actual, + // The window closes a Claude turn's held stdin; no spawner answers here. + sessionWriteExecStdin: async () => ({ ok: true }), sessionGetExecCheckpoint: async () => null, sessionPutExecCheckpoint: async () => undefined, drainSessionExecResilient: async ( From 0098f24e07a6e417db1ac58ada67ef6234b6a623 Mon Sep 17 00:00:00 2001 From: yannickmonney Date: Thu, 8 Oct 2026 12:52:14 +0200 Subject: [PATCH 2/8] perf(platform): skip harvest re-reads when the exec already exited Every settle re-listed an empty delivery box three times 500 ms apart before accepting it as empty, as a read-after-write guard. A task or automation turn that delivered no file (most review and answer turns) therefore spent 1.5 s in the harvest before it could settle. The listing goes through runnerd in the same container, on the same filesystem the exec wrote to; once the exec has exited nothing is left to land, so its first listing is final. The hosts now tell the harvest when the turn's exec exited on its own (the drain window's `exited`; a script step that ran to completion), and only then skip the re-reads. A reaped linger and a script cut at its timeout, whose processes may still be inside their kill grace, keep the three re-reads. --- .../agent_host.empty_answer.test.ts | 15 ++++- .../backend/core/automations/agent_host.ts | 5 ++ .../core/automations/script_host.test.ts | 23 +++++++ .../backend/core/automations/script_host.ts | 3 + .../node_only/sandbox/session_exec.test.ts | 27 ++++++++ .../core/node_only/sandbox/session_exec.ts | 18 ++++-- .../core/tasks/agent_run_host.reap.test.ts | 61 ++++++++++++++++++- .../backend/core/tasks/agent_run_host.ts | 5 ++ 8 files changed, 149 insertions(+), 8 deletions(-) diff --git a/services/platform/backend/core/automations/agent_host.empty_answer.test.ts b/services/platform/backend/core/automations/agent_host.empty_answer.test.ts index 5430babed8..343f255549 100644 --- a/services/platform/backend/core/automations/agent_host.empty_answer.test.ts +++ b/services/platform/backend/core/automations/agent_host.empty_answer.test.ts @@ -23,6 +23,8 @@ const io = vi.hoisted(() => ({ /** What the exec printed, replayed into every drain window. */ stdout: '', cancelled: [] as string[], + /** What each settle asked the output harvest for. */ + harvests: [] as Array>, })); vi.mock('../node_only/sandbox/helpers/session_client', async (importActual) => { @@ -59,7 +61,13 @@ vi.mock('../node_only/sandbox/helpers/session_client', async (importActual) => { }; }); vi.mock('../node_only/sandbox/session_exec', () => ({ - harvestSessionOutput: async () => ({ files: [], harvestSkipped: [] }), + harvestSessionOutput: async ( + _ctx: unknown, + args: Record, + ) => { + io.harvests.push(args); + return { files: [], harvestSkipped: [] }; + }, })); const { driveWorkflowAgentTurnImpl } = await import('./agent_host'); @@ -162,6 +170,7 @@ const SILENT_RESULT = { beforeEach(() => { io.stdout = ''; io.cancelled = []; + io.harvests = []; vi.spyOn(console, 'warn').mockImplementation(() => {}); vi.spyOn(console, 'error').mockImplementation(() => {}); }); @@ -192,6 +201,10 @@ describe('an automation agent turn', () => { .filter((text): text is string => typeof text === 'string'); expect(progress.length).toBeGreaterThan(0); expect(progress.every((text) => text.length <= 64 * 1024)).toBe(true); + // The exec exited before the settle: the harvest takes its first listing. + expect(io.harvests).toEqual([ + expect.objectContaining({ execId: KEYS.execId, execExited: true }), + ]); }); it('fails, retryably, when the model answered nothing', async () => { diff --git a/services/platform/backend/core/automations/agent_host.ts b/services/platform/backend/core/automations/agent_host.ts index efd9354b83..2a8c2926f0 100644 --- a/services/platform/backend/core/automations/agent_host.ts +++ b/services/platform/backend/core/automations/agent_host.ts @@ -2636,6 +2636,7 @@ async function continueOrSettle( ? { agentResultStatus: ended.status } : {}), harvest: true, + execExited: window.exited, }, ); } @@ -2654,6 +2655,9 @@ async function settleWorkflowAgentTurn( exitCode?: number; agentResultStatus?: string; harvest?: boolean; + /** The turn's exec exited on its own before the settle, so the harvest + * reads a box nothing is still writing to. */ + execExited?: boolean; } = {}, ): Promise { const release = await releaseTurnKey(ctx, { @@ -2740,6 +2744,7 @@ async function settleWorkflowAgentTurn( organizationId: args.organizationId, sessionId: args.sessionId, execId: args.execId, + ...(opts.execExited === true ? { execExited: true } : {}), }); files = harvested.files.map((file) => ({ name: file.path.split('/').at(-1) ?? file.path, diff --git a/services/platform/backend/core/automations/script_host.test.ts b/services/platform/backend/core/automations/script_host.test.ts index 41141db4c3..f427b1f085 100644 --- a/services/platform/backend/core/automations/script_host.test.ts +++ b/services/platform/backend/core/automations/script_host.test.ts @@ -145,9 +145,11 @@ describe('workflowScriptRunner', () => { packagesByLang: { python: ['openpyxl'] }, timeoutMs: DEFAULT_SCRIPT_TIMEOUT_MS, }); + // The script's exec has exited: the harvest takes its first listing. expect(harvestSessionOutput).toHaveBeenCalledWith(ctx, { organizationId: ORG, sessionId: 'wf-session', + execExited: true, }); expect(sessionReadFile).toHaveBeenCalledWith( 'wf-session', @@ -309,4 +311,25 @@ describe('workflowScriptRunner', () => { { path: '/agent/output/huge.bin', reason: 'over cap' }, ]); }); + + it('lets the harvest re-read an empty box after a script cut at its timeout', async () => { + // A cut exec's processes can still be inside their kill grace. + runStepsInSession.mockResolvedValue({ + status: 'cancelled', + exitCode: null, + stdout: '', + stderr: '', + }); + await workflowScriptRunner(ctx)({ + organizationId: ORG, + runId: RUN, + skill: 's', + entry: 'a.py', + }); + expect(harvestSessionOutput).toHaveBeenCalledWith(ctx, { + organizationId: ORG, + sessionId: 'wf-session', + execExited: false, + }); + }); }); diff --git a/services/platform/backend/core/automations/script_host.ts b/services/platform/backend/core/automations/script_host.ts index 79c02985fb..3dd144de8a 100644 --- a/services/platform/backend/core/automations/script_host.ts +++ b/services/platform/backend/core/automations/script_host.ts @@ -134,6 +134,9 @@ export function workflowScriptRunner( const { files, harvestSkipped } = await harvestSessionOutput(ctx, { organizationId: run.organizationId, sessionId, + // Every step's exec has ended by now; one cut at its timeout may still + // have processes inside their kill grace. + execExited: exec.status !== 'cancelled', }); // The result file is the script's own verdict — read it back directly so diff --git a/services/platform/backend/core/node_only/sandbox/session_exec.test.ts b/services/platform/backend/core/node_only/sandbox/session_exec.test.ts index 859414b052..35019b6432 100644 --- a/services/platform/backend/core/node_only/sandbox/session_exec.test.ts +++ b/services/platform/backend/core/node_only/sandbox/session_exec.test.ts @@ -243,6 +243,33 @@ describe('harvestSessionOutput — per-file harvest skips', () => { ); }); + it('re-reads an empty listing while the exec may still be writing', async () => { + // A reaped linger can still be inside its kill grace: a deliverable it + // was writing lands on a later listing. + sessionListFiles + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([outputEntry('late.txt')]); + sessionReadFile.mockResolvedValue(textBytes('late')); + const { files } = await harvestSessionOutput(harvestCtx({}), harvestArgs); + expect(files.map((file) => file.path)).toEqual(['/agent/output/late.txt']); + expect(sessionListFiles).toHaveBeenCalledTimes(2); + }); + + it('takes an empty listing at once when the exec already exited', async () => { + // Nothing is left writing to the box, so a turn that delivered no file + // settles without the 1.5 s of re-reads. + sessionListFiles.mockResolvedValue([]); + const started = Date.now(); + const { files, harvestSkipped } = await harvestSessionOutput( + harvestCtx({}), + { ...harvestArgs, execExited: true }, + ); + expect(files).toEqual([]); + expect(harvestSkipped).toEqual([]); + expect(sessionListFiles).toHaveBeenCalledTimes(1); + expect(Date.now() - started).toBeLessThan(400); + }); + it('skips an over-cap output without reading it and keeps the harvest green', async () => { sessionListFiles.mockResolvedValue([ outputEntry('big.bin', 25 * 1024 * 1024), diff --git a/services/platform/backend/core/node_only/sandbox/session_exec.ts b/services/platform/backend/core/node_only/sandbox/session_exec.ts index 79bfe6ba71..1acf2627ae 100644 --- a/services/platform/backend/core/node_only/sandbox/session_exec.ts +++ b/services/platform/backend/core/node_only/sandbox/session_exec.ts @@ -272,6 +272,11 @@ export async function harvestSessionOutput( * bumps the recovery sweep would read a working settle as a dead chain * and re-attach a second one. */ execId?: string; + /** The exec that wrote the outputs had exited before this harvest, so + * nothing is left writing to the box: its first listing is final. Leave + * it unset when the exec may still be running (a reaped linger, a window + * cut on a live exec) — an empty listing is then re-read briefly. */ + execExited?: boolean; }, ): Promise<{ files: HarvestedOutputFile[]; @@ -316,11 +321,14 @@ export async function harvestSessionOutput( `sandbox output listing came back 404 for ${outputDir} — the session or its delivery box disappeared before harvest`, ); } - if (entries.length === 0) { - // An EMPTY listing right after an exec that reported success is usually a - // read-after-write race on the session's delivery box, not a script that - // wrote nothing (scripts that write nothing are rare and retrying costs - // milliseconds). Re-list briefly before accepting emptiness as truth. + if (entries.length === 0 && args.execExited !== true) { + // An EMPTY listing while the exec may still be running (reaped after its + // turn ended, still inside its kill grace) can be a write that has not + // landed yet. Re-list briefly before accepting emptiness as truth. An + // exec that already exited skips this: the listing reads the same + // filesystem its writer used, so once the writer is gone there is + // nothing left to land, and the 1.5 s of re-reads would only delay + // every turn that delivered no file. for (let attempt = 0; attempt < 3 && entries.length === 0; attempt++) { await new Promise((resolve) => setTimeout(resolve, 500)); const again = await sessionListFiles(sessionId, outputDir); diff --git a/services/platform/backend/core/tasks/agent_run_host.reap.test.ts b/services/platform/backend/core/tasks/agent_run_host.reap.test.ts index 2a0780dbb9..56a5f53a62 100644 --- a/services/platform/backend/core/tasks/agent_run_host.reap.test.ts +++ b/services/platform/backend/core/tasks/agent_run_host.reap.test.ts @@ -10,7 +10,10 @@ * new CLI launches on the same workspace and delivery box; * - a steer's restart cancels the old exec as a rotation, keeping what the * turn started outside its own processes for the restarted turn, while - * every other cancel (a Stop, a crash) ends everything. + * every other cancel (a Stop, a crash) ends everything; + * - the settle's harvest takes the first listing of a turn whose exec + * exited on its own, and re-reads an empty box after a reaped linger, + * whose processes may still be writing. */ import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -28,6 +31,10 @@ const io = vi.hoisted(() => ({ predecessorRunningPolls: 0, drainThrows: false, afterDrain: undefined as (() => void) | undefined, + /** A terminal window the drain answers instead of `running`. */ + terminal: undefined as Record | undefined, + /** Every directory the harvest listed, in order. */ + listings: [] as string[], })); vi.mock('../chat/external_turn_shared', async (importActual) => { @@ -51,6 +58,7 @@ vi.mock('../chat/external_turn_shared', async (importActual) => { ended: { finalText: 'Late completion', isError: false }, }; } + if (io.terminal !== undefined) return io.terminal; if (args.start !== undefined) { io.starts.push({ execId: args.execId, @@ -104,10 +112,17 @@ vi.mock('../node_only/sandbox/helpers/session_client', async (importActual) => { return { state: 'exited', exitCode: 137 }; }, sessionDeleteFiles: async () => undefined, - sessionListFiles: async () => [], + sessionListFiles: async (_sessionId: string, dir: string) => { + io.listings.push(dir); + return []; + }, sessionStageFiles: async () => ({ staged: [], skipped: [] }), }; }); +// The settle's harvest stores into the organization's own bucket. +vi.mock('../lib/helpers/org_slug', () => ({ + orgSlugFromIdOrNull: async () => 'acme', +})); vi.mock('../node_only/sandbox/agent_session', () => ({ ensureAgentSession: async () => ({ liveCreatedAt: 1000 }), })); @@ -233,6 +248,8 @@ beforeEach(() => { io.predecessorRunningPolls = 0; io.drainThrows = false; io.afterDrain = undefined; + io.terminal = undefined; + io.listings = []; vi.spyOn(console, 'warn').mockImplementation(() => {}); vi.spyOn(console, 'error').mockImplementation(() => {}); }); @@ -280,6 +297,46 @@ describe('drive window failure', () => { }); }); +describe('settle harvest', () => { + const reported = (exited: boolean) => ({ + kind: 'terminal', + text: 'Reviewed, nothing to change.', + timeline: [], + ended: { + type: 'turn-ended', + status: 'completed', + finalText: 'Reviewed, nothing to change.', + }, + exited, + }); + + it('takes the first empty listing of a turn whose exec exited on its own', async () => { + io.terminal = reported(true); + const run: RunState = { status: 'running', execId: 'exec-old' }; + const { ctx, mutations } = makeCtx(run); + + await driveTaskAgentTurnImpl(ctx, KEYS as never); + + expect(io.listings).toEqual(['/agent/output/task-1']); + expect( + mutations.some((m) => m.name.endsWith(':completeTaskAgentRun')), + ).toBe(true); + }); + + it('re-reads an empty delivery box after a reaped linger', async () => { + io.terminal = reported(false); + const run: RunState = { status: 'running', execId: 'exec-old' }; + const { ctx, mutations } = makeCtx(run); + + await driveTaskAgentTurnImpl(ctx, KEYS as never); + + expect(io.listings).toEqual(Array(4).fill('/agent/output/task-1')); + expect( + mutations.some((m) => m.name.endsWith(':completeTaskAgentRun')), + ).toBe(true); + }); +}); + describe('start after a harness switch', () => { it('reaps the predecessor and waits for it to be gone before launching', async () => { const run: RunState = { status: 'queued', execId: 'exec-new' }; diff --git a/services/platform/backend/core/tasks/agent_run_host.ts b/services/platform/backend/core/tasks/agent_run_host.ts index 29875c1160..c3f2096c3b 100644 --- a/services/platform/backend/core/tasks/agent_run_host.ts +++ b/services/platform/backend/core/tasks/agent_run_host.ts @@ -1984,6 +1984,7 @@ async function continueOrSettle( ...(ended?.usageTotals !== undefined ? { usageTotals: ended.usageTotals } : {}), + ...(window.exited ? { execExited: true } : {}), }); } @@ -2095,6 +2096,9 @@ async function settleTaskAgentTurn( retryAtMs?: number; /** The harness's own token totals, booked alongside the gateway spend. */ usageTotals?: { inputTokens: number; outputTokens: number }; + /** The turn's exec exited on its own before the settle, so the harvest + * reads a box nothing is still writing to. */ + execExited?: boolean; }, ): Promise { const current = await ctx.runQuery( @@ -2221,6 +2225,7 @@ async function settleTaskAgentTurn( sessionId: args.sessionId, outputDir: taskOutputDir(args.taskId), execId: args.execId, + ...(result.execExited === true ? { execExited: true } : {}), }); // Outputs the harvest could not bring back (caps, unreadable, storage // rejection) go into the settle comment — the reviewer must see WHAT is From d5d0dc555488cfad5986054f7b0424f0d2730ed1 Mon Sep 17 00:00:00 2001 From: yannickmonney Date: Thu, 8 Oct 2026 13:02:20 +0200 Subject: [PATCH 3/8] fix(platform): remove a staged Gemini sign-in when the turn ends A Gemini subscription turn stages the member's Google OAuth blob (with its refresh token) as .runtime/home/.gemini/oauth_creds.json, and nothing ever removed it. Every later exec in that session HOME - other tasks of the same standing agent session, member-confined runs, connector calls - could read another member's credential. The file now leaves the session with the turn: the task lane's settle and orphan reap, and the automation lane's settle (whoever won the finalize claim), question park and orphan reap delete it through the idempotent files/delete route. A turn superseded by a steer restart, or an automation run that moved on to a newer exec, leaves it to that exec, which staged its own. A Gemini turn that runs without the subscription removes a leftover before it starts, the backstop for a removal that failed. Removal is best-effort and logged; only harnesses with a staged-file subscription delivery (today Gemini) make the extra call. --- .../system/harnesses/gemini/harness.yml | 4 +- docs/de/platform/agents/harnesses.md | 2 +- docs/en/platform/agents/harnesses.md | 2 +- docs/fr/platform/agents/harnesses.md | 2 +- .../agent_host.empty_answer.test.ts | 40 +++++- .../backend/core/automations/agent_host.ts | 37 +++++- .../core/chat/external_turn_shared.test.ts | 114 +++++++++++++++++- .../backend/core/chat/external_turn_shared.ts | 60 +++++++++ .../core/tasks/agent_run_host.reap.test.ts | 54 ++++++++- .../backend/core/tasks/agent_run_host.ts | 27 +++++ 10 files changed, 328 insertions(+), 14 deletions(-) diff --git a/configs/platform/system/harnesses/gemini/harness.yml b/configs/platform/system/harnesses/gemini/harness.yml index c59f02e4af..5673d7b10a 100644 --- a/configs/platform/system/harnesses/gemini/harness.yml +++ b/configs/platform/system/harnesses/gemini/harness.yml @@ -139,7 +139,9 @@ exec: TALE_GATEWAY_TOKEN: '${gateway.token}' # A subscription-key credential (the Gemini CLI's Google-account sign-in / # Code Assist license) is the CLI's OAuth credentials blob, staged verbatim -# as its credentials file under the session HOME (/agent/.runtime/home). +# as its credentials file under the session HOME (/agent/.runtime/home). The +# platform removes the file when the turn settles, and before a turn that +# runs without the subscription, so no later exec in the session reads it. subscription: kind: staged-file path: .runtime/home/.gemini/oauth_creds.json diff --git a/docs/de/platform/agents/harnesses.md b/docs/de/platform/agents/harnesses.md index 16df95f1a5..8f905dfc55 100644 --- a/docs/de/platform/agents/harnesses.md +++ b/docs/de/platform/agents/harnesses.md @@ -37,7 +37,7 @@ Gemini CLI ist die Ausnahme: Tale setzt eine seiner Unterhaltungen nie fort. Ein Bei einem gespeicherten API-Schlüssel oder einer Deployment-Umgebungsvariable stellt Tale einen sitzungsgebundenen Gateway-Schlüssel bereit. Der ursprüngliche Modell-Provider-Schlüssel bleibt bei der Plattform. Gateway-Aufrufe werden gemessen und unterliegen den geltenden Ausgabenregeln. Bereits an andere laufende Durchläufe vergebene Beträge werden berücksichtigt. -Provider-Abonnements verwenden ihre unterstützte Agent-Laufzeit und erhalten den Abonnement-Zugang in der Sitzungsumgebung. Sie dienen weder als normale Chat-Zugangsdaten noch für inkompatible Agent-Laufzeiten, weil die Anbieter Abo-Tokens nur in ihrer eigenen Laufzeit erlauben; mehr dazu unter [KI-Anbieter](/de/platform/admin/providers#abos-in-aufgaben-nutzen-nicht-im-chat). Ihre direkten Aufrufe umgehen die Kostenmessung und Ausgabengrenzen des Tale-Gateways. Prüfe die Nutzung beim Abonnement-Provider. +Provider-Abonnements verwenden ihre unterstützte Agent-Laufzeit und erhalten den Abonnement-Zugang in der Sitzungsumgebung. Gemini ist die Ausnahme: Seine Google-Anmeldung wird für den Durchlauf als Datei im Home-Ordner der Sitzung abgelegt und am Ende des Durchlaufs wieder entfernt. Sie dienen weder als normale Chat-Zugangsdaten noch für inkompatible Agent-Laufzeiten, weil die Anbieter Abo-Tokens nur in ihrer eigenen Laufzeit erlauben; mehr dazu unter [KI-Anbieter](/de/platform/admin/providers#abos-in-aufgaben-nutzen-nicht-im-chat). Ihre direkten Aufrufe umgehen die Kostenmessung und Ausgabengrenzen des Tale-Gateways. Prüfe die Nutzung beim Abonnement-Provider. Diese Regeln für Modellzugänge bedeuten nicht, dass die Sandbox keinerlei Geheimnisse enthält. Ausdrücklich vergebene **Secrets** sowie ein Token für einen zugeordneten GitHub-Zugang können darin verfügbar sein. Vergib nur den für die Aufgabe nötigen Zugriff. diff --git a/docs/en/platform/agents/harnesses.md b/docs/en/platform/agents/harnesses.md index 7a23a9aa6b..3d9495d0c7 100644 --- a/docs/en/platform/agents/harnesses.md +++ b/docs/en/platform/agents/harnesses.md @@ -37,7 +37,7 @@ Gemini CLI is the exception: Tale never continues one of its conversations. A la With a stored API key or deployment-environment credential, Tale supplies a session-scoped gateway key. The original model-provider key stays at the platform. Calls through this gateway are metered and subject to the applicable spending rules, including allowance already assigned to other running turns. -Vendor subscriptions use their supported agent runtime and receive the subscription credential in the session environment. They cannot be used as ordinary chat credentials or with an incompatible agent runtime, because the vendors permit subscription tokens only in their own runtime; [AI providers](/platform/admin/providers#use-subscriptions-in-tasks-not-in-chat) explains this. Their direct provider calls bypass Tale’s gateway metering and spending caps; review usage with the subscription provider. +Vendor subscriptions use their supported agent runtime and receive the subscription credential in the session environment. Gemini is the exception: its Google sign-in is written to a file in the session’s home folder for the turn, and removed when the turn ends. They cannot be used as ordinary chat credentials or with an incompatible agent runtime, because the vendors permit subscription tokens only in their own runtime; [AI providers](/platform/admin/providers#use-subscriptions-in-tasks-not-in-chat) explains this. Their direct provider calls bypass Tale’s gateway metering and spending caps; review usage with the subscription provider. These model-credential rules do not mean the sandbox contains no secrets. Explicitly granted **Secrets**, and the token supplied for an equipped GitHub connection, can be available inside it. Grant only the access the task needs. diff --git a/docs/fr/platform/agents/harnesses.md b/docs/fr/platform/agents/harnesses.md index dfe2ef7458..7ba72c0a00 100644 --- a/docs/fr/platform/agents/harnesses.md +++ b/docs/fr/platform/agents/harnesses.md @@ -37,7 +37,7 @@ Gemini CLI fait exception : Tale ne poursuit jamais une de ses conversations. Un Avec une clé API stockée ou fournie par l’environnement du déploiement, Tale remet une clé de passerelle limitée à la session. La clé d’origine du fournisseur de modèle reste dans la plateforme. Les appels de passerelle sont mesurés et soumis aux règles de dépense applicables, en tenant compte des montants déjà attribués aux autres échanges en cours. -Les abonnements fournisseurs utilisent leur environnement d’agent compatible et reçoivent les identifiants d’abonnement dans l’environnement de la session. Ils ne servent pas d’identifiants de chat ordinaire et ne fonctionnent pas avec un environnement d’agent incompatible, car les fournisseurs n’autorisent les jetons d’abonnement que dans leur propre environnement ; la page [Fournisseurs IA](/fr/platform/admin/providers#utiliser-les-abonnements-dans-les-taches-pas-dans-le-chat) l’explique. Leurs appels directs échappent à la mesure et aux plafonds de la passerelle Tale. Examine la consommation auprès du fournisseur d’abonnement. +Les abonnements fournisseurs utilisent leur environnement d’agent compatible et reçoivent les identifiants d’abonnement dans l’environnement de la session. Gemini fait exception : sa connexion Google est écrite dans un fichier du dossier personnel de la session pour l’échange, puis supprimée à la fin de l’échange. Ils ne servent pas d’identifiants de chat ordinaire et ne fonctionnent pas avec un environnement d’agent incompatible, car les fournisseurs n’autorisent les jetons d’abonnement que dans leur propre environnement ; la page [Fournisseurs IA](/fr/platform/admin/providers#utiliser-les-abonnements-dans-les-taches-pas-dans-le-chat) l’explique. Leurs appels directs échappent à la mesure et aux plafonds de la passerelle Tale. Examine la consommation auprès du fournisseur d’abonnement. Ces règles sur les identifiants de modèle ne signifient pas que la sandbox ne contient aucun secret. Les **Secrets** explicitement accordés et le jeton d’un accès GitHub équipé peuvent y être disponibles. Limite ces accès aux besoins de la tâche. diff --git a/services/platform/backend/core/automations/agent_host.empty_answer.test.ts b/services/platform/backend/core/automations/agent_host.empty_answer.test.ts index 343f255549..d45da3c3a8 100644 --- a/services/platform/backend/core/automations/agent_host.empty_answer.test.ts +++ b/services/platform/backend/core/automations/agent_host.empty_answer.test.ts @@ -10,6 +10,10 @@ * workflow went on as if the agent had deliberately changed nothing. A turn * that only called a tool — the question tool included, which parks the run * instead — still ends as before. + * + * The same drive also takes a Gemini turn's staged subscription credential + * out of the run's session when the turn settles, and leaves it to a newer + * exec the run moved on to. */ import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -25,6 +29,8 @@ const io = vi.hoisted(() => ({ cancelled: [] as string[], /** What each settle asked the output harvest for. */ harvests: [] as Array>, + /** Every path set the session was asked to delete, in order. */ + deletes: [] as string[][], })); vi.mock('../node_only/sandbox/helpers/session_client', async (importActual) => { @@ -58,6 +64,10 @@ vi.mock('../node_only/sandbox/helpers/session_client', async (importActual) => { io.cancelled.push(execId); return true; }, + sessionDeleteFiles: async (_sessionId: string, paths: string[]) => { + io.deletes.push(paths); + return { deleted: paths, skipped: [] }; + }, }; }); vi.mock('../node_only/sandbox/session_exec', () => ({ @@ -99,7 +109,9 @@ interface Call { args: Record; } -function makeCtx(opts: { pendingAsk?: typeof ASK } = {}) { +function makeCtx( + opts: { pendingAsk?: typeof ASK; cursorExecId?: string } = {}, +) { const mutations: Call[] = []; const ctx = { runQuery: async (ref: unknown) => { @@ -110,7 +122,7 @@ function makeCtx(opts: { pendingAsk?: typeof ASK } = {}) { cursor: { node: KEYS.nodeId, agent: { - execId: KEYS.execId, + execId: opts.cursorExecId ?? KEYS.execId, sessionId: KEYS.sessionId, deadlineAt: KEYS.deadlineAt, providerSlug: KEYS.providerSlug, @@ -171,6 +183,7 @@ beforeEach(() => { io.stdout = ''; io.cancelled = []; io.harvests = []; + io.deletes = []; vi.spyOn(console, 'warn').mockImplementation(() => {}); vi.spyOn(console, 'error').mockImplementation(() => {}); }); @@ -330,3 +343,26 @@ describe('an automation agent turn', () => { }); }); }); + +describe('a Gemini automation turn’s staged subscription credential', () => { + const GEMINI = { ...KEYS, harness: 'gemini' }; + const CREDENTIAL = ['.runtime/home/.gemini/oauth_creds.json']; + + it('leaves the session when the turn settles', async () => { + const { ctx, mutations } = makeCtx(); + + await driveWorkflowAgentTurnImpl(ctx, GEMINI); + + expect(called(mutations, 'recordAgentTurnSettled')).toHaveLength(1); + expect(io.deletes).toEqual([CREDENTIAL]); + }); + + it('stays for the newer exec the run moved on to', async () => { + const { ctx } = makeCtx({ cursorExecId: 'exec-2' }); + + await driveWorkflowAgentTurnImpl(ctx, GEMINI); + + expect(io.cancelled).toEqual([KEYS.execId]); + expect(io.deletes).toEqual([]); + }); +}); diff --git a/services/platform/backend/core/automations/agent_host.ts b/services/platform/backend/core/automations/agent_host.ts index 2a8c2926f0..2e95fb7a74 100644 --- a/services/platform/backend/core/automations/agent_host.ts +++ b/services/platform/backend/core/automations/agent_host.ts @@ -48,6 +48,7 @@ import { harnessMountsMcp, harnessResumesConversations, isManagedHarness, + removeStagedSubscription, resolveHarnessTurnContextWindow, SKILLS_DIR, } from '../chat/external_turn_shared'; @@ -1721,6 +1722,23 @@ export function isWorkflowTurnLive( ); } +/** Whether a newer agent exec of a live run holds its session now (a retry + * of the node, or the next agent node): that exec stages its own + * subscription credential, so an older turn must leave the file alone. */ +function heldByNewerExec( + state: AgentCursorState | null, + execId: string, +): boolean { + const agent = state?.cursor?.agent; + return ( + state !== null && + LIVE_RUN_STATUSES.has(state.status) && + agent !== undefined && + agent.execId !== execId && + agent.result === undefined + ); +} + /** * Why a scheduled start must not run, or null when it may. Deliberately * LENIENT where `isWorkflowTurnLive` is strict: the kick enqueues the start @@ -1841,6 +1859,9 @@ export async function driveWorkflowAgentTurnImpl( execId: args.execId, status: 'cancelled', }); + if (!heldByNewerExec(state, args.execId)) { + await removeStagedSubscription(args.sessionId, args.harness); + } // The run went terminal while this turn was live, so the terminal // hooks' hibernate skipped past it — the op is terminal now. await ctx @@ -2572,6 +2593,8 @@ async function continueOrSettle( ? { exitCode: window.execResult.exitCode } : {}), }); + // The turn that answers the question stages its credential again. + await removeStagedSubscription(args.sessionId, args.harness); return; } if (pendingAsk !== null) { @@ -2678,23 +2701,27 @@ async function settleWorkflowAgentTurn( // write. The record has its own first-wins gate, so finishing the dead // winner's job is safe: when the result is already there this returns // without touching anything. - const state = await ctx.runQuery( - internal.automations.queries.readAgentCursor, - // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- carried verbatim from the turn's own args - { organizationId: args.organizationId, runId: args.runId as never }, - ); + const state = await readCursorState(ctx, args); const agent = state?.cursor?.agent; if ( agent === undefined || agent.execId !== args.execId || agent.result !== undefined ) { + // Settled by someone else (a cancel, the watchdog), but this turn's + // exec is over all the same. + if (!heldByNewerExec(state, args.execId)) { + await removeStagedSubscription(args.sessionId, args.harness); + } return; } console.warn( `[agent-host] finalize claim for ${args.execId} was burned with no recorded result — completing the dead settle's record`, ); } + // The turn is over: its staged subscription credential leaves the session + // with it. + await removeStagedSubscription(args.sessionId, args.harness); // The broker account that served this exec, when one did: a 429 cools it // down, and a 401 on it is the broker refreshing the account under the diff --git a/services/platform/backend/core/chat/external_turn_shared.test.ts b/services/platform/backend/core/chat/external_turn_shared.test.ts index 81b673ebaf..0b92ceeea5 100644 --- a/services/platform/backend/core/chat/external_turn_shared.test.ts +++ b/services/platform/backend/core/chat/external_turn_shared.test.ts @@ -62,13 +62,28 @@ const transport = vi.hoisted(() => ({ * the write itself failing. */ onEof: 'exit' as 'exit' | 'ignore' | 'fail', exitOnEof: undefined as (() => void) | undefined, + /** Every session operation in order, as `verb:detail` lines. */ + sessionOps: [] as string[], + deleteFailure: undefined as Error | undefined, + deleteSkips: [] as Array<{ path: string; reason: string }>, })); vi.mock('../node_only/sandbox/helpers/session_client', () => ({ SessionNotFoundError: class SessionNotFoundError extends Error {}, ExecReplayGapError: transport.Gap, ExecStreamProtocolError: class ExecStreamProtocolError extends Error {}, - sessionStageFiles: async () => ({ staged: [], skipped: [] }), + sessionStageFiles: async ( + _sessionId: string, + files: Array<{ path: string }>, + ) => { + transport.sessionOps.push(`stage:${files.map((f) => f.path).join(',')}`); + return { staged: [], skipped: [] }; + }, + sessionDeleteFiles: async (_sessionId: string, paths: string[]) => { + transport.sessionOps.push(`delete:${paths.join(',')}`); + if (transport.deleteFailure !== undefined) throw transport.deleteFailure; + return { deleted: paths, skipped: transport.deleteSkips }; + }, sessionGetExecCheckpoint: async () => { if (transport.checkpointAfterGap !== 'none') { await new Promise((resolve) => setTimeout(resolve, 1600)); @@ -120,6 +135,7 @@ vi.mock('../node_only/sandbox/helpers/session_client', () => ({ opts: { cursor: { lastSeq: number }; resumeSinceSeq?: number }, ) => { if (signal.aborted) throw signal.reason; + if (opts.resumeSinceSeq === undefined) transport.sessionOps.push('launch'); callbacks.onReplayStarted?.(); if (opts.resumeSinceSeq !== undefined) transport.resumedAt.push(opts.resumeSinceSeq); @@ -175,8 +191,11 @@ const { classifyHarnessEnd, harnessOutputTail, isSpendRefusal, + removeStagedSubscription, spendRefusalReason, } = await import('./external_turn_shared'); +const { SessionNotFoundError } = + await import('../node_only/sandbox/helpers/session_client'); /** One NDJSON stream from event objects. */ function ndjson(lines: Array>): string { @@ -250,6 +269,9 @@ describe('drainHarnessWindow end-of-turn rules', () => { transport.stdinWrites = []; transport.onEof = 'exit'; transport.exitOnEof = undefined; + transport.sessionOps = []; + transport.deleteFailure = undefined; + transport.deleteSkips = []; vi.spyOn(console, 'warn').mockImplementation(() => {}); }); @@ -1120,6 +1142,93 @@ describe('drainHarnessWindow end-of-turn rules', () => { }); }); +/** Where the Gemini harness stages a member's Google sign-in. */ +const GEMINI_CREDENTIAL = '.runtime/home/.gemini/oauth_creds.json'; + +describe('a staged subscription credential', () => { + beforeEach(() => { + transport.replayComplete = true; + transport.stdout = ''; + transport.cancelled = []; + transport.exitAfterStdout = true; + transport.checkpoint = null; + transport.sessionOps = []; + transport.deleteFailure = undefined; + transport.deleteSkips = []; + vi.spyOn(console, 'warn').mockImplementation(() => {}); + }); + + const geminiStart = (stagedFiles: Array<{ path: string; content: string }>) => + ({ + argv: ['gemini'], + env: {}, + cwd: '/agent/workspace', + stagedFiles, + }) as const; + + it('is removed before a Gemini turn that runs without it can start', async () => { + await drainHarnessWindow({ + sessionId: 'sandbox', + execId: 'gemini-managed', + harness: 'gemini', + start: geminiStart([ + { path: '.runtime/home/.gemini/settings.json', content: '{}' }, + ]), + }); + expect(transport.sessionOps).toEqual([ + `delete:${GEMINI_CREDENTIAL}`, + 'stage:.runtime/home/.gemini/settings.json', + 'launch', + ]); + }); + + it('stays for the Gemini turn that stages it', async () => { + await drainHarnessWindow({ + sessionId: 'sandbox', + execId: 'gemini-subscription', + harness: 'gemini', + start: geminiStart([ + { path: GEMINI_CREDENTIAL, content: '{"refresh_token":"member"}' }, + ]), + }); + expect(transport.sessionOps).toEqual([ + `stage:${GEMINI_CREDENTIAL}`, + 'launch', + ]); + }); + + it('is never looked for on a harness that takes its subscription through the environment', async () => { + transport.stdout = readFixture('pi', 'shell-turn'); + await drainHarnessWindow({ + sessionId: 'sandbox', + execId: 'claude-turn', + harness: 'claude-code', + start: { argv: ['claude'], env: {}, cwd: '/agent/workspace' }, + }); + await removeStagedSubscription('sandbox', 'codex'); + expect(transport.sessionOps).toEqual(['launch']); + }); + + it('is removed at a settle, and a failed removal is logged, never thrown', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + warn.mockClear(); + await removeStagedSubscription('sandbox', 'gemini'); + expect(transport.sessionOps).toEqual([`delete:${GEMINI_CREDENTIAL}`]); + + transport.deleteSkips = [{ path: GEMINI_CREDENTIAL, reason: 'EACCES' }]; + await removeStagedSubscription('sandbox', 'gemini'); + transport.deleteSkips = []; + transport.deleteFailure = new Error('spawner unreachable'); + await removeStagedSubscription('sandbox', 'gemini'); + expect(warn).toHaveBeenCalledTimes(2); + + // A session that is gone took the credential with it: nothing to log. + transport.deleteFailure = new SessionNotFoundError('sandbox'); + await removeStagedSubscription('sandbox', 'gemini'); + expect(warn).toHaveBeenCalledTimes(2); + }); +}); + /** The reason an empty answer settles with — the words a run shows. */ const EMPTY_ANSWER = 'The model returned an empty answer, so the agent did nothing this turn.'; @@ -1176,6 +1285,9 @@ describe('classifyHarnessEnd', () => { transport.stdinWrites = []; transport.onEof = 'exit'; transport.exitOnEof = undefined; + transport.sessionOps = []; + transport.deleteFailure = undefined; + transport.deleteSkips = []; vi.spyOn(console, 'warn').mockImplementation(() => {}); }); diff --git a/services/platform/backend/core/chat/external_turn_shared.ts b/services/platform/backend/core/chat/external_turn_shared.ts index b4ad22d363..eee0af773d 100644 --- a/services/platform/backend/core/chat/external_turn_shared.ts +++ b/services/platform/backend/core/chat/external_turn_shared.ts @@ -46,6 +46,7 @@ import { ExecStreamProtocolError, SessionNotFoundError, sessionCancelExec, + sessionDeleteFiles, sessionGetExecCheckpoint, sessionPutExecCheckpoint, sessionWriteExecStdin, @@ -127,6 +128,50 @@ function harnessHoldsStdin(harness: string): boolean { return def?.exec.stdin.mode === 'ndjson-user-message'; } +/** The session-relative file a harness's subscription credential is staged + * to (the `staged-file` delivery: Gemini's OAuth credentials under the + * session HOME), when the harness delivers it that way. */ +function stagedSubscriptionPath(harness: string): string | undefined { + if (!isHarnessSlug(harness)) return undefined; + const def = loadHarnesses().find((h) => h.slug === harness); + return def?.subscription?.kind === 'staged-file' + ? def.subscription.path + : undefined; +} + +/** + * Remove a harness's staged subscription credential from the session. The + * file sits in the session HOME every later exec shares — another task's + * turn, a member-confined run, a connector call — so a member's refresh + * token must leave with the turn that was handed it, and must not be there + * for a turn that runs without it. A no-op for a harness that takes its + * subscription through the environment. Best-effort: a failure is logged, + * and the next turn of the harness that does not stage the credential + * removes it again before it starts. + */ +export async function removeStagedSubscription( + sessionId: string, + harness: string, +): Promise { + const path = stagedSubscriptionPath(harness); + if (path === undefined) return; + try { + const removed = await sessionDeleteFiles(sessionId, [path]); + for (const skipped of removed.skipped) { + console.warn( + `[harness-turn] ${sessionId}: the staged subscription credential ${skipped.path} could not be removed: ${skipped.reason}`, + ); + } + } catch (err) { + // A session that is gone took its HOME, and the credential, with it. + if (err instanceof SessionNotFoundError) return; + console.warn( + `[harness-turn] ${sessionId}: removing the staged subscription credential ${path} failed:`, + err, + ); + } +} + /** Whether a harness mounts MCP servers — and so the platform bridge every * workspace tool rides. A harness whose YAML declares `capabilities.mcp: * false` (Pi, Hermes, Cursor) can call none of them, so a tool that exists @@ -716,6 +761,21 @@ export async function drainHarnessWindow(args: { timeoutMs: EXTERNAL_TURN_DEADLINE_MS, }; + // A turn that runs without the subscription must not find an earlier + // turn's staged credential in the session HOME (its settle's removal is + // best-effort): take it out before this exec can read it. + if (args.start !== undefined) { + const credentialPath = stagedSubscriptionPath(args.harness); + if ( + credentialPath !== undefined && + !(args.start.stagedFiles ?? []).some( + (file) => file.path === credentialPath, + ) + ) { + await removeStagedSubscription(args.sessionId, args.harness); + } + } + // On the start window we STAGE the exec's input files, then start it; drain // windows restore parser state and continue after its acknowledged cursor. // An old runtime without checkpoints retains the seq-0 compatibility lane. diff --git a/services/platform/backend/core/tasks/agent_run_host.reap.test.ts b/services/platform/backend/core/tasks/agent_run_host.reap.test.ts index 56a5f53a62..2c406c8e5f 100644 --- a/services/platform/backend/core/tasks/agent_run_host.reap.test.ts +++ b/services/platform/backend/core/tasks/agent_run_host.reap.test.ts @@ -13,7 +13,10 @@ * every other cancel (a Stop, a crash) ends everything; * - the settle's harvest takes the first listing of a turn whose exec * exited on its own, and re-reads an empty box after a reaped linger, - * whose processes may still be writing. + * whose processes may still be writing; + * - a Gemini turn's staged subscription credential leaves the session when + * the turn settles or is orphaned, unless a steer moved the run onto a + * newer exec that staged its own. */ import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -35,6 +38,8 @@ const io = vi.hoisted(() => ({ terminal: undefined as Record | undefined, /** Every directory the harvest listed, in order. */ listings: [] as string[], + /** Every path set the session was asked to delete, in order. */ + deletes: [] as string[][], })); vi.mock('../chat/external_turn_shared', async (importActual) => { @@ -111,7 +116,10 @@ vi.mock('../node_only/sandbox/helpers/session_client', async (importActual) => { } return { state: 'exited', exitCode: 137 }; }, - sessionDeleteFiles: async () => undefined, + sessionDeleteFiles: async (_sessionId: string, paths: string[]) => { + io.deletes.push(paths); + return { deleted: paths, skipped: [] }; + }, sessionListFiles: async (_sessionId: string, dir: string) => { io.listings.push(dir); return []; @@ -250,6 +258,7 @@ beforeEach(() => { io.afterDrain = undefined; io.terminal = undefined; io.listings = []; + io.deletes = []; vi.spyOn(console, 'warn').mockImplementation(() => {}); vi.spyOn(console, 'error').mockImplementation(() => {}); }); @@ -337,6 +346,47 @@ describe('settle harvest', () => { }); }); +describe('a Gemini turn’s staged subscription credential', () => { + const GEMINI = { ...KEYS, harness: 'gemini' }; + const CREDENTIAL = ['.runtime/home/.gemini/oauth_creds.json']; + + it('leaves the session when the turn settles', async () => { + io.terminal = { + kind: 'terminal', + text: 'Done.', + timeline: [], + ended: { type: 'turn-ended', status: 'completed', finalText: 'Done.' }, + exited: true, + }; + const run: RunState = { status: 'running', execId: 'exec-old' }; + const { ctx } = makeCtx(run); + + await driveTaskAgentTurnImpl(ctx, GEMINI as never); + + expect(io.deletes).toEqual([CREDENTIAL]); + }); + + it('leaves the session when a Stop orphans the turn', async () => { + const run: RunState = { status: 'cancelled', execId: 'exec-old' }; + const { ctx } = makeCtx(run); + + await driveTaskAgentTurnImpl(ctx, GEMINI as never); + + expect(io.cancels).toEqual(['exec-old']); + expect(io.deletes).toEqual([CREDENTIAL]); + }); + + it('stays for the exec a steer restarted the run onto', async () => { + const run: RunState = { status: 'running', execId: 'exec-rotated' }; + const { ctx } = makeCtx(run); + + await driveTaskAgentTurnImpl(ctx, GEMINI as never); + + expect(io.cancels).toEqual(['exec-old']); + expect(io.deletes).toEqual([]); + }); +}); + describe('start after a harness switch', () => { it('reaps the predecessor and waits for it to be gone before launching', async () => { const run: RunState = { status: 'queued', execId: 'exec-new' }; diff --git a/services/platform/backend/core/tasks/agent_run_host.ts b/services/platform/backend/core/tasks/agent_run_host.ts index c3f2096c3b..2312f6895a 100644 --- a/services/platform/backend/core/tasks/agent_run_host.ts +++ b/services/platform/backend/core/tasks/agent_run_host.ts @@ -36,6 +36,7 @@ import { connectorsBridgeUrlForSessions, harnessMountsMcp, harnessResumesConversations, + removeStagedSubscription, resolveHarnessTurnContextWindow, type ExternalTurnServing, } from '../chat/external_turn_shared'; @@ -1657,6 +1658,9 @@ export async function startTaskAgentTurnImpl( releaseErr, ); }); + // The refused exec never ran, but its inputs were staged: the + // start that gets room stages its credential again. + await removeStagedSubscription(args.sessionId, args.harness); } return null; } @@ -1708,6 +1712,9 @@ export async function driveTaskAgentTurnImpl( execId: args.execId, status: 'cancelled', }); + if (!heldByAnotherExec(run, args.execId)) { + await removeStagedSubscription(args.sessionId, args.harness); + } await releaseProjectAgentSlotAfterSettle(ctx, args); return null; } @@ -2068,6 +2075,20 @@ export function buildSettleComments(args: { }; } +/** Whether a live run has moved on to another exec of the same session (a + * steer's restart): that exec staged its own subscription credential, so a + * superseded turn must leave the file alone. */ +function heldByAnotherExec( + run: { status: string; execId: string } | null, + execId: string, +): boolean { + return ( + run !== null && + run.execId !== execId && + (run.status === 'queued' || run.status === 'running') + ); +} + /** * Settle exactly once (the session-op finalize claim elects the winner): * harvest `/agent/output`, then on success post the agent's report as a task @@ -2119,6 +2140,9 @@ async function settleTaskAgentTurn( execId: args.execId, status: 'cancelled', }); + if (!heldByAnotherExec(current, args.execId)) { + await removeStagedSubscription(args.sessionId, args.harness); + } await releaseProjectAgentSlotAfterSettle(ctx, args); return; } @@ -2131,6 +2155,9 @@ async function settleTaskAgentTurn( ? { usageTotals: result.usageTotals } : {}), }); + // The turn is over, whoever won the finalize claim: its staged + // subscription credential leaves the session with it. + await removeStagedSubscription(args.sessionId, args.harness); if (!release.won) { // The finalize claim keys on the op row — a start that died BEFORE // writing one (model unresolvable, spawner error, staging failure) loses From e1a9017891f2e6542a0a2f0fb2bcb9a7a3aff715 Mon Sep 17 00:00:00 2001 From: yannickmonney Date: Thu, 8 Oct 2026 13:05:41 +0200 Subject: [PATCH 4/8] fix(platform): reap orphaned crawler render sessions within minutes A render session was created with the deployment's idle window (30 min by default). When the crawl action died between create and destroy (a killed action, a redeploy), the orphan held one of the organization's 2 render slots for that whole half hour, and the scan parked on capacity it did not have. Render creates now ask for a 3-minute idle window. A batch never sits idle that long: its exec (capped at 4 min by the crawl) keeps the session from being reaped whatever its length, and the create, staging, read-back and destroy around it are seconds apart. The spawner clamps the request to its own ceiling, so a deployment with a shorter idle window keeps it. --- .../core/node_only/sandbox/render_fetch.test.ts | 16 ++++++++++++++++ .../core/node_only/sandbox/render_fetch.ts | 11 +++++++++++ 2 files changed, 27 insertions(+) diff --git a/services/platform/backend/core/node_only/sandbox/render_fetch.test.ts b/services/platform/backend/core/node_only/sandbox/render_fetch.test.ts index 8d6a924099..5c37489448 100644 --- a/services/platform/backend/core/node_only/sandbox/render_fetch.test.ts +++ b/services/platform/backend/core/node_only/sandbox/render_fetch.test.ts @@ -288,6 +288,22 @@ describe('renderUrlsInSandbox — the session lifecycle', () => { expect(spawner.sessionDestroyIfIdle).not.toHaveBeenCalled(); }); + it('asks for a short idle window, so a batch that died before its destroy frees its render slot within minutes', async () => { + const run = renderRun('row_1'); + scriptSpawner(run.events); + + await run.render(); + + // Three minutes: well above the seconds between a batch's steps, its + // exec holding the session whatever its length. + expect(spawner.sessionCreate).toHaveBeenCalledWith( + expect.objectContaining({ + placement: 'server', + idleTimeoutMs: 3 * 60_000, + }), + ); + }); + it('a failure after the create still tears the session down', async () => { const run = renderRun('row_1'); scriptSpawner(run.events); diff --git a/services/platform/backend/core/node_only/sandbox/render_fetch.ts b/services/platform/backend/core/node_only/sandbox/render_fetch.ts index 83d4d2ab59..b000b452d5 100644 --- a/services/platform/backend/core/node_only/sandbox/render_fetch.ts +++ b/services/platform/backend/core/node_only/sandbox/render_fetch.ts @@ -68,6 +68,16 @@ const RENDER_IDLE_TIMEOUT_MS = 5_000; /** The worker stops STARTING pages this far before the exec hard kill, so * it always exits cleanly with its partial results on disk. */ const WORKER_EXIT_MARGIN_MS = 20_000; +/** How long a render session may sit with nothing running before the + * spawner reaps it. A batch keeps it busy throughout: its exec (at most + * four minutes, the crawl's own cap) holds the session whatever its length, + * and the create, staging, read-back and destroy around it are seconds + * apart. The window only matters for a session whose batch died before its + * destroy (the action was killed): such an orphan holds one of the + * organization's render slots until the spawner reaps it, which would + * otherwise take the deployment's whole idle window (30 minutes by + * default). */ +const RENDER_SESSION_IDLE_MS = 3 * 60_000; /** Rendered-DOM caps in BYTES: per page, and for the whole output file — * `sessionReadFile` serves at most 20MB, and an output the host cannot read * back fails the batch deterministically (the crawl would retry the same @@ -255,6 +265,7 @@ export async function renderUrlsInSandbox( // Knowledge crawling is the deployment's own work: renders never // leave the server for an organization's device. placement: 'server', + idleTimeoutMs: RENDER_SESSION_IDLE_MS, }); created = true; } catch (error) { From 509b97c45d3bda0fcc318e4d018e3323d0b2d7ea Mon Sep 17 00:00:00 2001 From: yannickmonney Date: Thu, 8 Oct 2026 13:11:28 +0200 Subject: [PATCH 5/8] fix(platform): retry transient spawner answers on acquire and create The spawner answers 503 session_unavailable with retry-after: 1 while a sibling create of the id is in flight, when its docker ps listing times out, or while a container is still starting; its docs say callers retry. The platform's acquire threw a generic error on it and create retried only the deploy-time 503 draining, so the turn's start failed on an answer that meant "ask again in a second". A spawner restart (ECONNREFUSED, ECONNRESET, EAI_AGAIN) failed starts the same way as SpawnerUnreachableError. Acquire and create now ask again at the spawner's retry-after for session_unavailable, and wait out a refused, reset or unresolved connection on a doubling backoff (250 ms to 5 s), both inside one 20 s budget per call; past it the call fails with the error it fails with today. Any other 503 or connection failure, and a timed-out request, still fail at once, and a 429 still parks the start on capacity. --- .../helpers/session_client.activity.test.ts | 106 ++++++++++++ .../sandbox/helpers/session_client.test.ts | 90 ++++++++++ .../sandbox/helpers/session_client.ts | 162 ++++++++++++++++-- services/sandbox/docs/sessions.md | 4 +- 4 files changed, 344 insertions(+), 18 deletions(-) diff --git a/services/platform/backend/core/node_only/sandbox/helpers/session_client.activity.test.ts b/services/platform/backend/core/node_only/sandbox/helpers/session_client.activity.test.ts index 547a3b8862..0455c2765f 100644 --- a/services/platform/backend/core/node_only/sandbox/helpers/session_client.activity.test.ts +++ b/services/platform/backend/core/node_only/sandbox/helpers/session_client.activity.test.ts @@ -10,6 +10,7 @@ import { sessionReleaseIdle, sessionReleaseTicket, SpawnerBusyError, + SpawnerUnreachableError, } from './session_client.ts'; beforeEach(() => { @@ -17,10 +18,28 @@ beforeEach(() => { vi.stubEnv('SANDBOX_URL', 'http://sandbox.test'); }); afterEach(() => { + vi.useRealTimers(); vi.unstubAllGlobals(); vi.unstubAllEnvs(); }); +/** The spawner's answer while a create of the id is in flight, its backend + * listing timed out, or the container is still starting. */ +function sessionUnavailable(): Response { + return Response.json( + { error: 'session_unavailable' }, + { status: 503, headers: { 'retry-after': '1' } }, + ); +} + +/** What the runtime's fetch throws when the spawner does not take the + * connection: the syscall's code rides the failure's `cause`. */ +function connectionFailure(code: string): TypeError { + return new TypeError('fetch failed', { + cause: Object.assign(new Error(`connect ${code} 10.0.0.7:8003`), { code }), + }); +} + describe('runtime acquisition and release transport', () => { it('forwards watchdog cancellation into the signed cleanup request', async () => { const fetcher = vi @@ -238,3 +257,90 @@ it('reads liveness and runtime pin in one GET, preserving unknown older shapes', expect(await sessionObserve('gone')).toBeNull(); expect(fetcher).toHaveBeenCalledTimes(4); }); + +describe('a warm acquire across transient spawner answers', () => { + it('asks again at the retry-after while the spawner answers session_unavailable', async () => { + vi.useFakeTimers(); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); + const fetcher = vi + .fn() + .mockResolvedValueOnce(sessionUnavailable()) + .mockResolvedValueOnce(sessionUnavailable()) + .mockResolvedValueOnce(Response.json({ generation: 'use-1' })); + vi.stubGlobal('fetch', fetcher); + + const acquired = sessionAcquire('session-1'); + await vi.advanceTimersByTimeAsync(999); + expect(fetcher).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1_001); + + await expect(acquired).resolves.toBe(true); + expect(fetcher).toHaveBeenCalledTimes(3); + }); + + it('fails once session_unavailable outlasts its 20-second budget', async () => { + vi.useFakeTimers(); + const fetcher = vi + .fn() + .mockImplementation(async () => sessionUnavailable()); + vi.stubGlobal('fetch', fetcher); + + const acquired = sessionAcquire('session-1').catch( + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(19_000); + expect(fetcher).toHaveBeenCalledTimes(20); + await vi.advanceTimersByTimeAsync(1_000); + + const error = await acquired; + expect(error).toBeInstanceOf(Error); + expect(String(error)).toMatch(/Sandbox acquisition unavailable \(503\)/); + expect(fetcher).toHaveBeenCalledTimes(21); + }); + + it.each(['ECONNREFUSED', 'ECONNRESET', 'EAI_AGAIN'])( + 'waits out a restarting spawner that answers %s', + async (code) => { + vi.useFakeTimers(); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); + const fetcher = vi + .fn() + .mockRejectedValueOnce(connectionFailure(code)) + .mockRejectedValueOnce(connectionFailure(code)) + .mockResolvedValueOnce(Response.json({ generation: 'use-1' })); + vi.stubGlobal('fetch', fetcher); + + const acquired = sessionAcquire('session-1'); + // A doubling backoff: 250 ms, then 500 ms. + await vi.advanceTimersByTimeAsync(750); + + await expect(acquired).resolves.toBe(true); + expect(fetcher).toHaveBeenCalledTimes(3); + }, + ); + + it('fails a spawner that stays away past the budget, and any other connection failure at once', async () => { + vi.useFakeTimers(); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); + const refused = vi + .fn() + .mockRejectedValue(connectionFailure('ECONNREFUSED')); + vi.stubGlobal('fetch', refused); + const acquired = sessionAcquire('session-1').catch( + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(20_000); + expect(await acquired).toBeInstanceOf(SpawnerUnreachableError); + // 250 + 500 + 1000 + 2000 + 4000 + 5000 + 5000 ms of waits fit in 20 s. + expect(refused).toHaveBeenCalledTimes(8); + + const misnamed = vi + .fn() + .mockRejectedValue(connectionFailure('ENOTFOUND')); + vi.stubGlobal('fetch', misnamed); + await expect(sessionAcquire('session-1')).rejects.toBeInstanceOf( + SpawnerUnreachableError, + ); + expect(misnamed).toHaveBeenCalledTimes(1); + }); +}); diff --git a/services/platform/backend/core/node_only/sandbox/helpers/session_client.test.ts b/services/platform/backend/core/node_only/sandbox/helpers/session_client.test.ts index 5c3209493c..4090438671 100644 --- a/services/platform/backend/core/node_only/sandbox/helpers/session_client.test.ts +++ b/services/platform/backend/core/node_only/sandbox/helpers/session_client.test.ts @@ -1218,6 +1218,96 @@ describe('sessionCreate drain-retry', () => { }, 10_000); }); +describe('sessionCreate across transient spawner answers', () => { + afterEach(() => { + vi.useRealTimers(); + }); + const create = () => + sessionCreate({ + sessionId: 'ses-t', + organizationId: 'org-1', + profile: 'agent', + }); + + test('asks again at the retry-after while a create of the id is in flight', async () => { + vi.useFakeTimers(); + let n = 0; + // oxlint-disable-next-line typescript-eslint/no-explicit-any + globalThis.fetch = (async () => { + n += 1; + return n === 1 + ? new Response(JSON.stringify({ error: 'session_unavailable' }), { + status: 503, + headers: { 'retry-after': '1' }, + }) + : createdResponse('ses-t'); + // oxlint-disable-next-line typescript-eslint/no-explicit-any + }) as any; + + const created = create(); + await vi.advanceTimersByTimeAsync(1_000); + + expect((await created).session.sessionId).toBe('ses-t'); + expect(n).toBe(2); + }); + + test('waits out a spawner restart that refuses the connection', async () => { + vi.useFakeTimers(); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); + let n = 0; + // oxlint-disable-next-line typescript-eslint/no-explicit-any + globalThis.fetch = (async () => { + n += 1; + if (n === 1) + throw new TypeError('fetch failed', { + cause: Object.assign(new Error('connect ECONNREFUSED'), { + code: 'ECONNREFUSED', + }), + }); + return createdResponse('ses-t'); + // oxlint-disable-next-line typescript-eslint/no-explicit-any + }) as any; + + const created = create(); + await vi.advanceTimersByTimeAsync(250); + + expect((await created).session.sessionId).toBe('ses-t'); + expect(n).toBe(2); + }); + + test('fails once session_unavailable outlasts the budget, and on any other 503 at once', async () => { + vi.useFakeTimers(); + let n = 0; + // oxlint-disable-next-line typescript-eslint/no-explicit-any + globalThis.fetch = (async () => { + n += 1; + return new Response(JSON.stringify({ error: 'session_unavailable' }), { + status: 503, + headers: { 'retry-after': '1' }, + }); + // oxlint-disable-next-line typescript-eslint/no-explicit-any + }) as any; + const created = create().catch((error: unknown) => error); + await vi.advanceTimersByTimeAsync(20_000); + expect(String(await created)).toMatch( + /sandbox session create failed \(503\)/, + ); + expect(n).toBe(21); + + n = 0; + // oxlint-disable-next-line typescript-eslint/no-explicit-any + globalThis.fetch = (async () => { + n += 1; + return new Response(JSON.stringify({ error: 'host_unhealthy' }), { + status: 503, + }); + // oxlint-disable-next-line typescript-eslint/no-explicit-any + }) as any; + await expect(create()).rejects.toThrow(/host_unhealthy/); + expect(n).toBe(1); + }); +}); + describe('sessionCreate at host capacity', () => { function refuse(body: string, retryAfter?: string): void { // oxlint-disable-next-line typescript-eslint/no-explicit-any diff --git a/services/platform/backend/core/node_only/sandbox/helpers/session_client.ts b/services/platform/backend/core/node_only/sandbox/helpers/session_client.ts index a809d002c6..fce5e947f9 100644 --- a/services/platform/backend/core/node_only/sandbox/helpers/session_client.ts +++ b/services/platform/backend/core/node_only/sandbox/helpers/session_client.ts @@ -271,6 +271,102 @@ function parseRetryAfterMs(res: Response): number | undefined { return Number.isFinite(seconds) && seconds >= 0 ? seconds * 1000 : undefined; } +/** How long one acquire or create keeps asking again after answers that say + * "in a moment": a 503 `session_unavailable` (a sibling create of the id in + * flight, a backend listing that timed out, a container still starting) and + * a connection the spawner did not take (it is restarting). Long enough for + * a spawner restart or a slow `docker ps`; short against the turn it delays, + * so a spawner that stays away still fails the start. */ +const SPAWNER_TRANSIENT_RETRY_BUDGET_MS = 20_000; +/** The first wait after a refused connection; it doubles per retry. */ +const SPAWNER_TRANSIENT_RETRY_MIN_MS = 250; +/** The longest single wait, whatever the backoff or a `retry-after` says. */ +const SPAWNER_TRANSIENT_RETRY_MAX_MS = 5_000; + +/** The connection failures that say the spawner is not there right now + * rather than that the request went wrong: refused (restarting), reset + * (killed mid-connect) and a name lookup that timed out (its DNS entry is + * being replaced). */ +const TRANSIENT_CONNECTION_CODES: ReadonlySet = new Set([ + 'ECONNREFUSED', + 'ECONNRESET', + 'EAI_AGAIN', +]); + +function errorCodeOf(error: unknown): string | undefined { + if (typeof error !== 'object' || error === null || !('code' in error)) + return undefined; + return typeof error.code === 'string' ? error.code : undefined; +} + +/** Whether a call failed because the spawner did not take the connection + * (see {@link TRANSIENT_CONNECTION_CODES}). The runtime's fetch reports the + * syscall on the failure's `cause`, or on each of an `AggregateError`'s + * errors when it tried several addresses. */ +function isTransientConnectionFailure(error: unknown): boolean { + if (!(error instanceof SpawnerUnreachableError)) return false; + const fetchFailure = error.cause; + const syscall = + fetchFailure instanceof Error ? fetchFailure.cause : undefined; + const codes = + syscall instanceof AggregateError + ? syscall.errors.map(errorCodeOf) + : [errorCodeOf(syscall)]; + return codes.some( + (code) => code !== undefined && TRANSIENT_CONNECTION_CODES.has(code), + ); +} + +/** One acquire's or create's allowance of retries of transient spawner + * answers ({@link SPAWNER_TRANSIENT_RETRY_BUDGET_MS}). */ +class SpawnerRetryBudget { + private readonly deadline = Date.now() + SPAWNER_TRANSIENT_RETRY_BUDGET_MS; + private retries = 0; + + /** How long to wait before asking again — the spawner's own hint when it + * gave one, else a doubling backoff — or undefined once that wait would + * end past the budget. */ + nextWaitMs(hintMs: number | undefined): number | undefined { + const backoff = SPAWNER_TRANSIENT_RETRY_MIN_MS * 2 ** this.retries; + const waitMs = Math.min( + SPAWNER_TRANSIENT_RETRY_MAX_MS, + Math.max(SPAWNER_TRANSIENT_RETRY_MIN_MS, hintMs ?? backoff), + ); + if (Date.now() + waitMs > this.deadline) return undefined; + this.retries += 1; + return waitMs; + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** Send one acquire or create request, waiting out a refused connection + * within the call's retry budget; any other failure, or one past the + * budget, is thrown as it came. */ +async function spawnerFetchRetryingConnection( + budget: SpawnerRetryBudget, + method: string, + path: string, + opts: () => { body?: string; signal?: AbortSignal }, +): Promise { + for (;;) { + try { + return await spawnerFetch(method, path, opts()); + } catch (error) { + const waitMs = isTransientConnectionFailure(error) + ? budget.nextWaitMs(undefined) + : undefined; + if (waitMs === undefined) throw error; + console.warn( + `[sandbox] ${method} ${path}: the spawner did not take the connection; asking again in ${waitMs} ms`, + ); + await sleep(waitMs); + } + } +} + /** The exec SSE went silent past the idle-read deadline (no events AND no * keepalive — a half-open/wedged connection). The resilient drain re-attaches * via sinceSeq WITHOUT consuming its consecutive-failure budget: a genuinely @@ -486,12 +582,29 @@ export async function sandboxDeviceDisconnect( * finishes first; a definitive 404 then allows the caller to recreate it. */ export async function sessionAcquire(sessionId: string): Promise { return traceSandboxPhase('acquire', async () => { - const response = await spawnerFetch( - 'POST', - `/v1/sessions/${encodeURIComponent(sessionId)}/acquire`, - { signal: AbortSignal.timeout(15_000) }, - ); - await throwIfDeviceOffline(response); + const path = `/v1/sessions/${encodeURIComponent(sessionId)}/acquire`; + const budget = new SpawnerRetryBudget(); + let response: Response; + for (;;) { + response = await spawnerFetchRetryingConnection( + budget, + 'POST', + path, + () => ({ signal: AbortSignal.timeout(15_000) }), + ); + await throwIfDeviceOffline(response); + // `session_unavailable`: a create of the id is in flight, the backend + // listing timed out, or the container is still starting — the + // spawner says when to ask again (`retry-after`). + if (response.status !== 503) break; + const body = await safeText(response); + const waitMs = body.includes('session_unavailable') + ? budget.nextWaitMs(parseRetryAfterMs(response)) + : undefined; + if (waitMs === undefined) + throw new Error(`Sandbox acquisition unavailable (503): ${body}`); + await sleep(waitMs); + } if (response.status === 429) throw await spawnerBusyErrorOf(response); if (response.status === 404) { // Rolling upgrade: an old spawner has neither activity routes nor the @@ -599,28 +712,43 @@ async function createSession( ): Promise { const path = '/v1/sessions'; const bodyJson = JSON.stringify(body); + const budget = new SpawnerRetryBudget(); + let drainRetries = 0; // Re-sign per attempt: each retry needs a fresh timestamp (clock-skew window) // and a fresh nonce (spawner replay cache) — see signedHeaders. - for (let attempt = 0; ; attempt++) { - const res = await spawnerFetch('POST', path, { - body: bodyJson, - signal: AbortSignal.timeout(CREATE_TIMEOUT_MS), - }); + for (;;) { + const res = await spawnerFetchRetryingConnection( + budget, + 'POST', + path, + () => ({ + body: bodyJson, + signal: AbortSignal.timeout(CREATE_TIMEOUT_MS), + }), + ); if (res.status === 409) throw new SessionDuplicateError(body.sessionId); if (res.status === 429) throw await spawnerBusyErrorOf(res); // 503 "draining": the targeted colour is mid-flip. Re-POST so the bare // `sandbox` alias re-resolves onto the now-active colour. A 503 for an // offline device is final for this create: the session's workspace lives - // there. A non-draining 503 (or exhausted retries) falls through to the - // generic failure below. + // there. A 503 `session_unavailable` (a create of the id already in + // flight, the backend listing timed out) is asked again at its + // `retry-after`, within the create's retry budget. Any other 503, or + // exhausted retries, falls through to the generic failure below. if (res.status === 503) { const peek = await safeText(res); const offline = deviceOfflineIn(res, peek); if (offline) throw offline; - if (peek.includes('draining') && attempt < CREATE_DRAIN_RETRY_MAX) { - await new Promise((resolve) => - setTimeout(resolve, CREATE_DRAIN_RETRY_DELAY_MS), - ); + if (peek.includes('draining') && drainRetries < CREATE_DRAIN_RETRY_MAX) { + drainRetries += 1; + await sleep(CREATE_DRAIN_RETRY_DELAY_MS); + continue; + } + const waitMs = peek.includes('session_unavailable') + ? budget.nextWaitMs(parseRetryAfterMs(res)) + : undefined; + if (waitMs !== undefined) { + await sleep(waitMs); continue; } throw new Error(`sandbox session create failed (503): ${peek}`); diff --git a/services/sandbox/docs/sessions.md b/services/sandbox/docs/sessions.md index f927a06b14..ae83db632f 100644 --- a/services/sandbox/docs/sessions.md +++ b/services/sandbox/docs/sessions.md @@ -198,7 +198,9 @@ A session absent from this spawner's registry is resolved from the backend. If that inventory or endpoint lookup fails, or an existing nonterminal runtime is still starting, session routes return `503 session_unavailable` with `Retry-After: 1`. A local create still in progress answers the same way. The caller retries without -declaring the running session lost or recreating it. A confirmed missing or +declaring the running session lost or recreating it. The platform's acquire and create ask again at +the `Retry-After` for up to 20 seconds, and wait out a refused, reset or unresolved connection to the +spawner (a restart) within the same budget, before the turn's start fails. A confirmed missing or stopped session still returns 404 so its preserved workspace can be resumed. The in-memory session registry is a **cache, not the source of truth**: the From cd841c8f6eab3c70acfd94fb47abc5c0b1b0bf1a Mon Sep 17 00:00:00 2001 From: yannickmonney Date: Thu, 8 Oct 2026 13:15:46 +0200 Subject: [PATCH 6/8] perf(platform): read a task turn's launch context concurrently After the run flips to running, a task agent launch (and a steer restart) read the organization's Custom instructions, the run's confinement, the task's language and the serving model's window one after another - four round trips, the window itself two more queries - before the exec could start. None of them depends on another, so they now run together. The credential env (agent secrets, the GitHub token) is still resolved after all of them answered and only for a run that is not confined, so the mint keeps its place and its gate. The confinement check after the launch flip stays a fresh read rather than reusing the subscription lane's earlier answer: it is the gate on handing out credentials, and now costs no extra latency. --- .../tasks/agent_run_host.member_run.test.ts | 64 ++++++++- .../backend/core/tasks/agent_run_host.ts | 126 +++++++++++------- 2 files changed, 138 insertions(+), 52 deletions(-) diff --git a/services/platform/backend/core/tasks/agent_run_host.member_run.test.ts b/services/platform/backend/core/tasks/agent_run_host.member_run.test.ts index 75ef752c34..3ff7398872 100644 --- a/services/platform/backend/core/tasks/agent_run_host.member_run.test.ts +++ b/services/platform/backend/core/tasks/agent_run_host.member_run.test.ts @@ -5,7 +5,9 @@ * and the turn is told so, by name, with what to do instead (an editor * starts the agent when the work needs them). A run an editor started gets * them as before. The test drives the REAL start and steer hosts with only - * external I/O replaced, and reads the exec each one builds. + * external I/O replaced, and reads the exec each one builds. A launch asks + * for its instructions, confinement and language together, and resolves + * the credentials only once all of them answered. */ import { createHash } from 'node:crypto'; @@ -24,8 +26,25 @@ const io = vi.hoisted(() => ({ execs: [] as Record[], /** The rotations a steer claimed. */ rotations: [] as Record[], + /** Once set, the launch reads named here answer only after all of them + * were asked — reads made one after another would never answer. */ + overlap: null as { + asked: Set; + all: Promise; + release: () => void; + } | null, + /** The launch reads answered and the equipment resolved, in order. */ + events: [] as string[], + armOverlap: false, })); +/** The reads a launch makes once its run is flipped running. */ +const LAUNCH_READS = [ + 'tasks/agent_runs:getTaskAgentRunAuthority', + 'tasks/agent_runs:getAgentLanguageContext', + 'governance/internal_queries:getPolicyConfigInternal', +]; + const { resolveTurnEquipmentEnv } = vi.hoisted(() => ({ resolveTurnEquipmentEnv: vi.fn(), })); @@ -109,6 +128,21 @@ function makeCtx() { const ctx = { runQuery: async (ref: unknown) => { const name = functionRefName(ref); + const overlap = io.overlap; + if (overlap !== null && LAUNCH_READS.includes(name)) { + overlap.asked.add(name); + if (overlap.asked.size === LAUNCH_READS.length) overlap.release(); + await Promise.race([ + overlap.all, + new Promise((_resolve, reject) => + setTimeout( + () => reject(new Error(`${name} was read on its own`)), + 2_000, + ), + ), + ]); + io.events.push(name); + } if (name === 'tasks/agent_runs:getTaskAgentRunForDrive') { return { status: run.status, @@ -157,6 +191,13 @@ function makeCtx() { } if (name === 'tasks/agent_runs:setTaskAgentRunRunning') { run.status = 'running'; + if (io.armOverlap) { + let release = () => {}; + const all = new Promise((resolve) => { + release = resolve; + }); + io.overlap = { asked: new Set(), all, release }; + } return true; } if (name === 'tasks/agent_runs:rotateTaskAgentRunExec') { @@ -218,6 +259,9 @@ beforeEach(() => { io.tokens.clear(); io.execs = []; io.rotations = []; + io.overlap = null; + io.armOverlap = false; + io.events = []; resolveTurnEquipmentEnv.mockResolvedValue({ VAT_API_KEY: 'vat-secret', GITHUB_TOKEN: 'gh-token', @@ -286,3 +330,21 @@ describe('a run an editor started', () => { expect(exec.instructions).not.toContain('started by a member'); }); }); + +describe('a launch', () => { + it('reads its instructions, confinement and language together, and resolves credentials only after them', async () => { + io.armOverlap = true; + resolveTurnEquipmentEnv.mockImplementation(async () => { + io.events.push('equipment'); + return { VAT_API_KEY: 'vat-secret' }; + }); + + await startTaskAgentTurnImpl(makeCtx(), { ...KEYS, sweep: true } as never); + + expect(io.events.slice(0, LAUNCH_READS.length).sort()).toEqual( + [...LAUNCH_READS].sort(), + ); + expect(io.events.at(-1)).toBe('equipment'); + expect(lastExec().extraEnv).toEqual({ VAT_API_KEY: 'vat-secret' }); + }); +}); diff --git a/services/platform/backend/core/tasks/agent_run_host.ts b/services/platform/backend/core/tasks/agent_run_host.ts index 2312f6895a..27840f0dd4 100644 --- a/services/platform/backend/core/tasks/agent_run_host.ts +++ b/services/platform/backend/core/tasks/agent_run_host.ts @@ -19,7 +19,10 @@ import { randomBytes } from 'node:crypto'; import { buildStdinUserMessage } from '../../../lib/harnesses/parsers/claude-stream-json'; import { isHarnessSlug } from '../../../lib/harnesses/types'; -import { agentLanguageGuidance } from '../../../lib/shared/agent-language'; +import { + agentLanguageGuidance, + type AgentLanguageContext, +} from '../../../lib/shared/agent-language'; import type { TaskCommentBodies } from '../../../lib/shared/schemas/task-comment'; import { liveProgressSink, @@ -160,6 +163,54 @@ function withheldCredentials(args: { ]; } +/** + * The reads a launching turn builds its instructions and exec from — the + * organization's Custom instructions, whether the run is confined (a run a + * member started holds none of the agent's credentials), the task's + * language, and the serving model's window (so the harness compacts before + * the prompt outgrows what the model serves; unknown leaves it to the + * harness). None depends on another, so they run together. The credential + * env is resolved after them by the caller, as before: only once every read + * succeeded and only for a run that is not confined. + */ +async function readTurnLaunchContext( + ctx: ActionCtx, + args: { + organizationId: string; + runId: Id<'projectAgentRuns'>; + taskId: Id<'tasks'>; + sessionId: string; + execId: string; + providerSlug: string; + modelId: string; + }, +): Promise<{ + mandatoryInstructions: string | undefined; + confined: boolean; + language: AgentLanguageContext; + contextWindow: number | undefined; +}> { + const languageRead: Promise = ctx.runQuery( + internal.tasks.agent_runs.getAgentLanguageContext, + { organizationId: args.organizationId, taskId: args.taskId }, + ); + const [mandatoryInstructions, confined, language, contextWindow] = + await Promise.all([ + readMandatoryInstructions(ctx, args.organizationId, '[task-agent]'), + isTurnConfined(ctx, args.runId), + languageRead, + resolveHarnessTurnContextWindow(ctx, { + organizationId: args.organizationId, + providerSlug: args.providerSlug, + modelId: args.modelId, + sessionId: args.sessionId, + execId: args.execId, + kind: 'task-agent', + }), + ]); + return { mandatoryInstructions, confined, language, contextWindow }; +} + interface TurnKeys { organizationId: string; runId: Id<'projectAgentRuns'>; @@ -1445,28 +1496,23 @@ export async function startTaskAgentTurnImpl( const toolsGuidance = grantedToolsGuidance( normalizeToolGrants(args.tools), ); - const mandatoryInstructions = await readMandatoryInstructions( - ctx, - args.organizationId, - '[task-agent]', - ); - // A run a member started holds none of the agent's credentials. - const confined = await isTurnConfined(ctx, args.runId); + const { mandatoryInstructions, confined, language, contextWindow } = + await readTurnLaunchContext(ctx, { + organizationId: args.organizationId, + runId: args.runId, + taskId: args.taskId, + sessionId: args.sessionId, + execId: args.execId, + providerSlug: resolved.providerSlug, + modelId: resolved.modelId, + }); const instructions = [ // The organization's Custom instructions lead, as on a chat turn. ...(mandatoryInstructions !== undefined ? [mandatoryInstructions] : []), ...(args.instructions !== undefined && args.instructions !== '' ? [args.instructions] : []), - agentLanguageGuidance( - await ctx.runQuery( - internal.tasks.agent_runs.getAgentLanguageContext, - { - organizationId: args.organizationId, - taskId: args.taskId, - }, - ), - ), + agentLanguageGuidance(language), ...(skillsAddendum !== '' ? [skillsAddendum] : []), `Write every file you produce to ${outputDir}/ (this task's own delivery box — never plain /agent/output/) — files there are collected when your turn ends and attached to the task.`, confined ? MEMBER_WORKSPACE_GUIDANCE : STANDING_WORKSPACE_GUIDANCE, @@ -1492,18 +1538,6 @@ export async function startTaskAgentTurnImpl( secrets: args.secrets, }); - // The serving model's window, so the harness compacts before the - // prompt outgrows what the model serves; unknown leaves it to the - // harness. - const contextWindow = await resolveHarnessTurnContextWindow(ctx, { - organizationId: args.organizationId, - providerSlug: resolved.providerSlug, - modelId: resolved.modelId, - sessionId: args.sessionId, - execId: args.execId, - kind: 'task-agent', - }); - // Everything of the exec except the prompt/resume pair, shared by the // resume attempt and its same-execId fresh fallback so the two can // never drift. @@ -2718,24 +2752,24 @@ export async function steerTaskAgentTurnImpl( }); const toolsGuidance = grantedToolsGuidance(normalizeToolGrants(args.tools)); - const mandatoryInstructions = await readMandatoryInstructions( - ctx, - args.organizationId, - '[task-agent]', - ); - const confined = await isTurnConfined(ctx, args.runId); + // Same reads as the fresh start, under the rotated exec. + const { mandatoryInstructions, confined, language, contextWindow } = + await readTurnLaunchContext(ctx, { + organizationId: args.organizationId, + runId: args.runId, + taskId: args.taskId, + sessionId: args.sessionId, + execId, + providerSlug: resolved.providerSlug, + modelId: resolved.modelId, + }); const instructions = [ // The organization's Custom instructions lead, as on a chat turn. ...(mandatoryInstructions !== undefined ? [mandatoryInstructions] : []), ...(args.instructions !== undefined && args.instructions !== '' ? [args.instructions] : []), - agentLanguageGuidance( - await ctx.runQuery(internal.tasks.agent_runs.getAgentLanguageContext, { - organizationId: args.organizationId, - taskId: args.taskId, - }), - ), + agentLanguageGuidance(language), ...(skillsAddendum !== '' ? [skillsAddendum] : []), `Write every file you produce to ${outputDir}/ (this task's own delivery box — never plain /agent/output/) — files there are collected when your turn ends and attached to the task.`, confined ? MEMBER_WORKSPACE_GUIDANCE : STANDING_WORKSPACE_GUIDANCE, @@ -2758,16 +2792,6 @@ export async function steerTaskAgentTurnImpl( connectors: args.connectors, secrets: args.secrets, }); - - // Same window resolution as the fresh start, under the rotated exec. - const contextWindow = await resolveHarnessTurnContextWindow(ctx, { - organizationId: args.organizationId, - providerSlug: resolved.providerSlug, - modelId: resolved.modelId, - sessionId: args.sessionId, - execId, - kind: 'task-agent', - }); const exec = buildExternalTurnExec({ harness: args.harness, gatewayModel: prepared.execModel, From e6c325fac76abe40e89dc9d513ff50960c195c3f Mon Sep 17 00:00:00 2001 From: yannickmonney Date: Thu, 8 Oct 2026 13:20:05 +0200 Subject: [PATCH 7/8] test(platform): type the Gemini start fixture as a harness exec The staged-credential tests built their start exec with `as const`, whose readonly argv the HarnessExec type refuses; the workspace typecheck failed on it while vitest, which does not typecheck, passed. --- .../core/chat/external_turn_shared.test.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/services/platform/backend/core/chat/external_turn_shared.test.ts b/services/platform/backend/core/chat/external_turn_shared.test.ts index 0b92ceeea5..1eef7ba802 100644 --- a/services/platform/backend/core/chat/external_turn_shared.test.ts +++ b/services/platform/backend/core/chat/external_turn_shared.test.ts @@ -30,6 +30,7 @@ import { ReplayBudget, } from '../../../../sandbox-runtime/daemon/src/exec-replay'; import { readFixture } from '../../../lib/harnesses/test-helpers'; +import type { HarnessExec } from '../../../lib/harnesses/types'; const transport = vi.hoisted(() => ({ replayComplete: true, @@ -1158,13 +1159,14 @@ describe('a staged subscription credential', () => { vi.spyOn(console, 'warn').mockImplementation(() => {}); }); - const geminiStart = (stagedFiles: Array<{ path: string; content: string }>) => - ({ - argv: ['gemini'], - env: {}, - cwd: '/agent/workspace', - stagedFiles, - }) as const; + const geminiStart = ( + stagedFiles: Array<{ path: string; content: string }>, + ): HarnessExec => ({ + argv: ['gemini'], + env: {}, + cwd: '/agent/workspace', + stagedFiles, + }); it('is removed before a Gemini turn that runs without it can start', async () => { await drainHarnessWindow({ From eb77bf69f0b8b5fee45cf17864fe529c1510ada8 Mon Sep 17 00:00:00 2001 From: yannickmonney Date: Fri, 9 Oct 2026 04:40:41 +0200 Subject: [PATCH 8/8] docs(manual): move this change's automation rows to the tasks file The automation coverage map now keeps one file per suite, so parallel changes stop appending to the same lines; this change's rows move there, after their own anchor row. --- .../platform/tests/manual/reference/automation/tasks.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/platform/tests/manual/reference/automation/tasks.md b/services/platform/tests/manual/reference/automation/tasks.md index 1880275909..925633aa6c 100644 --- a/services/platform/tests/manual/reference/automation/tasks.md +++ b/services/platform/tests/manual/reference/automation/tasks.md @@ -17,6 +17,12 @@ Rows for [`tasks`](../../suites/tasks.md) boxes that moved out of the suite. Rea | [tasks](../../suites/tasks.md) | The shared reviewer picker stays inside a 375px viewport with a long help footer or an oversized trigger, keeps search, the last option and a localized footer reachable at short heights in default and switcher variants, preserves a desktop field's width, and returns keyboard focus after selection or Escape. | ✅ real Chromium component | `packages/ui/src/components/forms/searchable-select.browser.test.tsx`; localized task and project layouts remain manual in `TASK-A15` | | [tasks](../../suites/tasks.md) | Authenticated REST custom-source close/reopen keeps captured agent review ownership. The strict workflow batch rejects legacy externalState before storage; a defensive internal-store batch control continues without a verdict. Immutable GitHub source closure is an unchanged control. A member’s real app POST cannot change or clear the reviewer, ordinary own-task edits still work, and the archive-only public REST PATCH rejects reviewer fields. | ✅ unit + real Postgres + HTTP | `backend/domains/tasks/external-ref.test.ts`, `backend/domains/tasks/service.reviewer.test.ts`, `backend/domains/tasks/agent-review-routing.integration.ts`, `lib/connectors/natives/platform-tasks.test.ts` | | [tasks](../../suites/tasks.md) | Validated custom-source status (contract 3.16.0): exact external binding and existing work permission, native lifecycle compare-and-swap including a concurrent move, monotonic source ordering, lost-reply replay, metadata refresh preserving accepted Done, atomic archival, active verified native actor provenance, preserved discussion/assignee, source evidence without human approval and captured native agent review protection. | ✅ unit + real Postgres + HTTP | `backend/domains/tasks/external-status.test.ts`, `backend/domains/tasks/external-status.integration.ts`, `backend/domains/tasks/external-ref.test.ts` | +| [tasks](../../suites/tasks.md) / [automations](../../suites/automations.md) | A Claude Code turn (held-open stdin) whose reply is in and whose background-task ledger is empty gets its stdin closed at once and ends on the CLI's own exit, with its real exit code and no reap; stdin stays open while a background task runs, and a close-stdin harness is never sent an EOF; a CLI that ignores the EOF, or an EOF write that fails, is still cut and reaped after the 1.5 s grace | ✅ backend | `backend/core/chat/external_turn_shared.test.ts` (real parsers and harness registry, sandbox transport stood in: natural exit, ignored and failed EOF, open ledger, close-stdin Pi turn, ledger restored across windows); the pinned CLI's own exit on EOF stays a deployment check | +| [tasks](../../suites/tasks.md) / [automations](../../suites/automations.md) | The output harvest takes the first listing of a delivery box whose exec exited on its own (a task or automation agent turn that exited, a script step that finished), so a turn that delivered no file settles without 1.5 s of re-reads; after a reaped linger or a script cut at its timeout, whose processes may still be writing, an empty box is still re-read three times 500 ms apart | ✅ backend | `backend/core/node_only/sandbox/session_exec.test.ts` (both listing paths), `backend/core/tasks/agent_run_host.reap.test.ts` (the task settle passes the window's exit), `backend/core/automations/agent_host.empty_answer.test.ts` (the automation settle), `backend/core/automations/script_host.test.ts` (finished and cut scripts) | +| [tasks](../../suites/tasks.md) / [automations](../../suites/automations.md) | A Gemini turn's staged subscription credential (`.runtime/home/.gemini/oauth_creds.json`, a member's Google sign-in) leaves the session when the turn settles — a task run's settle and orphan reap, an automation turn's settle (whoever won its finalize claim), its question park and its orphan reap — unless a steer or the run moved on to a newer exec that staged its own; a Gemini turn that runs without the subscription removes a leftover file before it starts; a harness that takes its subscription through the environment never touches the session for it; a failed removal is logged and never fails the turn | ✅ backend | `backend/core/chat/external_turn_shared.test.ts` (removal before staging and launch, kept for a subscription turn, env-delivered harnesses untouched, failures logged), `backend/core/tasks/agent_run_host.reap.test.ts` (settle, Stop, steer successor), `backend/core/automations/agent_host.empty_answer.test.ts` (settle, newer exec); the pinned Gemini CLI reading its credentials only at start stays a deployment check | +| [settings](../../suites/settings.md) | A crawler render session asks the spawner for a 3-minute idle window, so a session whose batch died before its destroy (the action killed) frees its organization's render slot within minutes instead of the deployment's 30-minute default; a batch's own exec holds the session whatever its length | ✅ backend | `backend/core/node_only/sandbox/render_fetch.test.ts` (the create's idle window); the spawner's idle reap itself is `services/sandbox/src/session/session-routes.test.ts` | +| [tasks](../../suites/tasks.md) / [automations](../../suites/automations.md) | A session acquire or create the spawner answers with `503 session_unavailable` (a create of the id in flight, a backend listing that timed out, a container still starting) asks again at its `Retry-After`, and one whose connection the spawner refused, reset or could not be resolved for (a restart) asks again on a doubling backoff from 250 ms to 5 s, both within 20 seconds per call, before the start fails; any other 503 or connection failure fails at once, and a `429` still parks the start on capacity without a retry | ✅ backend | `backend/core/node_only/sandbox/helpers/session_client.activity.test.ts` (acquire: retry-after, budget, each connection code, a non-transient code), `backend/core/node_only/sandbox/helpers/session_client.test.ts` (create: retry-after, refused connection, budget, other 503s, the drain retry and 429 unchanged) | +| [tasks](../../suites/tasks.md) | A task agent launch and a steer restart read the organization's Custom instructions, the run's confinement, the task's language and the serving model's window together rather than one after another; the agent's secrets and GitHub token are resolved only after all of them answered, and never for a confined run | ✅ backend | `backend/core/tasks/agent_run_host.member_run.test.ts` (the reads answer only once all were asked; equipment last; confined runs without it), `backend/core/tasks/agent_run_host.context_window.test.ts` (window and Custom instructions on start and restart) | | [tasks](../../suites/tasks.md) | Native source forms: complete field validation, verified session actor, same-column intent, one pending request, immutable replay/decision, older-decision preservation, visible refusal and archived reopening; Chromium covers controls, retry identity, keyboard dismissal and EN/DE/FR phone layout. | ✅ unit + real Postgres/HTTP + browser | `packages/shared/src/schemas/task-external-status.test.ts`, `backend/domains/tasks/external-status.integration.ts`, `app/features/tasks/components/task-external-status-card.browser.test.tsx`; two-session live source outcome remains `TASK-F65` | | [tasks](../../suites/tasks.md) | Task discovery returns one batch of captured pending-review summaries per page. The signed reviewerAgentId filter follows the recorded reviewer, not the implementation assignee or project default; deleted reviewers remain discoverable in the manager’s full review scan. Native task_get carries the same current blocker as the review reader. | ✅ unit + real Postgres + native HTTP | `backend/domains/tasks/agent-review-discovery.test.ts`, `backend/domains/tasks/service.agent-list.test.ts`, `backend/core/node_only/sandbox/workspace_tools_bridge.task_reads.test.ts`, `backend/domains/tasks/agent-read-tools.integration.ts` (`checkAgentTaskReadTools`) | | [tasks](../../suites/tasks.md) | A captured reviewer can stage one listed file of at most 20 MiB with its exact review identity. The 50-entry manifest cursor binds reviewer and source evidence. Current issuer, grant, task/source membership and policy are rechecked; document bindings retain their ACL, invalid or ambiguous metadata stays unavailable. Signed v2 caps and exact-byte checks reject overflow, clean short EOF and body errors before the existing daemon writes, preserving prior bytes and leaving no new partial file. Ordinary v1 remains unchanged. Native PG/MinIO plus actual runnerd prove exact bytes, repeats and concurrent handoff refusal; historical authorized bytes may remain, and staging is not a claim of model inspection. | ✅ unit + real Postgres + native HTTP + actual daemon | `backend/core/lib/storage/sandbox_stage_token.test.ts`, `backend/domains/files/sandbox-blob-routes.test.ts`, `backend/domains/tasks/agent-review-files.test.ts`, `backend/core/node_only/sandbox/workspace_tools_bridge.test.ts`, `backend/domains/tasks/agent-review-files.integration.ts` (through `checkAgentTaskReviews`) |