From 975b9de8579de130a0b040651338c0dcb6cd5aa0 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 6 Sep 2026 02:22:49 +0800 Subject: [PATCH 01/32] feat(runtime): give a turn's system notes a RuntimeEvent of their own A note the runtime writes during a turn -- context compacted, step cap reached, the turn aborted -- is a fact of that invocation, but the only place it could be written was the Session transcript. That left the ledger unable to state part of what a run did, and left the importer dropping those rows on the floor. Notes that happen between turns belong to no invocation, so they stay Session transcript rows. The split is by owner, not by how they render. Generated-by: Claude Code --- packages/core/src/runtime-event.ts | 35 +++++++- packages/core/src/session.ts | 53 ++++++++---- .../runtime-event-read-model.test.ts | 84 +++++++++++++++++++ .../runtime/src/runtime-event-backfill.ts | 39 +++++++-- .../runtime/src/runtime-event-read-model.ts | 26 ++++++ 5 files changed, 212 insertions(+), 25 deletions(-) diff --git a/packages/core/src/runtime-event.ts b/packages/core/src/runtime-event.ts index 8323d289b4..e586b2df4e 100644 --- a/packages/core/src/runtime-event.ts +++ b/packages/core/src/runtime-event.ts @@ -61,7 +61,11 @@ import { type OrchestrationMode, } from './orchestration.js'; import { isToolMode, type ToolMode } from './tool-mode.js'; -import type { PersistedBackendKind } from './session.js'; +import { + isTurnScopedSystemNoteKind, + type PersistedBackendKind, + type TurnScopedSystemNoteKind, +} from './session.js'; import { decodeTurnOrigin, type TurnOrigin } from './turn-origin.js'; import type { UserQuestionRequest } from './user-question.js'; import { @@ -217,6 +221,22 @@ export interface RuntimeEventFunctionResponseContent { modelProjection?: DurableToolResultProjection; } +/** + * A note the runtime wrote about what happened during an invocation — context + * was compacted, the step cap was reached, the turn was aborted. + * + * It is a transcript row, not a model-facing payload: nothing replays it to a + * provider. It lives here because it is a fact of the invocation, and the + * invocation's events are the only record of those. Notes that happen between + * turns have no invocation, so they stay Session transcript rows. + */ +export interface RuntimeEventSystemNoteContent { + kind: 'system_note'; + note: TurnScopedSystemNoteKind; + /** Shape depends on `note`, exactly as it does on the transcript row. */ + data?: unknown; +} + export interface RuntimeEventErrorContent { kind: 'error'; code?: string; @@ -337,6 +357,7 @@ export type RuntimeEventContent = | RuntimeEventFunctionCallContent | RuntimeEventFunctionResponseContent | RuntimeEventErrorContent + | RuntimeEventSystemNoteContent | RuntimeEventInvocationOpenedContent; export const RUNTIME_EVENT_CONTENT_KINDS = [ @@ -345,6 +366,7 @@ export const RUNTIME_EVENT_CONTENT_KINDS = [ 'function_call', 'function_response', 'error', + 'system_note', 'invocation_opened', ] as const; export type RuntimeEventContentKind = (typeof RUNTIME_EVENT_CONTENT_KINDS)[number]; @@ -696,6 +718,10 @@ const ERROR_CONTENT_SHAPE = defineObjectShape()( ['kind', 'message'], ['code', 'reason', 'details'], ); +const SYSTEM_NOTE_CONTENT_SHAPE = defineObjectShape()( + ['kind', 'note'], + ['data'], +); const INVOCATION_OPENED_CONTENT_SHAPE = defineObjectShape()( ['kind', 'protocol', 'route', 'configuration', 'root', 'source'], ['lineage'], @@ -1022,6 +1048,12 @@ function isRuntimeEventContent(value: unknown): value is RuntimeEventContent { typeof value.message === 'string' && (value.details === undefined || isStringArray(value.details) || isRecord(value.details)) ); + case 'system_note': + return ( + hasExactShape(value, SYSTEM_NOTE_CONTENT_SHAPE) && + typeof value.note === 'string' && + isTurnScopedSystemNoteKind(value.note) + ); case 'invocation_opened': return isRuntimeInvocationOpened(value); default: @@ -1485,6 +1517,7 @@ export function runtimeEventHasModelVisibleContent(event: RuntimeEvent): boolean case 'function_response': return true; case 'error': + case 'system_note': case 'invocation_opened': return false; } diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts index c440de253b..997d8dcc84 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session.ts @@ -1147,27 +1147,50 @@ export interface TurnRecord { partialOutputRetained: boolean; } +/** + * Every note kind a transcript can carry. + * + * The list is split by who owns the fact, not by how it renders. A turn-scoped + * note is part of what one invocation did, so the RuntimeEvent ledger records + * it; a session-scoped note happens between turns, where there is no invocation + * to belong to, and the Session transcript records it. + */ +export const TURN_SCOPED_SYSTEM_NOTE_KINDS = [ + 'context_compacted', + 'context_compaction_failed_open', + 'context_provider_dropping', + 'context_window_suggestion', + 'context_window_overrun', + 'context_reported_window_exceeded', + 'context_overflow_after_compaction', + 'step_limit', + 'error', + 'abort', +] as const; + +export const SESSION_SCOPED_SYSTEM_NOTE_KINDS = [ + 'session_start', + 'session_resume', + 'mode_change', + 'model_change', +] as const; + +export type TurnScopedSystemNoteKind = (typeof TURN_SCOPED_SYSTEM_NOTE_KINDS)[number]; +export type SystemNoteKind = + | TurnScopedSystemNoteKind + | (typeof SESSION_SCOPED_SYSTEM_NOTE_KINDS)[number]; + +export function isTurnScopedSystemNoteKind(kind: string): kind is TurnScopedSystemNoteKind { + return (TURN_SCOPED_SYSTEM_NOTE_KINDS as readonly string[]).includes(kind); +} + export interface SystemNoteMessage { type: 'system_note'; id: string; /** Session-level notes omit turnId. */ turnId?: string; ts: number; - kind: - | 'session_start' - | 'session_resume' - | 'mode_change' - | 'model_change' - | 'context_compacted' - | 'context_compaction_failed_open' - | 'context_provider_dropping' - | 'context_window_suggestion' - | 'context_window_overrun' - | 'context_reported_window_exceeded' - | 'context_overflow_after_compaction' - | 'step_limit' - | 'error' - | 'abort'; + kind: SystemNoteKind; /** * Shape depends on `kind`. `context_compaction_failed_open` carries * `{ failOpenReason?: string }` — the reason the fold was refused (e.g. diff --git a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts index c47760680f..f5f9a660d4 100644 --- a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts +++ b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts @@ -1995,6 +1995,90 @@ const ACTION_COVERAGE_SAMPLES: ActionCoverageSamples = { runtimeProtocol: { action: { toolBoundary: 't1_after_preflight_v1' } }, }; +describe('system note projection', () => { + test('projects a turn-scoped note back into its transcript row', () => { + const out = projectRuntimeEventsToStoredMessages( + [ + ev({ + id: 'evt-note', + content: { + kind: 'system_note', + note: 'context_compacted', + data: { removedMessages: 12 }, + }, + modelVisibility: 'hidden', + refs: { storedMessageId: 'legacy-note' }, + }), + ], + { invocations: [invocation] }, + ); + + assert.deepStrictEqual(out.diagnostics, []); + assert.deepStrictEqual(out.messages, [ + { + type: 'system_note', + id: 'legacy-note', + turnId, + ts, + kind: 'context_compacted', + data: { removedMessages: 12 }, + }, + ]); + }); + + test('converts a legacy turn-scoped note and reads back the same row', () => { + const note: StoredMessage = { + type: 'system_note', + id: 'legacy-step-limit', + turnId, + ts, + kind: 'step_limit', + data: { steps: 40 }, + }; + + const backfilled = backfillRuntimeEventsFromStoredMessages({ + run: { sessionId, invocationId, runId, turnId }, + outcome: { status: 'completed', ts }, + messages: [note], + modelHistory: 'full', + now: () => ts, + }); + + assert.deepStrictEqual(backfilled.diagnostics, []); + const projected = projectRuntimeEventsToStoredMessages(backfilled.events, { + invocations: [invocation], + }); + assert.deepStrictEqual( + projected.messages.filter((message) => message.type === 'system_note'), + [note], + ); + }); + + test('leaves a session-level note out of the run ledger', () => { + const backfilled = backfillRuntimeEventsFromStoredMessages({ + run: { sessionId, invocationId, runId, turnId }, + messages: [ + { + type: 'system_note', + id: 'legacy-mode-change', + turnId, + ts, + kind: 'mode_change', + data: { from: 'ask', to: 'bypass' }, + }, + ], + modelHistory: 'full', + now: () => ts, + }); + + assert.deepStrictEqual( + backfilled.events.filter((event) => event.content?.kind === 'system_note'), + [], + ); + assert.partialDeepStrictEqual(backfilled.diagnostics, [{ code: 'skipped_high_risk_message' }]); + }); +}); + describe('RuntimeEventActions projection coverage', () => { for (const [field, sample] of Object.entries(ACTION_COVERAGE_SAMPLES)) { test(`actions.${field} projects without an unclaimed-event diagnostic`, () => { diff --git a/packages/runtime/src/runtime-event-backfill.ts b/packages/runtime/src/runtime-event-backfill.ts index 2b5ab6137c..9ff40070bb 100644 --- a/packages/runtime/src/runtime-event-backfill.ts +++ b/packages/runtime/src/runtime-event-backfill.ts @@ -19,6 +19,7 @@ import type { RuntimeInvocationOutcome } from '@maka/core/runtime-invocation'; import type { RunIdentity } from './terminal-run-commit.js'; +import { isTurnScopedSystemNoteKind } from '@maka/core/session'; import type { PermissionDecisionMessage, StoredMessage, @@ -356,18 +357,38 @@ export function backfillRuntimeEventsFromStoredMessages( case 'turn_state': break; + // A note that names a turn is that invocation's own fact, so it converts. + // A session-level kind that somehow carries a turnId is not: it says + // something about the Session, and the Session transcript keeps it. case 'system_note': if (conversationTextOnly) break; - diagnostics.push({ - code: 'skipped_high_risk_message', - message: - 'system_note is not recovered into a run ledger because session-level notes may not belong to this run', - detail: { - messageId: message.id, - kind: message.kind, - runId: input.run.runId, - turnId: input.run.turnId, + if (!isTurnScopedSystemNoteKind(message.kind)) { + diagnostics.push({ + code: 'skipped_high_risk_message', + message: + 'session-level system_note is not recovered into a run ledger because it does not belong to this run', + detail: { + messageId: message.id, + kind: message.kind, + runId: input.run.runId, + turnId: input.run.turnId, + }, + }); + break; + } + events.push({ + ...base, + id: newId(), + role: 'system', + author: 'system', + modelVisibility: 'hidden', + content: { + kind: 'system_note', + note: message.kind, + ...(message.data !== undefined ? { data: structuredClone(message.data) } : {}), }, + actions: { stateDelta: recoveryState(now, message) }, + refs: { storedMessageId: message.id }, }); break; } diff --git a/packages/runtime/src/runtime-event-read-model.ts b/packages/runtime/src/runtime-event-read-model.ts index d3cbba2a6e..914b714ec0 100644 --- a/packages/runtime/src/runtime-event-read-model.ts +++ b/packages/runtime/src/runtime-event-read-model.ts @@ -251,6 +251,9 @@ export function projectRuntimeEventsToStoredMessages( case 'thinking': projected = projectThinking(event, state, messages) || projected; break; + case 'system_note': + projected = projectSystemNote(event, state, messages) || projected; + break; case 'invocation_opened': // The opening fact records route, configuration and lineage once per // invocation. Every reader joins it by invocationId; it has no chat row. @@ -1211,6 +1214,29 @@ function projectTerminalTurnState( return true; } +/** + * The note row of an invocation that wrote one. + * + * There is nothing to reconcile: the event carries the kind and the payload the + * row is made of, so the row is the event said back in the transcript's shape. + */ +function projectSystemNote( + event: RuntimeEvent, + state: ProjectionState, + messages: StoredMessage[], +): boolean { + if (event.content?.kind !== 'system_note') return false; + messages.push({ + type: 'system_note', + id: stableMessageId(event, state, 'system_note'), + turnId: event.turnId, + ts: event.ts, + kind: event.content.note, + ...(event.content.data !== undefined ? { data: structuredClone(event.content.data) } : {}), + }); + return true; +} + function attachPendingThinking( event: RuntimeEvent, state: ProjectionState, From a1b58961190bd278a24dc8171104bd3b06208453 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 6 Sep 2026 02:27:36 +0800 Subject: [PATCH 02/32] feat(runtime): convert every legacy transcript row instead of dropping some The converter used to answer "this row cannot be recovered losslessly" by writing nothing, which loses conversation the user can still read today. Losslessness is a model-replay property, not a transcript one, so the rows it could not replay now convert hidden: the card stays in the transcript and no provider request is ever built from it. A permission decision names its own tool, so it no longer needs a matching call in the same turn; it carries the prompt's hint when nothing else records it. A turn whose transcript never said how it ended now ends as the failure it was, because an invocation left open is not a legal ledger state and would strand the turn in recovery forever. Generated-by: Claude Code --- packages/core/src/runtime-event.ts | 11 +- .../runtime-event-read-model.test.ts | 103 ++++++++++++++++++ .../runtime/src/runtime-event-backfill.ts | 98 +++++++++-------- .../runtime/src/runtime-event-read-model.ts | 5 +- 4 files changed, 168 insertions(+), 49 deletions(-) diff --git a/packages/core/src/runtime-event.ts b/packages/core/src/runtime-event.ts index e586b2df4e..3344b310e0 100644 --- a/packages/core/src/runtime-event.ts +++ b/packages/core/src/runtime-event.ts @@ -387,6 +387,12 @@ export interface RuntimeEventTokenUsage extends TokenUsageFields {} */ export interface RuntimeEventPermissionDecision extends PermissionResponse { toolName?: string; + /** + * What the prompt told the user they were approving. Normally read off the + * paired request; carried here when the decision is the only surviving + * evidence that the prompt happened. + */ + hint?: string; } export const TOOL_BOUNDARY_PROTOCOL_V1 = 't1_after_preflight_v1' as const; @@ -829,7 +835,7 @@ const PERMISSION_CLOSURE_ACCEPTED_SHAPE = defineObjectShape()(['requestId', 'reason'], []); const RUNTIME_PERMISSION_DECISION_SHAPE = defineObjectShape()( ['requestId', 'decision'], - ['rememberForTurn', 'reviewer', 'rationale', 'riskLevel', 'toolName'], + ['rememberForTurn', 'reviewer', 'rationale', 'riskLevel', 'toolName', 'hint'], ); const UTF8 = new TextEncoder(); const RUNTIME_TOOL_DISPATCH_SHAPE = defineObjectShape()( @@ -1298,7 +1304,8 @@ function isRuntimeEventPermissionDecision(value: unknown): value is RuntimeEvent (value.toolName === undefined || (typeof value.toolName === 'string' && value.toolName.length > 0 && - UTF8.encode(value.toolName).byteLength <= INTERACTION_TOOL_NAME_MAX_BYTES)) + UTF8.encode(value.toolName).byteLength <= INTERACTION_TOOL_NAME_MAX_BYTES)) && + isOptionalString(value.hint) ); } diff --git a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts index f5f9a660d4..68044efb3f 100644 --- a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts +++ b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts @@ -22,6 +22,7 @@ import { describe, test } from 'node:test'; import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { CreateSessionInput, SessionListFilter } from '@maka/core/runtime-inputs'; import type { RuntimeEvent, RuntimeEventActions } from '@maka/core/runtime-event'; +import { runtimeEventHasModelVisibleContent } from '@maka/core/runtime-event'; import type { SessionHeader, SessionSummary, StoredMessage, TurnRecord } from '@maka/core/session'; import { deriveTurnRecords } from '@maka/core/session'; import { @@ -2079,6 +2080,108 @@ describe('system note projection', () => { }); }); +describe('legacy transcript conversion keeps every row', () => { + const convert = (messages: readonly StoredMessage[]) => + backfillRuntimeEventsFromStoredMessages({ + run: { sessionId, invocationId, runId, turnId }, + outcome: { status: 'completed', ts }, + messages, + modelHistory: 'full', + now: () => ts, + }); + + test('keeps a tool result whose call is not in the turn, out of model replay', () => { + const orphan: StoredMessage = { + type: 'tool_result', + id: 'legacy-orphan-result', + turnId, + ts, + toolUseId: 'tool-gone', + isError: false, + content: { kind: 'text', text: 'done' }, + }; + + const converted = convert([orphan]); + const response = converted.events.find((event) => event.content?.kind === 'function_response'); + assert.strictEqual(response?.modelVisibility, 'hidden'); + assert.strictEqual(runtimeEventHasModelVisibleContent(response as RuntimeEvent), false); + + const projected = projectRuntimeEventsToStoredMessages(converted.events, { + invocations: [invocation], + }); + assert.partialDeepStrictEqual( + projected.messages.filter((message) => message.type === 'tool_result'), + [{ id: 'legacy-orphan-result', toolUseId: 'tool-gone' }], + ); + }); + + test('keeps a provider-native call whose opaque output was not retained', () => { + const converted = convert([ + { + type: 'tool_call', + id: 'tool-native', + turnId, + ts, + toolName: 'WebSearch', + args: { query: 'maka' }, + providerExecuted: true, + }, + ]); + + const call = converted.events.find((event) => event.content?.kind === 'function_call'); + assert.strictEqual(call?.modelVisibility, 'hidden'); + assert.partialDeepStrictEqual(converted.diagnostics, [ + { code: 'skipped_provider_native_replay_gap' }, + ]); + + const projected = projectRuntimeEventsToStoredMessages(converted.events, { + invocations: [invocation], + }); + assert.partialDeepStrictEqual( + projected.messages.filter((message) => message.type === 'tool_call'), + [{ id: 'tool-native', toolName: 'WebSearch' }], + ); + }); + + test('converts a permission decision on its own evidence', () => { + const decision: StoredMessage = { + type: 'permission_decision', + id: 'request-1', + turnId, + ts, + toolUseId: 'tool-1', + toolName: 'Bash', + decision: 'allow', + hint: 'rm -rf build', + }; + + const converted = convert([decision]); + assert.deepStrictEqual(converted.diagnostics, []); + + const projected = projectRuntimeEventsToStoredMessages(converted.events, { + invocations: [invocation], + }); + assert.deepStrictEqual( + projected.messages.filter((message) => message.type === 'permission_decision'), + [decision], + ); + }); + + test('ends a turn whose transcript never said how it ended', () => { + const converted = backfillRuntimeEventsFromStoredMessages({ + run: { sessionId, invocationId, runId, turnId }, + messages: [{ type: 'user', id: 'legacy-user', turnId, ts, text: 'hello' }], + modelHistory: 'full', + now: () => ts, + }); + + const terminal = converted.events.filter((event) => event.actions?.endInvocation); + assert.partialDeepStrictEqual(terminal, [{ status: 'failed' }]); + assert.strictEqual(terminal[0]?.actions?.stateDelta?.failureClass, 'missing_terminal_event'); + assert.partialDeepStrictEqual(converted.diagnostics, [{ code: 'synthesized_terminal_event' }]); + }); +}); + describe('RuntimeEventActions projection coverage', () => { for (const [field, sample] of Object.entries(ACTION_COVERAGE_SAMPLES)) { test(`actions.${field} projects without an unclaimed-event diagnostic`, () => { diff --git a/packages/runtime/src/runtime-event-backfill.ts b/packages/runtime/src/runtime-event-backfill.ts index 9ff40070bb..a48b0c5e98 100644 --- a/packages/runtime/src/runtime-event-backfill.ts +++ b/packages/runtime/src/runtime-event-backfill.ts @@ -37,8 +37,7 @@ export type RuntimeEventBackfillDiagnosticCode = | 'skipped_high_risk_message' | 'skipped_provider_native_replay_gap' | 'skipped_unmatched_tool_result' - | 'skipped_unmatched_permission_decision' - | 'skipped_unsafe_terminal_state'; + | 'synthesized_terminal_event'; export interface RuntimeEventBackfillDiagnostic { code: RuntimeEventBackfillDiagnosticCode; @@ -194,14 +193,18 @@ export function backfillRuntimeEventsFromStoredMessages( case 'tool_call': { if (conversationTextOnly) break; - if (message.providerExecuted === true && !replayableProviderToolUseIds.has(message.id)) { + // A provider-native call whose opaque output was not retained can never + // be replayed to a provider again, so it is converted hidden: the + // transcript keeps the card, and no model request is built from it. + const unreplayable = + message.providerExecuted === true && !replayableProviderToolUseIds.has(message.id); + if (unreplayable) { diagnostics.push({ code: 'skipped_provider_native_replay_gap', message: 'provider-native tool history requires the opaque provider output for lossless recovery', detail: { messageId: message.id, toolUseId: message.id }, }); - break; } const stateDelta = toolCallStateDelta(message); events.push({ @@ -210,6 +213,7 @@ export function backfillRuntimeEventsFromStoredMessages( role: 'model', author: 'agent', ...storedToolActivityIdentity(message), + ...(unreplayable ? { modelVisibility: 'hidden' as const } : {}), content: { kind: 'function_call', id: message.id, @@ -243,15 +247,18 @@ export function backfillRuntimeEventsFromStoredMessages( case 'tool_result': { if (conversationTextOnly) break; - if (message.providerExecuted === true && message.providerOutput === undefined) { - break; - } + // Same rule as the call it answers: a result the provider cannot be + // shown again is kept as a transcript row and hidden from replay. A + // result whose call is not in this turn is hidden for the same reason — + // a lone result is not a request a provider would accept. const call = safePriorToolCall(toolCalls, message); + const unreplayable = + !call || (message.providerExecuted === true && message.providerOutput === undefined); if (!call) { diagnostics.push({ code: 'skipped_unmatched_tool_result', message: - 'tool_result requires an earlier same-turn tool_call to recover RuntimeEvent function_response', + 'tool_result has no earlier same-turn tool_call, so its RuntimeEvent stays out of model replay', detail: { messageId: message.id, toolUseId: message.toolUseId, @@ -259,18 +266,18 @@ export function backfillRuntimeEventsFromStoredMessages( turnId: input.run.turnId, }, }); - break; } events.push({ ...base, id: newId(), role: 'tool', author: 'tool', - ...storedToolActivityIdentity(call), + ...storedToolActivityIdentity(call ?? message), + ...(unreplayable ? { modelVisibility: 'hidden' as const } : {}), content: { kind: 'function_response', id: message.toolUseId, - name: call.toolName, + name: call?.toolName ?? '', result: message.content, isError: message.isError, ...(message.providerExecuted !== undefined @@ -286,10 +293,10 @@ export function backfillRuntimeEventsFromStoredMessages( refs: { storedMessageId: message.id, toolCallId: message.toolUseId, - ...(call.parentToolCallId !== undefined + ...(call?.parentToolCallId !== undefined ? { parentToolCallId: call.parentToolCallId } : {}), - ...(call.parentOperationId !== undefined + ...(call?.parentOperationId !== undefined ? { parentOperationId: call.parentOperationId } : {}), }, @@ -297,23 +304,11 @@ export function backfillRuntimeEventsFromStoredMessages( break; } - case 'permission_decision': { + // The decision names the tool it answered for, so it converts on its own + // evidence; a matching call in the same turn is confirmation, not a + // requirement. + case 'permission_decision': if (conversationTextOnly) break; - const call = safePriorToolCall(toolCalls, message); - if (!call) { - diagnostics.push({ - code: 'skipped_unmatched_permission_decision', - message: - 'permission_decision requires an earlier same-turn tool_call to recover RuntimeEvent permissionDecision', - detail: { - messageId: message.id, - toolUseId: message.toolUseId, - runId: input.run.runId, - turnId: input.run.turnId, - }, - }); - break; - } events.push({ ...base, id: newId(), @@ -324,15 +319,19 @@ export function backfillRuntimeEventsFromStoredMessages( permissionDecision: { requestId: message.id, decision: message.decision, + toolName: message.toolName, ...(message.rememberForTurn !== undefined ? { rememberForTurn: message.rememberForTurn } : {}), + ...(message.reviewer !== undefined ? { reviewer: message.reviewer } : {}), + ...(message.rationale !== undefined ? { rationale: message.rationale } : {}), + ...(message.riskLevel !== undefined ? { riskLevel: message.riskLevel } : {}), + ...(message.hint !== undefined ? { hint: message.hint } : {}), }, }, - refs: { storedMessageId: message.id, toolCallId: call.id }, + refs: { storedMessageId: message.id, toolCallId: message.toolUseId }, }); break; - } case 'token_usage': if (conversationTextOnly) break; @@ -354,7 +353,11 @@ export function backfillRuntimeEventsFromStoredMessages( }); break; + // Both are already accounted for elsewhere: the turn's ending becomes the + // terminal RuntimeEvent below, and a coordination record is the WorkHub's + // own durable proof, which no run ledger owns a copy of. case 'turn_state': + case 'workhub_coordination': break; // A note that names a turn is that invocation's own fact, so it converts. @@ -402,11 +405,8 @@ export function backfillRuntimeEventsFromStoredMessages( newId, now, }); - if (terminal.event) { - events.push(terminal.event); - } else if (terminal.diagnostic) { - diagnostics.push(terminal.diagnostic); - } + if (terminal.event) events.push(terminal.event); + if (terminal.diagnostic) diagnostics.push(terminal.diagnostic); return { events, diagnostics }; } @@ -469,25 +469,30 @@ function terminalRuntimeEvent(input: { now: () => number; }): { event?: RuntimeEvent; diagnostic?: RuntimeEventBackfillDiagnostic } { const turnState = latestTurnState(input.turnMessages); - const status = terminalStatus(input.outcome, turnState); - if (!status) { - return { - diagnostic: { - code: 'skipped_unsafe_terminal_state', + const readStatus = terminalStatus(input.outcome, turnState); + // An invocation with no ending is not a legal ledger state, and leaving one + // open would strand the turn in recovery forever. Incomplete legacy evidence + // does not get to claim the turn completed, so it ends as the failure it + // actually was, marked so a reader can tell it apart from a recorded one. + const status = readStatus ?? 'failed'; + const diagnostic: RuntimeEventBackfillDiagnostic | undefined = readStatus + ? undefined + : { + code: 'synthesized_terminal_event', message: - 'terminal RuntimeEvent was not recovered because legacy terminal evidence is incomplete', + 'terminal RuntimeEvent was synthesized because legacy terminal evidence is incomplete', detail: { runId: input.run.runId, turnId: input.run.turnId, declaredStatus: input.outcome?.status, turnStatus: turnState?.status, }, - }, - }; - } + }; const ts = turnState?.ts ?? input.outcome?.ts ?? input.now(); const failureClass = - status === 'failed' ? (turnState?.errorClass ?? input.outcome?.failureClass) : undefined; + status === 'failed' + ? (turnState?.errorClass ?? input.outcome?.failureClass ?? 'missing_terminal_event') + : undefined; const abortSource = status === 'aborted' ? (turnState?.abortSource ?? @@ -516,6 +521,7 @@ function terminalRuntimeEvent(input: { }, ...(turnState ? { refs: { storedMessageId: turnState.id } } : {}), }, + ...(diagnostic ? { diagnostic } : {}), }; } diff --git a/packages/runtime/src/runtime-event-read-model.ts b/packages/runtime/src/runtime-event-read-model.ts index 914b714ec0..4dacb25268 100644 --- a/packages/runtime/src/runtime-event-read-model.ts +++ b/packages/runtime/src/runtime-event-read-model.ts @@ -1019,6 +1019,9 @@ function projectPermissionDecision( ); return false; } + // The prompt's own wording when the request survived, and the decision's copy + // of it when the decision is all that is left. + const hint = request?.hint ?? decision.hint; messages.push({ type: 'permission_decision', id: decision.requestId, @@ -1033,7 +1036,7 @@ function projectPermissionDecision( ...(decision.reviewer !== undefined ? { reviewer: decision.reviewer } : {}), ...(decision.rationale !== undefined ? { rationale: decision.rationale } : {}), ...(decision.riskLevel !== undefined ? { riskLevel: decision.riskLevel } : {}), - ...(request?.hint !== undefined ? { hint: request.hint } : {}), + ...(hint !== undefined ? { hint } : {}), }); return true; } From 64fa31243c28a9875423575770b01ae37abc7b15 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 6 Sep 2026 02:30:34 +0800 Subject: [PATCH 03/32] feat(runtime): make the transcript conversion whole and resumable Every event id is now derived from the run it belongs to and its position in that run, so converting the same transcript twice writes the same events and the store keeps one copy. That is what lets an interrupted conversion resume: a turn is skipped once its invocation has ended, and re-derived until then. Before, a turn was skipped as soon as its opening existed, which froze a half-converted turn in that state forever. Maka's own history now converts whole. Only a foreign transcript stays conversation-text: another runtime's tool calls belong to its protocol, not to the provider this Session talks to next. Generated-by: Claude Code --- .../__tests__/runtime-ledger-repair.test.ts | 84 +++++++++++++++++++ .../src/__tests__/session-manager.test.ts | 6 +- packages/runtime/src/runtime-ledger-repair.ts | 48 +++++++---- 3 files changed, 121 insertions(+), 17 deletions(-) diff --git a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts index 3e42ad0777..908e6f4905 100644 --- a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts +++ b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts @@ -412,6 +412,90 @@ test('an imported turn with no terminal state is repaired to failed', async () = } }); +test("converts Maka's own legacy transcript whole, and resumes an interrupted conversion", async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-native-transcript-')); + const sessions = createSessionStore(root); + const runtimeEvents = createSqliteRuntimeStore(join(root, 'runtime.sqlite')); + + try { + const ts = Date.now(); + const session = await sessions.create({ + cwd: '/repo', + llmConnectionSlug: 'anthropic', + model: 'claude-opus-5', + permissionMode: 'ask', + }); + await sessions.appendMessages(session.id, [ + { type: 'user', id: 'n-user', turnId: 'turn-1', ts, text: 'run the tests' }, + { + type: 'tool_call', + id: 'n-tool', + turnId: 'turn-1', + ts: ts + 1, + toolName: 'Bash', + args: { command: 'npm test' }, + }, + { + type: 'tool_result', + id: 'n-result', + turnId: 'turn-1', + ts: ts + 2, + toolUseId: 'n-tool', + isError: false, + content: { kind: 'text', text: 'ok' }, + }, + { + type: 'system_note', + id: 'n-note', + turnId: 'turn-1', + ts: ts + 3, + kind: 'step_limit', + }, + { + type: 'assistant', + id: 'n-assistant', + turnId: 'turn-1', + ts: ts + 4, + text: 'All green.', + modelId: 'claude-opus-5', + }, + { + type: 'turn_state', + id: 'n-state', + turnId: 'turn-1', + ts: ts + 5, + status: 'completed', + partialOutputRetained: true, + }, + ]); + + const repair = new RuntimeLedgerRepair({ + runtimeEventStore: runtimeEvents, + readMessages: (sessionId) => sessions.readMessages(sessionId), + appendMessage: (sessionId, message) => sessions.appendMessage(sessionId, message), + newId: () => 'unused', + now: () => 100, + }); + + await repair.materializeTranscriptLedger(await sessions.readHeader(session.id)); + // The same transcript converts to the same events, so a second pass — the + // retry after an interrupted one — adds nothing. + await repair.materializeTranscriptLedger(await sessions.readHeader(session.id)); + + const [run] = await runtimeEvents.listSessionInvocations(session.id); + assert.ok(run); + assert.equal(runtimeInvocationOutcome(run), 'completed'); + const events = await runtimeEvents.readRuntimeEvents(session.id, run.runId); + assert.deepEqual( + events.flatMap((event) => (event.content ? [event.content.kind] : [])), + ['invocation_opened', 'text', 'function_call', 'function_response', 'system_note', 'text'], + ); + } finally { + await runtimeEvents.close?.(); + await rm(root, { recursive: true, force: true }); + } +}); + test('a resolved Claude transcript replays as the conversation the user kept', async () => { // The whole path, end to end: raw records → lineage resolution → conversion // → Ledger materialization → the replay a continuation would be given. diff --git a/packages/runtime/src/__tests__/session-manager.test.ts b/packages/runtime/src/__tests__/session-manager.test.ts index b059a7e8af..e950c2bf04 100644 --- a/packages/runtime/src/__tests__/session-manager.test.ts +++ b/packages/runtime/src/__tests__/session-manager.test.ts @@ -13198,7 +13198,11 @@ class MemoryAgentRunStore this.options.failRuntimeEventAppendAfter = undefined; throw new Error('runtime event append failed'); } - assertDoubleRunNotSealed(this.runtimeEvents.get(key(sessionId, runId)) ?? [], event); + const existing = this.runtimeEvents.get(key(sessionId, runId)) ?? []; + // Same identity, same event: the store writes an id once, so a retry of an + // interrupted append lands on what is already there instead of a copy. + if (event.partial !== true && existing.some((candidate) => candidate.id === event.id)) return; + assertDoubleRunNotSealed(existing, event); this.seedRuntimeEvent(sessionId, runId, event); } diff --git a/packages/runtime/src/runtime-ledger-repair.ts b/packages/runtime/src/runtime-ledger-repair.ts index 8ab8949c99..e736ee25f5 100644 --- a/packages/runtime/src/runtime-ledger-repair.ts +++ b/packages/runtime/src/runtime-ledger-repair.ts @@ -72,12 +72,13 @@ export class RuntimeLedgerRepair { constructor(private readonly deps: RuntimeLedgerRepairDeps) {} /** - * Give an imported transcript a runtime spine: one invocation per turn, opened - * by its own opening fact and closed by its own terminal event. + * Give a transcript a runtime spine: one invocation per turn, opened by its + * own opening fact and closed by its own terminal event. * - * The transcript is the only evidence there is, so a turn it cannot close is - * refused rather than imported half-formed. Re-running is a no-op: a turn - * whose invocation already exists is left exactly as it is. + * Every event id is derived from the run it belongs to and its position in + * that run, so importing the same transcript twice writes the same events and + * the store dedupes them. That is what makes an interrupted import resumable: + * a turn is skipped once its invocation has ended, and re-derived until then. */ async materializeTranscriptLedger(header: SessionHeader): Promise { const sessionId = header.id; @@ -86,8 +87,10 @@ export class RuntimeLedgerRepair { const ledgerMessages = messages.filter( (message) => message.type !== 'user' || message.steeringEventId === undefined, ); - const openedTurnIds = new Set( - (await this.listInlineInvocations(sessionId)).map((invocation) => invocation.turnId), + const endedTurnIds = new Set( + (await this.listInlineInvocations(sessionId)) + .filter((invocation) => invocation.terminalEvent) + .map((invocation) => invocation.turnId), ); const messagesByTurn = groupMessagesByTurn(ledgerMessages); const turns = deriveTurnRecords(ledgerMessages).filter((turn) => @@ -98,25 +101,26 @@ export class RuntimeLedgerRepair { const firstOpenedAt = Math.max(0, header.createdAt - turns.length); for (const [index, turn] of turns.entries()) { - if (openedTurnIds.has(turn.turnId)) continue; + if (endedTurnIds.has(turn.turnId)) continue; const turnMessages = messagesByTurn.get(turn.turnId) ?? []; const runId = transcriptRunId(sessionId, turn.turnId); const openedAt = firstOpenedAt + index; const run = { sessionId, runId, turnId: turn.turnId, invocationId: runId }; const events = [ - transcriptOpeningEvent({ header, run, openedAt, newId: this.deps.newId }), + transcriptOpeningEvent({ header, run, openedAt }), ...backfillRuntimeEventsFromStoredMessages({ run, outcome: transcriptOutcome(turn, turnMessages, openedAt), messages: turnMessages, - modelHistory: 'conversation_text', - newId: this.deps.newId, + // Another runtime's tool calls belong to its protocol, not to the + // provider this Session will talk to next, so a foreign transcript + // converts as the conversation it is. Maka's own history converts + // whole: its tool calls are the ones it would replay. + modelHistory: header.externalOrigin ? 'conversation_text' : 'full', + newId: transcriptEventIds(runId), now: this.deps.now, }).events, ]; - if (!events.some(isTerminalRuntimeEvent)) { - throw new Error(`Imported transcript Run ${runId} has no terminal RuntimeEvent`); - } for (const event of events) { await this.deps.runtimeEventStore.appendRuntimeEvent(sessionId, runId, event); } @@ -178,6 +182,19 @@ function transcriptRunId(sessionId: string, turnId: string): string { return `transcript-${digest.slice(0, 48)}`; } +/** + * Ids for one run's converted events, numbered in the order the converter + * emits them. The run id is already derived from the Session and turn, so the + * same transcript always produces the same ids and a re-run appends nothing. + */ +function transcriptEventIds(runId: string): () => string { + let seq = 0; + return () => { + seq += 1; + return `${runId}-e${seq}`; + }; +} + /** * The opening fact of an imported turn. * @@ -189,7 +206,6 @@ function transcriptOpeningEvent(input: { header: SessionHeader; run: { sessionId: string; runId: string; turnId: string; invocationId: string }; openedAt: number; - newId: () => string; }): RuntimeEvent { const opening: RuntimeEventInvocationOpenedContent = { kind: 'invocation_opened', @@ -212,7 +228,7 @@ function transcriptOpeningEvent(input: { source: { kind: 'fresh' }, }; return buildInvocationOpenedEvent({ - id: input.newId(), + id: `${input.run.runId}-opened`, run: input.run, openedAt: input.openedAt, opening, From fa058d910c95b202705e63663bd3725793ff6364 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 6 Sep 2026 02:40:01 +0800 Subject: [PATCH 04/32] refactor(runtime): read a running turn from its own ledger A running turn's rows came from the Session transcript store while every finished turn's came from the RuntimeEvent ledger. That is the double write: the same execution facts written twice so a reader could find them in whichever place it looked. Now one place answers. An open invocation is read the way the Host's active overlay already read it -- arriving text presented as settled, a step that has only thought given the empty assistant row that thinking hangs on -- and that reading moves next to the projection so both readers share it instead of keeping a copy each. "Still running" is the absence of the terminal event, so it is stated on the turn record where it belongs rather than as a transcript row. Generated-by: Claude Code --- .../src/server/execution-composition.ts | 1 - .../src/server/session-transcript-reader.ts | 66 +------ .../src/__tests__/session-manager.test.ts | 108 ++++++------ .../runtime/src/runtime-event-read-model.ts | 62 +++++++ packages/runtime/src/runtime-read-model.ts | 165 ++++-------------- packages/runtime/src/session-manager.ts | 15 +- 6 files changed, 159 insertions(+), 258 deletions(-) diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index c0e503699c..9e7a2c298b 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -971,7 +971,6 @@ export async function createExecutionRuntimeHostComposition( }), readModel: new RuntimeReadModel({ runtimeEventStore: stores.runtimeEventStore, - projectionCache: stores.sessionStore, canonicalPermissionOutcomes, }), artifacts: openedArtifactStore, diff --git a/packages/runtime-host/src/server/session-transcript-reader.ts b/packages/runtime-host/src/server/session-transcript-reader.ts index cc51d9bd4a..6eb0259ea5 100644 --- a/packages/runtime-host/src/server/session-transcript-reader.ts +++ b/packages/runtime-host/src/server/session-transcript-reader.ts @@ -20,6 +20,7 @@ import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { StoredMessage } from '@maka/core/session'; import { + activePresentationRuntimeEvents, affectsRuntimeEventStoredMessageProjection, isHardRuntimeEventReadModelDiagnostic, projectRuntimeEventsToStoredMessages, @@ -66,10 +67,13 @@ export function createSessionTranscriptReader(input: { events, input.canonicalPermissionOutcomes, ); - const projected = projectRuntimeEventsToStoredMessages(activePresentationEvents(events), { - invocations: invocations.filter((invocation) => invocation.runId === rootTurn.runId), - canonicalPermissionOutcomes, - }); + const projected = projectRuntimeEventsToStoredMessages( + activePresentationRuntimeEvents(events), + { + invocations: invocations.filter((invocation) => invocation.runId === rootTurn.runId), + canonicalPermissionOutcomes, + }, + ); if (projected.diagnostics.some(isHardRuntimeEventReadModelDiagnostic)) { throw new Error('Active RuntimeEvent transcript projection is incomplete'); } @@ -144,37 +148,6 @@ async function readCanonicalPermissionOutcomes( return outcomes; } -function activePresentationEvents(events: readonly RuntimeEvent[]): RuntimeEvent[] { - const textMessages = new Set(); - const lastThinkingByMessage = new Map(); - - for (const event of events) { - const content = event.content; - if (event.role !== 'model' || (content?.kind !== 'text' && content?.kind !== 'thinking')) { - continue; - } - const messageKey = activeMessageKey(event); - if (content.kind === 'text') textMessages.add(messageKey); - else lastThinkingByMessage.set(messageKey, event); - } - - const syntheticAfter = new Map(); - for (const [messageKey, thinking] of lastThinkingByMessage) { - if (textMessages.has(messageKey)) continue; - const existing = syntheticAfter.get(thinking) ?? []; - existing.push(emptyAssistantText(thinking)); - syntheticAfter.set(thinking, existing); - } - - const presented: RuntimeEvent[] = []; - for (const event of events) { - presented.push(presentationEvent(event)); - const synthetic = syntheticAfter.get(event); - if (synthetic) presented.push(...synthetic); - } - return presented; -} - async function readActiveProjectionEvents( stores: ExecutionStoresWriter<'interactive'>, sessionId: string, @@ -215,29 +188,6 @@ async function readActiveProjectionEvents( return events; } -function activeMessageKey(event: RuntimeEvent): string { - const messageId = event.refs?.providerEventId ?? event.refs?.storedMessageId ?? event.id; - return `${event.runId}\0${messageId}`; -} - -function presentationEvent(event: RuntimeEvent): RuntimeEvent { - const content = event.content; - return event.partial && - event.role === 'model' && - (content?.kind === 'text' || content?.kind === 'thinking') - ? { ...event, partial: false } - : event; -} - -function emptyAssistantText(thinking: RuntimeEvent): RuntimeEvent { - return { - ...thinking, - id: `${thinking.id}:active-transcript-empty-text`, - partial: false, - content: { kind: 'text', text: '' }, - }; -} - function isTerminalTurn(turn: TurnSnapshot): boolean { return turn.status === 'completed' || turn.status === 'failed' || turn.status === 'cancelled'; } diff --git a/packages/runtime/src/__tests__/session-manager.test.ts b/packages/runtime/src/__tests__/session-manager.test.ts index e950c2bf04..577e640b15 100644 --- a/packages/runtime/src/__tests__/session-manager.test.ts +++ b/packages/runtime/src/__tests__/session-manager.test.ts @@ -7589,17 +7589,6 @@ describe('SessionManager permission mode updates', () => { hint: 'write approval', }, ); - - const cachedView = await new RuntimeReadModel({ - runtimeEventStore: runStore, - projectionCache: { - readMessages: async () => - messages.filter((message) => message.type !== 'permission_decision'), - }, - canonicalPermissionOutcomes, - }).getSessionView(header.sessionId); - - assert.deepStrictEqual(cachedView.diagnostics, []); }); test('SessionManager joins a canonical hosted permission without a ledger request', async () => { @@ -8221,7 +8210,7 @@ describe('SessionManager permission mode updates', () => { ); }); - test('getMessages includes in-flight projection cache rows for an active RuntimeEvent run', async () => { + test('getMessages reads an active run from its own ledger', async () => { const store = new MemorySessionStore(); const runStore = new MemoryAgentRunStore(); const manager = makeManagerForReadCutover(store, runStore); @@ -8236,7 +8225,51 @@ describe('SessionManager permission mode updates', () => { assistantText: 'completed answer', legacyIdPrefix: 'legacy', }); - const activeMessages: StoredMessage[] = [ + const activeHeader = makeRunHeader({ + sessionId: session.id, + runId: 'run-2', + turnId: 'turn-2', + status: 'running', + createdAt: 200, + updatedAt: 203, + }); + await seedInvocationFromHeader(runStore, activeHeader); + await runStore.appendRuntimeEvent( + session.id, + 'run-2', + runtimeEvent({ + id: 'active-user-event', + sessionId: session.id, + runId: 'run-2', + turnId: 'turn-2', + ts: 201, + role: 'user', + author: 'user', + content: { kind: 'text', text: 'active question' }, + refs: { storedMessageId: 'active-user' }, + }), + ); + // Still arriving: the row a reader sees now, with more of it to come. + await runStore.appendRuntimeEvent( + session.id, + 'run-2', + runtimeEvent({ + id: 'active-assistant-event', + sessionId: session.id, + runId: 'run-2', + turnId: 'turn-2', + ts: 202, + partial: true, + role: 'model', + author: 'agent', + content: { kind: 'text', text: 'partial active answer' }, + refs: { storedMessageId: 'active-assistant' }, + }), + ); + + const messages = await manager.getMessages(session.id); + assert.deepStrictEqual(messages, [ + ...completed.projectedMessages, { type: 'user', id: 'active-user', turnId: 'turn-2', ts: 201, text: 'active question' }, { type: 'assistant', @@ -8246,30 +8279,7 @@ describe('SessionManager permission mode updates', () => { text: 'partial active answer', modelId: 'fake-model', }, - { - type: 'turn_state', - id: 'active-state', - turnId: 'turn-2', - ts: 203, - status: 'running', - partialOutputRetained: true, - }, - ]; - await store.appendMessages(session.id, activeMessages); - await seedInvocationFromHeader( - runStore, - makeRunHeader({ - sessionId: session.id, - runId: 'run-2', - turnId: 'turn-2', - status: 'running', - createdAt: 200, - updatedAt: 203, - }), - ); - - const messages = await manager.getMessages(session.id); - assert.deepStrictEqual(messages, [...completed.projectedMessages, ...activeMessages]); + ]); assert.deepStrictEqual(await manager.listTurns(session.id), [ { turnId: 'turn-1', @@ -8284,19 +8294,6 @@ describe('SessionManager permission mode updates', () => { partialOutputRetained: true, }, ]); - - const view = await new RuntimeReadModel({ - runtimeEventStore: runStore, - projectionCache: store, - }).getSessionView(session.id); - assert.strictEqual( - view.diagnostics.some( - (diagnostic) => - diagnostic.code === 'incomplete_event' && - diagnostic.message.includes('in-flight projection cache'), - ), - true, - ); }); test('getMessages overlays a canonical permission acceptance from a running ledger', async () => { @@ -8480,7 +8477,6 @@ describe('SessionManager permission mode updates', () => { const view = await new RuntimeReadModel({ runtimeEventStore: runStore, - projectionCache: store, }).getSessionView(session.id); const readRequestId = (value: unknown): string[] => @@ -8677,8 +8673,8 @@ describe('SessionManager permission mode updates', () => { userText: 'runtime regenerate text', assistantText: 'runtime answer', legacyIdPrefix: 'legacy', + legacyUserText: 'stale transcript text', }); - store.failNextReadMessagesFor.set(session.id, 1); await drain(manager.regenerateTurn(session.id, { sourceTurnId: 'source', turnId: 'regen-1' })); @@ -8812,8 +8808,8 @@ describe('SessionManager permission mode updates', () => { }), ], ); - store.failNextReadMessagesFor.set(session.id, 1); - + // The transcript store holds no source rows at all, so what regenerate + // finds can only have come from the ledger. await drain( manager.regenerateTurn(session.id, { sourceTurnId: 'source', turnId: 'regen-aborted' }), ); @@ -14228,6 +14224,8 @@ async function seedRuntimeReadTurn(input: { userText: string; assistantText: string; legacyIdPrefix: string; + /** Says something else in the transcript store, so a reader proves its source. */ + legacyUserText?: string; }): Promise<{ legacyMessages: StoredMessage[]; projectedMessages: StoredMessage[] }> { const header = makeRunHeader({ sessionId: input.sessionId, @@ -14279,7 +14277,7 @@ async function seedRuntimeReadTurn(input: { id: `${input.legacyIdPrefix}-user`, turnId: input.turnId, ts: 101, - text: input.userText, + text: input.legacyUserText ?? input.userText, }, { type: 'assistant', diff --git a/packages/runtime/src/runtime-event-read-model.ts b/packages/runtime/src/runtime-event-read-model.ts index 4dacb25268..9aa5f21deb 100644 --- a/packages/runtime/src/runtime-event-read-model.ts +++ b/packages/runtime/src/runtime-event-read-model.ts @@ -440,6 +440,68 @@ export function projectRuntimeEventsToStoredMessages( return { messages, diagnostics: state.diagnostics }; } +/** + * A running invocation's events as the transcript should show them right now. + * + * Two things separate a live run from a finished one. Its last text or thinking + * event is still arriving, so it is presented as settled rather than withheld; + * and a step that has only thought so far has no assistant row to hang that + * thinking on, so an empty one is opened for it. Neither changes the ledger: + * both are how the same events read before the run ends. + */ +export function activePresentationRuntimeEvents(events: readonly RuntimeEvent[]): RuntimeEvent[] { + const textMessages = new Set(); + const lastThinkingByMessage = new Map(); + + for (const event of events) { + const content = event.content; + if (event.role !== 'model' || (content?.kind !== 'text' && content?.kind !== 'thinking')) { + continue; + } + const messageKey = activeMessageKey(event); + if (content.kind === 'text') textMessages.add(messageKey); + else lastThinkingByMessage.set(messageKey, event); + } + + const syntheticAfter = new Map(); + for (const [messageKey, thinking] of lastThinkingByMessage) { + if (textMessages.has(messageKey)) continue; + const existing = syntheticAfter.get(thinking) ?? []; + existing.push(emptyAssistantText(thinking)); + syntheticAfter.set(thinking, existing); + } + + const presented: RuntimeEvent[] = []; + for (const event of events) { + presented.push(settledPresentationEvent(event)); + presented.push(...(syntheticAfter.get(event) ?? [])); + } + return presented; +} + +function activeMessageKey(event: RuntimeEvent): string { + const messageId = event.refs?.providerEventId ?? event.refs?.storedMessageId ?? event.id; + return `${event.runId}\0${messageId}`; +} + +function settledPresentationEvent(event: RuntimeEvent): RuntimeEvent { + const content = event.content; + return event.partial && + event.role === 'model' && + (content?.kind === 'text' || content?.kind === 'thinking') + ? { ...event, partial: false } + : event; +} + +function emptyAssistantText(thinking: RuntimeEvent): RuntimeEvent { + return { + ...thinking, + id: `${thinking.id}:active-transcript-empty-text`, + partial: false, + content: { kind: 'text', text: '' }, + }; +} + export function projectRuntimeEventsToStoredMessagesWithArchiveStatuses( events: readonly RuntimeEvent[], options: ProjectRuntimeEventsToStoredMessagesOptions & { diff --git a/packages/runtime/src/runtime-read-model.ts b/packages/runtime/src/runtime-read-model.ts index e103ae0dd5..eb9b8ae98d 100644 --- a/packages/runtime/src/runtime-read-model.ts +++ b/packages/runtime/src/runtime-read-model.ts @@ -28,6 +28,7 @@ import type { CanonicalPermissionOutcomeRecord, } from './interaction-authority.js'; import { + activePresentationRuntimeEvents, classifyRuntimeEventTerminalFact, compareRuntimeReadModelMessages, isHardRuntimeEventReadModelDiagnostic, @@ -42,13 +43,8 @@ import { const CANONICAL_PERMISSION_READ_CONCURRENCY = 8; -export interface RuntimeReadModelProjectionCache { - readMessages(sessionId: string): Promise; -} - export interface RuntimeReadModelDeps { runtimeEventStore: RuntimeEventStore; - projectionCache?: RuntimeReadModelProjectionCache; canonicalPermissionOutcomes?: CanonicalPermissionOutcomeReader; } @@ -132,29 +128,12 @@ export class RuntimeReadModel { } // No terminal event yet: the invocation is still open, or the process died - // holding it. Either way the ledger is the whole truth about it, so the - // in-flight projection cache supplies the rows a live turn has not - // committed instead of a status field claiming otherwise. + // holding it. Either way its own events are the whole truth about it, read + // as a running turn reads — the arriving text presented as settled. No + // durable ordinals exist for them yet, so they keep ledger order. if (!invocation.terminalEvent) { - diagnostics.push( - readModelDiagnostic( - 'incomplete_event', - 'active run is using the in-flight projection cache', - { runId: invocation.runId, turnId: invocation.turnId }, - ), - ); inFlightTurnIds.add(invocation.turnId); - if (!this.deps.projectionCache) { - throw new RuntimeReadModelError('RuntimeEvent ledger is incomplete for an active run', [ - readModelDiagnostic( - 'incomplete_event', - 'active run has no stable RuntimeEvent read projection', - { runId: invocation.runId, turnId: invocation.turnId }, - ), - ]); - } - const overlayEvents = runEvents.flatMap(activeInteractionOverlayEvent); - appendOrderedEvents(ordered, overlayEvents, runIndex); + appendOrderedEvents(ordered, activePresentationRuntimeEvents(runEvents), runIndex); continue; } @@ -222,46 +201,12 @@ export class RuntimeReadModel { throw new RuntimeReadModelError('RuntimeEvent read projection is incomplete', diagnostics); } - const sessionId = input.invocations[0]?.sessionId; - let cachedMessages: StoredMessage[] | undefined; - if (sessionId && this.deps.projectionCache) { - try { - cachedMessages = await this.deps.projectionCache.readMessages(sessionId); - } catch (error) { - const diagnostic = readModelDiagnostic( - 'unsupported_event', - 'SessionProjectionCache.readMessages failed', - { - error: errorMessage(error), - }, - ); - diagnostics.push(diagnostic); - if (input.inFlightTurnIds && input.inFlightTurnIds.size > 0) { - throw new RuntimeReadModelError( - 'RuntimeEvent active projection cache read failed', - diagnostics, - ); - } - } - } - - const messages = - input.inFlightTurnIds && input.inFlightTurnIds.size > 0 - ? mergeInFlightProjectionCache( - projected.messages, - cachedMessages ?? [], - input.inFlightTurnIds, - ) - : projected.messages; - - diagnostics.push( - ...this.compareProjectionCache(messages, cachedMessages, canonicalPermissionRead.outcomes), - ); + const messages = projected.messages; return { source: 'runtime_events', messages, - turns: deriveTurnRecords(messages), + turns: runningTurnRecords(deriveTurnRecords(messages), input.inFlightTurnIds), events: input.events, invocations: input.invocations, diagnostics, @@ -315,82 +260,36 @@ export class RuntimeReadModel { ); return { outcomes, diagnostics }; } - - private compareProjectionCache( - messages: readonly StoredMessage[], - cached: readonly StoredMessage[] | undefined, - canonicalPermissionOutcomes: ReadonlyMap, - ): RuntimeEventReadModelDiagnostic[] { - if (!cached) return []; - const canonicalRequestIds = new Set(canonicalPermissionOutcomes.keys()); - const excludesCanonicalPermission = (message: StoredMessage): boolean => - message.type === 'permission_decision' && canonicalRequestIds.has(message.id); - return compareRuntimeReadModelMessages( - messages.filter((message) => !excludesCanonicalPermission(message)), - cached.filter((message) => !excludesCanonicalPermission(message)), - ).diagnostics; - } } /** - * The interaction facts an active run must keep even while its messages come - * from the in-flight projection cache. Permission prompts were always carried - * here; sandbox boundary requests and decisions belong for the same reason - * (#1612): they are the only durable record that a prompt was raised and how - * it settled, so dropping them makes a pending request invisible to anything - * reading the view instead of the live backend. + * A turn whose invocation has not ended is running. + * + * The transcript has no row that says so, and it should not: "still running" is + * the absence of the terminal event, read off the invocation itself. Rows are + * what the turn produced, and a turn that has produced an answer but not ended + * would otherwise read as finished. */ -function activeInteractionOverlayEvent(event: RuntimeEvent): RuntimeEvent[] { - const permissionRequest = event.actions?.permissionRequest; - const permissionAnswerAccepted = event.actions?.permissionAnswerAccepted; - const permissionClosureAccepted = event.actions?.permissionClosureAccepted; - const sandboxBoundaryRequest = event.actions?.stateDelta?.sandboxBoundaryRequest; - const sandboxBoundaryDecision = event.actions?.stateDelta?.sandboxBoundaryDecision; - if ( - !permissionRequest && - !permissionAnswerAccepted && - !permissionClosureAccepted && - sandboxBoundaryRequest === undefined && - sandboxBoundaryDecision === undefined - ) { - return []; - } - const overlay = { ...event }; - delete overlay.content; - delete overlay.status; - const stateDelta = { - ...(sandboxBoundaryRequest !== undefined ? { sandboxBoundaryRequest } : {}), - ...(sandboxBoundaryDecision !== undefined ? { sandboxBoundaryDecision } : {}), - }; - overlay.actions = { - ...(permissionRequest ? { permissionRequest } : {}), - ...(permissionAnswerAccepted ? { permissionAnswerAccepted } : {}), - ...(permissionClosureAccepted ? { permissionClosureAccepted } : {}), - ...(Object.keys(stateDelta).length > 0 ? { stateDelta } : {}), - }; - return [overlay]; -} - -function mergeInFlightProjectionCache( - runtimeMessages: readonly StoredMessage[], - cachedMessages: readonly StoredMessage[], - inFlightTurnIds: ReadonlySet, -): StoredMessage[] { - const merged = runtimeMessages.map((message, index) => ({ message, index })); - const seenIds = new Set(runtimeMessages.map((message) => message.id)); - for (const cached of cachedMessages) { - const turnId = messageTurnId(cached); - if (!turnId || !inFlightTurnIds.has(turnId) || seenIds.has(cached.id)) continue; - seenIds.add(cached.id); - merged.push({ message: cached, index: merged.length }); +function runningTurnRecords( + turns: readonly TurnRecord[], + inFlightTurnIds: ReadonlySet | undefined, +): TurnRecord[] { + if (!inFlightTurnIds || inFlightTurnIds.size === 0) return [...turns]; + const running = new Set(inFlightTurnIds); + const marked = turns.map((turn) => { + if (!running.delete(turn.turnId)) return turn; + return { ...turn, status: 'running' as const, statusSource: 'recorded' as const }; + }); + // An invocation that has opened but produced nothing yet still has a turn. + for (const turnId of running) { + marked.push({ + turnId, + status: 'running', + statusSource: 'recorded', + partialOutputRetained: false, + }); } - return merged - .sort((a, b) => a.message.ts - b.message.ts || a.index - b.index) - .map((entry) => entry.message); -} - -function messageTurnId(message: StoredMessage): string | undefined { - return 'turnId' in message && typeof message.turnId === 'string' ? message.turnId : undefined; + return marked; } function readModelDiagnostic( diff --git a/packages/runtime/src/session-manager.ts b/packages/runtime/src/session-manager.ts index 38ac4474e2..408e8b2a11 100644 --- a/packages/runtime/src/session-manager.ts +++ b/packages/runtime/src/session-manager.ts @@ -150,7 +150,6 @@ import { import { RuntimeReadModel, RuntimeReadModelError, - type RuntimeReadModelProjectionCache, type RuntimeReadModelSessionView, } from './runtime-read-model.js'; import { inspectAgentRunReadModel, type AgentRunInspectModel } from './agent-run-inspect.js'; @@ -3989,7 +3988,7 @@ export class SessionManager { throw new Error('Conversation copy requires a side-effect-free message snapshot'); } const readMessages = readMessagesSnapshot.bind(this.deps.store); - const view = await this.getSessionView(sessionId, { readMessages }); + const view = await this.getSessionView(sessionId); if (view.invocations.length > 0 || view.messages.length > 0) return view; const messages = await readMessages(sessionId); if (messages.length === 0) return view; @@ -4341,22 +4340,16 @@ export class SessionManager { return turn; } - private async getSessionView( - sessionId: string, - projectionCache: RuntimeReadModelProjectionCache = this.deps.store, - ): Promise { - return this.readModel(projectionCache).getSessionView(sessionId); + private async getSessionView(sessionId: string): Promise { + return this.readModel().getSessionView(sessionId); } - private readModel( - projectionCache: RuntimeReadModelProjectionCache = this.deps.store, - ): RuntimeReadModel { + private readModel(): RuntimeReadModel { if (!this.deps.runStore || !this.deps.runtimeEventStore) { throw new Error('RuntimeReadModel requires AgentRunStore and RuntimeEventStore'); } return new RuntimeReadModel({ runtimeEventStore: this.deps.runtimeEventStore, - projectionCache, ...(this.deps.canonicalPermissionOutcomes ? { canonicalPermissionOutcomes: this.deps.canonicalPermissionOutcomes } : {}), From 88228a9faf535392265832ff3b8d3b857d7fa96c Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 6 Sep 2026 03:10:45 +0800 Subject: [PATCH 05/32] refactor(runtime): move runtime notes onto the ledger and retire the rest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A system note stated one of two things. The ones that describe what happened inside an invocation — compaction, context pressure, the step cap — are facts of that invocation, and now live where its facts live: the RuntimeEvent ledger, through `AgentRun.recordSystemNote` and a `recordSystemNote` hook the backend reaches like its other recorders. They stay `modelVisibility: 'hidden'`, so the reader sees them and the provider never replays them. The others said something that already had an owner. The Session header carries the mode, the model and the copy lineage; the invocation's opening fact carries its own configuration; the terminal event carries the abort and its source. `session_start`, `session_resume`, `mode_change`, `model_change`, `error` and `abort` only wrote those facts a second time, into rows nothing rendered. Their write sites are gone; the kinds stay decodable so legacy transcripts still read. Two readers depended on `session_start` as a position marker for "this revision copy admitted a turn of its own". The admission ledger answers that directly — a copy clones history but never admissions — and the Host revision coordinator, which already reads it, settles every `preparing` copy at recovery before SessionManager's duplicate check ever ran. Generated-by: Claude Code --- packages/cli/src/pi-transcript.ts | 15 +-- packages/core/src/runtime-event.ts | 8 +- packages/core/src/session.ts | 74 +++++------ .../session-revision-two-client-uds.test.ts | 2 +- .../src/server/execution-model-composition.ts | 3 + .../server/session-revision-coordinator.ts | 73 +---------- .../mid-turn-capacity-backend.test.ts | 11 ++ .../overflow-reactive-recovery.test.ts | 11 ++ .../runtime-kernel-interaction.test.ts | 10 -- .../src/__tests__/session-manager.test.ts | 38 +----- packages/runtime/src/agent-run.ts | 40 +++--- packages/runtime/src/ai-sdk-backend.ts | 12 +- packages/runtime/src/ai-sdk-turn.ts | 120 +++++++----------- .../runtime/src/runtime-event-backfill.ts | 4 +- packages/runtime/src/runtime-kernel.ts | 37 ++---- packages/runtime/src/session-manager.ts | 104 +++------------ 16 files changed, 182 insertions(+), 380 deletions(-) diff --git a/packages/cli/src/pi-transcript.ts b/packages/cli/src/pi-transcript.ts index d32a711b72..d7af4da7df 100644 --- a/packages/cli/src/pi-transcript.ts +++ b/packages/cli/src/pi-transcript.ts @@ -31,6 +31,7 @@ import type { } from '@maka/core/events'; import { deriveTurnRecords, + isRuntimeSystemNoteKind, STEP_LIMIT_NOTICE_TEXT, type StoredMessage, type SystemNoteMessage, @@ -1344,14 +1345,10 @@ function tokenDelta(before: number | undefined, after: number | undefined): numb } function systemNoteText(message: SystemNoteMessage): string | undefined { + // Retired kinds are still decoded off legacy transcript rows, and none of + // them ever had a line here worth reading. + if (!isRuntimeSystemNoteKind(message.kind)) return undefined; switch (message.kind) { - case 'session_start': - case 'session_resume': - return undefined; - case 'mode_change': - return 'Permission mode changed.'; - case 'model_change': - return 'Model changed.'; case 'context_compacted': return 'Context compacted to keep this task within the model window.'; case 'context_compaction_failed_open': @@ -1408,10 +1405,6 @@ function systemNoteText(message: SystemNoteMessage): string | undefined { } case 'step_limit': return STEP_LIMIT_NOTICE_TEXT; - case 'error': - return 'Session recorded an error.'; - case 'abort': - return 'Session was stopped.'; } } diff --git a/packages/core/src/runtime-event.ts b/packages/core/src/runtime-event.ts index 3344b310e0..863679e82e 100644 --- a/packages/core/src/runtime-event.ts +++ b/packages/core/src/runtime-event.ts @@ -62,9 +62,9 @@ import { } from './orchestration.js'; import { isToolMode, type ToolMode } from './tool-mode.js'; import { - isTurnScopedSystemNoteKind, + isRuntimeSystemNoteKind, type PersistedBackendKind, - type TurnScopedSystemNoteKind, + type RuntimeSystemNoteKind, } from './session.js'; import { decodeTurnOrigin, type TurnOrigin } from './turn-origin.js'; import type { UserQuestionRequest } from './user-question.js'; @@ -232,7 +232,7 @@ export interface RuntimeEventFunctionResponseContent { */ export interface RuntimeEventSystemNoteContent { kind: 'system_note'; - note: TurnScopedSystemNoteKind; + note: RuntimeSystemNoteKind; /** Shape depends on `note`, exactly as it does on the transcript row. */ data?: unknown; } @@ -1058,7 +1058,7 @@ function isRuntimeEventContent(value: unknown): value is RuntimeEventContent { return ( hasExactShape(value, SYSTEM_NOTE_CONTENT_SHAPE) && typeof value.note === 'string' && - isTurnScopedSystemNoteKind(value.note) + isRuntimeSystemNoteKind(value.note) ); case 'invocation_opened': return isRuntimeInvocationOpened(value); diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts index 997d8dcc84..282d355bb8 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session.ts @@ -789,20 +789,15 @@ export function userFacingText(message: Pick()( ['text'], ['signature', 'providerOptions', 'parts'], ); -const SYSTEM_NOTE_KINDS = new Set([ - 'session_start', - 'session_resume', - 'mode_change', - 'model_change', - 'context_compacted', - 'context_compaction_failed_open', - 'context_provider_dropping', - 'context_window_suggestion', - 'context_window_overrun', - 'context_reported_window_exceeded', - 'context_overflow_after_compaction', - 'step_limit', - 'error', - 'abort', +const SYSTEM_NOTE_KINDS = new Set([ + ...RUNTIME_SYSTEM_NOTE_KINDS, + ...RETIRED_SYSTEM_NOTE_KINDS, ]); export function decodeCanonicalMessage(value: unknown): StoredMessage { diff --git a/packages/runtime-host/src/__tests__/session-revision-two-client-uds.test.ts b/packages/runtime-host/src/__tests__/session-revision-two-client-uds.test.ts index 8e039340ca..790ba8ba68 100644 --- a/packages/runtime-host/src/__tests__/session-revision-two-client-uds.test.ts +++ b/packages/runtime-host/src/__tests__/session-revision-two-client-uds.test.ts @@ -1712,7 +1712,7 @@ async function verifyDurableBranch( // projects the copied turn as ended, exactly as the source reads. assert.deepEqual( messages.map((message) => message.type), - ['user', 'assistant', 'tool_call', 'tool_result', 'system_note', 'turn_state'], + ['user', 'assistant', 'tool_call', 'tool_result', 'turn_state'], ); const user = messages.find((message) => message.type === 'user'); assert.ok(user?.attachments?.[0]); diff --git a/packages/runtime-host/src/server/execution-model-composition.ts b/packages/runtime-host/src/server/execution-model-composition.ts index f06cb71872..b2812d74e1 100644 --- a/packages/runtime-host/src/server/execution-model-composition.ts +++ b/packages/runtime-host/src/server/execution-model-composition.ts @@ -346,6 +346,9 @@ async function buildHostAiSdkBackend( appendMessage: input.context.appendMessage ?? ((message) => input.context.store.appendMessage(input.context.sessionId, message)), + ...(input.context.recordSystemNote + ? { recordSystemNote: input.context.recordSystemNote } + : {}), readExecutionBoundary: () => input.context.store.readExecutionBoundary(input.context.sessionId), ...(input.context.store.createSandboxBoundaryRequest diff --git a/packages/runtime-host/src/server/session-revision-coordinator.ts b/packages/runtime-host/src/server/session-revision-coordinator.ts index 3972b9baf8..829872cc8c 100644 --- a/packages/runtime-host/src/server/session-revision-coordinator.ts +++ b/packages/runtime-host/src/server/session-revision-coordinator.ts @@ -611,10 +611,6 @@ export class HostSessionRevisionCoordinator { if (copiedMessages.length > 0) { await this.#stores.sessionStore.appendMessages(input.targetSessionId, [...copiedMessages]); } - await this.#stores.sessionStore.appendMessage( - input.targetSessionId, - conversationCopyStartNote(kind, input, createInput), - ); await this.#stores.sessionStore.updateHeader(input.targetSessionId, { conversationCopy: { ...createInput.conversationCopy!, @@ -890,25 +886,15 @@ export class HostSessionRevisionCoordinator { ); } + /** + * A revision copy that admitted a turn of its own. The admission ledger is + * the whole answer: a copy clones the source's history but never its + * admissions, so every row it holds was admitted on this session. + */ async #hasAdmittedRevisionTurn(sessionId: string): Promise { - if ( + return ( (await this.#stores.agentRunStore.listRootTurnAdmissionsForRecovery(sessionId)).length > 0 - ) { - return true; - } - const messages = await this.#stores.sessionStore.readMessagesForRecovery(sessionId); - let boundary = -1; - for (let index = 0; index < messages.length; index += 1) { - const message = messages[index]!; - if ( - message.type === 'system_note' && - message.kind === 'session_start' && - isRevisionStartData(message.data) - ) { - boundary = index; - } - } - return boundary >= 0 && messages.slice(boundary + 1).some((message) => message.type === 'user'); + ); } async #hasCommittedConversationCopyDependent(sessionId: string): Promise { @@ -948,42 +934,6 @@ function conversationCopyFingerprint( return `sha256:${createHash('sha256').update(JSON.stringify(identity)).digest('hex')}`; } -function conversationCopyStartNote( - kind: ConversationCopySemanticKind, - input: SessionConversationCopyInput, - createInput: ConversationCopyCreateInput, -): StoredMessage { - const base = { - type: 'system_note' as const, - id: randomUUID(), - ts: Date.now(), - kind: 'session_start' as const, - }; - if (kind !== 'revision') { - // Empty copies record provenance without a branch turn. - return { - ...base, - data: { - parentSessionId: input.sourceSessionId, - ...(input.sourceTurnId === undefined ? {} : { branchOfTurnId: input.sourceTurnId }), - }, - }; - } - if (input.sourceTurnId === undefined) { - throw new Error('Session revision copy requires a turn boundary'); - } - return { - ...base, - data: { - revisionRootSessionId: createInput.revisionRootSessionId, - revisionParentSessionId: input.sourceSessionId, - revisionOfTurnId: input.sourceTurnId, - revisionIndex: createInput.revisionIndex, - revisionState: 'preparing', - }, - }; -} - function conversationCopySemanticKind( kind: ConversationCopyKind, input: SessionConversationCopyInput, @@ -995,15 +945,6 @@ function persistedConversationCopyKind(kind: ConversationCopySemanticKind): Conv return kind === 'revision' ? 'revision' : 'branch'; } -function isRevisionStartData(value: unknown): boolean { - return ( - !!value && - typeof value === 'object' && - !Array.isArray(value) && - 'revisionRootSessionId' in value - ); -} - function collectArchivedToolResultPlaceholders( events: readonly RuntimeEvent[], messages: readonly StoredMessage[], diff --git a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts index 8e0eff790d..a1f9df154b 100644 --- a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts +++ b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts @@ -471,6 +471,17 @@ function buildFixture(options: MidTurnFixtureOptions = {}): MidTurnFixture { appendMessage: async (message) => { messages.push(message); }, + // A note is a runtime event now. The fixture records it in the shape the + // read model projects back, so these assertions still read the row a + // transcript would show. + recordSystemNote: async (kind, turnId, data) => { + messages.push({ + type: 'system_note', + kind, + turnId, + ...(data !== undefined ? { data } : {}), + }); + }, connection: { ...connection(), ...(options.providerNative diff --git a/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts b/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts index 36a3b8e2e9..f8de7b5087 100644 --- a/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts +++ b/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts @@ -587,6 +587,17 @@ function buildReactiveFixture(options: ReactiveFixtureOptions): ReactiveFixture if (!options.slowAppendMessage) return; for (let i = 0; i < 5; i += 1) await flushMacrotask(); }, + // A note is a runtime event now. The fixture records it in the shape the + // read model projects back, so these assertions still read the row a + // transcript would show. + recordSystemNote: async (kind, turnId, data) => { + messages.push({ + type: 'system_note', + kind, + turnId, + ...(data !== undefined ? { data } : {}), + }); + }, connection: { ...connection(), ...(options.providerNative diff --git a/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts b/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts index 2216a52160..4bfb344305 100644 --- a/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts +++ b/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts @@ -260,11 +260,6 @@ describe('RuntimeKernel Interaction close cleanup', () => { ).length, 1, ); - assert.equal( - messages.filter((message) => message.type === 'system_note' && message.kind === 'abort') - .length, - 1, - ); const blockedActivation = fixture.kernel .startTurn(SESSION_ID, { turnId: 'turn-before-runner-settled', text: 'must not send' }) @@ -376,11 +371,6 @@ describe('RuntimeKernel Interaction close cleanup', () => { ).length, 1, ); - assert.equal( - firstMessages.filter((message) => message.type === 'system_note' && message.kind === 'abort') - .length, - 1, - ); const second = kernel .startTurn(SESSION_ID, { turnId: 'turn-generation-2', text: 'second' }) diff --git a/packages/runtime/src/__tests__/session-manager.test.ts b/packages/runtime/src/__tests__/session-manager.test.ts index 577e640b15..78aa131334 100644 --- a/packages/runtime/src/__tests__/session-manager.test.ts +++ b/packages/runtime/src/__tests__/session-manager.test.ts @@ -2932,7 +2932,9 @@ describe('SessionManager child-session runtime primitive', () => { runtimeEventStore: runStore, backends, childTools: [testTool('Read'), testTool('Glob'), testTool('Grep')], - newId: nextId(), + // Its own id space: a restarted host mints fresh ids, it does not replay + // the sequence the dead process was on. + newId: nextId('restarted'), now: nextNow(196), }); await drain( @@ -3453,7 +3455,7 @@ describe('SessionManager manual compaction and quiescent session changes', () => if (complete?.type !== 'complete') throw new Error('expected complete'); assert.strictEqual(complete.contextCompactionOutcome?.kind, 'compacted'); - const messages = await store.readMessages(session.id); + const messages = await manager.getMessages(session.id); assert.strictEqual( messages.some((message) => message.type === 'user' && message.text.includes('compact')), false, @@ -3618,7 +3620,7 @@ describe('SessionManager manual compaction and quiescent session changes', () => await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' })); await drain(manager.compactSession(session.id, { turnId: 'turn-compact' })); - const warnings = (await store.readMessages(session.id)).filter( + const warnings = (await manager.getMessages(session.id)).filter( (message) => message.type === 'system_note' && message.turnId === 'turn-compact' && @@ -4591,13 +4593,6 @@ describe('SessionManager permission mode updates', () => { assert.strictEqual(summary.permissionMode, 'ask'); assert.deepStrictEqual(summary.labels, ['kept']); assert.deepStrictEqual((await store.readHeader(session.id)).labels, ['kept']); - - const messages = await store.readMessages(session.id); - const modeNote = messages.find( - (message) => message.type === 'system_note' && message.kind === 'mode_change', - ); - if (modeNote?.type !== 'system_note') throw new Error('mode_change note was not written'); - assert.deepStrictEqual(modeNote.data, { from: 'explore', to: 'ask' }); }); test('starts a new turn without workspace identity when safety inspection fails', async () => { @@ -9446,11 +9441,6 @@ describe('SessionManager permission mode updates', () => { assert.strictEqual(backend?.stopCalls, 1); const messages = await store.readMessages(session.id); - assert.strictEqual( - messages.filter((message) => message.type === 'system_note' && message.kind === 'abort') - .length, - 1, - ); assert.strictEqual( messages.filter( (message) => @@ -9649,7 +9639,7 @@ describe('SessionManager permission mode updates', () => { [Symbol.asyncIterator](); await turn.next(); store.failAfterNextAppendMessage = (message) => - message.type === 'system_note' && message.kind === 'abort'; + message.type === 'turn_state' && message.status === 'aborted'; await expectRejects( manager.stopSession(session.id, { source: 'stop_button' }), @@ -9668,11 +9658,6 @@ describe('SessionManager permission mode updates', () => { ).length, 1, ); - assert.strictEqual( - messages.filter((message) => message.type === 'system_note' && message.kind === 'abort') - .length, - 1, - ); sendGate.release(); while (!(await turn.next()).done) {} }); @@ -9723,11 +9708,6 @@ describe('SessionManager permission mode updates', () => { ).length, 1, ); - assert.strictEqual( - messages.filter((message) => message.type === 'system_note' && message.kind === 'abort') - .length, - 1, - ); }); test('agent projections list catalog definitions separately from child runs and read output artifacts by child turn', async () => { @@ -10616,12 +10596,6 @@ describe('SessionManager permission mode updates', () => { const [turn] = await store.listTurns(session.id); assert.strictEqual(turn?.status, 'aborted'); assert.strictEqual(turn?.abortSource, 'renderer.stop_button'); - const abortNote = (await store.readMessages(session.id)).find( - (message) => message.type === 'system_note' && message.kind === 'abort', - ); - assert.strictEqual(abortNote?.type, 'system_note'); - if (abortNote?.type !== 'system_note') throw new Error('abort note missing'); - assert.deepStrictEqual(abortNote.data, { source: 'renderer.stop_button' }); }); test('stopSession persists abortSource on a terminal RuntimeEvent emitted during backend stop', async () => { diff --git a/packages/runtime/src/agent-run.ts b/packages/runtime/src/agent-run.ts index ff919edf1d..b63db06080 100644 --- a/packages/runtime/src/agent-run.ts +++ b/packages/runtime/src/agent-run.ts @@ -59,8 +59,8 @@ import type { SessionHeader, SessionHeaderPatch, SessionStatus, + RuntimeSystemNoteKind, StoredMessage, - SystemNoteMessage, TurnRecord, UserMessage, } from '@maka/core/session'; @@ -843,10 +843,29 @@ export class AgentRun { }; } - async recordStoredSessionEvent(ev: SessionEvent): Promise { - if (ev.type === 'token_usage') { - await this.input.store.appendMessage(this.sessionId, { ...ev } satisfies StoredMessage); - } + /** + * Record something the runtime needs to tell the reader about this turn. + * + * It is a fact of the invocation, so it goes where the invocation's facts go. + * Never model-visible: the note describes what happened to the conversation, + * it is not part of it. + */ + async recordSystemNote(kind: RuntimeSystemNoteKind, data?: unknown): Promise { + await this.recordRuntimeEvents([ + { + id: this.input.newId(), + invocationId: this.invocationId, + runId: this.runId, + sessionId: this.sessionId, + turnId: this.turnId, + ts: this.input.now(), + partial: false, + role: 'system', + author: 'system', + modelVisibility: 'hidden', + content: { kind: 'system_note', note: kind, ...(data !== undefined ? { data } : {}) }, + }, + ]); } async recordSessionEvent( @@ -1084,17 +1103,6 @@ export class AgentRun { } catch { // The user-visible turn already completed; preserve existing behavior. } - if (this.sawCompletion) { - await this.input.store - .appendMessage(this.sessionId, { - type: 'system_note', - id: this.input.newId(), - turnId: this.turnId, - ts: lastTs, - kind: 'session_resume', - } satisfies SystemNoteMessage) - .catch(() => {}); - } await this.finishRun(this.finalStatus, lastTs); } diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index 2a065fffb2..da0cff59f7 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -25,7 +25,12 @@ */ import type { SessionEvent } from '@maka/core/events'; -import type { BackendKind, SessionHeader, StoredMessage } from '@maka/core/session'; +import type { + BackendKind, + RuntimeSystemNoteKind, + SessionHeader, + StoredMessage, +} from '@maka/core/session'; import type { AgentBackend, BackendCompactHistoryInput, @@ -173,6 +178,11 @@ export interface AiSdkBackendInput extends AiSdkCompactionCapabilities { readChildAgentOutput?: ToolRuntimeInput['readChildAgentOutput']; /** Optional diagnostic trace hook for explaining a runtime turn without changing renderer events. */ recordRunTrace?: RunTraceRecorder; + /** + * Writes one runtime note — something that happened inside this invocation — + * to the invocation's RuntimeEvent ledger, which is where its record lives. + */ + recordSystemNote?: (kind: RuntimeSystemNoteKind, turnId: string, data?: unknown) => Promise; /** * Commits one settled provider request: the canonical attempt and, when it * is the completed main call, the derived latest-context row it authorises. diff --git a/packages/runtime/src/ai-sdk-turn.ts b/packages/runtime/src/ai-sdk-turn.ts index 8ae5270101..57ddbe3d65 100644 --- a/packages/runtime/src/ai-sdk-turn.ts +++ b/packages/runtime/src/ai-sdk-turn.ts @@ -43,8 +43,8 @@ import type { AssistantMessage, AssistantStepContentKind, AssistantThinkingPart, + RuntimeSystemNoteKind, SessionHeader, - SystemNoteMessage, TokenUsageMessage, } from '@maka/core/session'; import type { BackendSendInput } from '@maka/core/backend-types'; @@ -852,6 +852,24 @@ export class AiSdkTurn { }; } + /** + * A note about what happened inside this invocation, written to the + * invocation's own ledger. Fail-open: the note explains a turn, it is not + * what the turn did, so losing it must never end a send that is otherwise + * fine. Returns whether the note landed. + */ + private async recordSystemNote( + kind: RuntimeSystemNoteKind, + turnId: string, + data?: unknown, + ): Promise { + if (!this.deps.backend.recordSystemNote) return false; + return await this.deps.backend + .recordSystemNote(kind, turnId, data) + .then(() => true) + .catch(() => false); + } + // -------------------------------------------------------------------------- // manual history compaction // -------------------------------------------------------------------------- @@ -1025,33 +1043,16 @@ export class AiSdkTurn { decision.boundaryKind === 'historyCompact' && decision.decision === 'failedOpen', ) .at(-1)?.failOpenReason; - const note: SystemNoteMessage = { - type: 'system_note', - id: this.deps.newId(), - turnId, - ts: this.deps.now(), - kind: 'context_compaction_failed_open', - ...(failOpenReason !== undefined ? { data: { failOpenReason } } : {}), - }; // Mark written only after the append lands: a failed write must leave // the flag down so the settlement fallback can still record the note. - contextCompactionFailedOpenNoteWritten = await this.deps.backend - .appendMessage(note) - .then(() => true) - .catch(() => false); + contextCompactionFailedOpenNoteWritten = await this.recordSystemNote( + 'context_compaction_failed_open', + turnId, + failOpenReason !== undefined ? { failOpenReason } : undefined, + ); } if (!contextCompactedNoteWritten && shouldAppendContextCompactedNote(contextBudget)) { - const note: SystemNoteMessage = { - type: 'system_note', - id: this.deps.newId(), - turnId, - ts: this.deps.now(), - kind: 'context_compacted', - }; - contextCompactedNoteWritten = await this.deps.backend - .appendMessage(note) - .then(() => true) - .catch(() => false); + contextCompactedNoteWritten = await this.recordSystemNote('context_compacted', turnId); } }; // Request index (0-based) at which the active prune last rewrote the @@ -1723,15 +1724,10 @@ export class AiSdkTurn { : stepUsage.inputTokens <= priorInput) ) { this.deps.session.contextProviderDroppingReported = true; - const note: SystemNoteMessage = { - type: 'system_note', - id: this.deps.newId(), - turnId, - ts: this.deps.now(), - kind: 'context_provider_dropping', - data: { inputTokens: stepUsage.inputTokens, priorInputTokens: priorInput }, - }; - await this.deps.backend.appendMessage(note).catch(() => {}); + await this.recordSystemNote('context_provider_dropping', turnId, { + inputTokens: stepUsage.inputTokens, + priorInputTokens: priorInput, + }); } // Fail closed: reset on every step boundary so a missing final // step's usage does not leave a stale value from an earlier step. @@ -1752,18 +1748,10 @@ export class AiSdkTurn { stepUsage.inputTokens + stepUsage.outputTokens > midTurnState.capacity ) { contextWindowOverrunNoteWritten = true; - const note: SystemNoteMessage = { - type: 'system_note', - id: this.deps.newId(), - turnId, - ts: this.deps.now(), - kind: 'context_window_overrun', - data: { - usedTokens: stepUsage.inputTokens + stepUsage.outputTokens, - declaredContextWindow: midTurnState.capacity, - }, - }; - await this.deps.backend.appendMessage(note).catch(() => {}); + await this.recordSystemNote('context_window_overrun', turnId, { + usedTokens: stepUsage.inputTokens + stepUsage.outputTokens, + declaredContextWindow: midTurnState.capacity, + }); } // Nothing declared, and the provider accepted a request past // the window this model reports. Every other signal in this @@ -1805,15 +1793,10 @@ export class AiSdkTurn { (previousTotal === undefined || previousTotal <= reported); if (reported !== undefined && crossedNow) { contextReportedWindowNoteWritten = true; - const note: SystemNoteMessage = { - type: 'system_note', - id: this.deps.newId(), - turnId, - ts: this.deps.now(), - kind: 'context_reported_window_exceeded', - data: { usedTokens: used, reportedContextWindow: reported }, - }; - await this.deps.backend.appendMessage(note).catch(() => {}); + await this.recordSystemNote('context_reported_window_exceeded', turnId, { + usedTokens: used, + reportedContextWindow: reported, + }); } } lastStepInputTokens = stepUsage?.inputTokens; @@ -2132,20 +2115,12 @@ export class AiSdkTurn { (midTurnState.capacity === undefined || acceptedTotal < midTurnState.capacity) ) { contextWindowSuggestionNoteWritten = true; - const note: SystemNoteMessage = { - type: 'system_note', - id: this.deps.newId(), - turnId, - ts: this.deps.now(), - kind: 'context_window_suggestion', - data: { - suggestedContextWindow: acceptedTotal, - ...(midTurnState.capacity !== undefined - ? { declaredContextWindow: midTurnState.capacity } - : {}), - }, - }; - await this.deps.backend.appendMessage(note).catch(() => {}); + await this.recordSystemNote('context_window_suggestion', turnId, { + suggestedContextWindow: acceptedTotal, + ...(midTurnState.capacity !== undefined + ? { declaredContextWindow: midTurnState.capacity } + : {}), + }); } // A folded projection was selected in this send and the provider // still rejects the request. That is worth saying, because the @@ -2161,14 +2136,7 @@ export class AiSdkTurn { midTurnState?.compactionAppliedThisSend === true ) { contextOverflowAfterCompactionNoteWritten = true; - const note: SystemNoteMessage = { - type: 'system_note', - id: this.deps.newId(), - turnId, - ts: this.deps.now(), - kind: 'context_overflow_after_compaction', - }; - await this.deps.backend.appendMessage(note).catch(() => {}); + await this.recordSystemNote('context_overflow_after_compaction', turnId); } const idleWatchdogRecovery = settledWatchdogTimeout?.phase === 'idle' && diff --git a/packages/runtime/src/runtime-event-backfill.ts b/packages/runtime/src/runtime-event-backfill.ts index a48b0c5e98..4c33befd1f 100644 --- a/packages/runtime/src/runtime-event-backfill.ts +++ b/packages/runtime/src/runtime-event-backfill.ts @@ -19,7 +19,7 @@ import type { RuntimeInvocationOutcome } from '@maka/core/runtime-invocation'; import type { RunIdentity } from './terminal-run-commit.js'; -import { isTurnScopedSystemNoteKind } from '@maka/core/session'; +import { isRuntimeSystemNoteKind } from '@maka/core/session'; import type { PermissionDecisionMessage, StoredMessage, @@ -365,7 +365,7 @@ export function backfillRuntimeEventsFromStoredMessages( // something about the Session, and the Session transcript keeps it. case 'system_note': if (conversationTextOnly) break; - if (!isTurnScopedSystemNoteKind(message.kind)) { + if (!isRuntimeSystemNoteKind(message.kind)) { diagnostics.push({ code: 'skipped_high_risk_message', message: diff --git a/packages/runtime/src/runtime-kernel.ts b/packages/runtime/src/runtime-kernel.ts index d18cce8d65..358b314dbd 100644 --- a/packages/runtime/src/runtime-kernel.ts +++ b/packages/runtime/src/runtime-kernel.ts @@ -46,7 +46,6 @@ import type { SessionHeaderPatch, SessionStatus, StoredMessage, - SystemNoteMessage, TurnRecord, TurnStateMessage, } from '@maka/core/session'; @@ -332,8 +331,6 @@ interface StopOperation { projected: boolean; } >; - abortNote: SystemNoteMessage; - abortNoteProjected: boolean; targets: Map; queue: Promise; } @@ -1085,24 +1082,18 @@ export class RuntimeKernel implements RuntimeKernelLike { runId: run.runId, turnId: run.turnId, }); + // Ahead of the usage row, because the ledger seals on its terminal fact: + // a note queued behind one this compaction may already own would be + // refused, and the reader would never learn the summary was skipped. + if (result.outcome.kind === 'failed') { + await run.recordSystemNote('context_compaction_failed_open').catch(() => {}); + } await run.acceptMappedEvent( tokenUsageEvent, mapSessionEventToRuntimeEvent(tokenUsageEvent, eventContext), { requireTerminalWrite: true }, ); if (run.isStopped()) return; - await run.recordStoredSessionEvent(tokenUsageEvent); - if (run.isStopped()) return; - if (result.outcome.kind === 'failed') { - const note: SystemNoteMessage = { - type: 'system_note', - id: this.deps.newId(), - turnId: run.turnId, - ts: this.deps.now(), - kind: 'context_compaction_failed_open', - }; - await this.deps.store.appendMessage(sessionId, note).catch(() => {}); - } yield tokenUsageEvent; if (run.isStopped()) return; await run.acceptMappedEvent( @@ -1847,14 +1838,6 @@ export class RuntimeKernel implements RuntimeKernelLike { ts, statusProjected: false, turnProjections: new Map(), - abortNote: { - type: 'system_note', - id: this.deps.newId(), - ts, - kind: 'abort', - ...(abortSource ? { data: { source: abortSource } } : {}), - }, - abortNoteProjected: false, targets: new Map(), queue: Promise.resolve(), }; @@ -1948,10 +1931,6 @@ export class RuntimeKernel implements RuntimeKernelLike { await this.appendStopProjection(sessionId, projection.message); projection.projected = true; } - if (!operation.abortNoteProjected) { - await this.appendStopProjection(sessionId, operation.abortNote); - operation.abortNoteProjected = true; - } // The Session projection above now reads as aborted. The ledger has to say // the same thing before this stop reports success: a Run left non-terminal // here stays that way, because the stream that would have finalized it is @@ -1977,7 +1956,6 @@ export class RuntimeKernel implements RuntimeKernelLike { } const completed = operation.statusProjected && - operation.abortNoteProjected && [...operation.turnProjections.values()].every((projection) => projection.projected) && [...operation.targets.values()].every( (target) => @@ -2298,6 +2276,7 @@ export class RuntimeKernel implements RuntimeKernelLike { }): Pick< BackendFactoryContext, | 'recordRunTrace' + | 'recordSystemNote' | 'recordModelCallAttempt' | 'recordRunComposition' | 'loadHistoryCompactCheckpoint' @@ -2316,6 +2295,8 @@ export class RuntimeKernel implements RuntimeKernelLike { recordRunTrace: (event) => { runFor(event.turnId)?.recordRunTrace(event); }, + recordSystemNote: (kind, turnId, data) => + runFor(turnId)?.recordSystemNote(kind, data) ?? Promise.resolve(), ...(this.deps.runStore ? { // Resolved by runId rather than turnId: the canonical record names diff --git a/packages/runtime/src/session-manager.ts b/packages/runtime/src/session-manager.ts index 408e8b2a11..ddfbcddb9e 100644 --- a/packages/runtime/src/session-manager.ts +++ b/packages/runtime/src/session-manager.ts @@ -52,11 +52,11 @@ import type { SessionStatus, SessionSummary, StoredMessage, + RuntimeSystemNoteKind, SubagentSessionParent, TurnRecord, UserMessage, PermissionDecisionMessage, - SystemNoteMessage, PersistedBackendKind, } from '@maka/core/session'; import type { @@ -697,6 +697,11 @@ export interface BackendFactoryContext { * provider call, including metering and its prepared-request observation. */ recordModelCallAttempt?: (commit: ModelCallCommit) => Promise; + /** + * Writes one runtime note — something that happened inside the running + * invocation — to that invocation's RuntimeEvent ledger. + */ + recordSystemNote?: (kind: RuntimeSystemNoteKind, turnId: string, data?: unknown) => Promise; /** Immutable Run policy snapshot; provider dispatch waits for this durable commit. */ recordRunComposition?: (runId: string, snapshot: RunCompositionSnapshot) => Promise; loadHistoryCompactCheckpoint?: () => Promise; @@ -1478,15 +1483,10 @@ export class SessionManager { messagesReadable = false; } - if (session.revisionState === 'preparing' && messagesReadable) { - if (hasRevisionUserMessage(messages)) { - await recoverOr(policy, () => this.commitRevisionVersion(session.id), undefined); - } else { - await recoverOr(policy, () => this.remove(session.id), undefined); - recovered.add(session.id); - continue; - } - } + // A revision copy still `preparing` is settled by the Host's revision + // coordinator, which reads the admission ledger and runs before this + // recovery. Deciding it a second time here — off a transcript scan, and + // ending in `remove()` — could only ever contradict it. let continuationClaimRecovered = false; const continuationAuthority = runtimeContinuationAuthority(this.deps.runtimeEventStore); @@ -1638,15 +1638,6 @@ export class SessionManager { }); const next = await this.deps.store.readHeader(sessionId); this.runtimeKernel.updateCachedHeader(sessionId, next); - await this.deps.store - .appendMessage(sessionId, { - type: 'system_note', - id: this.deps.newId(), - ts: this.deps.now(), - kind: 'mode_change', - data: { from: previous.permissionMode, to: mode }, - } satisfies SystemNoteMessage) - .catch(() => undefined); return headerToSummary(next); } @@ -1863,13 +1854,6 @@ export class SessionManager { const next = await this.deps.store.updateHeader(sessionId, { collaborationMode: mode, }); - await this.deps.store.appendMessage(sessionId, { - type: 'system_note', - id: this.deps.newId(), - ts: this.deps.now(), - kind: 'mode_change', - data: { dimension: 'collaboration', from, to: mode }, - } satisfies SystemNoteMessage); this.runtimeKernel.updateCachedHeader(sessionId, next); await this.runtimeKernel.disposeBackend(sessionId); return headerToSummary(next); @@ -1886,13 +1870,6 @@ export class SessionManager { throw new Error('Cannot change orchestration mode while a tool call awaits confirmation.'); } const next = await this.deps.store.updateHeader(sessionId, { orchestrationMode: mode }); - await this.deps.store.appendMessage(sessionId, { - type: 'system_note', - id: this.deps.newId(), - ts: this.deps.now(), - kind: 'mode_change', - data: { dimension: 'orchestration', from, to: mode }, - } satisfies SystemNoteMessage); this.runtimeKernel.updateCachedHeader(sessionId, next); return headerToSummary(next); } @@ -1935,7 +1912,7 @@ export class SessionManager { } if (!replay) await this.runtimeKernel.disposeBackend(sessionId); const result = await this.requirePlanStore().abandonProposal(input); - await this.finalizePlanAbandonment(sessionId, operationId, replay); + await this.finalizePlanAbandonment(sessionId); return result; } @@ -4206,41 +4183,13 @@ export class SessionManager { this.runtimeKernel.updateCachedHeader(sessionId, next); } - private async finalizePlanAbandonment( - sessionId: string, - operationId: string | undefined, - replay: boolean, - ): Promise { + private async finalizePlanAbandonment(sessionId: string): Promise { const header = await this.deps.store.readHeader(sessionId); - const from = header.collaborationMode ?? 'agent'; - const changed = from !== 'agent'; - const next = changed - ? await this.deps.store.updateHeader(sessionId, { collaborationMode: 'agent' }) - : header; + const next = + (header.collaborationMode ?? 'agent') === 'agent' + ? header + : await this.deps.store.updateHeader(sessionId, { collaborationMode: 'agent' }); this.runtimeKernel.updateCachedHeader(sessionId, next); - - if (!changed && !replay) return; - if (!operationId) { - await this.deps.store.appendMessage(sessionId, { - type: 'system_note', - id: this.deps.newId(), - ts: this.deps.now(), - kind: 'mode_change', - data: { dimension: 'collaboration', from, to: 'agent' }, - } satisfies SystemNoteMessage); - return; - } - - const noteId = planAbandonmentNoteId(operationId); - const messages = await this.deps.store.readMessages(sessionId); - if (messages.some((message) => message.id === noteId)) return; - await this.deps.store.appendMessage(sessionId, { - type: 'system_note', - id: noteId, - ts: this.deps.now(), - kind: 'mode_change', - data: { dimension: 'collaboration', from: 'plan', to: 'agent' }, - } satisfies SystemNoteMessage); } private requirePlanStore(): PlanStore { @@ -4711,10 +4660,6 @@ export class SessionManager { } } -function planAbandonmentNoteId(operationId: string): string { - return `plan-abandonment-${createHash('sha256').update(operationId).digest('hex')}`; -} - function resumeFeatureDisabledPlan(): SafeBoundaryContinuationPlan { return { disposition: 'park', @@ -5188,23 +5133,6 @@ interface InterruptedTurnRecovery { >; } -function hasRevisionUserMessage(messages: readonly StoredMessage[]): boolean { - let boundary = -1; - for (let index = 0; index < messages.length; index += 1) { - const message = messages[index]!; - if ( - message.type === 'system_note' && - message.kind === 'session_start' && - message.data && - typeof message.data === 'object' && - 'revisionRootSessionId' in message.data - ) { - boundary = index; - } - } - return boundary >= 0 && messages.slice(boundary + 1).some((message) => message.type === 'user'); -} - function interruptedTurnRecoveries(messages: readonly StoredMessage[]): InterruptedTurnRecovery[] { const byTurn = new Map< string, From 08cbf54962b3a719393a370394adeeef3d58defd Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 6 Sep 2026 06:01:11 +0800 Subject: [PATCH 06/32] refactor(runtime): make the RuntimeEvent ledger the only transcript authority MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every execution fact was written twice: once as a RuntimeEvent and once as a `session_messages` row. Two authorities for the same fact means every writer has to keep them in step, every reader has to pick one, and a crash between the two writes leaves a Session that disagrees with itself. This cuts the second write. The ledger is the durable record; `session_messages` survives only as input to the one-way importer that converts a pre-ledger transcript on first read, and as the WorkHub Coordination Session's own store, which is out of scope here. What moved: - `markMessagesHandedOff` no longer projects admitted Messages into transcript rows. It validates the admission and retires it; the durable proof the rows used to carry already lives in the agent-run admission's `sourceMessages` and in the RuntimeEvent steering proof. - Catalog projection (`lastMessagePreview`, `lastMessageAt`, `connectionLocked`) is committed by `AgentRun` through `commitMessageCatalogProjection` instead of falling out of a transcript insert. It is fail-closed for a user message, because that write also takes the Session's one-way connection lock, and fail-open for the assistant preview, which costs a stale sidebar line at worst. - The read marker no longer needs an ordered index of visible transcript rows. `lastReadMessageId` has no consumer, so `hasUnread` is the only decision left: it clears when the client has caught up with the ledger's newest visible message, read off a bounded tail of the last run. - Startup recovery writes a crashed Turn's admitted prompt into the invocation that had already opened for it. A Root folded from several queued Messages has no single admitted Message identity, so the prompt is durable under a derived `${runId}-admitted-prompt`, which makes recovering the same crash twice a no-op append. A sealed Run takes nothing: it is immutable, and a Run that reached its terminal fact has a prompt the crash did not eat. - WorkHub target linkage (#4699) enumerated a delegated Message's identity from three lifecycle tables, one of which was the transcript row this change stops writing. Its handed-off arm now reads `core_root_source_message_proofs` — the Root admission that consumed the Message, in the same database and as durable as the Session. Deleted with their last caller: `markSessionReadThroughMessage`, `SessionReadMarkerMessageNotFoundError`, `readMessagesForRecovery` (identical to `readMessages`), `listForRecovery`'s separate query, the transcript-ordering privates in the SQLite store, and `buildTurnStateMessage` with its lineage types. Ablations kept out: an `existing.some(role !== 'system')` guard in recovery on top of the terminal-event check (the terminal check alone is exact), and removing the singular `appendMessage` (pure test churn for no production gain). Verified on this base: storage 1092 pass, runtime-host 1709 pass, runtime 3129 pass, core 821 pass, cli 805 pass; 0 failures. Closes #4791 Generated-by: Claude Code --- packages/core/src/session.ts | 15 +- ...t-capability-admission-integration.test.ts | 1 - .../__tests__/execution-composition.test.ts | 6 +- .../__tests__/execution-host-queue.test.ts | 3 +- .../__tests__/execution-host-recovery.test.ts | 84 +- .../execution-model-composition.test.ts | 7 +- .../fixtures/execution-host-suite.ts | 30 +- .../__tests__/fixtures/ledger-transcript.ts | 39 + .../fixtures/session-transcript-reader.ts | 44 + .../src/__tests__/goal-coordinator.test.ts | 11 + .../src/__tests__/goal-root-authority.test.ts | 36 +- .../__tests__/root-turn-coordinator.test.ts | 80 +- .../session-catalog-coordinator.test.ts | 70 +- .../session-catalog-two-client-uds.test.ts | 45 +- .../session-revision-two-client-uds.test.ts | 145 +- .../session-transcript-reader.test.ts | 54 +- .../src/server/execution-composition.ts | 39 +- .../src/server/execution-model-composition.ts | 3 - .../src/server/goal-coordinator.ts | 4 +- .../src/server/hosted-execution-recovery.ts | 231 ++-- .../src/server/session-catalog-coordinator.ts | 61 +- .../server/session-revision-coordinator.ts | 5 +- .../src/server/session-transcript-reader.ts | 387 +++++- .../src/__tests__/admission-limiter.test.ts | 1 - .../agent-run-steering-recovery.test.ts | 180 --- .../src/__tests__/ai-sdk-backend.test.ts | 227 +--- .../src/__tests__/ask-user-question.test.ts | 2 - .../src/__tests__/code-mode-backend.test.ts | 1 - .../computer-use-privacy-boundary.test.ts | 1 - .../computer-use-provider-protocol.test.ts | 7 - .../__tests__/deferred-tools-backend.test.ts | 1 - .../execution-boundary-test-helpers.ts | 147 +- .../src/__tests__/fake-backend.test.ts | 28 +- .../__tests__/interaction-authority.test.ts | 1 - .../__tests__/latest-context-commit.test.ts | 2 - .../pre-dispatch-refusal-ledger.test.ts | 1 - .../runtime-continuation-crash.test.ts | 22 +- .../runtime-kernel-interaction.test.ts | 27 - .../__tests__/runtime-ledger-repair.test.ts | 12 - .../sandbox-boundary-restart-recovery.test.ts | 52 +- .../session-manager-terminal-ledger.test.ts | 28 - .../src/__tests__/session-manager.test.ts | 192 ++- .../session-projection-helpers.test.ts | 77 -- .../__tests__/shell-run-tool-result.test.ts | 2 +- .../src/__tests__/subagent-tools.test.ts | 1 - .../src/__tests__/tool-args-violation.test.ts | 1 - .../src/__tests__/tool-artifacts.test.ts | 1 - ...-result-archive-capability-backend.test.ts | 1 - .../tool-runtime-argument-ownership.test.ts | 6 +- .../tool-runtime-durable-boundary.test.ts | 3 +- .../tool-runtime-form-interaction.test.ts | 3 - .../__tests__/tool-runtime-progress.test.ts | 1 - .../tool-runtime-sandbox-boundary.test.ts | 15 - .../__tests__/tool-runtime-settlement.test.ts | 1 - .../tool-runtime-sqlite-boundary.test.ts | 6 - packages/runtime/src/agent-run.ts | 177 +-- packages/runtime/src/ai-sdk-backend.ts | 4 - packages/runtime/src/ai-sdk-turn.ts | 40 - packages/runtime/src/runtime-kernel.ts | 117 +- packages/runtime/src/runtime-ledger-repair.ts | 85 +- packages/runtime/src/runtime-read-model.ts | 1 - packages/runtime/src/session-manager.ts | 369 ++--- .../runtime/src/session-projection-helpers.ts | 56 - .../runtime/src/test-only/fake-backend.ts | 81 +- packages/runtime/src/tool-runtime.ts | 63 - packages/storage/package.json | 1 + .../src/__tests__/session-store.test.ts | 502 +------ .../sqlite-session-metadata-store.test.ts | 590 +------- .../workhub-message-assignment.test.ts | 62 +- packages/storage/src/execution-stores.ts | 36 +- .../storage/src/runtime-event-persistence.ts | 6 + .../storage/src/session-message-projection.ts | 41 + packages/storage/src/session-store.ts | 204 +-- .../src/sqlite-session-metadata-store.ts | 1202 +---------------- 74 files changed, 1771 insertions(+), 4316 deletions(-) create mode 100644 packages/runtime-host/src/__tests__/fixtures/ledger-transcript.ts diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts index 282d355bb8..f97427bbc4 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session.ts @@ -283,7 +283,7 @@ export interface SessionHeader { /** Immutable Connection entity identity. Optional only on legacy Session records. */ llmConnectionId?: string; llmConnectionSlug: string; - /** True after first UserMessage is flushed. Storage self-heals (§5.2). */ + /** True once the Session's first UserMessage is durable. One-way. */ connectionLocked: boolean; /** Sticky session default model id, captured when the session is created. */ model: string; @@ -790,11 +790,8 @@ export function userFacingText(message: Pick undefined, readExecutionBoundary: async () => createManagedExecutionBoundary(createWorkspaceWritePermissionProfile(), 0), newId: nextId(), diff --git a/packages/runtime-host/src/__tests__/execution-composition.test.ts b/packages/runtime-host/src/__tests__/execution-composition.test.ts index 4c7cc83f84..64aef45990 100644 --- a/packages/runtime-host/src/__tests__/execution-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-composition.test.ts @@ -65,6 +65,7 @@ import { stopReplacedWorkHubRoot, } from '../server/execution-composition.js'; import { waitFor as pollFor } from '@maka/core/test-only/async-primitives'; +import { readLedgerMessages } from './fixtures/ledger-transcript.js'; const require = createRequire(import.meta.url); const FAKE_CONNECTION_ID = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'; @@ -462,6 +463,9 @@ test('production recovery preserves legacy Automation history and closes an orph const composition = await createExecutionRuntimeHostComposition(compositionContext(owner)); try { await composition.recover(); + // The legacy transcript itself, as the converter reads it: recovery must + // leave a pre-ledger Automation's origin intact for the import that + // follows on the Session's first read. const history = await stores.sessionStore.readMessages(historical.id); assert.deepEqual(history[0]?.type === 'user' ? history[0].origin : undefined, { kind: 'legacy_automation', @@ -2019,7 +2023,7 @@ async function assertUniqueGraphExecutionFacts( ): Promise { const [runs, messages, runtimeEvents] = await Promise.all([ stores.runtimeEventStore.listSessionInvocations(claim.targetSessionId), - stores.sessionStore.readMessages(claim.targetSessionId), + readLedgerMessages(stores.runtimeEventStore, claim.targetSessionId), stores.runtimeEventStore.readImmutableRuntimeEvents(claim.targetSessionId, claim.targetRunId), ]); assert.deepEqual( diff --git a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts index 88d4da5e37..0d62d9a3b3 100644 --- a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts @@ -90,6 +90,7 @@ import { } from '../protocol/index.js'; import { SessionAdmissionGate } from '../server/session-admission-gate.js'; import { FramedTransport } from '../transport/framed-transport.js'; +import { readLedgerMessages } from './fixtures/ledger-transcript.js'; import { CONNECTION_EFFECT_MODEL_IDS, @@ -773,7 +774,7 @@ test('startup recovery canonically closes pending linked child admissions withou assert.equal(terminal.fact.failureClass, 'app_restarted'); } const userMessages: StoredMessage[] = ( - await stores.sessionStore.readMessages(recovered.sessionId) + await readLedgerMessages(stores.runtimeEventStore, recovered.sessionId) ).filter((message) => message.type === 'user' && message.turnId === recovered.turnId); assert.equal(userMessages.length, recovered.kind === 'linked_child_provider_retry' ? 0 : 1); if (recovered.kind !== 'linked_child_provider_retry') { diff --git a/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts b/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts index 7288617e0d..7b6c40030a 100644 --- a/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts @@ -209,47 +209,42 @@ test('startup recovery replays an admitted regenerate with its source lineage', }); }); -test('startup recovery materializes legacy terminal Root sources exactly once', async () => { +// A Root folded from several queued Messages ran as one prompt, so the ledger +// carries that one prompt — under an id derived from its Run, which is what +// makes a second recovery pass write nothing new. +function legacyRootPrompt(legacy: { + runId: string; + turnId: string; + sources: readonly { content: { text: string }; admittedAt: number }[]; +}) { + return { + id: `${legacy.runId}-admitted-prompt`, + turnId: legacy.turnId, + ts: legacy.sources[0]!.admittedAt, + text: legacy.sources.map((source) => source.content.text).join('\n\n'), + }; +} + +test('startup recovery retires a legacy terminal Root without reopening its sealed Run', async () => { await withExecutionRoot(async (fixture) => { const legacy = await fixture.seedLegacyRootWithoutSourceTranscripts(); - assert.deepEqual( - (await fixture.readSessionUserMessages()).filter((message) => - legacy.sources.some((source) => source.messageId === message.id), - ), - [], - ); + assert.deepEqual(await fixture.readSessionUserMessages(), []); const firstHost = await fixture.startHost(); await fixture.stopHost(firstHost); - assert.deepEqual( - (await fixture.readSessionUserMessages()) - .filter((message) => legacy.sources.some((source) => source.messageId === message.id)) - .map(({ id, turnId, ts, text }) => ({ id, turnId, ts, text })), - legacy.sources.map((source) => ({ - id: source.messageId, - turnId: legacy.turnId, - ts: source.admittedAt, - text: source.content.text, - })), - ); - const secondHost = await fixture.startHost(); await fixture.stopHost(secondHost); - assert.deepEqual( - (await fixture.readSessionUserMessages()) - .filter((message) => legacy.sources.some((source) => source.messageId === message.id)) - .map(({ id, turnId, ts, text }) => ({ id, turnId, ts, text })), - legacy.sources.map((source) => ({ - id: source.messageId, - turnId: legacy.turnId, - ts: source.admittedAt, - text: source.content.text, - })), - ); + + // A sealed Run is immutable, so its ledger stays exactly as the crash left + // it; recovery's job here is only to retire the admission it outlived. + assert.deepEqual(await fixture.readSessionUserMessages(), []); + const ledger = await fixture.readTurn(legacy.turnId); + assert.equal(ledger.runs.length, 1); + assert.equal(ledger.terminalEvents.length, 1); }); }); -test('startup recovery replays a legacy Root without a Run before materializing its sources', async () => { +test('startup recovery replays a legacy Root without a Run before recording its prompt', async () => { await withExecutionRoot(async (fixture) => { const legacy = await fixture.seedLegacyRootWithoutSourceTranscripts('missing'); @@ -259,15 +254,8 @@ test('startup recovery replays a legacy Root without a Run before materializing await fixture.stopHost(secondHost); assert.deepEqual( - (await fixture.readSessionUserMessages()) - .filter((message) => legacy.sources.some((source) => source.messageId === message.id)) - .map(({ id, turnId, ts, text }) => ({ id, turnId, ts, text })), - legacy.sources.map((source) => ({ - id: source.messageId, - turnId: legacy.turnId, - ts: source.admittedAt, - text: source.content.text, - })), + (await fixture.readSessionUserMessages()).map(({ turnId, text }) => ({ turnId, text })), + [{ turnId: legacy.turnId, text: legacyRootPrompt(legacy).text }], ); const ledger = await fixture.readTurn(legacy.turnId); assert.equal(ledger.runs.length, 1); @@ -275,7 +263,7 @@ test('startup recovery replays a legacy Root without a Run before materializing }); }); -test('startup recovery closes a legacy non-terminal Run before materializing its sources', async () => { +test('startup recovery closes a legacy non-terminal Run before recording its prompt', async () => { await withExecutionRoot(async (fixture) => { const legacy = await fixture.seedLegacyRootWithoutSourceTranscripts('created'); @@ -285,15 +273,13 @@ test('startup recovery closes a legacy non-terminal Run before materializing its await fixture.stopHost(secondHost); assert.deepEqual( - (await fixture.readSessionUserMessages()) - .filter((message) => legacy.sources.some((source) => source.messageId === message.id)) - .map(({ id, turnId, ts, text }) => ({ id, turnId, ts, text })), - legacy.sources.map((source) => ({ - id: source.messageId, - turnId: legacy.turnId, - ts: source.admittedAt, - text: source.content.text, + (await fixture.readSessionUserMessages()).map(({ id, turnId, ts, text }) => ({ + id, + turnId, + ts, + text, })), + [legacyRootPrompt(legacy)], ); const ledger = await fixture.readTurn(legacy.turnId); assert.equal(ledger.runs.length, 1); diff --git a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts index bfae661190..398934d65c 100644 --- a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts @@ -116,6 +116,7 @@ import { } from '../server/oauth-execution-authority.js'; import type { HostSkillCatalogCoordinator } from '../server/skill-catalog-coordinator.js'; import { AgentGraphProviderScenario } from './fixtures/agent-graph-provider-scenario.js'; +import { readLedgerMessages } from './fixtures/ledger-transcript.js'; const MODEL_ID = 'hosted-real-model'; const API_KEY = 'hosted-provider-key'; @@ -1979,7 +1980,7 @@ test('production Host executes a canonical ai-sdk Session against a real provide ]); assert.match(JSON.stringify(compactRequests[0]?.body), /context summarization assistant/); - const messages = await execution.sessionStore.readMessagesSnapshot(session.id); + const messages = await readLedgerMessages(execution.runtimeEventStore, session.id); const assistant = messages.find( (message) => message.type === 'assistant' && message.turnId === turnIds[0], ); @@ -2445,7 +2446,7 @@ test('production Host executes a durable runnable child with an exact tool ceili assert.equal(childRuns.length, 1); assert.equal(childRuns[0] && runtimeInvocationOutcome(childRuns[0]), 'completed'); assert.equal(childRuns[0]?.opening.lineage?.parentRunId, undefined); - const childMessages = await execution.sessionStore.readMessagesSnapshot(child.id); + const childMessages = await readLedgerMessages(execution.runtimeEventStore, child.id); assert.equal( childMessages.find((message) => message.type === 'assistant')?.text, CHILD_AGENT_RESULT_TEXT, @@ -2661,7 +2662,7 @@ test('production Host publishes and retires an implementation child patch', asyn assert.equal(childRuns.length, 1); assert.equal(childRuns[0] && runtimeInvocationOutcome(childRuns[0]), 'completed'); assert.equal(childRuns[0]?.opening.lineage?.parentRunId, undefined); - const childMessages = await execution.sessionStore.readMessagesSnapshot(child.id); + const childMessages = await readLedgerMessages(execution.runtimeEventStore, child.id); assert.equal( childMessages.find((message) => message.type === 'assistant')?.text, CHILD_AGENT_RESULT_TEXT, diff --git a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts index 520c689a22..2c9d4f98ba 100644 --- a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts +++ b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts @@ -55,6 +55,7 @@ import type { StoredMessage } from '@maka/core/session'; import { isTerminalRuntimeEvent } from '@maka/core/runtime-event'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { BackendRegistry, SessionManager } from '@maka/runtime/session-manager'; +import { readLedgerMessages } from './ledger-transcript.js'; import { buildRecoveredTerminalRuntimeEvent, classifyTerminalRuntimeLedger, @@ -302,16 +303,7 @@ export class ExecutionFixture { try { stores = await openInteractiveExecutionStoresForWrite(owner.lease); const backends = new BackendRegistry(); - backends.register( - 'ai-sdk', - (ctx) => - new FakeBackend({ - sessionId: ctx.sessionId, - header: ctx.header, - store: ctx.store, - appendMessage: ctx.appendMessage, - }), - ); + backends.register('ai-sdk', (ctx) => new FakeBackend({ sessionId: ctx.sessionId })); const workspace = await resolveWorkspaceIdentity({ path: this.root }); let markReached!: () => void; const reached = new Promise((resolve) => { @@ -973,7 +965,7 @@ export class ExecutionFixture { let stores: Awaited> | undefined; try { stores = await openInteractiveExecutionStoresForWrite(owner.lease); - const messages = await stores.sessionStore.readMessages(this.sessionId); + const messages = await readLedgerMessages(stores.runtimeEventStore, this.sessionId); const source = messages.find( (message): message is Extract => message.type === 'user' && message.turnId === sourceTurnId, @@ -1058,12 +1050,18 @@ export class ExecutionFixture { } assert.ok(result.admission.userMessageId); if (createUserMessage) { - await stores.sessionStore.appendMessage(this.sessionId, { - type: 'user', + assert.ok(createRun, 'a seeded UserMessage needs the invocation that carries it'); + await stores.runtimeEventStore.appendRuntimeEvent(this.sessionId, result.admission.runId, { id: result.admission.userMessageId, + sessionId: this.sessionId, + invocationId: result.admission.runId, + runId: result.admission.runId, turnId, ts: admittedAt, - ...content, + partial: false, + role: 'user', + author: 'user', + content: { kind: 'text', ...content }, }); } return { @@ -1149,11 +1147,11 @@ export class ExecutionFixture { const runs = invocations.filter((candidate) => candidate.turnId === turnId); const run = invocations.find((candidate) => candidate.runId === admission.runId); assert.ok(run); - const messages = await stores.sessionStore.readMessages(this.sessionId); const runtimeEvents = await stores.runtimeEventStore.readImmutableRuntimeEvents( this.sessionId, admission.runId, ); + const messages = await readLedgerMessages(stores.runtimeEventStore, this.sessionId); return { runs, userMessages: messages.filter( @@ -1201,7 +1199,7 @@ export class ExecutionFixture { let stores: Awaited> | undefined; try { stores = await openInteractiveExecutionStoresForRead(reader.lease); - return (await stores.sessionStore.readMessages(this.sessionId)).filter( + return (await readLedgerMessages(stores.runtimeEventStore, this.sessionId)).filter( (message): message is Extract => message.type === 'user', ); } finally { diff --git a/packages/runtime-host/src/__tests__/fixtures/ledger-transcript.ts b/packages/runtime-host/src/__tests__/fixtures/ledger-transcript.ts new file mode 100644 index 0000000000..b5d87a0e3d --- /dev/null +++ b/packages/runtime-host/src/__tests__/fixtures/ledger-transcript.ts @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; +import type { StoredMessage } from '@maka/core/session'; +import type { ExecutionRuntimeEventReader } from '@maka/storage/execution-stores'; +import { RuntimeReadModel } from '@maka/runtime/runtime-read-model'; + +/** + * A Session's transcript as the ledger tells it, for tests that used to read + * `session_messages` directly. This is the read model itself, without a + * SessionManager to host it — so ordering, inline-invocation scope and running + * turns read exactly as the product presents them. + */ +export async function readLedgerMessages( + runtimeEventStore: Readonly, + sessionId: string, +): Promise { + // The read model only reads; the reader fragment carries every method it uses. + const store = runtimeEventStore as unknown as RuntimeEventStore; + return (await new RuntimeReadModel({ runtimeEventStore: store }).getSessionView(sessionId)) + .messages; +} diff --git a/packages/runtime-host/src/__tests__/fixtures/session-transcript-reader.ts b/packages/runtime-host/src/__tests__/fixtures/session-transcript-reader.ts index 8d2dbecb96..916989da53 100644 --- a/packages/runtime-host/src/__tests__/fixtures/session-transcript-reader.ts +++ b/packages/runtime-host/src/__tests__/fixtures/session-transcript-reader.ts @@ -18,6 +18,8 @@ */ import type { StoredMessage } from '@maka/core/session'; +import type { SessionTurnContribution, SessionTurnLandmark } from '@maka/storage/execution-stores'; +import { foldTurnContribution } from '@maka/storage/session-message-projection'; import type { SessionTranscriptReader } from '../../server/session-transcript-reader.js'; export function transcriptReader( @@ -142,6 +144,48 @@ export function transcriptReader( (message, sequence) => sequence <= request.throughSequence! && request.messageIds.includes(message.id), ), + readDurableTurnContributions: async ( + _sessionId, + throughSequence, + position, + maxContributions, + ) => { + const watermark = throughSequence ?? (durable.length === 0 ? null : durable.length - 1); + if (watermark === null) + return { throughSequence: null, contributions: [], nextPosition: null }; + const folded = new Map(); + for (const [sequence, message] of durable.entries()) { + const turnId = message.turnId; + if (turnId === undefined || sequence < position || sequence > watermark) continue; + if (!folded.has(turnId) && folded.size >= maxContributions) { + return { + throughSequence: watermark, + contributions: [...folded.values()], + nextPosition: sequence, + }; + } + folded.set(turnId, foldTurnContribution(folded.get(turnId), turnId, sequence, message)); + } + return { + throughSequence: watermark, + contributions: [...folded.values()], + nextPosition: null, + }; + }, + readDurableTurnLandmarks: async (_sessionId, maxLandmarks) => { + const watermark = durable.length === 0 ? null : durable.length - 1; + if (watermark === null) return { throughSequence: null, landmarks: [] }; + const seen = new Set(); + const landmarks: SessionTurnLandmark[] = []; + for (const [sequence, message] of durable.entries()) { + if (landmarks.length >= maxLandmarks) break; + const turnId = message.turnId; + if (message.type !== 'user' || turnId === undefined || seen.has(turnId)) continue; + seen.add(turnId); + landmarks.push({ turnId, sequence, label: message.displayText ?? message.text }); + } + return { throughSequence: watermark, landmarks }; + }, readActiveOverlay: async () => overlay, }; } diff --git a/packages/runtime-host/src/__tests__/goal-coordinator.test.ts b/packages/runtime-host/src/__tests__/goal-coordinator.test.ts index b69e4e0538..16c619d717 100644 --- a/packages/runtime-host/src/__tests__/goal-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/goal-coordinator.test.ts @@ -63,6 +63,7 @@ test('one Host Goal is shared across clients with CAS control and crash-clear re const coordinator = new HostGoalCoordinator({ store: goalStore, stores, + readSessionMessages: (sessionId) => stores.sessionStore.readMessagesSnapshot(sessionId), executions: { reconcile: async () => assert.fail('No Goal execution recovery is expected'), subscribe: () => () => undefined, @@ -201,6 +202,7 @@ test('one Host Goal is shared across clients with CAS control and crash-clear re const recovered = new HostGoalCoordinator({ store: goalStore, stores, + readSessionMessages: (sessionId) => stores.sessionStore.readMessagesSnapshot(sessionId), executions: { reconcile: async () => assert.fail('Recovered Goal has no current execution'), subscribe: () => () => undefined, @@ -275,6 +277,7 @@ test('session retirement forgets a terminal Goal without recreating deleted auth const coordinator = new HostGoalCoordinator({ store: goalStore, stores, + readSessionMessages: (sessionId) => stores.sessionStore.readMessagesSnapshot(sessionId), executions: { reconcile: async () => assert.fail('A terminal Goal has no execution to recover'), subscribe: () => () => undefined, @@ -399,6 +402,7 @@ test('restart settles the durable current Goal execution through Hosted Executio const coordinator = new HostGoalCoordinator({ store: goalStore, stores, + readSessionMessages: (sessionId) => stores.sessionStore.readMessagesSnapshot(sessionId), executions: { reconcile: (requested) => executionProjection.read(requested), subscribe: () => () => undefined, @@ -485,6 +489,7 @@ test('restart replaces a stale current execution with the current durable Goal i const coordinator = new HostGoalCoordinator({ store: goalStore, stores, + readSessionMessages: (sessionId) => stores.sessionStore.readMessagesSnapshot(sessionId), executions: { reconcile: async () => assert.fail('A stale execution must not be reconciled'), subscribe: () => () => undefined, @@ -573,6 +578,7 @@ test('goal.arm creates one Goal per Session and refuses a second while it is unf const coordinator = new HostGoalCoordinator({ store: goalStore, stores, + readSessionMessages: (sessionId) => stores.sessionStore.readMessagesSnapshot(sessionId), executions: { reconcile: async () => assert.fail('Arming alone has no execution to recover'), subscribe: () => () => undefined, @@ -687,6 +693,7 @@ test('a Goal armed but never carried by a Turn does not start itself after a res const armingHost = new HostGoalCoordinator({ store: goalStore, stores, + readSessionMessages: (sessionId) => stores.sessionStore.readMessagesSnapshot(sessionId), executions: { reconcile: async () => assert.fail('Arming alone has no execution to recover'), subscribe: () => () => undefined, @@ -726,6 +733,7 @@ test('a Goal armed but never carried by a Turn does not start itself after a res const restarted = new HostGoalCoordinator({ store: goalStore, stores, + readSessionMessages: (sessionId) => stores.sessionStore.readMessagesSnapshot(sessionId), executions: { reconcile: async () => assert.fail('An armed Goal has no execution to recover'), subscribe: () => () => undefined, @@ -789,6 +797,7 @@ test('resuming an armed Goal drives it, and a restart puts that drive back', asy const host = new HostGoalCoordinator({ store: goalStore, stores, + readSessionMessages: (sessionId) => stores.sessionStore.readMessagesSnapshot(sessionId), executions: { reconcile: async () => assert.fail('Arming alone has no execution to recover'), subscribe: () => () => undefined, @@ -875,6 +884,7 @@ test('resuming an armed Goal drives it, and a restart puts that drive back', asy const restarted = new HostGoalCoordinator({ store: goalStore, stores, + readSessionMessages: (sessionId) => stores.sessionStore.readMessagesSnapshot(sessionId), executions: { reconcile: async () => assert.fail('A busy admission left no execution to recover'), subscribe: () => () => undefined, @@ -932,6 +942,7 @@ test('an arm admitted before the drain creates no Goal after it', async () => { const host = new HostGoalCoordinator({ store: goalStore, stores, + readSessionMessages: (sessionId) => stores.sessionStore.readMessagesSnapshot(sessionId), executions: { reconcile: async () => assert.fail('A refused arm has no execution'), subscribe: () => () => undefined, diff --git a/packages/runtime-host/src/__tests__/goal-root-authority.test.ts b/packages/runtime-host/src/__tests__/goal-root-authority.test.ts index abd2ddd8e5..451518f030 100644 --- a/packages/runtime-host/src/__tests__/goal-root-authority.test.ts +++ b/packages/runtime-host/src/__tests__/goal-root-authority.test.ts @@ -60,6 +60,7 @@ import { RootAdmissionOwner } from '../server/root-admission-owner.js'; import { RootTurnCoordinator } from '../server/root-turn-coordinator.js'; import { SessionAdmissionGate } from '../server/session-admission-gate.js'; import { SessionContinuityCoordinator } from '../server/session-continuity-coordinator.js'; +import { readLedgerMessages } from './fixtures/ledger-transcript.js'; test('Goal continuation uses the canonical root admission and durable origin', { timeout: 10_000, @@ -105,9 +106,9 @@ test('Goal continuation uses the canonical root admission and durable origin', { if (!durableAdmission) return; const run = await readInvocation(fixture, durableAdmission.runId); assert.deepEqual(run?.opening.root, { kind: 'goal', goalId: created.id }); - const user = (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).find( - (message) => message.type === 'user' && message.turnId === admission.turnId, - ); + const user = ( + await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId) + ).find((message) => message.type === 'user' && message.turnId === admission.turnId); assert.deepEqual(user?.type === 'user' ? user.origin : undefined, { kind: 'goal', goalId: created.id, @@ -172,7 +173,7 @@ test('queued Goal control revokes a prepared root before durable admission', asy false, ); assert.equal( - (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).some( + (await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId)).some( (message) => message.type === 'user' && message.turnId === admission.turnId, ), false, @@ -430,9 +431,9 @@ test('restart closes an admitted Goal without a Run instead of replaying it', as assert.deepEqual(run?.opening.root, { kind: 'goal', goalId: 'goal-restart' }); assert.equal(run && runtimeInvocationOutcome(run), 'failed'); assert.equal(run && runtimeInvocationFailureClass(run), 'app_restarted'); - const user = (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).find( - (message) => message.type === 'user' && message.turnId === turnId, - ); + const user = ( + await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId) + ).find((message) => message.type === 'user' && message.turnId === turnId); assert.deepEqual(user?.type === 'user' ? user.origin : undefined, { kind: 'goal', goalId: 'goal-restart', @@ -447,11 +448,12 @@ test('restart rejects an admitted Goal whose existing UserMessage lost its origi const fixture = await createFixture({ recoverAdmissions: false }); try { const turnId = randomUUID(); + const runId = randomUUID(); const userMessageId = randomUUID(); await fixture.stores.agentRunStore.admitRootTurn({ sessionId: fixture.sessionId, turnId, - proposedRunId: randomUUID(), + proposedRunId: runId, proposedUserMessageId: userMessageId, execution: { kind: 'goal', goalId: 'goal-corrupt-origin' }, previousRootTurnId: null, @@ -459,12 +461,23 @@ test('restart rejects an admitted Goal whose existing UserMessage lost its origi sourceMessages: [], admittedAt: 1, }); - await fixture.stores.sessionStore.appendMessage(fixture.sessionId, { - type: 'user', + const seeded = await seedInvocation(fixture.stores.runtimeEventStore, { + sessionId: fixture.sessionId, + turnId, + runId, + opening: { root: { kind: 'goal', goalId: 'goal-corrupt-origin' } }, + }); + await fixture.stores.runtimeEventStore.appendRuntimeEvent(fixture.sessionId, runId, { id: userMessageId, + sessionId: fixture.sessionId, + invocationId: seeded.invocationId, + runId, turnId, ts: 1, - text: 'Preserve durable Goal provenance', + partial: false, + role: 'user', + author: 'user', + content: { kind: 'text', text: 'Preserve durable Goal provenance' }, }); await assert.rejects( @@ -669,6 +682,7 @@ async function createFixture(options: { recoverAdmissions?: boolean } = {}): Pro goal = new HostGoalCoordinator({ store: goalStore, stores, + readSessionMessages: (sessionId) => manager.getMessages(sessionId), executions: rootCoordinator, sessionAdmission: admission, evaluator: { diff --git a/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts b/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts index d2c5e1e8c6..65cb7330cd 100644 --- a/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts @@ -106,6 +106,7 @@ import type { SessionContinuityFrameSink } from '../server/session-continuity-se import { HostTurnControlCoordinator } from '../server/turn-control-coordinator.js'; import { RuntimePolicyActivationGate } from '../server/runtime-policy-activation-gate.js'; import { PROCESS_TIMEOUT_MS } from './fixtures/execution-host-suite.js'; +import { readLedgerMessages } from './fixtures/ledger-transcript.js'; import { waitFor } from '@maka/core/test-only/async-primitives'; const HOLD_EXTERNAL_PROMPT = 'hold external root before follow-up'; @@ -404,9 +405,9 @@ test('uses the submitted Turn identity for the canonical external user message', assertStartedTurn(started); await fixture.coordinator.whenIdle(fixture.sessionId); - const user = (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).find( - (message) => message.type === 'user' && message.turnId === turnId, - ); + const user = ( + await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId) + ).find((message) => message.type === 'user' && message.turnId === turnId); assert.equal(user?.id, turnId); } finally { await fixture.coordinator.close(); @@ -474,7 +475,7 @@ test('startup recovery replays one admitted safe-boundary continuation without a 'completed', ); assert.equal( - (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).some( + (await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId)).some( (message) => message.type === 'user' && message.turnId === pending.targetTurnId, ), false, @@ -515,14 +516,6 @@ test('startup recovery closes a ScheduledTask Run after its pending fire was set admittedAt, }); assert.equal(admission.kind, 'admitted'); - await fixture.stores.sessionStore.appendMessage(fixture.sessionId, { - type: 'user', - id: userMessageId, - turnId, - ts: admittedAt, - text: 'Continue the scheduled work.', - origin: { kind: 'scheduled_task', scheduledTaskId: 'task-settled-fire' }, - }); await seedInvocation(fixture.stores.runtimeEventStore, { sessionId: fixture.sessionId, invocationId: runId, @@ -548,6 +541,22 @@ test('startup recovery closes a ScheduledTask Run after its pending fire was set root: { kind: 'scheduled_task', scheduledTaskId: 'task-settled-fire' }, }, }); + await fixture.stores.runtimeEventStore.appendRuntimeEvent(fixture.sessionId, runId, { + id: userMessageId, + sessionId: fixture.sessionId, + invocationId: runId, + runId, + turnId, + ts: admittedAt, + partial: false, + role: 'user', + author: 'host', + content: { + kind: 'text', + text: 'Continue the scheduled work.', + origin: { kind: 'scheduled_task', scheduledTaskId: 'task-settled-fire' }, + }, + }); recovery = fixture.createRecoveryCoordinator(); await recovery.prepareRecovery(); @@ -692,7 +701,7 @@ test('a failed exact Capability retry does not poison the parked continuation bi 1, ); assert.equal( - (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).filter( + (await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId)).filter( (message) => message.type === 'user' && message.turnId === pending.targetTurnId, ).length, 0, @@ -1299,7 +1308,10 @@ test('idle Skill admission persists a canonical draft without history before roo displayText: '/skill:writer Draft this.', inlineReferences: [], }); - assert.deepEqual(await fixture.stores.sessionStore.readMessages(fixture.sessionId), []); + assert.deepEqual( + await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId), + [], + ); } finally { await fixture.dispose(); } @@ -2110,9 +2122,9 @@ test('Agent Graph supervisor wake waits for root idle and binds one durable exec }); assert.equal(graphRun.opening.configuration.orchestrationMode, 'graph'); assert.equal(graphRun.opening.configuration.orchestrationSource, 'turn_override'); - const userMessage = (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).find( - (message) => message.id === graphAdmission?.userMessageId, - ); + const userMessage = ( + await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId) + ).find((message) => message.id === graphAdmission?.userMessageId); assert.ok(userMessage?.type === 'user'); if (userMessage?.type === 'user') { assert.deepEqual(userMessage.origin, { @@ -2297,7 +2309,7 @@ test('manual context compact uses durable root query, stop, and exact retry auth assert.deepEqual(admission?.execution, { kind: 'context_compact' }); assert.equal(admission?.userMessageId, null); assert.equal( - (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).some( + (await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId)).some( (message) => message.type === 'user' && message.turnId === turnId, ), false, @@ -2518,7 +2530,10 @@ test('Agent Graph supervisor wake revalidates freshness before durable root admi await fixture.stores.runtimeEventStore.listSessionInvocations(fixture.sessionId), [], ); - assert.deepEqual(await fixture.stores.sessionStore.readMessages(fixture.sessionId), []); + assert.deepEqual( + await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId), + [], + ); assert.equal(fixture.drainRequested(), false); } finally { await fixture.coordinator.close(); @@ -2576,9 +2591,9 @@ test('Agent Graph supervisor recovery closes a durable admission that has no Run }); assert.equal(run.opening.configuration.orchestrationMode, 'graph'); assert.equal(run.opening.configuration.orchestrationSource, 'turn_override'); - const message = (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).find( - (candidate) => candidate.id === userMessageId, - ); + const message = ( + await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId) + ).find((candidate) => candidate.id === userMessageId); assert.ok(message?.type === 'user'); if (message?.type === 'user') { assert.deepEqual(message.origin, { @@ -3697,7 +3712,7 @@ test('mixed-Client queued follow-ups use separate Session successors without con [[], ['followup-from-provider-b'], ['followup-from-provider-a']], ); assert.deepEqual( - (await fixture.stores.sessionStore.readMessages(fixture.sessionId)) + (await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId)) .filter((message) => message.type === 'user' && message.id.startsWith('followup-from-')) .map((message) => message.id), ['followup-from-provider-b', 'followup-from-provider-a'], @@ -5475,9 +5490,9 @@ test('directory references enforce Host identity without reading the filesystem' ); assert.equal(accepted.ok, true, JSON.stringify(accepted)); await fixture.coordinator.whenIdle(fixture.sessionId); - const user = (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).find( - (message) => message.type === 'user' && message.id === 'local-directory', - ); + const user = ( + await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId) + ).find((message) => message.type === 'user' && message.id === 'local-directory'); assert.equal(user?.type, 'user'); if (user?.type !== 'user') throw new Error('Expected directory user message'); assert.equal(user.text, 'inspect local directory'); @@ -5538,7 +5553,7 @@ test('turn start and regeneration preserve one Host-bound directory reference', assert.deepEqual(input.directoryReferences, [reference]); } const regeneratedUser = ( - await fixture.stores.sessionStore.readMessages(fixture.sessionId) + await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId) ).find((message) => message.type === 'user' && message.turnId === 'directory-regenerated'); assert.equal(regeneratedUser?.type, 'user'); if (regeneratedUser?.type !== 'user') throw new Error('Expected regenerated user message'); @@ -5615,16 +5630,19 @@ test('queued directory references survive text editing and next-Turn delivery', release.resolve(); await fixture.coordinator.whenIdle(fixture.sessionId); await waitUntil(async () => - (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).some( + (await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId)).some( (message) => message.type === 'user' && message.text === 'edited inspection', ), ); - const user = (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).find( - (message) => message.type === 'user' && message.text === 'edited inspection', - ); + const user = ( + await readLedgerMessages(fixture.stores.runtimeEventStore, fixture.sessionId) + ).find((message) => message.type === 'user' && message.text === 'edited inspection'); assert.equal(user?.type, 'user'); if (user?.type !== 'user') throw new Error('Expected queued directory user message'); assert.deepEqual(user.directoryReferences, [reference]); + // The ledger carries the delivered message before its Turn ends; close only + // once that Turn has, so shutdown does not race its terminal fact. + await fixture.coordinator.whenIdle(fixture.sessionId); } finally { release.resolve(); await fixture.coordinator.close(); diff --git a/packages/runtime-host/src/__tests__/session-catalog-coordinator.test.ts b/packages/runtime-host/src/__tests__/session-catalog-coordinator.test.ts index 578a4f9c30..a4c8983619 100644 --- a/packages/runtime-host/src/__tests__/session-catalog-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/session-catalog-coordinator.test.ts @@ -61,6 +61,7 @@ import { import { SessionAdmissionGate } from '../server/session-admission-gate.js'; type CatalogStores = HostSessionCatalogCoordinatorOptions['stores']; +type CatalogTurnIndex = HostSessionCatalogCoordinatorOptions['turnIndex']; type RuntimePolicy = HostSessionCatalogCoordinatorOptions['runtimePolicy']; type ConfigurationAuthority = HostSessionCatalogCoordinatorOptions['manager']; type SessionContinuity = HostSessionCatalogCoordinatorOptions['continuity']; @@ -104,8 +105,8 @@ test('reduces turn pages to their encoded wire budget without skipping contribut })); const requestedLimits: number[] = []; const fixture = createFixture({ - stores: { - readTurnContributionsSnapshot: async (_sessionId, _watermark, position, limit) => { + turnIndex: { + readDurableTurnContributions: async (_sessionId, _watermark, position, limit) => { requestedLimits.push(limit); const end = Math.min(position + limit, contributions.length); return { @@ -148,6 +149,52 @@ test('reduces turn pages to their encoded wire budget without skipping contribut assert.ok(requestedLimits.some((limit) => limit < 128)); }); +test('read marker clears unread only at the ledger transcript tail', async () => { + const fixture = createFixture({ + header: { hasUnread: true }, + turnIndex: { + readDurableRecords: async () => ({ + throughSequence: 1, + records: [ + { + sequence: 1, + message: { + type: 'assistant', + id: 'message-2', + turnId: 'turn-1', + ts: 20, + text: 'answer', + modelId: 'fake-model', + }, + }, + { + sequence: 0, + message: { type: 'user', id: 'message-1', turnId: 'turn-1', ts: 10, text: 'ask' }, + }, + ], + nextPosition: null, + }), + }, + }); + const setReadMarker = async (readThroughMessageId: string) => { + const outcome = await fixture.coordinator.handlers['session.read_marker.set']( + { sessionId: fixture.sessionId, readThroughMessageId }, + context, + ); + assert.equal(outcome.ok, true); + if (!outcome.ok || !('hasUnread' in outcome.result)) assert.fail('Read marker failed'); + return outcome.result; + }; + + const behind = await setReadMarker('message-1'); + assert.equal(behind.hasUnread, true); + assert.equal(behind.lastReadMessageId, undefined); + + const caughtUp = await setReadMarker('message-2'); + assert.equal(caughtUp.hasUnread, false); + assert.equal(caughtUp.lastReadMessageId, 'message-2'); +}); + test('metadata replacement preserves execution-semantic labels and ignores injected ones', async () => { const fixture = createFixture({ labels: ['old-user-label', DEEP_RESEARCH_SESSION_LABEL], @@ -1596,6 +1643,7 @@ function createFixture( readonly labels?: readonly string[]; readonly cwd?: string; readonly stores?: Partial; + readonly turnIndex?: Partial; readonly manager?: Partial; readonly continuity?: Partial; readonly connection?: FixtureConnection; @@ -1628,17 +1676,10 @@ function createFixture( records: [catalogRecord(header, revision)], hasMore: false, }), - markSessionReadThroughMessage: async () => headerSnapshot(header, revision), probeStableSessionCreate: async () => ({ kind: 'absent' }), readCatalogRecord: async () => catalogRecord(header, revision), readExecutionBoundary: async () => createGenesisExecutionBoundary('ask'), readHeaderRecordSnapshot: async () => headerSnapshot(header, revision), - readTurnContributionsSnapshot: async () => ({ - throughSequence: null, - contributions: [], - nextPosition: null, - }), - readTurnLandmarksSnapshot: async () => ({ throughSequence: null, landmarks: [] }), updateHeaderVersioned: async (_sessionId, patch, expectedRevision) => { if (expectedRevision !== revision) { throw new SessionMetadataVersionConflictError(sessionId, expectedRevision, revision); @@ -1649,6 +1690,16 @@ function createFixture( }, ...options.stores, }; + const turnIndex: CatalogTurnIndex = { + readDurableRecords: async () => ({ throughSequence: null, records: [], nextPosition: null }), + readDurableTurnContributions: async () => ({ + throughSequence: null, + contributions: [], + nextPosition: null, + }), + readDurableTurnLandmarks: async () => ({ throughSequence: null, landmarks: [] }), + ...options.turnIndex, + }; const runtimePolicy = options.runtimePolicy ?? runtimePolicyFixture(options.connection ?? {}); const manager: ConfigurationAuthority = { runningTurnIds: () => [], @@ -1677,6 +1728,7 @@ function createFixture( }; const coordinator = new HostSessionCatalogCoordinator({ stores, + turnIndex, runtimePolicy, manager, admission: new SessionAdmissionGate(), diff --git a/packages/runtime-host/src/__tests__/session-catalog-two-client-uds.test.ts b/packages/runtime-host/src/__tests__/session-catalog-two-client-uds.test.ts index 560bf3a3f5..dc689b52d9 100644 --- a/packages/runtime-host/src/__tests__/session-catalog-two-client-uds.test.ts +++ b/packages/runtime-host/src/__tests__/session-catalog-two-client-uds.test.ts @@ -31,6 +31,7 @@ import { DatabaseSync } from 'node:sqlite'; import { DEEP_RESEARCH_SESSION_LABEL, DEEP_RESEARCH_SESSION_NAME } from '@maka/core/deep-research'; import { openInteractiveArtifactStoreForWrite } from '@maka/storage/artifact-stores'; import { openInteractiveExecutionStoresForWrite } from '@maka/storage/execution-stores'; +import { seedInvocation } from '@maka/runtime/test-only/invocation-fixture'; import { openInteractiveRuntimePolicyStoresForWrite } from '@maka/storage/runtime-policy-stores'; import { resolveRootControlNamespace, @@ -824,25 +825,45 @@ async function seedAuthority( model: 'fake-model', permissionMode: 'ask', }); - await execution.sessionStore.appendMessages(unread.id, [ - { type: 'user', id: 'message-1', turnId: 'turn-1', ts: 1, text: 'one' }, + await seedInvocation(execution.runtimeEventStore, { + sessionId: unread.id, + runId: 'run-1', + turnId: 'turn-1', + openedAt: 1, + }); + for (const event of [ + { + id: 'message-1', + ts: 1, + role: 'user' as const, + author: 'user' as const, + content: { kind: 'text' as const, text: 'one' }, + }, { - type: 'assistant', id: 'message-2', - turnId: 'turn-1', ts: 2, - text: 'two', - modelId: 'fake-model', + role: 'model' as const, + author: 'agent' as const, + content: { kind: 'text' as const, text: 'two' }, }, { - type: 'tool_call', - id: 'tool-1', - turnId: 'turn-1', + id: 'run-1-terminal', ts: 3, - toolName: 'Read', - args: {}, + role: 'system' as const, + author: 'system' as const, + status: 'completed' as const, + actions: { endInvocation: true }, }, - ]); + ]) { + await execution.runtimeEventStore.appendRuntimeEvent(unread.id, 'run-1', { + sessionId: unread.id, + invocationId: 'run-1', + runId: 'run-1', + turnId: 'turn-1', + partial: false, + ...event, + }); + } await execution.sessionStore.updateHeader(unread.id, { hasUnread: true, lastMessageAt: 2, diff --git a/packages/runtime-host/src/__tests__/session-revision-two-client-uds.test.ts b/packages/runtime-host/src/__tests__/session-revision-two-client-uds.test.ts index 790ba8ba68..4a490c001a 100644 --- a/packages/runtime-host/src/__tests__/session-revision-two-client-uds.test.ts +++ b/packages/runtime-host/src/__tests__/session-revision-two-client-uds.test.ts @@ -52,6 +52,7 @@ import { import { openInteractiveSessionTodoStoreForWrite } from '@maka/storage/session-todo-authority'; import { removePosixEndpointDirectories } from './fixtures/endpoint-hygiene.js'; import { requireStartedTurn } from './fixtures/execution-host-suite.js'; +import { readLedgerMessages } from './fixtures/ledger-transcript.js'; import { connectRuntimeHost, RuntimeHostOperationError, @@ -858,13 +859,6 @@ async function seedSource( model: 'fake-model', permissionMode: 'ask', }); - await execution.sessionStore.appendMessage(continuationSource.id, { - type: 'user', - id: 'continuation-parent-user', - turnId: 'continuation-parent-turn', - ts: 1, - text: 'retain the child continuation closure', - }); const continuationParent = agentRunHeader( root, continuationSource.id, @@ -969,45 +963,6 @@ async function seedSource( source: 'tool_result', now: 2, }); - await execution.sessionStore.appendMessages(source.id, [ - { - type: 'user', - id: 'user-1', - turnId: 'turn-1', - ts: 1, - text: 'first', - attachments: [ - { - kind: 'code', - name: 'source.txt', - mimeType: 'text/plain', - bytes: 14, - ref: { - kind: 'session_file', - sessionId: source.id, - relativePath: artifact.id, - }, - }, - ], - }, - { - type: 'assistant', - id: 'assistant-1', - turnId: 'turn-1', - ts: 2, - text: 'first response', - modelId: 'fake-model', - }, - { type: 'user', id: 'user-2', turnId: 'turn-2', ts: 3, text: 'second' }, - { - type: 'assistant', - id: 'assistant-2', - turnId: 'turn-2', - ts: 4, - text: 'second response', - modelId: 'fake-model', - }, - ]); await execution.sessionStore.updateHeader(source.id, { isFlagged: true, titleIsManual: true, @@ -1293,39 +1248,6 @@ async function seedSource( completedAt: 2, durationMs: 1, }; - await execution.sessionStore.appendMessages(linkedChildSource.id, [ - { - type: 'user', - id: 'linked-user', - turnId: 'linked-turn', - ts: 1, - text: 'delegate this', - }, - { - type: 'tool_result', - id: 'linked-result', - turnId: 'linked-turn', - ts: 2, - toolUseId: 'linked-call', - isError: false, - content: graphResult, - }, - { - type: 'user', - id: 'linked-after-user', - turnId: 'linked-after-turn', - ts: 3, - text: 'revise this later turn', - }, - { - type: 'assistant', - id: 'linked-after-assistant', - turnId: 'linked-after-turn', - ts: 4, - text: 'later response', - modelId: 'fake-model', - }, - ]); for (const run of [ agentRunHeader( root, @@ -1420,13 +1342,39 @@ async function seedSource( stop: [], finish: { resultIds: ['graph-item'], reason: 'complete' }, }); - await execution.sessionStore.appendMessage(metadataLinkedSource.id, { - type: 'user', - id: 'metadata-linked-user', - turnId: 'metadata-linked-turn', - ts: 1, - text: 'delegate without a committed result', - }); + await seedInvocation( + execution.runtimeEventStore, + agentRunHeader( + root, + metadataLinkedSource.id, + 'metadata-linked-run', + 'metadata-linked-invocation', + 'metadata-linked-turn', + ), + ); + for (const event of [ + runtimeEvent( + metadataLinkedSource.id, + 'metadata-linked-run', + 'metadata-linked-invocation', + 'metadata-linked-turn', + { + id: 'metadata-linked-user', + role: 'user', + author: 'user', + content: { kind: 'text', text: 'delegate without a committed result' }, + }, + ), + runtimeEvent( + metadataLinkedSource.id, + 'metadata-linked-run', + 'metadata-linked-invocation', + 'metadata-linked-turn', + { id: 'metadata-linked-terminal', ts: 2, status: 'completed' }, + ), + ]) { + await execution.runtimeEventStore.appendRuntimeEvent(event.sessionId, event.runId, event); + } const ordinaryLinkedChild = await execution.sessionStore.createSubagent({ cwd: root, name: 'Metadata-linked Child Session', @@ -1483,15 +1431,6 @@ async function seedSource( source: 'tool_result_archive', now: 1, }); - await execution.sessionStore.appendMessages(archivedOwnedSource.id, [ - { - type: 'user', - id: 'archived-owned-user', - turnId: 'archived-owned-turn', - ts: 1, - text: 'reuse the archived result', - }, - ]); const archivedOwnedRuns = [ agentRunHeader( root, @@ -1692,7 +1631,7 @@ async function verifyDurableBranch( // readable copy of the user-uploaded attachment (regression guard for the // turn-scoped-only artifact selection that dropped user uploads). const assertCopiedUpload = async (sessionId: string): Promise => { - const sessionMessages = await execution.sessionStore.readMessagesSnapshot(sessionId); + const sessionMessages = await readLedgerMessages(execution.runtimeEventStore, sessionId); const uploadMessage = sessionMessages.find( (message) => message.type === 'user' && message.attachments?.[0], ); @@ -1707,7 +1646,7 @@ async function verifyDurableBranch( text: 'retained bytes', }); }; - const messages = await execution.sessionStore.readMessagesSnapshot(branchSessionId); + const messages = await readLedgerMessages(execution.runtimeEventStore, branchSessionId); // The copied invocation opens on the branch's own spine, so its transcript // projects the copied turn as ended, exactly as the source reads. assert.deepEqual( @@ -1827,7 +1766,8 @@ async function verifyDurableBranch( ); assert.equal(sideConversationHeader.conversationCopy?.intent, 'side_conversation'); assert.ok(sideConversationHeader.labels.includes('mode:side_conversation')); - const sideConversationMessages = await execution.sessionStore.readMessagesSnapshot( + const sideConversationMessages = await readLedgerMessages( + execution.runtimeEventStore, graphSideConversationTargetId, ); const sideConversationResult = sideConversationMessages.find( @@ -1845,7 +1785,8 @@ async function verifyDurableBranch( assert.equal(sideConversationResult.content.items[0]?.runId, undefined); const sideConversationArtifactId = sideConversationResult.content.items[0]?.artifactIds[0]; assert.ok(sideConversationArtifactId); - const activeSourceSideConversationMessages = await execution.sessionStore.readMessagesSnapshot( + const activeSourceSideConversationMessages = await readLedgerMessages( + execution.runtimeEventStore, activeSourceSideConversationTargetId, ); assert.ok(activeSourceSideConversationMessages.some((message) => message.turnId === 'turn-2')); @@ -1929,8 +1870,10 @@ async function verifyDurableBranch( { offset: 0, limit: 10 }, ); assert.equal(archivedSideConversationArtifacts.total, 0); - const graphRevisionMessages = - await execution.sessionStore.readMessagesSnapshot(graphRevisionTargetId); + const graphRevisionMessages = await readLedgerMessages( + execution.runtimeEventStore, + graphRevisionTargetId, + ); const graphResult = graphRevisionMessages.find( (message) => message.type === 'tool_result' && message.content.kind === 'agent_swarm', ); diff --git a/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts b/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts index 1da0eab3ce..69922cc93f 100644 --- a/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts +++ b/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts @@ -25,6 +25,7 @@ import test from 'node:test'; import { seedInvocation, testInvocationOpening } from '@maka/runtime/test-only/invocation-fixture'; import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { RuntimeEvent } from '@maka/core/runtime-event'; +import type { StoredMessage } from '@maka/core/session'; import { type ExecutionStoresWriter, openInteractiveExecutionStoresForWrite, @@ -50,12 +51,43 @@ test('keeps durable history separate from the canonical active overlay', async ( model: 'fake-model', permissionMode: 'ask', }); - await stores.sessionStore.appendMessage(session.id, { - type: 'system_note', - id: 'history-1', - ts: 1, - kind: 'session_start', + // An ended Turn is what the durable half is made of; the running one below + // belongs to the overlay and must not appear in a durable page. + await seedInvocation(stores.runtimeEventStore, { + sessionId: session.id, + runId: 'run-0', + turnId: 'turn-0', + openedAt: 0, }); + await stores.runtimeEventStore.appendRuntimeEvent( + session.id, + 'run-0', + runtimeEvent(session.id, { + id: 'user-event-0', + invocationId: 'run-0', + runId: 'run-0', + turnId: 'turn-0', + ts: 0.1, + role: 'user', + author: 'user', + content: { kind: 'text', text: 'settled' }, + refs: { storedMessageId: 'user-0' }, + }), + ); + await stores.runtimeEventStore.appendRuntimeEvent( + session.id, + 'run-0', + runtimeEvent(session.id, { + id: 'terminal-0', + invocationId: 'run-0', + runId: 'run-0', + turnId: 'turn-0', + ts: 0.2, + role: 'system', + author: 'system', + status: 'completed', + }), + ); await seedInvocation(stores.runtimeEventStore, { sessionId: session.id, runId: 'run-1', @@ -219,10 +251,16 @@ test('keeps durable history separate from the canonical active overlay', async ( maxBytes: 1024, maxMessages: 10, }); - assert.equal(durable.throughSequence, 0); + assert.equal(durable.throughSequence, 1); assert.deepEqual( - durable.fragments.map((fragment) => JSON.parse(fragment.data.toString('utf8'))), - [{ type: 'system_note', id: 'history-1', ts: 1, kind: 'session_start' }], + durable.fragments.map((fragment) => { + const message = JSON.parse(fragment.data.toString('utf8')) as StoredMessage; + return { type: message.type, id: message.id }; + }), + [ + { type: 'turn_state', id: 'terminal-0' }, + { type: 'user', id: 'user-0' }, + ], ); } finally { await owner.close(); diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index 9e7a2c298b..3f9f3851ae 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -172,7 +172,10 @@ import { HostSessionRetirementCoordinator } from './session-retirement-coordinat import { HostSessionRevisionCoordinator } from './session-revision-coordinator.js'; import { HostSessionEffectCoordinator } from './session-effect-coordinator.js'; import { SessionContinuityCoordinator } from './session-continuity-coordinator.js'; -import { createSessionTranscriptReader } from './session-transcript-reader.js'; +import { + createSessionTranscriptReader, + type SessionTranscriptReader, +} from './session-transcript-reader.js'; import { HostSkillCatalogCoordinator } from './skill-catalog-coordinator.js'; import { SkillCatalogRepository } from './skill-catalog-repository.js'; import { HostSessionTodoCoordinator } from './session-todo-coordinator.js'; @@ -272,6 +275,7 @@ export async function createExecutionRuntimeHostComposition( let sessionEffects: HostSessionEffectCoordinator | undefined; let memoryExtraction: HostMemoryExtractionCoordinator | undefined; let unsubscribeTranscriptChanges: (() => void) | undefined; + let transcriptReader: SessionTranscriptReader | undefined; let unsubscribeUsageChanges: (() => void) | undefined; let workspaceExecution: RuntimeHostWorkspaceExecutionComposition | undefined; let goalExecutions: HostGoalExecutionCoordinator | undefined; @@ -613,12 +617,18 @@ export async function createExecutionRuntimeHostComposition( const canonicalPermissionOutcomes = new HostCanonicalPermissionOutcomeReader({ store: stores.interactionStore, }); + transcriptReader = createSessionTranscriptReader({ + stores, + canonicalPermissionOutcomes, + ensureTranscriptLedger: (sessionId) => + requireSessionManager(manager).ensureTranscriptLedgerForRead(sessionId), + }); continuity = new SessionContinuityCoordinator( context.hostEpoch, (sessionId) => canonicalProjectionReader.read(sessionId), sessionAdmission, context.requestDrain, - createSessionTranscriptReader({ stores, canonicalPermissionOutcomes }), + transcriptReader, (sessionId) => hostChanges.publishSessionCatalog(sessionId), context.sessionAccessAuthority, ); @@ -962,6 +972,10 @@ export async function createExecutionRuntimeHostComposition( if (!preview.ok) throw new Error(preview.message); return preview.value; }; + const recapReadModel = new RuntimeReadModel({ + runtimeEventStore: stores.runtimeEventStore, + canonicalPermissionOutcomes, + }); const sessionEffectCoordinator = new HostSessionEffectCoordinator({ model: createHostSessionEffectModel({ runtimePolicy: runtimePolicyStores, @@ -969,10 +983,14 @@ export async function createExecutionRuntimeHostComposition( usage: openedUsageStores, requestDrain: context.requestDrain, }), - readModel: new RuntimeReadModel({ - runtimeEventStore: stores.runtimeEventStore, - canonicalPermissionOutcomes, - }), + readModel: { + getSessionView: async (sessionId) => { + // A Session whose transcript predates the ledger projects an empty + // view, and a recap of nothing reads as a successful recap. + await requireSessionManager(manager).ensureTranscriptLedgerForRead(sessionId); + return recapReadModel.getSessionView(sessionId); + }, + }, artifacts: openedArtifactStore, sessions: stores.sessionStore, readSessionHeader: (sessionId) => stores.sessionStore.readHeaderSnapshot(sessionId), @@ -1314,6 +1332,7 @@ export async function createExecutionRuntimeHostComposition( goal = new HostGoalCoordinator({ store: openedGoalStore, stores, + readSessionMessages: (sessionId) => requireSessionManager(manager).getMessages(sessionId), executions: coordinator, sessionAdmission, evaluator: createHostGoalEvaluator({ @@ -1346,6 +1365,7 @@ export async function createExecutionRuntimeHostComposition( }); const sessionCatalog = new HostSessionCatalogCoordinator({ stores: stores.sessionStore, + turnIndex: requireTranscriptReader(transcriptReader), runtimePolicy: runtimePolicyStores, manager, admission: sessionAdmission, @@ -2319,6 +2339,13 @@ function requireSessionManager(manager: SessionManager | undefined): SessionMana return manager; } +function requireTranscriptReader( + reader: SessionTranscriptReader | undefined, +): SessionTranscriptReader { + if (!reader) throw new Error('Runtime Host transcript reader is not composed'); + return reader; +} + function requireGraphCoordinator( coordinator: AgentGraphCoordinator | undefined, ): AgentGraphCoordinator { diff --git a/packages/runtime-host/src/server/execution-model-composition.ts b/packages/runtime-host/src/server/execution-model-composition.ts index b2812d74e1..4eafd729ed 100644 --- a/packages/runtime-host/src/server/execution-model-composition.ts +++ b/packages/runtime-host/src/server/execution-model-composition.ts @@ -343,9 +343,6 @@ async function buildHostAiSdkBackend( permissionMode: input.context.header.permissionMode, }), }, - appendMessage: - input.context.appendMessage ?? - ((message) => input.context.store.appendMessage(input.context.sessionId, message)), ...(input.context.recordSystemNote ? { recordSystemNote: input.context.recordSystemNote } : {}), diff --git a/packages/runtime-host/src/server/goal-coordinator.ts b/packages/runtime-host/src/server/goal-coordinator.ts index 7104a83076..f8252a77c3 100644 --- a/packages/runtime-host/src/server/goal-coordinator.ts +++ b/packages/runtime-host/src/server/goal-coordinator.ts @@ -75,6 +75,8 @@ type GoalStores = Pick, 'sessionStore' | 'a export interface HostGoalCoordinatorOptions { readonly store: InteractiveGoalAuthorityWriter; readonly stores: GoalStores; + /** The Session transcript as its ledger projects it; the Goal reads its tail. */ + readonly readSessionMessages: (sessionId: string) => Promise; readonly sessionAdmission: SessionAdmissionGate; readonly evaluator: GoalEvaluatorResource; readonly executions: Pick; @@ -156,7 +158,7 @@ export class HostGoalCoordinator { goalManager: this.manager, evaluator: options.evaluator, getRecentContext: async (sessionId) => { - const messages = await this.#stores.sessionStore.readMessagesSnapshot(sessionId); + const messages = await options.readSessionMessages(sessionId); tokenCache.set(sessionId, tokenCount(messages)); return recentContext(messages); }, diff --git a/packages/runtime-host/src/server/hosted-execution-recovery.ts b/packages/runtime-host/src/server/hosted-execution-recovery.ts index 82133c7e9d..2369be21b8 100644 --- a/packages/runtime-host/src/server/hosted-execution-recovery.ts +++ b/packages/runtime-host/src/server/hosted-execution-recovery.ts @@ -21,13 +21,16 @@ import { isDeepStrictEqual } from 'node:util'; import { runtimeInvocationOutcome, type RootExecutionDescriptor, + type RuntimeInvocationRecord, } from '@maka/core/runtime-invocation'; import { messageContentsEqual, normalizeMessageContent, type MessageContent, } from '@maka/core/events'; +import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { StoredMessage } from '@maka/core/session'; +import { projectRuntimeEventUserMessage } from '@maka/runtime/runtime-event-read-model'; import { RuntimeMessageAuthorityInvariantError } from '@maka/runtime/message-authority'; import { type SessionManager } from '@maka/runtime/session-manager'; import type { ExecutionStoresWriter, RootTurnAdmission } from '@maka/storage/execution-stores'; @@ -56,32 +59,38 @@ export async function prepareHostedExecutionRecovery( input: PrepareHostedExecutionRecoveryInput, ): Promise { // listHeaders() avoids listForRecovery()'s discarded per-Session message - // pre-read; the per-Session readMessagesForRecovery below remains the single - // decode that validates durable messages before replay. + // pre-read. The messages an admission is checked against are the Session's + // own RuntimeEvents: the ledger is where a Turn's user message is committed, + // so it is also the only place a missing one can be detected. const sessions = await input.stores.sessionStore.listHeaders(); const prepared: PreparedRecoverySession[] = []; for (const session of sessions) { const admissions = await input.rootAdmissions.recoverSession(session.id); - const messages = await input.stores.sessionStore.readMessagesForRecovery(session.id); const runs = await input.stores.runtimeEventStore.listSessionInvocations(session.id); const runsById = new Map(runs.map((run) => [run.runId, run])); for (const run of runs) { await input.stores.agentRunStore.readEventsForRecovery(session.id, run.runId); await input.stores.runtimeEventStore.readRuntimeEvents(session.id, run.runId); } - const messageIndex = indexRecoveryMessages(messages); + const messageIndex = indexRecoveryMessages( + recoveryUserMessagesFromLedger( + await input.stores.runtimeEventStore.readSessionRuntimeEvents(session.id), + ), + ); const replayAdmissions: RootTurnAdmission[] = []; const rootReplayAdmissions: RootTurnAdmission[] = []; - const missingMessages: RecoveryUserMessage[] = []; - const pendingRecoveryClosures: RootTurnAdmission[] = []; + const pendingRecoveryClosures: PendingRecoveryClosure[] = []; for (const admission of admissions) { const run = runsById.get(admission.runId); + const admittedMessageId = admittedUserMessageId(admission); const rootUserMessages = ( messageIndex.userMessagesByTurnId.get(admission.turnId) ?? [] - ).filter((message) => message.id === admission.userMessageId); - const messageIdOwners = admission.userMessageId - ? (messageIndex.messagesById.get(admission.userMessageId) ?? []) - : []; + ).filter((message) => message.id === admittedMessageId); + const messageIdOwners = messageIndex.messagesById.get(admittedMessageId) ?? []; + if (messageIdOwners.length > 1) { + throw new Error(`Admitted Turn ${admission.turnId} has a duplicated UserMessage identity`); + } + const messageIdOwner = messageIdOwners[0]; const executionContract = recoveryExecutionContract(admission.execution); if ( admission.execution.kind === 'scheduled_task' && @@ -114,12 +123,9 @@ export async function prepareHostedExecutionRecovery( if (admission.sourceMessages.length > 0) { await verifyQueueSourceMessages(admission, messageIndex, input.stores.agentRunStore); } - if (rootUserMessages.length > 0) { - throw new Error(`Admitted Turn ${admission.turnId} must not record a UserMessage`); - } if (!run) { if (executionContract.pendingWithoutRun === 'host_recovery_closure') { - pendingRecoveryClosures.push(admission); + pendingRecoveryClosures.push({ admission }); } else { replayAdmissions.push(admission); if (executionContract.pendingWithoutRun === 'root_replay') { @@ -137,49 +143,33 @@ export async function prepareHostedExecutionRecovery( admission.execution, ); } + if ( + executionContract.requiresUserMessage && + !verifyUserMessage(admission, rootUserMessages, messageIdOwner) + ) { + await recordAdmittedUserMessage(input.stores, admission, run); + } continue; } - if (messageIdOwners.length > 1) { - throw new Error(`Admitted Turn ${admission.turnId} has a duplicated UserMessage identity`); - } - const messageIdOwner = messageIdOwners[0]; if (!run && executionContract.pendingWithoutRun === 'host_recovery_closure') { - verifyOrRecoverUserMessage( + // The closure below opens this Turn's invocation, so it is also what + // writes the message the crashed admission never got to record. + const recorded = verifyUserMessage(admission, rootUserMessages, messageIdOwner); + pendingRecoveryClosures.push({ admission, - rootUserMessages, - messageIdOwner, - missingMessages, - messageIndex, - ); - pendingRecoveryClosures.push(admission); - continue; - } - if (!run && executionContract.pendingWithoutRun === 'domain_replay') { - verifyOrRecoverUserMessage( - admission, - rootUserMessages, - messageIdOwner, - missingMessages, - messageIndex, - ); - replayAdmissions.push(admission); + ...(recorded ? {} : { writesUserMessage: true }), + }); continue; } if (!run) { - if (executionContract.pendingWithoutRun === 'root_replay') { - verifyOrRecoverUserMessage( - admission, - rootUserMessages, - messageIdOwner, - missingMessages, - messageIndex, - false, - ); - } else if (rootUserMessages.length > 0 || messageIdOwner) { - throw new Error(`Admitted Turn ${admission.turnId} has a UserMessage but no Run`); - } + // Every remaining path replays the admission, and a replay opens the + // Turn with the admission's own message id — writing the message here + // would only race the Run that owns it. + verifyUserMessage(admission, rootUserMessages, messageIdOwner); replayAdmissions.push(admission); - rootReplayAdmissions.push(admission); + if (executionContract.pendingWithoutRun !== 'domain_replay') { + rootReplayAdmissions.push(admission); + } continue; } await input.projection.assertRunIdentityAndContinuation( @@ -187,13 +177,9 @@ export async function prepareHostedExecutionRecovery( admission.turnId, admission.execution, ); - verifyOrRecoverUserMessage( - admission, - rootUserMessages, - messageIdOwner, - missingMessages, - messageIndex, - ); + if (!verifyUserMessage(admission, rootUserMessages, messageIdOwner)) { + await recordAdmittedUserMessage(input.stores, admission, run); + } } if (replayAdmissions.length > 1) { throw new Error(`Session ${session.id} has multiple admitted Turns without Runs`); @@ -205,25 +191,31 @@ export async function prepareHostedExecutionRecovery( sessionId: session.id, admissions, ...(rootReplayAdmissions[0] ? { rootReplayAdmission: rootReplayAdmissions[0] } : {}), - missingMessages, pendingRecoveryClosures, }); } for (const plan of prepared) { - for (const message of plan.missingMessages) { - await input.stores.sessionStore.appendMessage(plan.sessionId, message); - } - for (const admission of plan.pendingRecoveryClosures) { + for (const { admission, writesUserMessage } of plan.pendingRecoveryClosures) { if (!usesHostRecoveryClosure(admission.execution)) { throw new Error('Execution domain cannot use Host recovery closure'); } + const origin = hostedExecutionMessageOrigin(admission.execution); await input.runtime.closePendingHostedAdmission({ sessionId: admission.sessionId, turnId: admission.turnId, runId: admission.runId, admittedAt: admission.admittedAt, execution: admission.execution, + ...(writesUserMessage && admission.userMessageId + ? { + userMessage: { + id: admission.userMessageId, + content: requireHostedExecutionMessageContent(admission), + ...(origin ? { origin } : {}), + }, + } + : {}), }); } } @@ -234,6 +226,48 @@ export async function prepareHostedExecutionRecovery( })); } +/** + * The message a crashed Turn was admitted with, written into the Run that had + * already opened for it. + * + * A Run records its own user message right after its opening fact, so a Run + * that exists without one crashed between those two writes. Nothing else will + * write it now: the terminal fact recovery is about to append would seal the + * Turn without ever saying what the user asked for. + * + * The admission's normalized input is what goes in, not its queue sources: a + * Root folded from several Messages ran as one prompt, and that is the prompt + * the Turn was executed with. + */ +async function recordAdmittedUserMessage( + stores: ExecutionStoresWriter<'interactive'>, + admission: RootTurnAdmission, + run: RuntimeInvocationRecord, +): Promise { + if (run.terminalEvent) return; + const content = requireHostedExecutionMessageContent(admission); + const origin = hostedExecutionMessageOrigin(admission.execution); + const event: RuntimeEvent = { + id: admittedUserMessageId(admission), + sessionId: admission.sessionId, + invocationId: run.invocationId, + runId: run.runId, + turnId: admission.turnId, + ts: admission.admittedAt, + partial: false, + role: 'user', + author: origin ? 'host' : 'user', + content: { kind: 'text', ...content, ...(origin ? { origin } : {}) }, + }; + await stores.runtimeEventStore.appendRuntimeEvent(admission.sessionId, run.runId, event); + // The Turn never reached the commit that carries these, and no later path + // recomputes them: the connection lock is one-way and the preview is a write. + const message = projectRuntimeEventUserMessage(event, event.id); + if (message) { + await stores.sessionStore.commitMessageCatalogProjection(admission.sessionId, message); + } +} + export function requireHostedExecutionMessageContent(admission: RootTurnAdmission): MessageContent { if (admission.normalizedInput === null) { throw new RuntimeMessageAuthorityInvariantError( @@ -270,8 +304,13 @@ export function hostedExecutionMessageOrigin(execution: RootExecutionDescriptor) } interface PreparedRecoverySession extends HostedExecutionRecoveryPlan { - readonly missingMessages: readonly RecoveryUserMessage[]; - readonly pendingRecoveryClosures: readonly RootTurnAdmission[]; + readonly pendingRecoveryClosures: readonly PendingRecoveryClosure[]; +} + +interface PendingRecoveryClosure { + readonly admission: RootTurnAdmission; + /** The ledger has no message for this admission; the closure records it. */ + readonly writesUserMessage?: true; } type RecoveryUserMessage = Extract; @@ -287,14 +326,25 @@ interface RecoveryExecutionContract { readonly pendingWithoutRun: 'root_replay' | 'domain_replay' | 'host_recovery_closure'; } -function verifyOrRecoverUserMessage( +/** + * The id the admitted prompt is durable under. A Root folded from several + * queued Messages carries no single admitted Message identity, so recovery + * derives one from the Run — the same crash recovered twice writes the same + * event, and the store dedupes it. + */ +function admittedUserMessageId(admission: RootTurnAdmission): string { + return admission.userMessageId ?? `${admission.runId}-admitted-prompt`; +} + +/** + * Whether the ledger already carries this admission's message, throwing when + * what it carries contradicts the admission. + */ +function verifyUserMessage( admission: RootTurnAdmission, rootUserMessages: readonly RecoveryUserMessage[], messageIdOwner: StoredMessage | undefined, - missingMessages: RecoveryUserMessage[], - index: RecoveryMessageIndex, - materializeMissing = true, -): void { +): boolean { if (rootUserMessages.length > 1) { throw new Error(`Admitted Turn ${admission.turnId} has multiple UserMessages`); } @@ -302,7 +352,7 @@ function verifyOrRecoverUserMessage( if (userMessage) { if ( messageIdOwner !== userMessage || - userMessage.id !== admission.userMessageId || + userMessage.id !== admittedUserMessageId(admission) || !recoveryUserMessageOriginMatches(userMessage, admission.execution) || !messageContentsEqual( normalizeMessageContent(userMessage), @@ -311,15 +361,33 @@ function verifyOrRecoverUserMessage( ) { throw new Error(`Admitted Turn ${admission.turnId} does not match its UserMessage`); } - return; + return true; } if (messageIdOwner) { throw new Error(`Admitted Turn ${admission.turnId} reuses another message identity`); } - if (!materializeMissing) return; - const recoveredMessage = recoveryUserMessage(admission); - missingMessages.push(recoveredMessage); - indexRecoveryMessage(index, recoveredMessage); + return false; +} + +/** + * The user messages a Session's ledger holds, as the transcript presents them. + * + * Recovery reads the raw events rather than the read model: a Session it is + * about to repair may be exactly the one whose projection is still incomplete. + */ +function recoveryUserMessagesFromLedger( + events: readonly RuntimeEvent[], +): readonly RecoveryUserMessage[] { + const messages: RecoveryUserMessage[] = []; + for (const event of events) { + if (event.role !== 'user' || event.content?.kind !== 'text' || event.partial) continue; + const projected: RecoveryUserMessage | undefined = projectRuntimeEventUserMessage( + event, + event.id, + ); + if (projected) messages.push(projected); + } + return messages; } async function verifyQueueSourceMessages( @@ -364,21 +432,6 @@ async function verifyQueueSourceMessages( } } -function recoveryUserMessage(admission: RootTurnAdmission): RecoveryUserMessage { - if (!admission.userMessageId || !admission.normalizedInput) { - throw new Error(`Admitted Turn ${admission.turnId} does not own a UserMessage`); - } - const origin = hostedExecutionMessageOrigin(admission.execution); - return { - type: 'user', - id: admission.userMessageId, - turnId: admission.turnId, - ts: admission.admittedAt, - ...normalizeMessageContent(admission.normalizedInput), - ...(origin ? { origin } : {}), - }; -} - function recoveryUserMessageOriginMatches( message: RecoveryUserMessage, execution: RootExecutionDescriptor, diff --git a/packages/runtime-host/src/server/session-catalog-coordinator.ts b/packages/runtime-host/src/server/session-catalog-coordinator.ts index 0c08a2f029..4c3a50135f 100644 --- a/packages/runtime-host/src/server/session-catalog-coordinator.ts +++ b/packages/runtime-host/src/server/session-catalog-coordinator.ts @@ -44,13 +44,13 @@ import { isSessionNotFoundError, SessionMetadataConflictError, SessionMetadataVersionConflictError, - SessionReadMarkerMessageNotFoundError, type SessionCatalogPageCursor, type SessionCatalogRecord, type SessionHeaderSnapshot, type ExecutionStoresWriter, } from '@maka/storage/execution-stores'; import type { CreateStableSessionRequest } from '@maka/storage/session-store'; +import { isVisibleSessionMessage } from '@maka/storage/session-message-projection'; import type { RuntimePolicyStoresWriter } from '@maka/storage/runtime-policy-stores'; import { SessionConfigurationRevisionConflictError, @@ -94,22 +94,34 @@ import type { SessionCatalogOperationHandlerMap } from './operation-dispatcher.j import type { RuntimeHostAccessAuthority } from './access-authority.js'; import { type SessionAdmissionLease, SessionAdmissionGate } from './session-admission-gate.js'; import type { SessionContinuityCoordinator } from './session-continuity-coordinator.js'; +import type { SessionTranscriptReader } from './session-transcript-reader.js'; import { type HostWorkspaceResolver, WorkspaceResolutionError } from './workspace-resolver.js'; type SessionCatalogStores = Pick< ExecutionStoresWriter<'interactive'>['sessionStore'], | 'createStableSession' | 'listCatalogPage' - | 'markSessionReadThroughMessage' | 'probeStableSessionCreate' | 'readCatalogRecord' | 'readExecutionBoundary' | 'readHeaderRecordSnapshot' - | 'readTurnContributionsSnapshot' - | 'readTurnLandmarksSnapshot' | 'updateHeaderVersioned' >; +/** The Turn index a Session catalog page is built from, read off the ledger. */ +type SessionTurnIndexReader = Pick< + SessionTranscriptReader, + 'readDurableRecords' | 'readDurableTurnContributions' | 'readDurableTurnLandmarks' +>; + +/** + * How far back a read marker looks for the newest visible message. A Turn ends + * on its assistant text, so the tail of one run is enough; the bound only keeps + * a run of pure tool traffic from walking the whole ledger. + */ +const SESSION_READ_MARKER_TAIL_MAX_MESSAGES = 64; +const SESSION_READ_MARKER_TAIL_MAX_BYTES = 256 * 1024; + type SessionRuntimePolicyStores = { readonly connectionCatalog: Pick; readonly runtimePolicy: Pick; @@ -166,6 +178,7 @@ export class NoUsableImportModelError extends SessionOperationFailure { export interface HostSessionCatalogCoordinatorOptions { readonly stores: SessionCatalogStores; + readonly turnIndex: SessionTurnIndexReader; readonly runtimePolicy: SessionRuntimePolicyStores; readonly manager: SessionConfigurationAuthority; readonly admission: SessionAdmissionGate; @@ -276,6 +289,7 @@ export class HostSessionCatalogCoordinator { }; readonly #stores: SessionCatalogStores; + readonly #turnIndex: SessionTurnIndexReader; readonly #runtimePolicy: SessionRuntimePolicyStores; readonly #manager: SessionConfigurationAuthority; readonly #admission: SessionAdmissionGate; @@ -288,6 +302,7 @@ export class HostSessionCatalogCoordinator { constructor(options: HostSessionCatalogCoordinatorOptions) { this.#stores = options.stores; + this.#turnIndex = options.turnIndex; this.#runtimePolicy = options.runtimePolicy; this.#manager = options.manager; this.#admission = options.admission; @@ -487,7 +502,7 @@ export class HostSessionCatalogCoordinator { let maxContributions = input.maxContributions; let throughSequence = input.throughSequence; while (true) { - const page = await this.#stores.readTurnContributionsSnapshot( + const page = await this.#turnIndex.readDurableTurnContributions( input.sessionId, throughSequence, input.position, @@ -527,7 +542,7 @@ export class HostSessionCatalogCoordinator { input: SessionTurnLandmarksQueryInput, ): Promise> { try { - const snapshot = await this.#stores.readTurnLandmarksSnapshot( + const snapshot = await this.#turnIndex.readDurableTurnLandmarks( input.sessionId, input.maxLandmarks, ); @@ -819,10 +834,7 @@ export class HostSessionCatalogCoordinator { 'WorkHub Coordination Session read state requires WorkHub authority', ); } - await this.#stores.markSessionReadThroughMessage( - input.sessionId, - input.readThroughMessageId, - ); + await this.#clearUnreadAtTranscriptTail(current, input.readThroughMessageId); await this.#continuity.refreshCanonical(input.sessionId, lease); return { ok: true, @@ -832,9 +844,6 @@ export class HostSessionCatalogCoordinator { }; } catch (error) { if (isNotFound(error)) return readMarkerFailure('not_found', 'Session does not exist'); - if (error instanceof SessionReadMarkerMessageNotFoundError) { - return readMarkerFailure('invalid_request', error.message); - } if (error instanceof SessionMetadataVersionConflictError) { return readMarkerFailure( 'operation_conflict', @@ -850,6 +859,32 @@ export class HostSessionCatalogCoordinator { }); } + /** + * A Session is read once the client has caught up with the ledger's newest + * visible message. `hasUnread` is the only thing the marker decides and every + * Turn raises it again, so a client still behind the tail changes nothing. + */ + async #clearUnreadAtTranscriptTail( + record: SessionHeaderSnapshot, + readThroughMessageId: string, + ): Promise { + const tail = await this.#turnIndex.readDurableRecords(record.header.id, { + direction: 'older', + maxMessages: SESSION_READ_MARKER_TAIL_MAX_MESSAGES, + maxStoredBytes: SESSION_READ_MARKER_TAIL_MAX_BYTES, + }); + const latest = tail.records.find(({ message }) => isVisibleSessionMessage(message)); + if (latest?.message.id !== readThroughMessageId) return; + if (record.header.lastReadMessageId === readThroughMessageId && !record.header.hasUnread) { + return; + } + await this.#stores.updateHeaderVersioned( + record.header.id, + { lastReadMessageId: readThroughMessageId, hasUnread: false }, + record.revision, + ); + } + async #committedUpdate( sessionId: string, lease: SessionAdmissionLease, diff --git a/packages/runtime-host/src/server/session-revision-coordinator.ts b/packages/runtime-host/src/server/session-revision-coordinator.ts index 829872cc8c..4b517a25c2 100644 --- a/packages/runtime-host/src/server/session-revision-coordinator.ts +++ b/packages/runtime-host/src/server/session-revision-coordinator.ts @@ -608,9 +608,8 @@ export class HostSessionRevisionCoordinator { copyCurrent: kind === 'branch' && slice.beforeTs === undefined && input.sourceTurnId !== undefined, }); - if (copiedMessages.length > 0) { - await this.#stores.sessionStore.appendMessages(input.targetSessionId, [...copiedMessages]); - } + // `cloneConversationRuntimeLedger` already wrote the copy's own spine, and + // the copy reads back off that: nothing here writes a second transcript. await this.#stores.sessionStore.updateHeader(input.targetSessionId, { conversationCopy: { ...createInput.conversationCopy!, diff --git a/packages/runtime-host/src/server/session-transcript-reader.ts b/packages/runtime-host/src/server/session-transcript-reader.ts index 6eb0259ea5..19c5eec280 100644 --- a/packages/runtime-host/src/server/session-transcript-reader.ts +++ b/packages/runtime-host/src/server/session-transcript-reader.ts @@ -29,17 +29,29 @@ import { type CanonicalPermissionOutcomeReader, type CanonicalPermissionOutcomeRecord, } from '@maka/runtime/interaction-authority'; +import { + isSessionInlineInvocation, + type RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; import type { ExecutionStoresWriter, SessionTranscriptMessageLookupRequest, SessionTranscriptPageRequest, SessionTranscriptRecordScanPage, SessionTranscriptRecordScanRequest, + SessionTranscriptStorageFragment, SessionTranscriptStoragePage, + SessionTurnContribution, + SessionTurnContributionPage, + SessionTurnLandmark, + SessionTurnLandmarkSnapshot, } from '@maka/storage/execution-stores'; +import { foldTurnContribution } from '@maka/storage/session-message-projection'; import { SESSION_TRANSCRIPT_OVERLAY_MAX_MESSAGES, type TurnSnapshot } from '../protocol/index.js'; const PERMISSION_OUTCOME_READ_CONCURRENCY = 8; +/** Sequence room reserved for one invocation's projected transcript rows. */ +const RUN_SEQUENCE_STRIDE = 1 << 20; export const ACTIVE_TRANSCRIPT_OVERLAY_MAX_MESSAGES = SESSION_TRANSCRIPT_OVERLAY_MAX_MESSAGES; export const ACTIVE_TRANSCRIPT_OVERLAY_MAX_BYTES = 16 * 1024 * 1024; const ACTIVE_TRANSCRIPT_SOURCE_MAX_EVENTS = ACTIVE_TRANSCRIPT_OVERLAY_MAX_MESSAGES * 2; @@ -48,16 +60,47 @@ const ACTIVE_TRANSCRIPT_SCAN_BATCH_MAX_BYTES = 256 * 1024; export function createSessionTranscriptReader(input: { stores: ExecutionStoresWriter<'interactive'>; canonicalPermissionOutcomes: CanonicalPermissionOutcomeReader; + /** + * Converts a Session whose transcript predates the ledger, before this reader + * looks for invocations that only the conversion can create. Omitted only by + * tests that seed the ledger themselves. + */ + ensureTranscriptLedger?: (sessionId: string) => Promise; }): SessionTranscriptReader { + const durable = createDurableLedgerTranscriptReader(input); + const prepared = async (sessionId: string): Promise => { + await input.ensureTranscriptLedger?.(sessionId); + }; return { - readDurableHighWater: (sessionId) => - input.stores.sessionStore.readTranscriptHighWaterSnapshot(sessionId), - readDurablePage: (sessionId, request) => - input.stores.sessionStore.readTranscriptPageSnapshot(sessionId, request), - readDurableRecords: (sessionId, request) => - input.stores.sessionStore.readTranscriptRecordsSnapshot(sessionId, request), - readDurableMessagesById: (sessionId, request) => - input.stores.sessionStore.readTranscriptMessagesSnapshot(sessionId, request), + readDurableHighWater: async (sessionId) => { + await prepared(sessionId); + return durable.readHighWater(sessionId); + }, + readDurablePage: async (sessionId, request) => { + await prepared(sessionId); + return durable.readPage(sessionId, request); + }, + readDurableRecords: async (sessionId, request) => { + await prepared(sessionId); + return durable.readRecords(sessionId, request); + }, + readDurableMessagesById: async (sessionId, request) => { + await prepared(sessionId); + return durable.readMessagesById(sessionId, request); + }, + readDurableTurnContributions: async ( + sessionId, + throughSequence, + position, + maxContributions, + ) => { + await prepared(sessionId); + return durable.readTurnContributions(sessionId, throughSequence, position, maxContributions); + }, + readDurableTurnLandmarks: async (sessionId, maxLandmarks) => { + await prepared(sessionId); + return durable.readTurnLandmarks(sessionId, maxLandmarks); + }, readActiveOverlay: async (sessionId, rootTurn) => { if (!rootTurn || isTerminalTurn(rootTurn)) return []; @@ -97,12 +140,340 @@ export interface SessionTranscriptReader { sessionId: string, request: SessionTranscriptMessageLookupRequest, ): Promise; + readDurableTurnContributions( + sessionId: string, + throughSequence: number | null, + position: number, + maxContributions: number, + ): Promise; + readDurableTurnLandmarks( + sessionId: string, + maxLandmarks: number, + ): Promise; readActiveOverlay( sessionId: string, rootTurn: TurnSnapshot | null, ): Promise; } +/** + * The settled part of a Session's transcript, read off the RuntimeEvent ledger. + * + * A page is bounded by reading one ended invocation at a time: a Session with a + * thousand Turns costs the same per page as one with three. Sequences are + * `runIndex * RUN_SEQUENCE_STRIDE + indexWithinRun`, which is monotone in the + * order the read model presents runs and derivable from the invocation list + * alone — so locating a page never has to project the Turns before it. They are + * stable for as long as a subscription lives, which is exactly as long as the + * signed cursors that carry them. + */ +function createDurableLedgerTranscriptReader(input: { + stores: ExecutionStoresWriter<'interactive'>; + canonicalPermissionOutcomes: CanonicalPermissionOutcomeReader; +}) { + const endedInvocations = async (sessionId: string): Promise => + (await input.stores.runtimeEventStore.listSessionInvocations(sessionId)).filter( + (invocation) => isSessionInlineInvocation(invocation.opening) && invocation.terminalEvent, + ); + + const projectRun = async ( + sessionId: string, + invocation: RuntimeInvocationRecord, + ): Promise => { + const events = await input.stores.runtimeEventStore.readRuntimeEvents( + sessionId, + invocation.runId, + ); + const projected = projectRuntimeEventsToStoredMessages(events, { + invocations: [invocation], + canonicalPermissionOutcomes: await readCanonicalPermissionOutcomes( + events, + input.canonicalPermissionOutcomes, + ), + }); + if (projected.diagnostics.some(isHardRuntimeEventReadModelDiagnostic)) { + throw new Error('Durable RuntimeEvent transcript projection is incomplete'); + } + if (projected.messages.length > RUN_SEQUENCE_STRIDE) { + throw new Error('Durable Session Turn exceeds its transcript sequence stride'); + } + return projected.messages; + }; + + /** + * The runs a page must visit, in traversal order, already clipped to the + * watermark and the caller's position. + */ + const traversal = ( + invocations: readonly RuntimeInvocationRecord[], + direction: 'older' | 'newer', + throughSequence: number, + position: number, + ): Array<{ runIndex: number; invocation: RuntimeInvocationRecord }> => { + const highestRunIndex = Math.min(runIndexOf(throughSequence), invocations.length - 1); + const startRunIndex = Math.min(runIndexOf(position), highestRunIndex); + const runs: Array<{ runIndex: number; invocation: RuntimeInvocationRecord }> = []; + if (direction === 'older') { + for (let index = startRunIndex; index >= 0; index -= 1) { + const invocation = invocations[index]; + if (invocation) runs.push({ runIndex: index, invocation }); + } + return runs; + } + for (let index = Math.max(0, startRunIndex); index <= highestRunIndex; index += 1) { + const invocation = invocations[index]; + if (invocation) runs.push({ runIndex: index, invocation }); + } + return runs; + }; + + const highWaterOf = async ( + sessionId: string, + invocations: readonly RuntimeInvocationRecord[], + ): Promise => { + for (let index = invocations.length - 1; index >= 0; index -= 1) { + const invocation = invocations[index]!; + const messages = await projectRun(sessionId, invocation); + if (messages.length > 0) return index * RUN_SEQUENCE_STRIDE + messages.length - 1; + } + return null; + }; + + /** Every projected record of the requested page, ordered for its direction. */ + const scan = async function* ( + sessionId: string, + request: { + direction: 'older' | 'newer'; + throughSequence?: number | null; + position?: number; + }, + ): AsyncGenerator<{ sequence: number; message: StoredMessage }> { + const invocations = await endedInvocations(sessionId); + const throughSequence = + request.throughSequence === undefined + ? await highWaterOf(sessionId, invocations) + : request.throughSequence; + if (throughSequence === null) return; + const position = request.position ?? (request.direction === 'older' ? throughSequence : 0); + for (const { runIndex, invocation } of traversal( + invocations, + request.direction, + throughSequence, + position, + )) { + const messages = await projectRun(sessionId, invocation); + const indexed = messages.map((message, index) => ({ + sequence: runIndex * RUN_SEQUENCE_STRIDE + index, + message, + })); + const selected = indexed.filter( + ({ sequence }) => + sequence <= throughSequence && + (request.direction === 'older' ? sequence <= position : sequence >= position), + ); + if (request.direction === 'older') selected.reverse(); + yield* selected; + } + }; + + return { + async readHighWater(sessionId: string): Promise { + return highWaterOf(sessionId, await endedInvocations(sessionId)); + }, + + async readPage( + sessionId: string, + request: SessionTranscriptPageRequest, + ): Promise { + const throughSequence = + request.throughSequence === undefined + ? await this.readHighWater(sessionId) + : request.throughSequence; + if (throughSequence === null) { + return { throughSequence: null, fragments: [], rawBytes: 0, next: null }; + } + const fragments: SessionTranscriptStorageFragment[] = []; + let rawBytes = 0; + let next: SessionTranscriptStoragePage['next'] = null; + let truncated = false; + for await (const record of scan(sessionId, { ...request, throughSequence })) { + if (fragments.length >= request.maxMessages || rawBytes >= request.maxBytes) { + truncated = true; + next = { position: record.sequence, byteOffset: null }; + break; + } + const data = Buffer.from(JSON.stringify(record.message), 'utf8'); + // A message larger than the remaining budget is served in byte slices, + // from the edge the traversal is moving away from, so the next page + // resumes inside the same record instead of skipping it. + const continued = record.sequence === request.position && request.byteOffset !== undefined; + const edge = continued + ? request.byteOffset! + : request.direction === 'older' + ? data.byteLength + : 0; + const available = request.maxBytes - rawBytes; + const byteOffset = request.direction === 'older' ? Math.max(0, edge - available) : edge; + const end = + request.direction === 'older' ? edge : Math.min(data.byteLength, edge + available); + fragments.push({ + sequence: record.sequence, + byteOffset, + totalBytes: data.byteLength, + payloadDigest: null, + data: data.subarray(byteOffset, end), + }); + rawBytes += end - byteOffset; + const complete = request.direction === 'older' ? byteOffset === 0 : end === data.byteLength; + if (!complete) { + truncated = true; + next = { + position: record.sequence, + byteOffset: request.direction === 'older' ? byteOffset : end, + }; + break; + } + } + if (!truncated) next = null; + return { throughSequence, fragments, rawBytes, next }; + }, + + async readRecords( + sessionId: string, + request: SessionTranscriptRecordScanRequest, + ): Promise { + const throughSequence = + request.throughSequence === undefined + ? await this.readHighWater(sessionId) + : request.throughSequence; + if (throughSequence === null) { + return { throughSequence: null, records: [], nextPosition: null }; + } + const records: Array<{ sequence: number; message: StoredMessage }> = []; + let storedBytes = 0; + let nextPosition: number | null = null; + for await (const record of scan(sessionId, { ...request, throughSequence })) { + if (records.length >= request.maxMessages || storedBytes >= request.maxStoredBytes) { + nextPosition = record.sequence; + break; + } + records.push(record); + storedBytes += Buffer.byteLength(JSON.stringify(record.message), 'utf8'); + } + return { throughSequence, records, nextPosition }; + }, + + /** + * What each Turn contributed, one ended invocation at a time. + * + * An invocation is a Turn, so the run listing is the index: a page costs the + * runs it actually summarizes, never a scan of the Turns before them. + */ + async readTurnContributions( + sessionId: string, + throughSequence: number | null, + position: number, + maxContributions: number, + ): Promise { + const invocations = await endedInvocations(sessionId); + const watermark = throughSequence ?? (await highWaterOf(sessionId, invocations)); + if (watermark === null) { + return { throughSequence: null, contributions: [], nextPosition: null }; + } + const contributions: SessionTurnContribution[] = []; + const lastRunIndex = Math.min(runIndexOf(watermark), invocations.length - 1); + let runIndex = Math.max(0, runIndexOf(position)); + for (; runIndex <= lastRunIndex; runIndex += 1) { + if (contributions.length >= maxContributions) { + return { + throughSequence: watermark, + contributions, + nextPosition: runIndex * RUN_SEQUENCE_STRIDE, + }; + } + const invocation = invocations[runIndex]; + if (!invocation) continue; + const messages = await projectRun(sessionId, invocation); + let contribution: SessionTurnContribution | undefined; + for (const [index, message] of messages.entries()) { + const sequence = runIndex * RUN_SEQUENCE_STRIDE + index; + if (sequence > watermark || sequence < position) continue; + contribution = foldTurnContribution(contribution, invocation.turnId, sequence, message); + } + if (contribution) contributions.push(contribution); + } + return { throughSequence: watermark, contributions, nextPosition: null }; + }, + + /** Evenly spaced Turn starts, sampled from the run listing itself. */ + async readTurnLandmarks( + sessionId: string, + maxLandmarks: number, + ): Promise { + const invocations = await endedInvocations(sessionId); + const throughSequence = await highWaterOf(sessionId, invocations); + if (throughSequence === null) return { throughSequence: null, landmarks: [] }; + const lastRunIndex = Math.min(runIndexOf(throughSequence), invocations.length - 1); + const count = Math.min(maxLandmarks, lastRunIndex + 1); + const sampled = + count <= 0 + ? [] + : Array.from({ length: count }, (_, index) => + count === 1 ? lastRunIndex : Math.floor((lastRunIndex * index) / (count - 1)), + ); + const landmarks: SessionTurnLandmark[] = []; + for (const runIndex of [...new Set(sampled)]) { + const invocation = invocations[runIndex]; + if (!invocation) continue; + const messages = await projectRun(sessionId, invocation); + const index = messages.findIndex((message) => message.type === 'user'); + const message = index < 0 ? undefined : messages[index]; + if (message?.type !== 'user') continue; + const label = (message.displayText ?? message.text).trim(); + if (!label) continue; + landmarks.push({ + turnId: invocation.turnId, + sequence: runIndex * RUN_SEQUENCE_STRIDE + index, + label, + }); + } + return { throughSequence, landmarks }; + }, + + async readMessagesById( + sessionId: string, + request: SessionTranscriptMessageLookupRequest, + ): Promise { + if (request.throughSequence === null || request.messageIds.length === 0) return []; + const wanted = new Set(request.messageIds); + const found: StoredMessage[] = []; + let bytes = 0; + // Callers look up streams that were active a moment ago, so a durable copy + // can only be in the run that just sealed. Without this bound the ordinary + // miss — the run is still open — reprojects every Turn in the Session. + let newestRunIndex: number | undefined; + for await (const record of scan(sessionId, { + direction: 'older', + throughSequence: request.throughSequence, + })) { + newestRunIndex ??= runIndexOf(record.sequence); + if (runIndexOf(record.sequence) < newestRunIndex) break; + if (!wanted.delete(record.message.id)) continue; + bytes += Buffer.byteLength(JSON.stringify(record.message), 'utf8'); + if (found.length >= request.maxMessages || bytes > request.maxBytes) break; + found.push(record.message); + if (wanted.size === 0) break; + } + // Restore transcript order: the scan walked backwards to find them. + return found.reverse(); + }, + }; +} + +function runIndexOf(sequence: number): number { + return Math.floor(sequence / RUN_SEQUENCE_STRIDE); +} + function assertActiveOverlayBounded(messages: readonly StoredMessage[]): void { if (messages.length > ACTIVE_TRANSCRIPT_OVERLAY_MAX_MESSAGES) { throw new Error('Active Session transcript overlay exceeds its message limit'); diff --git a/packages/runtime/src/__tests__/admission-limiter.test.ts b/packages/runtime/src/__tests__/admission-limiter.test.ts index 25380bba18..25d2372ec3 100644 --- a/packages/runtime/src/__tests__/admission-limiter.test.ts +++ b/packages/runtime/src/__tests__/admission-limiter.test.ts @@ -318,7 +318,6 @@ function buildRuntime( header: testHeader(), connection: testConnection(), modelId: 'mock-model', - appendMessage: async () => {}, newId: nextId(), now: () => 1, getPermissionPauseTarget: () => null, diff --git a/packages/runtime/src/__tests__/agent-run-steering-recovery.test.ts b/packages/runtime/src/__tests__/agent-run-steering-recovery.test.ts index 0c1d48965b..7d88a24454 100644 --- a/packages/runtime/src/__tests__/agent-run-steering-recovery.test.ts +++ b/packages/runtime/src/__tests__/agent-run-steering-recovery.test.ts @@ -29,7 +29,6 @@ import { createSqliteAgentRunStore } from '@maka/storage/agent-run-store'; import { createWorkspaceRuntimeStore } from '@maka/storage/runtime-event-persistence'; import { createSessionStore } from '@maka/storage/session-store'; import { AgentRun } from '../agent-run.js'; -import { RuntimeLedgerRepair } from '../runtime-ledger-repair.js'; import { buildStatusPatch } from '../session-projection-helpers.js'; import { waitFor as pollFor } from '@maka/core/test-only/async-primitives'; import { seedInvocation } from './invocation-fixture.js'; @@ -55,7 +54,6 @@ test('rejects an invalid tool mode before a durable AgentRun can be created', as userInput: { turnId: 'turn-invalid-mode', text: 'invalid', toolMode: 'typo' as never }, runStore, runtimeEventStore, - store, newId: () => 'unused', now: () => 1, hooks: { @@ -65,7 +63,6 @@ test('rejects an invalid tool mode before a durable AgentRun can be created', as unregisterRun: () => {}, updateHeader: async () => session, updateStatus: async () => {}, - appendTurnState: async () => {}, }, }), /invalid tool mode/i, @@ -94,7 +91,6 @@ test('does not re-append atomically committed tool facts through the generic eve header: session, userInput: { turnId, text: 'run a durable tool' }, runId, - store, runtimeEventStore, toolBoundaryProtocol: 't1_after_preflight_v1', newId: () => 'unused-id', @@ -106,7 +102,6 @@ test('does not re-append atomically committed tool facts through the generic eve unregisterRun: () => {}, updateHeader: (sessionId, patch) => store.updateHeader(sessionId, patch), updateStatus: async () => {}, - appendTurnState: async () => {}, }, }); const sessionEvent: SessionEvent = { @@ -168,7 +163,6 @@ test('acks a steering event whose canonical append preceded proof publication fa header: session, userInput: { turnId, text: 'start' }, runId, - store, runStore, runtimeEventStore, newId: () => 'unused-id', @@ -180,7 +174,6 @@ test('acks a steering event whose canonical append preceded proof publication fa unregisterRun: () => {}, updateHeader: (sessionId, patch) => store.updateHeader(sessionId, patch), updateStatus: async () => {}, - appendTurnState: async () => {}, }, }); const sessionEvent: SessionEvent = { @@ -224,177 +217,6 @@ test('acks a steering event whose canonical append preceded proof publication fa } }); -test('materializes a durable steering event into the transcript exactly once', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-agent-run-steering-transcript-')); - try { - const store = createSessionStore(root); - const session = await store.create({ - cwd: '/tmp/cwd', - llmConnectionSlug: 'fake', - model: 'fake-model', - permissionMode: 'ask', - }); - const runtimeEventStore = createWorkspaceRuntimeStore(root); - const turnId = 'turn-steering-transcript'; - const sessionEvent: SessionEvent = { - type: 'steering_message', - id: 'runtime-steering-transcript', - turnId, - ts: 2, - messageId: 'message-steering-transcript', - content: { text: 'persist this interjection' }, - }; - const run = new AgentRun({ - sessionId: session.id, - header: session, - userInput: { turnId, text: 'start' }, - runId: 'run-steering-transcript', - store, - runtimeEventStore, - newId: () => 'unused-id', - now: () => 10, - hooks: { - reserveRun: async () => { - throw new Error('reserveRun should not be called'); - }, - unregisterRun: () => {}, - updateHeader: (sessionId, patch) => store.updateHeader(sessionId, patch), - updateStatus: async () => {}, - appendTurnState: async () => {}, - }, - }); - const runtimeEvent: RuntimeEvent = { - id: sessionEvent.id, - invocationId: run.invocationId, - runId: 'run-steering-transcript', - sessionId: session.id, - turnId, - ts: sessionEvent.ts, - partial: false, - role: 'user', - author: 'user', - content: { - kind: 'text', - text: sessionEvent.content.text, - displayText: '/skill:writer persist this interjection', - inlineReferences: [{ kind: 'skill', value: '/skill:writer', label: 'Writer', start: 0 }], - steering: true, - }, - refs: { providerEventId: sessionEvent.messageId }, - }; - - await run.acceptMappedEvent(sessionEvent, runtimeEvent); - await run.acceptMappedEvent(sessionEvent, runtimeEvent); - - assert.deepEqual(await store.readMessages(session.id), [ - { - type: 'user', - id: sessionEvent.messageId, - turnId, - ts: sessionEvent.ts, - text: sessionEvent.content.text, - displayText: '/skill:writer persist this interjection', - inlineReferences: [{ kind: 'skill', value: '/skill:writer', label: 'Writer', start: 0 }], - steeringEventId: sessionEvent.id, - }, - ]); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('recovers a steering transcript message from the committed RuntimeEvent ledger', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-agent-run-steering-crash-cut-')); - try { - const store = createSessionStore(root); - const session = await store.create({ - cwd: '/tmp/cwd', - llmConnectionSlug: 'fake', - model: 'fake-model', - permissionMode: 'ask', - }); - const runId = 'run-steering-crash-cut'; - const turnId = 'turn-steering-crash-cut'; - const runStore = createSqliteAgentRunStore(root); - const runtimeEventStore = createWorkspaceRuntimeStore(root); - const steeringContent = { - kind: 'text' as const, - text: 'canonical steering envelope', - displayText: '/skill:writer recover this interjection', - attachments: [ - { - kind: 'pdf' as const, - name: 'evidence.pdf', - mimeType: 'application/pdf', - bytes: 2048, - ref: { - kind: 'session_file' as const, - sessionId: session.id, - relativePath: 'attachments/evidence.pdf', - }, - }, - ], - quotes: [{ text: 'quoted evidence', label: 'Assistant', sourceTurnId: 'turn-source' }], - inlineReferences: [ - { kind: 'skill' as const, value: '/skill:writer', label: 'Writer', start: 0 }, - ], - steering: true as const, - }; - await seedInvocation(runtimeEventStore, { - sessionId: session.id, - invocationId: 'invocation-steering-crash-cut', - runId, - turnId, - openedAt: 1, - }); - const runtimeEvent: RuntimeEvent = { - id: 'runtime-steering-crash-cut', - invocationId: 'invocation-steering-crash-cut', - runId, - sessionId: session.id, - turnId, - ts: 2, - partial: false, - role: 'user', - author: 'user', - content: steeringContent, - refs: { providerEventId: 'message-steering-crash-cut' }, - }; - await runtimeEventStore.appendRuntimeEvent(session.id, runId, runtimeEvent); - assert.deepEqual(await store.readMessages(session.id), []); - - const recoveredStore = createSessionStore(root); - const recoveredRunStore = createSqliteAgentRunStore(root); - const recoveredRuntimeEventStore = createWorkspaceRuntimeStore(root); - const repair = new RuntimeLedgerRepair({ - runtimeEventStore: recoveredRuntimeEventStore, - readMessages: (sessionId) => recoveredStore.readMessages(sessionId), - appendMessage: (sessionId, message) => recoveredStore.appendMessage(sessionId, message), - newId: () => 'unused-id', - now: () => 10, - }); - - assert.equal(await repair.repairSteeringMessagesOnce(session.id), 1); - assert.equal(await repair.repairSteeringMessagesOnce(session.id), 0); - assert.deepEqual(await recoveredStore.readMessages(session.id), [ - { - type: 'user', - id: 'message-steering-crash-cut', - turnId, - ts: 2, - text: 'canonical steering envelope', - displayText: '/skill:writer recover this interjection', - attachments: steeringContent.attachments, - quotes: steeringContent.quotes, - inlineReferences: steeringContent.inlineReferences, - steeringEventId: runtimeEvent.id, - }, - ]); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - test('awaits the durable settlement fact before accepting an interaction resume', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-agent-run-status-barrier-')); try { @@ -437,7 +259,6 @@ test('awaits the durable settlement fact before accepting an interaction resume' userInput: { turnId, text: 'resume after answer' }, runId, durability: 'required', - store, runStore, runtimeEventStore: delayedRuntimeEventStore, newId: () => 'status-event', @@ -452,7 +273,6 @@ test('awaits the durable settlement fact before accepting an interaction resume' sessionUpdateStarted = true; await store.updateHeader(sessionId, buildStatusPatch(status, ts, blockedReason)); }, - appendTurnState: async () => {}, }, }); let accepted = false; diff --git a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts index cc3a65b7a0..6baf558e54 100644 --- a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts +++ b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts @@ -87,6 +87,7 @@ import { buildLlmHistorySummarizer } from '../history-compact-summarizer.js'; import { createToolResultArchiveCapability } from '../tool-result-archive-capability.js'; import { createTestAiSdkBackend, + projectedTranscriptOf, readExternalExecutionBoundary, testToolResultArchive, } from './execution-boundary-test-helpers.js'; @@ -108,7 +109,6 @@ describe('AiSdkBackend ApplyPatch routing', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: providerType === 'openai' ? { ...connection(), slug: 'openai', providerType } @@ -138,7 +138,6 @@ describe('AiSdkBackend ApplyPatch routing', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { ...connection(), slug: 'deepseek', @@ -170,7 +169,6 @@ describe('AiSdkBackend ApplyPatch routing', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { ...connection(), slug: 'openai', providerType: 'openai' }, apiKey: 'sk-test', modelId: 'gpt-5.4', @@ -242,7 +240,6 @@ describe('AiSdkBackend ApplyPatch routing', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: targetConnection, apiKey: 'sk-test', modelId, @@ -343,7 +340,6 @@ describe('AiSdkBackend ApplyPatch routing', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -415,7 +411,6 @@ describe('AiSdkBackend ApplyPatch routing', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { ...connection(), slug: 'openai', providerType: 'openai' }, apiKey: 'sk-test', modelId: 'gpt-5.4', @@ -499,7 +494,6 @@ describe('AiSdkBackend ApplyPatch routing', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { ...connection(), slug: 'openai', providerType: 'openai' }, apiKey: 'sk-test', modelId: 'gpt-5.4', @@ -632,7 +626,6 @@ describe('AiSdkBackend Memory Extraction triggers', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -713,7 +706,6 @@ describe('AiSdkBackend Memory Extraction triggers', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { ...connection(), providerType: 'openai' }, apiKey: 'sk-test', modelId: 'gpt-5.4', @@ -788,7 +780,6 @@ describe('AiSdkBackend Memory Extraction triggers', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -879,7 +870,6 @@ describe('AiSdkBackend Memory Extraction triggers', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -962,7 +952,6 @@ describe('AiSdkBackend Memory Extraction triggers', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -1079,7 +1068,6 @@ describe('AiSdkBackend sandbox boundary convergence', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: { ...header(), cwd, workspaceRoot: cwd }, - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -1229,7 +1217,6 @@ describe('AiSdkBackend sandbox boundary convergence', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -1352,7 +1339,6 @@ describe('AiSdkBackend sandbox boundary convergence', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -1559,7 +1545,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { slug: 'kimi-coding-plan', providerType: 'kimi-coding-plan', @@ -1590,7 +1575,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { slug: 'kimi-coding-plan', providerType: 'kimi-coding-plan', @@ -1627,7 +1611,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { slug: 'mistral', providerType: 'mistral', @@ -1657,7 +1640,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -1720,7 +1702,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -1765,7 +1746,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -1818,7 +1798,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -1878,7 +1857,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { ...connection(), providerType: 'openai' }, apiKey: 'sk-test', modelId: 'mock-model-id', @@ -1928,7 +1906,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2021,7 +1998,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2072,7 +2048,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2139,7 +2114,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2192,7 +2166,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2260,7 +2233,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2324,7 +2296,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2392,7 +2363,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2445,7 +2415,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2496,7 +2465,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2540,7 +2508,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2590,7 +2557,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2679,7 +2645,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2717,7 +2682,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2810,7 +2774,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2912,7 +2875,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -2979,7 +2941,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -3085,7 +3046,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -3168,7 +3128,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -3377,7 +3336,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { slug: 'deepseek', providerType: 'deepseek', @@ -3461,7 +3419,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { slug: 'deepseek', providerType: 'deepseek', @@ -3575,7 +3532,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -3701,7 +3657,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -3839,7 +3794,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -4000,7 +3954,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -4283,7 +4236,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -4333,7 +4285,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -4426,7 +4377,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: { ...header(), llmConnectionId: 'test-connection-id', model: 'mock-model-id' }, - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -4499,7 +4449,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -4569,7 +4518,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -4649,7 +4597,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -4735,7 +4682,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -4775,7 +4721,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -4828,7 +4773,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -4885,7 +4829,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -4986,7 +4929,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -5071,7 +5013,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -5183,7 +5124,6 @@ describe('AiSdkBackend model history', () => { const backendInput: AiSdkBackendInput = { sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -5248,7 +5188,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -5286,7 +5225,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -5351,7 +5289,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -5392,7 +5329,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -5772,7 +5708,6 @@ describe('AiSdkBackend model history', () => { backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -5806,8 +5741,9 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async (message) => { - if (message.type !== 'token_usage') return; + // The usage checkpoint is the persistence this turn awaits at its step + // boundary, so holding it here is the window the stop has to win. + recordUsageCheckpoint: async () => { usagePersistenceStarted = true; await gate.promise; }, @@ -6008,7 +5944,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: codexConnection, apiKey: 'codex-token', modelId: 'mock-model-id', @@ -6076,7 +6011,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: codexConnection, apiKey: 'codex-token', modelId: 'mock-model-id', @@ -6168,7 +6102,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: codexConnection, apiKey: 'codex-token', modelId: 'mock-model-id', @@ -6261,7 +6194,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: codexConnection, apiKey: 'codex-token', modelId: 'mock-model-id', @@ -6296,7 +6228,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -6368,7 +6299,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -6428,7 +6358,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: openAiConnection, apiKey: 'sk-test', modelId: 'mock-model-id', @@ -6499,7 +6428,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: { ...header(), llmConnectionId: 'connection-a', model: 'claude-b' }, - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'claude-b', @@ -6583,7 +6511,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: { ...header(), llmConnectionId: 'connection-a', model: 'claude-a' }, - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'claude-a', @@ -6636,7 +6563,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: { ...header(), llmConnectionId: 'connection-a', model: 'claude-a' }, - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'claude-a', @@ -6697,7 +6623,6 @@ describe('AiSdkBackend model history', () => { llmConnectionSlug: 'github-copilot', model: 'gpt-5.4', }, - appendMessage: async () => {}, connection: copilotConnection, apiKey: 'sk-test', modelId: 'gpt-5.4', @@ -6791,7 +6716,6 @@ describe('AiSdkBackend model history', () => { llmConnectionSlug: 'openai-main', model: 'gpt-5.4', }, - appendMessage: async () => {}, connection: openAiConnection, apiKey: 'sk-test', modelId: 'gpt-5.4', @@ -6844,7 +6768,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { ...connection(), providerType: 'openai' }, apiKey: 'sk-test', modelId: 'mock-model-id', @@ -6914,7 +6837,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { ...connection(), slug: 'kimi-main', @@ -6995,7 +6917,6 @@ describe('AiSdkBackend model history', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { ...connection(), providerType: 'openai' }, apiKey: 'sk-test', modelId: 'mock-model-id', @@ -7057,7 +6978,6 @@ describe('AiSdkBackend error surfaces', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-live-secret-token-value', modelId: 'claude-sonnet-4-5-20250929', @@ -7454,7 +7374,6 @@ describe('AiSdkBackend usage telemetry', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -7510,7 +7429,6 @@ describe('AiSdkBackend usage telemetry', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -7571,7 +7489,6 @@ describe('AiSdkBackend usage telemetry', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -7623,7 +7540,6 @@ describe('AiSdkBackend usage telemetry', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -7671,7 +7587,6 @@ describe('AiSdkBackend usage telemetry', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -7708,7 +7623,6 @@ describe('AiSdkBackend usage telemetry', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -7745,7 +7659,6 @@ describe('AiSdkBackend usage telemetry', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -7817,7 +7730,6 @@ describe('AiSdkBackend usage telemetry', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: { ...header(), collaborationMode: 'agent' }, - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -7883,7 +7795,6 @@ describe('AiSdkBackend usage telemetry', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -7980,7 +7891,6 @@ describe('AiSdkBackend usage telemetry', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -8092,7 +8002,6 @@ describe('AiSdkBackend usage telemetry', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -8382,7 +8291,6 @@ describe('AiSdkBackend usage telemetry', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -8428,7 +8336,6 @@ describe('AiSdkBackend usage telemetry', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'unpriced-model', @@ -8509,7 +8416,6 @@ describe('AiSdkBackend usage telemetry', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -9000,7 +8906,6 @@ describe('AiSdkBackend tool availability diagnostics', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -9076,7 +8981,6 @@ describe('AiSdkBackend tool availability diagnostics', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -9167,7 +9071,6 @@ describe('AiSdkBackend context budget and prompt attribution', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -9333,7 +9236,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -9405,7 +9307,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { ...connection(), models: [{ id: 'mock-model-id', contextWindow: 200_000 }], @@ -9582,7 +9483,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -9643,7 +9543,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -9906,7 +9805,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -10050,7 +9948,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -10124,7 +10021,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { ...connection(), slug: 'deepseek', @@ -10162,59 +10058,6 @@ describe('AiSdkBackend RunTrace', () => { assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'end_turn'); }); - test('does not report a consumed idle timeout for a later assistant append failure', async () => { - const timers = manualWatchdogTimer(); - let calls = 0; - const model = new MockLanguageModelV4({ - doStream: async (options) => { - calls += 1; - return { - stream: hangingProviderStream( - [ - { type: 'stream-start', warnings: [] }, - { type: 'reasoning-start', id: 'reasoning-1' }, - { - type: 'reasoning-delta', - id: 'reasoning-1', - delta: 'partial thought', - }, - ], - options.abortSignal, - ), - }; - }, - }); - const backend = createTestAiSdkBackend({ - sessionId: 'session-1', - header: header(), - appendMessage: async () => { - throw new Error('assistant append failed'); - }, - connection: connection(), - apiKey: 'sk-test', - modelId: 'mock-model-id', - modelFactory: () => model, - tools: [], - newId: idGenerator(), - now: monotonicClock(), - streamWatchdogTimer: timers.clock, - providerRetrySleep: async () => {}, - }); - - const events: SessionEvent[] = []; - for await (const event of backend.send({ turnId: 'turn-1', text: 'hi', context: [] })) { - events.push(event); - if (event.type === 'thinking_delta' && event.text === 'partial thought') timers.fire(); - } - - assert.equal(calls, 1); - const error = events.find((event) => event.type === 'error'); - assert.equal(error?.type, 'error'); - assert.notEqual(error?.type === 'error' ? error.reason : undefined, 'timeout'); - assert.equal(error?.type === 'error' ? error.message : undefined, 'Operation failed'); - assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'error'); - }); - test('links a recovered tool call to the retry assistant step', async () => { const timers = manualWatchdogTimer(); const durable = durableTurnHarness('turn-1', 'read notes'); @@ -10290,7 +10133,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -10354,7 +10196,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -10434,7 +10275,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -10506,7 +10346,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -10564,7 +10403,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -10722,7 +10560,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -10782,7 +10619,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -10837,7 +10673,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -10874,7 +10709,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -10958,7 +10792,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -11003,7 +10836,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -11086,7 +10918,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -11163,7 +10994,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -11193,7 +11023,6 @@ describe('AiSdkBackend RunTrace', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'claude-sonnet-4-5-20250929', @@ -11231,7 +11060,6 @@ describe('AiSdkBackend tool execution', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header('bypass'), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'claude-sonnet-4-5-20250929', @@ -11328,7 +11156,6 @@ describe('AiSdkBackend tool execution', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header('ask'), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'claude-sonnet-4-5-20250929', @@ -11392,7 +11219,6 @@ describe('AiSdkBackend tool execution', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header('explore'), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'claude-sonnet-4-5-20250929', @@ -11455,7 +11281,6 @@ describe('AiSdkBackend tool execution', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header('explore'), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'claude-sonnet-4-5-20250929', @@ -11554,7 +11379,6 @@ describe('AiSdkBackend tool execution', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header('bypass'), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'claude-sonnet-4-5-20250929', @@ -11876,7 +11700,6 @@ describe('AiSdkBackend concurrent turns', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -11937,7 +11760,6 @@ describe('AiSdkBackend concurrent turns', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -11980,7 +11802,6 @@ describe('AiSdkBackend concurrent turns', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -12117,7 +11938,6 @@ describe('AiSdkBackend thinking persistence', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -12304,7 +12124,6 @@ describe('AiSdkBackend thinking persistence', () => { const firstBackend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: openCodeClaudeConnection, apiKey: 'sk-test', modelId: 'claude-opus-4-8', @@ -12351,7 +12170,6 @@ describe('AiSdkBackend thinking persistence', () => { const secondBackend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: openCodeClaudeConnection, apiKey: 'sk-test', modelId: 'claude-opus-4-8', @@ -12452,7 +12270,6 @@ describe('AiSdkBackend thinking persistence', () => { const secondBackend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -12572,7 +12389,6 @@ describe('AiSdkBackend thinking persistence', () => { const secondBackend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { slug: 'openai', providerType: 'openai', @@ -12724,7 +12540,6 @@ describe('AiSdkBackend thinking persistence', () => { const secondBackend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { slug: 'volcengine-agent-plan', providerType: 'volcengine-agent-plan', @@ -12835,7 +12650,6 @@ describe('AiSdkBackend thinking persistence', () => { const secondBackend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { slug: 'deepseek', providerType: 'deepseek', @@ -12987,7 +12801,6 @@ describe('AiSdkBackend thinking persistence', () => { const firstBackend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: tokenPlanConnection, apiKey: 'alibaba-token', modelId: 'qwen3.8-max', @@ -13055,7 +12868,6 @@ describe('AiSdkBackend thinking persistence', () => { const secondBackend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: tokenPlanConnection, apiKey: 'alibaba-token', modelId: 'qwen3.8-max', @@ -13190,7 +13002,6 @@ describe('AiSdkBackend thinking persistence', () => { const recoveryBackend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { slug: 'alibaba-token-plan-cn', providerType: 'alibaba-token-plan-cn', @@ -13401,7 +13212,6 @@ describe('AiSdkBackend thinking persistence', () => { const recoveryBackend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection, apiKey: 'alibaba-token', modelId: 'qwen3.8-max', @@ -13546,7 +13356,6 @@ describe('AiSdkBackend thinking persistence', () => { const recoveryBackend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection, apiKey: 'alibaba-token', modelId: 'qwen3.8-max', @@ -13586,7 +13395,6 @@ describe('AiSdkBackend thinking persistence', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { slug: 'alibaba-token-plan-cn', providerType: 'alibaba-token-plan-cn', @@ -13677,7 +13485,6 @@ describe('AiSdkBackend thinking persistence', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { slug: 'alibaba-token-plan-cn', providerType: 'alibaba-token-plan-cn', @@ -13753,7 +13560,6 @@ describe('AiSdkBackend thinking persistence', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: { ...header(), thinkingLevel: 'max' }, - appendMessage: async () => {}, connection: { slug: 'deepseek', providerType: 'deepseek', @@ -13935,7 +13741,6 @@ describe('AiSdkBackend thinking persistence', () => { const secondBackend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: planConnection, apiKey: 'ark-plan-token', modelId: 'ark-code-latest', @@ -14043,7 +13848,6 @@ describe('AiSdkBackend thinking persistence', () => { const secondBackend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -14154,7 +13958,6 @@ describe('AiSdkBackend thinking persistence', () => { const secondBackend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -14274,7 +14077,6 @@ describe('AiSdkBackend thinking persistence', () => { const secondBackend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -14584,7 +14386,6 @@ describe('AiSdkBackend thinking persistence', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: { slug: 'openai-main', providerType: 'openai', @@ -14625,7 +14426,6 @@ describe('AiSdkBackend steering durability and identity', () => { createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -15202,7 +15002,6 @@ describe('AiSdkBackend steering durability and identity', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -15299,7 +15098,6 @@ describe('AiSdkBackend steering durability and identity', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -15776,7 +15574,6 @@ describe('AiSdkBackend steering durability and identity', () => { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -15901,7 +15698,6 @@ function imageReplayBackend( return createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -16010,7 +15806,6 @@ async function runPlanToolBoundary(input: { const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -16194,7 +15989,6 @@ async function replayPrompt( const backend = createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -16587,9 +16381,18 @@ function runtimeExecute( eventSink: { push(event: SessionEvent): void }, ) { const runtime = turnScope(backend, turnId).toolRuntime; + // This drives the tool runtime beneath `send()`, so the stream that becomes + // the ledger is teed here instead. + const project = projectedTranscriptOf(backend); const durableEventSink: DurableSessionEventSink = { - push: (event) => eventSink.push(event), - pushAndWaitUntilConsumed: async (event) => eventSink.push(event), + push: (event) => { + eventSink.push(event); + void project?.(event, turnId); + }, + pushAndWaitUntilConsumed: async (event) => { + eventSink.push(event); + await project?.(event, turnId); + }, }; return async ( input: unknown, diff --git a/packages/runtime/src/__tests__/ask-user-question.test.ts b/packages/runtime/src/__tests__/ask-user-question.test.ts index 9eee1664cb..acc11f1976 100644 --- a/packages/runtime/src/__tests__/ask-user-question.test.ts +++ b/packages/runtime/src/__tests__/ask-user-question.test.ts @@ -137,7 +137,6 @@ describe('AskUserQuestion runtime round trip', () => { header: header(), connection: { providerType: 'openai', slug: 'c' } as never, modelId: 'm', - appendMessage: async () => {}, newId: () => `id-${++id}`, now: () => 1, getPermissionPauseTarget: () => null, @@ -191,7 +190,6 @@ describe('AskUserQuestion runtime round trip', () => { header: header(), connection: { providerType: 'openai', slug: 'c' } as never, modelId: 'm', - appendMessage: async () => {}, newId: () => `id-${++id}`, now: () => 1, getPermissionPauseTarget: () => null, diff --git a/packages/runtime/src/__tests__/code-mode-backend.test.ts b/packages/runtime/src/__tests__/code-mode-backend.test.ts index 61b37f8c83..226fbf66d3 100644 --- a/packages/runtime/src/__tests__/code-mode-backend.test.ts +++ b/packages/runtime/src/__tests__/code-mode-backend.test.ts @@ -974,7 +974,6 @@ function backend( return createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', diff --git a/packages/runtime/src/__tests__/computer-use-privacy-boundary.test.ts b/packages/runtime/src/__tests__/computer-use-privacy-boundary.test.ts index ee0904306e..d87630b7ad 100644 --- a/packages/runtime/src/__tests__/computer-use-privacy-boundary.test.ts +++ b/packages/runtime/src/__tests__/computer-use-privacy-boundary.test.ts @@ -354,7 +354,6 @@ test('the model reads its own call back in the names the tool accepts', async () header: header(), connection: connection(), modelId: 'mock-model', - appendMessage: async () => {}, newId: nextId(), now: () => 1, getPermissionPauseTarget: () => null, diff --git a/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts b/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts index fc1e335d1c..73a0cc617d 100644 --- a/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts +++ b/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts @@ -92,7 +92,6 @@ describe('Anthropic-compatible Computer Use product loops', () => { ...header('anthropic', 'claude-sonnet-4-5-20250929'), llmConnectionId: 'connection-anthropic', }, - appendMessage: async () => {}, connection: providerConnection, apiKey: 'test-key', providerStateIdentity: PROVIDER_STATE_IDENTITY, @@ -265,7 +264,6 @@ describe('Anthropic-compatible Computer Use product loops', () => { testProjectionArtifacts: true, sessionId, header: header(provider.providerType, provider.modelId), - appendMessage: async () => {}, connection: providerConnection, apiKey: 'test-key', modelId: provider.modelId, @@ -383,7 +381,6 @@ describe('Anthropic-compatible Computer Use product loops', () => { testProjectionArtifacts: true, sessionId, header: header(provider.providerType, provider.modelId), - appendMessage: async () => {}, connection: connection( provider.providerType, `${server.url}${provider.baseSuffix}`, @@ -465,7 +462,6 @@ describe('OpenAI-compatible product loops', () => { ...header('github-copilot', 'gpt-5.4'), llmConnectionId: 'connection-copilot', }, - appendMessage: async () => {}, connection: providerConnection, apiKey: 'test-key', providerStateIdentity: PROVIDER_STATE_IDENTITY, @@ -607,7 +603,6 @@ describe('OpenAI-compatible product loops', () => { testProjectionArtifacts: true, sessionId, header: header(provider.providerType, provider.modelId), - appendMessage: async () => {}, connection: providerConnection, apiKey: 'test-key', modelId: provider.modelId, @@ -725,7 +720,6 @@ describe('OpenAI-compatible product loops', () => { testProjectionArtifacts: true, sessionId, header: header('kimi-coding-plan', 'k3'), - appendMessage: async () => {}, connection: providerConnection, apiKey: 'test-key', providerStateIdentity: PROVIDER_STATE_IDENTITY, @@ -933,7 +927,6 @@ describe('OpenAI-compatible product loops', () => { testProjectionArtifacts: true, sessionId, header: header('kimi-coding-plan', 'k3'), - appendMessage: async () => {}, connection: providerConnection, apiKey: 'test-key', modelId: 'k3', diff --git a/packages/runtime/src/__tests__/deferred-tools-backend.test.ts b/packages/runtime/src/__tests__/deferred-tools-backend.test.ts index 086f717a27..9902b7d512 100644 --- a/packages/runtime/src/__tests__/deferred-tools-backend.test.ts +++ b/packages/runtime/src/__tests__/deferred-tools-backend.test.ts @@ -85,7 +85,6 @@ function backend(input: { return createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', diff --git a/packages/runtime/src/__tests__/execution-boundary-test-helpers.ts b/packages/runtime/src/__tests__/execution-boundary-test-helpers.ts index b715b10c73..890e2167a0 100644 --- a/packages/runtime/src/__tests__/execution-boundary-test-helpers.ts +++ b/packages/runtime/src/__tests__/execution-boundary-test-helpers.ts @@ -18,8 +18,22 @@ */ import { createExternalExecutionBoundary } from '@maka/core/sandbox-boundary'; +import type { SessionEvent } from '@maka/core/events'; +import type { RuntimeEvent } from '@maka/core/runtime-event'; +import type { StoredMessage } from '@maka/core/session'; +import { DEFAULT_TOOL_MODE } from '@maka/core/tool-mode'; +import { + buildInvocationOpenedEvent, + runtimeInvocationsFromSessionEvents, +} from '@maka/core/runtime-invocation'; import { AiSdkBackend, type AiSdkBackendInput } from '../ai-sdk-backend.js'; +import { + createSessionEventMapMemory, + isLiveBackendSessionEvent, + mapSessionEventToRuntimeEvent, +} from '../session-event-runtime-mapper.js'; +import { projectRuntimeEventsToStoredMessages } from '../runtime-event-read-model.js'; import { createToolResultArchiveCapability, type ToolResultArchiveCapability, @@ -34,10 +48,49 @@ export const readExternalExecutionBoundary: AiSdkBackendInput['readExecutionBoun type TestAiSdkBackendInput = Omit & Partial> & { testProjectionArtifacts?: boolean; + /** + * The transcript this backend's turn produces, row by row as it appears. + * + * The backend writes no transcript: it emits SessionEvents, an AgentRun + * maps them onto the ledger, and the read model projects the ledger back. + * This runs that same path over the stream so a fixture can read the + * transcript rows a turn yields without standing a whole Session up. + */ + appendMessage?: (message: StoredMessage) => Promise; }; +type ProjectedTranscriptSink = (event: SessionEvent, turnId: string) => Promise; + +const projectedTranscripts = new WeakMap(); + +/** + * The transcript sink of a backend built with `appendMessage`, for a fixture + * that drives the backend's tool runtime directly instead of through `send()`. + */ +export function projectedTranscriptOf(backend: AiSdkBackend): ProjectedTranscriptSink | undefined { + return projectedTranscripts.get(backend); +} + +/** Tee one live backend stream into projected transcript rows. */ +function teeProjectedTranscript( + backend: AiSdkBackend, + sessionId: string, + appendMessage: (message: StoredMessage) => Promise, +): AiSdkBackend { + const send = backend.send.bind(backend); + const project = projectedTranscriptSink(sessionId, appendMessage); + projectedTranscripts.set(backend, project); + backend.send = async function* (sendInput) { + for await (const event of send(sendInput) as AsyncIterable) { + yield event; + await project(event, sendInput.turnId); + } + } as AiSdkBackend['send']; + return backend; +} + export function createTestAiSdkBackend(input: TestAiSdkBackendInput): AiSdkBackend { - const { testProjectionArtifacts, ...backendInput } = input; + const { testProjectionArtifacts, appendMessage, ...backendInput } = input; const artifacts = new Map(); let nextArtifactId = 0; // A whole transition ledger by default, for the same reason the archive @@ -45,7 +98,7 @@ export function createTestAiSdkBackend(input: TestAiSdkBackendInput): AiSdkBacke // when it can be made durable, so a fixture without this seam would silently // disable pruning rather than exercise it (#4283). const transitions: ModelProjectionTransition[] = []; - return new AiSdkBackend({ + const backend = new AiSdkBackend({ readExecutionBoundary: readExternalExecutionBoundary, loadModelProjectionTransitions: async () => ({ transitions: [...transitions], @@ -85,6 +138,7 @@ export function createTestAiSdkBackend(input: TestAiSdkBackendInput): AiSdkBacke } : {}), }); + return appendMessage ? teeProjectedTranscript(backend, input.sessionId, appendMessage) : backend; } /** @@ -105,13 +159,96 @@ export function testToolResultArchive( } type TestToolRuntimeInput = Omit & - Partial>; + Partial> & { + /** The transcript rows this runtime's calls produce; see the backend helper. */ + appendMessage?: (message: StoredMessage) => Promise; + }; /** Defaults to the turn id nearly every ToolRuntime test already uses. */ export function createTestToolRuntime(input: TestToolRuntimeInput): ToolRuntime { - return new ToolRuntime({ + const { appendMessage, ...runtimeInput } = input; + const runtime = new ToolRuntime({ readExecutionBoundary: readExternalExecutionBoundary, turnId: 'turn-1', - ...input, + ...runtimeInput, }); + if (!appendMessage) return runtime; + const settleToolCall = runtime.settleToolCall.bind(runtime); + const project = projectedTranscriptSink(input.sessionId, appendMessage); + runtime.settleToolCall = (call) => + settleToolCall({ + ...call, + eventSink: { + push: (event) => { + call.eventSink.push(event); + void project(event, call.turnId); + }, + pushAndWaitUntilConsumed: async (event) => { + await call.eventSink.pushAndWaitUntilConsumed(event); + await project(event, call.turnId); + }, + }, + }); + return runtime; +} + +/** + * A stateful sink turning one live stream into projected transcript rows. + * + * Every row is derived by the production mapper and the production read model, + * so what a fixture observes is what a reader of the ledger would see — not a + * second copy written beside it. + */ +function projectedTranscriptSink( + sessionId: string, + appendMessage: (message: StoredMessage) => Promise, +): (event: SessionEvent, turnId: string) => Promise { + const memory = createSessionEventMapMemory(); + const events: RuntimeEvent[] = []; + let projected = 0; + return async (event, turnId) => { + if (!isLiveBackendSessionEvent(event)) return; + const run = { sessionId, invocationId: turnId, runId: turnId, turnId }; + // Nothing projects without the invocation it belongs to. A fixture drives + // the backend directly, so the opening fact an AgentRun would have + // committed is stated here once, on the run's first event. + if (events.length === 0) { + events.push( + buildInvocationOpenedEvent({ + id: `${turnId}-opened`, + run, + openedAt: event.ts, + opening: { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: { + provenance: 'unknown', + backendKind: 'ai-sdk', + llmConnectionSlug: 'test-connection', + modelId: 'test-model', + }, + configuration: { + cwd: '/', + permissionMode: 'bypass', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: DEFAULT_TOOL_MODE, + }, + root: { kind: 'user' }, + source: { kind: 'fresh' }, + }, + }), + ); + } + events.push(mapSessionEventToRuntimeEvent(event, run, memory)); + // Re-project the whole run: a row can only be completed by a later event + // (a step's thinking pairs with the text row that follows it), so the + // prefix is re-derived and only genuinely new rows are emitted. + const messages = projectRuntimeEventsToStoredMessages(events, { + invocations: runtimeInvocationsFromSessionEvents(sessionId, events), + }).messages; + for (const message of messages.slice(projected)) await appendMessage(message); + projected = messages.length; + }; } diff --git a/packages/runtime/src/__tests__/fake-backend.test.ts b/packages/runtime/src/__tests__/fake-backend.test.ts index c61f3447e5..afe9fac4ad 100644 --- a/packages/runtime/src/__tests__/fake-backend.test.ts +++ b/packages/runtime/src/__tests__/fake-backend.test.ts @@ -52,12 +52,7 @@ test('Fake question publication waits for exact hosted admission', async () => { }, { sessionId: 'session-1', turnId: 'turn-1', runId: 'run-1' }, ); - const backend = new FakeBackend({ - sessionId: 'session-1', - header: { model: 'fake-model' } as SessionHeader, - store: {} as SessionStore, - appendMessage: async () => {}, - }); + const backend = new FakeBackend({ sessionId: 'session-1' }); const iterator = backend .send({ turnId: 'turn-1', @@ -101,12 +96,7 @@ test('Fake question publication waits for exact hosted admission', async () => { }); test('pullSteering drains queued messages at step boundaries as steering events', async () => { - const backend = new FakeBackend({ - sessionId: 'session-1', - header: { model: 'fake-model' } as SessionHeader, - store: {} as SessionStore, - appendMessage: async () => {}, - }); + const backend = new FakeBackend({ sessionId: 'session-1' }); // Queue two steering messages, delivered one per step boundary, then dry up. const pending = [ { id: 'lease-1', messageId: 'message-1', content: { text: 'do X' } }, @@ -138,12 +128,7 @@ test('a batch of leases settles per lease: delivered ones ack, undelivered ones // while suspended at B's yield: A crossed its yield (delivered — the // consumer pulled past it), B did not. Batch settlement would nack both, // redelivering the already-delivered A. - const backend = new FakeBackend({ - sessionId: 'session-1', - header: { model: 'fake-model' } as SessionHeader, - store: {} as SessionStore, - appendMessage: async () => {}, - }); + const backend = new FakeBackend({ sessionId: 'session-1' }); let pulled = false; const acked: string[] = []; const nacked: string[] = []; @@ -183,12 +168,7 @@ test('a lease is acked only after its event is consumed, and nacked when the con // durable ledger, so its delivery boundary is the consumer receiving the // echoed event; acking at pull time marked messages delivered that a // detaching consumer never saw, silently dropping them. - const backend = new FakeBackend({ - sessionId: 'session-1', - header: { model: 'fake-model' } as SessionHeader, - store: {} as SessionStore, - appendMessage: async () => {}, - }); + const backend = new FakeBackend({ sessionId: 'session-1' }); const pending = [{ id: 'lease-1', messageId: 'message-1', content: { text: 'do X' } }]; const acked: string[] = []; const nacked: string[] = []; diff --git a/packages/runtime/src/__tests__/interaction-authority.test.ts b/packages/runtime/src/__tests__/interaction-authority.test.ts index ebca8c44c3..78ce640e16 100644 --- a/packages/runtime/src/__tests__/interaction-authority.test.ts +++ b/packages/runtime/src/__tests__/interaction-authority.test.ts @@ -621,7 +621,6 @@ function toolRuntime( header: header(), connection: { providerType: 'openai', slug: 'c' } as never, modelId: 'm', - appendMessage: async () => {}, newId: () => `runtime-${++id}`, now: () => 1, getPermissionPauseTarget: () => null, diff --git a/packages/runtime/src/__tests__/latest-context-commit.test.ts b/packages/runtime/src/__tests__/latest-context-commit.test.ts index af4149cda5..766af9193b 100644 --- a/packages/runtime/src/__tests__/latest-context-commit.test.ts +++ b/packages/runtime/src/__tests__/latest-context-commit.test.ts @@ -70,7 +70,6 @@ test('a real send seals its observation into SQLite and reconstructs it after re createTestAiSdkBackend({ sessionId: ctx.sessionId, header: ctx.header, - appendMessage: async () => {}, connection: { slug: 'mock-main', providerType: 'anthropic', @@ -208,7 +207,6 @@ test('a turn aborted before dispatch does not create a canonical sent attempt', backend = createTestAiSdkBackend({ sessionId: ctx.sessionId, header: ctx.header, - appendMessage: async () => {}, connection: { slug: 'mock-main', providerType: 'anthropic', diff --git a/packages/runtime/src/__tests__/pre-dispatch-refusal-ledger.test.ts b/packages/runtime/src/__tests__/pre-dispatch-refusal-ledger.test.ts index 5207802f61..1adbc14da7 100644 --- a/packages/runtime/src/__tests__/pre-dispatch-refusal-ledger.test.ts +++ b/packages/runtime/src/__tests__/pre-dispatch-refusal-ledger.test.ts @@ -142,7 +142,6 @@ function runtimeInput(h: LedgerHarness) { runId: RUN_ID, invocationId: INVOCATION_ID, runtimeCommitSink: h.sink, - appendMessage: async () => {}, newId: nextId(), now: () => 1, getPermissionPauseTarget: () => null, diff --git a/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts b/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts index a44272cda5..462c8b9e5a 100644 --- a/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts +++ b/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts @@ -160,16 +160,7 @@ async function runCrashChild(): Promise { const runStore = createSqliteAgentRunStore(workspaceRoot); const runtimeEventStore = createCrashRuntimeStore(workspaceRoot); const backends = new BackendRegistry(); - backends.register( - 'ai-sdk', - (ctx) => - new FakeBackend({ - sessionId: ctx.sessionId, - header: ctx.header, - store: ctx.store, - appendMessage: ctx.appendMessage, - }), - ); + backends.register('ai-sdk', (ctx) => new FakeBackend({ sessionId: ctx.sessionId })); let id = 0; let resolveSelectedFailpoint!: () => void; const selectedFailpointReached = new Promise((resolve) => { @@ -244,16 +235,7 @@ function createManager(workspaceRoot: string): { const runStore = createSqliteAgentRunStore(workspaceRoot); const runtimeEventStore = createCrashRuntimeStore(workspaceRoot); const backends = new BackendRegistry(); - backends.register( - 'ai-sdk', - (ctx) => - new FakeBackend({ - sessionId: ctx.sessionId, - header: ctx.header, - store: ctx.store, - appendMessage: ctx.appendMessage, - }), - ); + backends.register('ai-sdk', (ctx) => new FakeBackend({ sessionId: ctx.sessionId })); let id = 100; return { agentRunStore: runStore, diff --git a/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts b/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts index 4bfb344305..bbe96c8e1d 100644 --- a/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts +++ b/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts @@ -250,16 +250,6 @@ describe('RuntimeKernel Interaction close cleanup', () => { ); assert.equal(containsFailure(retryFailure, stopFailure), true); assert.equal(fixture.backend.stopCalls.length, 1); - const messages = await fixture.store.readMessages(SESSION_ID); - assert.equal( - messages.filter( - (message) => - message.type === 'turn_state' && - message.turnId === 'turn-blocked-send' && - message.status === 'aborted', - ).length, - 1, - ); const blockedActivation = fixture.kernel .startTurn(SESSION_ID, { turnId: 'turn-before-runner-settled', text: 'must not send' }) @@ -361,16 +351,6 @@ describe('RuntimeKernel Interaction close cleanup', () => { await drainIterator(first); assert.equal(built[0]?.disposeCalls, 1); assert.deepEqual(built[0]?.stopCalls, [{ reason: 'user_stop', mode: 'after_step' }]); - const firstMessages = await store.readMessages(SESSION_ID); - assert.equal( - firstMessages.filter( - (message) => - message.type === 'turn_state' && - message.turnId === 'turn-generation-1' && - message.status === 'aborted', - ).length, - 1, - ); const second = kernel .startTurn(SESSION_ID, { turnId: 'turn-generation-2', text: 'second' }) @@ -644,13 +624,6 @@ function memoryStore(): SessionStore { list: async () => [], readHeader: async () => header, readMessages: async () => [...messages], - listTurns: async () => [], - appendMessage: async (_sessionId, message) => { - messages.push(message); - }, - appendMessages: async (_sessionId, next) => { - messages.push(...next); - }, updateHeader: async (_sessionId, patch) => { header = { ...header, ...patch }; return header; diff --git a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts index 908e6f4905..3a0c41ce3c 100644 --- a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts +++ b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts @@ -104,8 +104,6 @@ test('repairs imported transcript turns into provider-neutral canonical history' const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), - appendMessage: (sessionId, message) => sessions.appendMessage(sessionId, message), - newId, now: () => 100, }); @@ -286,8 +284,6 @@ test('an imported snapshot cutoff survives materialization as aborted', async () const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), - appendMessage: (sessionId, message) => sessions.appendMessage(sessionId, message), - newId, now: () => 100, }); @@ -343,8 +339,6 @@ test('does not import Host-handed-off transcript messages as synthetic runs', as const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), - appendMessage: (sessionId, message) => sessions.appendMessage(sessionId, message), - newId: () => `host-repair-${++sequence}`, now: () => 100, }); @@ -395,8 +389,6 @@ test('an imported turn with no terminal state is repaired to failed', async () = const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), - appendMessage: (sessionId, message) => sessions.appendMessage(sessionId, message), - newId, now: () => 100, }); @@ -472,8 +464,6 @@ test("converts Maka's own legacy transcript whole, and resumes an interrupted co const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), - appendMessage: (sessionId, message) => sessions.appendMessage(sessionId, message), - newId: () => 'unused', now: () => 100, }); @@ -645,8 +635,6 @@ test('a resolved Claude transcript replays as the conversation the user kept', a const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), - appendMessage: (sessionId, message) => sessions.appendMessage(sessionId, message), - newId, now: () => 100, }); await repair.materializeTranscriptLedger(session); diff --git a/packages/runtime/src/__tests__/sandbox-boundary-restart-recovery.test.ts b/packages/runtime/src/__tests__/sandbox-boundary-restart-recovery.test.ts index 1ad13b4d54..17f1539f1c 100644 --- a/packages/runtime/src/__tests__/sandbox-boundary-restart-recovery.test.ts +++ b/packages/runtime/src/__tests__/sandbox-boundary-restart-recovery.test.ts @@ -27,6 +27,7 @@ import type { AgentRunEvent, EmittedAgentRunEvent } from '@maka/core/agent-run'; import { buildInvocationOpenedEvent } from '@maka/core/runtime-invocation'; import { runtimeInvocationFailureClass } from '../runtime-event-read-model.js'; import type { CreateSessionInput } from '@maka/core/runtime-inputs'; +import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { SessionHeader, StoredMessage } from '@maka/core/session'; import { type DurableAgentRunStore, @@ -68,9 +69,10 @@ describe('sandbox boundary restart recovery on durable stores', () => { await manager(stores).recoverInterruptedSessions(); }); - await withStores(root, async ({ sessions, runtimeEvents }) => { + await withStores(root, async (stores) => { + const { sessions, runtimeEvents } = stores; assert.deepEqual(await sessions.listPendingSandboxBoundaryRequests(session.id), []); - const [turn] = await sessions.listTurns(session.id); + const [turn] = await manager(stores).listTurns(session.id); assert.equal(turn?.status, 'failed'); assert.equal(turn?.errorClass, 'sandbox_boundary_closed_by_restart'); const [invocation] = await runtimeEvents.listSessionInvocations(session.id); @@ -115,15 +117,15 @@ describe('sandbox boundary restart recovery on durable stores', () => { await manager(stores).recoverInterruptedSessions(); }); - const failedStatesAfterFirst = await withStores(root, async ({ sessions, runtimeEvents }) => { - const [turn] = await sessions.listTurns(session.id); + const failedStatesAfterFirst = await withStores(root, async (stores) => { + const [turn] = await manager(stores).listTurns(session.id); assert.equal(turn?.errorClass, 'sandbox_boundary_closed_by_restart'); - const [invocation] = await runtimeEvents.listSessionInvocations(session.id); + const [invocation] = await stores.runtimeEvents.listSessionInvocations(session.id); assert.equal( invocation && runtimeInvocationFailureClass(invocation), 'sandbox_boundary_closed_by_restart', ); - return countFailedTurnStates(await sessions.readMessages(session.id)); + return countFailedTurnStates(await manager(stores).getMessages(session.id)); }); // A later restart re-reads the same durable closure and must change @@ -132,11 +134,12 @@ describe('sandbox boundary restart recovery on durable stores', () => { await manager(stores).recoverInterruptedSessions(); }); - await withStores(root, async ({ sessions, runtimeEvents }) => { - const [turn] = await sessions.listTurns(session.id); + await withStores(root, async (stores) => { + const { sessions, runtimeEvents } = stores; + const [turn] = await manager(stores).listTurns(session.id); assert.equal(turn?.errorClass, 'sandbox_boundary_closed_by_restart'); assert.equal( - countFailedTurnStates(await sessions.readMessages(session.id)), + countFailedTurnStates(await manager(stores).getMessages(session.id)), failedStatesAfterFirst, ); const [invocation] = await runtimeEvents.listSessionInvocations(session.id); @@ -208,28 +211,37 @@ function manager(stores: DurableStores): SessionManager { }); } +/** + * A turn whose invocation opened and never ended: the opening fact and the + * user's own event are on the ledger, and no terminal fact follows them. + */ async function seedInterruptedTurn( sessions: SessionAuthorityStore, runs: DurableAgentRunStore, runtimeEvents: DurableRuntimeEventStore, sessionId: string, ): Promise { - await sessions.appendMessages(sessionId, [ - { type: 'user', id: 'turn-1-user', turnId: 'turn-1', ts: 9, text: 'build it' }, - { - type: 'turn_state', - id: 'turn-1-state', - turnId: 'turn-1', - ts: 10, - status: 'running', - partialOutputRetained: false, - }, - ]); await sessions.updateHeader(sessionId, { status: 'waiting_for_user' }); await runtimeEvents.appendRuntimeEvent(sessionId, 'run-1', openingEvent(sessionId)); + await runtimeEvents.appendRuntimeEvent(sessionId, 'run-1', userEvent(sessionId)); await runs.appendEvent(sessionId, 'run-1', runEvent(sessionId)); } +function userEvent(sessionId: string): RuntimeEvent { + return { + id: 'run-1-user', + sessionId, + invocationId: 'run-1', + runId: 'run-1', + turnId: 'turn-1', + ts: 11, + partial: false, + role: 'user', + author: 'user', + content: { kind: 'text', text: 'build it' }, + }; +} + function countFailedTurnStates(messages: readonly StoredMessage[]): number { return messages.filter((message) => message.type === 'turn_state' && message.status === 'failed') .length; diff --git a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts index 3c0f693608..f2bc41d4fb 100644 --- a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts +++ b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts @@ -74,7 +74,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runtimeEventStore, newId: nextId(), now: nextNow(10_000), @@ -142,7 +141,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runtimeEventStore, newId: nextId(), now: nextNow(10_100), @@ -486,7 +484,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runStore, runtimeEventStore: runStore, newId: nextId(), @@ -546,7 +543,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runStore, runtimeEventStore: runStore, newId: nextId(), @@ -616,7 +612,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runStore, runtimeEventStore: runStore, newId: nextId(), @@ -688,7 +683,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runStore, runtimeEventStore: runStore, newId: nextId(), @@ -730,7 +724,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runStore, runtimeEventStore: runStore, newId, @@ -767,7 +760,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-2', text: 'again' }, - store, runStore, runtimeEventStore: runStore, newId, @@ -830,7 +822,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runStore, runtimeEventStore: runStore, newId: nextId(), @@ -1102,7 +1093,6 @@ describe('SessionManager terminal ledger invariants', () => { schemaVersion: 1, }, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runStore, newId: nextId(), now: nextNow(25_200), @@ -1113,7 +1103,6 @@ describe('SessionManager terminal ledger invariants', () => { unregisterRun: () => {}, updateHeader: (sessionId, patch) => store.updateHeader(sessionId, patch), updateStatus: async () => {}, - appendTurnState: async () => {}, }, }), /RuntimeEventStore/, @@ -1128,7 +1117,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runStore, runtimeEventStore: runStore, newId: nextId(), @@ -1140,7 +1128,6 @@ describe('SessionManager terminal ledger invariants', () => { unregisterRun: () => {}, updateHeader: (sessionId, patch) => store.updateHeader(sessionId, patch), updateStatus: async () => {}, - appendTurnState: async () => {}, }, }); const terminalEvent = runtimeEvent({ @@ -1180,7 +1167,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runStore, runtimeEventStore: runStore, newId: nextId(), @@ -1192,7 +1178,6 @@ describe('SessionManager terminal ledger invariants', () => { unregisterRun: () => {}, updateHeader: (sessionId, patch) => store.updateHeader(sessionId, patch), updateStatus: async () => {}, - appendTurnState: async () => {}, }, }); @@ -1224,7 +1209,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runStore, runtimeEventStore: runStore, newId: nextId(), @@ -1247,7 +1231,6 @@ describe('SessionManager terminal ledger invariants', () => { }, updateHeader: (sessionId, patch) => store.updateHeader(sessionId, patch), updateStatus: async () => {}, - appendTurnState: async () => {}, }, }); @@ -1286,7 +1269,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runStore, runtimeEventStore: runStore, newId: nextId(), @@ -1309,7 +1291,6 @@ describe('SessionManager terminal ledger invariants', () => { }, updateHeader: (sessionId, patch) => store.updateHeader(sessionId, patch), updateStatus: async () => {}, - appendTurnState: async () => {}, }, }); @@ -1342,7 +1323,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runStore, runtimeEventStore: runStore, newId: nextId(), @@ -1365,7 +1345,6 @@ describe('SessionManager terminal ledger invariants', () => { }, updateHeader: (sessionId, patch) => store.updateHeader(sessionId, patch), updateStatus: async () => {}, - appendTurnState: async () => {}, }, }); await run.begin(); @@ -1405,7 +1384,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runStore, runtimeEventStore: runStore, newId: nextId(), @@ -1428,7 +1406,6 @@ describe('SessionManager terminal ledger invariants', () => { }, updateHeader: (sessionId, patch) => store.updateHeader(sessionId, patch), updateStatus: async () => {}, - appendTurnState: async () => {}, }, }); await run.begin(); @@ -1483,7 +1460,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runStore, runtimeEventStore: runStore, newId: nextId(), @@ -1506,7 +1482,6 @@ describe('SessionManager terminal ledger invariants', () => { }, updateHeader: (sessionId, patch) => store.updateHeader(sessionId, patch), updateStatus: async () => {}, - appendTurnState: async () => {}, }, }); await run.begin(); @@ -1562,7 +1537,6 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: session.id, header: session, userInput: { turnId: 'turn-1', text: 'hello' }, - store, runStore, runtimeEventStore: runStore, newId: nextId(), @@ -1585,7 +1559,6 @@ describe('SessionManager terminal ledger invariants', () => { }, updateHeader: (sessionId, patch) => store.updateHeader(sessionId, patch), updateStatus: async () => {}, - appendTurnState: async () => {}, }, }); await run.begin(); @@ -2580,7 +2553,6 @@ function inertAgentRunHooks(store: TinySessionStore) { updateHeader: (sessionId: string, patch: Partial) => store.updateHeader(sessionId, patch), updateStatus: async () => {}, - appendTurnState: async () => {}, }; } diff --git a/packages/runtime/src/__tests__/session-manager.test.ts b/packages/runtime/src/__tests__/session-manager.test.ts index 78aa131334..9e8776e619 100644 --- a/packages/runtime/src/__tests__/session-manager.test.ts +++ b/packages/runtime/src/__tests__/session-manager.test.ts @@ -1311,7 +1311,7 @@ describe('SessionManager claimed graph intent execution', () => { }); assert.deepStrictEqual((executions[0] as { content?: unknown }).content, { text: prompt }); assert.partialDeepStrictEqual( - (await store.readMessages(child.id)).find( + (await manager.getMessages(child.id)).find( (message) => message.type === 'user' && message.turnId === claim.targetTurnId, ), { id: 'id-1', text: prompt }, @@ -1461,7 +1461,7 @@ describe('SessionManager claimed graph intent execution', () => { assert.strictEqual(result.status, 'completed'); assert.strictEqual(newIdCallsAtExecution, 0); assert.partialDeepStrictEqual( - (await store.readMessages(child.id)).find( + (await manager.getMessages(child.id)).find( (message) => message.type === 'user' && message.turnId === claim.targetTurnId, ), { @@ -1747,7 +1747,7 @@ describe('SessionManager claimed graph intent execution', () => { assert.strictEqual(run.opening.lineage?.parentRunId, undefined); assert.strictEqual(run.turnId, 'graph-turn'); assert.partialDeepStrictEqual( - (await store.readMessages(child.id)).find( + (await manager.getMessages(child.id)).find( (message) => message.type === 'user' && message.turnId === 'graph-turn', ), { text: 'summarize the routed records' }, @@ -2029,7 +2029,7 @@ describe('SessionManager claimed graph intent execution', () => { [firstClaim.targetTurnId], ); assert.deepStrictEqual( - (await store.readMessages(child.id)).filter( + (await manager.getMessages(child.id)).filter( (message) => 'turnId' in message && (message.turnId === secondClaim.targetTurnId || @@ -2394,8 +2394,8 @@ describe('SessionManager child-session runtime primitive', () => { false, ); - const parentMessages = await store.readMessages(parent.id); - const childMessages = await store.readMessages(result.childSessionId); + const parentMessages = await manager.getMessages(parent.id); + const childMessages = await manager.getMessages(result.childSessionId); assert.strictEqual( parentMessages.some( (message) => message.type === 'user' && message.text === 'inspect the storage boundary', @@ -3386,7 +3386,7 @@ describe('SessionManager child-session runtime primitive', () => { assert.strictEqual(runtimeInvocationOutcome(recoveredRun), 'failed'); assert.strictEqual(runtimeInvocationFailureClass(recoveredRun), 'app_restarted'); assert.strictEqual( - (await store.readMessages(child.id)).some( + (await manager.getMessages(child.id)).some( (message) => message.type === 'turn_state' && message.turnId === 'child-turn' && @@ -3522,7 +3522,6 @@ describe('SessionManager manual compaction and quiescent session changes', () => createTestAiSdkBackend({ sessionId: ctx.sessionId, header: ctx.header, - appendMessage: async () => {}, connection: { slug: 'mock-main', providerType: 'anthropic', @@ -4163,13 +4162,11 @@ describe('SessionManager manual compaction and quiescent session changes', () => const turn = drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'start' })); await new Promise((resolve) => setImmediate(resolve)); assert.deepEqual(activatedModels, []); - assert.equal((await store.readHeader(session.id)).transcriptLedgerVersion, undefined); releaseUpdate.release(); await transition; await turn; assert.deepEqual(activatedModels, ['new-model']); - assert.equal((await store.readHeader(session.id)).transcriptLedgerVersion, 1); }); test('backend refresh propagates delayed disposal failure after an active turn settles', async () => { @@ -5213,7 +5210,7 @@ describe('SessionManager permission mode updates', () => { { kind: 'workspace_file', value: '@accepted.ts', label: 'accepted.ts', start: 0 }, ], }); - const storedUserMessage = (await store.readMessages(session.id)).find( + const storedUserMessage = (await manager.getMessages(session.id)).find( (message) => message.type === 'user' && message.turnId === 'turn-snapshot', ); assert.deepStrictEqual( @@ -5259,7 +5256,6 @@ describe('SessionManager permission mode updates', () => { createTestAiSdkBackend({ sessionId: ctx.sessionId, header: ctx.header, - appendMessage: ctx.appendMessage ?? (async () => {}), connection: { slug: 'mock-main', providerType: 'anthropic', @@ -5543,10 +5539,6 @@ describe('SessionManager permission mode updates', () => { continuationEvents.some((event) => event.role === 'user'), false, ); - assert.strictEqual( - (await store.readMessages(session.id)).some((message) => message.type === 'user'), - false, - ); assert.deepStrictEqual( (await runStore.readRuntimeEvents(session.id, sourceRunId)).slice(1), sourceEvents, @@ -5610,7 +5602,6 @@ describe('SessionManager permission mode updates', () => { createTestAiSdkBackend({ sessionId: ctx.sessionId, header: ctx.header, - appendMessage: ctx.appendMessage ?? (async () => {}), connection: { slug: ctx.header.llmConnectionSlug, providerType: 'anthropic', @@ -5933,7 +5924,7 @@ describe('SessionManager permission mode updates', () => { await refresh; assert.strictEqual(store.disposeCount, 1); - const cachedMessages = await store.readMessages(session.id); + const cachedMessages = await manager.getMessages(session.id); assert.partialDeepStrictEqual( cachedMessages .filter( @@ -7291,6 +7282,7 @@ describe('SessionManager permission mode updates', () => { now: nextNow(7_025), }); const session = await manager.createSession(makeInput()); + await store.updateHeader(session.id, { transcriptLedgerVersion: 0 }); await store.appendMessages(session.id, [ { type: 'user', id: 'imported-user-1', turnId: 'turn-1', ts: 101, text: 'First question' }, { @@ -7328,10 +7320,19 @@ describe('SessionManager permission mode updates', () => { }, ]); + // The first conversion dies partway through. A staged import stays staged + // until one whole conversion lands, so the retry is another import — a live + // Turn is refused meanwhile, and the second pass re-derives the same event + // ids and appends only what the interrupted one never wrote. await expectRejects( manager.prepareImportedSessionHistory(session.id), /runtime event append failed/, ); + await expectRejects( + drain(manager.sendMessage(session.id, { turnId: 'turn-early', text: 'Too early' })), + /history is still being prepared/, + ); + await manager.prepareImportedSessionHistory(session.id); await seedRuntimeRun( runStore, makeRunHeader({ @@ -8673,7 +8674,7 @@ describe('SessionManager permission mode updates', () => { await drain(manager.regenerateTurn(session.id, { sourceTurnId: 'source', turnId: 'regen-1' })); - const messages = await store.readMessages(session.id); + const messages = await manager.getMessages(session.id); const regenUser = messages.find( (message) => message.type === 'user' && message.turnId === 'regen-1', ); @@ -8809,7 +8810,7 @@ describe('SessionManager permission mode updates', () => { manager.regenerateTurn(session.id, { sourceTurnId: 'source', turnId: 'regen-aborted' }), ); - const regenUser = (await store.readMessages(session.id)).find( + const regenUser = (await manager.getMessages(session.id)).find( (message) => message.type === 'user' && message.turnId === 'regen-aborted', ); assert.strictEqual( @@ -9440,7 +9441,7 @@ describe('SessionManager permission mode updates', () => { while (!(await turn.next()).done) {} assert.strictEqual(backend?.stopCalls, 1); - const messages = await store.readMessages(session.id); + const messages = await manager.getMessages(session.id); assert.strictEqual( messages.filter( (message) => @@ -9615,9 +9616,19 @@ describe('SessionManager permission mode updates', () => { assert.strictEqual(backend?.sendInputs?.length, 2); }); - test('stopSession retries only unfinished projections', async () => { + test('stopSession retries an unsettled abort without a second backend stop', async () => { const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); + // The stop's own terminal fact fails once. Nothing else records the abort, + // so the retry has to settle the ledger — and must not reach the backend a + // second time to do it. + let failAbortAppend = false; + const runStore = new MemoryAgentRunStore({ + beforeRuntimeEventAppend: (_sessionId, _runId, event) => { + if (!failAbortAppend || event.status !== 'aborted') return; + failAbortAppend = false; + throw new Error('append runtime event failed'); + }, + }); const backends = new BackendRegistry(); const sendGate = makeGate(); let backend: CountingStopBackend | undefined; @@ -9638,17 +9649,16 @@ describe('SessionManager permission mode updates', () => { .sendMessage(session.id, { turnId: 'turn-1', text: 'hello' }) [Symbol.asyncIterator](); await turn.next(); - store.failAfterNextAppendMessage = (message) => - message.type === 'turn_state' && message.status === 'aborted'; + failAbortAppend = true; await expectRejects( manager.stopSession(session.id, { source: 'stop_button' }), - /append message failed/, + /append runtime event failed/, ); await manager.stopSession(session.id, { source: 'stop_button' }); assert.strictEqual(backend?.stopCalls, 1); - const messages = await store.readMessages(session.id); + const messages = await manager.getMessages(session.id); assert.strictEqual( messages.filter( (message) => @@ -9698,7 +9708,7 @@ describe('SessionManager permission mode updates', () => { await manager.stopSession(session.id, { source: 'stop_button' }); assert.strictEqual(backend?.stopCalls, 1); - const messages = await store.readMessages(session.id); + const messages = await manager.getMessages(session.id); assert.strictEqual( messages.filter( (message) => @@ -10190,12 +10200,12 @@ describe('SessionManager permission mode updates', () => { const header = await store.readHeader(session.id); assert.strictEqual(header.status, 'blocked'); assert.strictEqual(header.blockedReason, 'unknown'); - const messages = await store.readMessages(session.id); + const messages = await manager.getMessages(session.id); assert.strictEqual( messages.some((message) => message.type === 'user' && message.turnId === 'turn-1'), true, ); - const turn = (await store.listTurns(session.id)).find( + const turn = (await manager.listTurns(session.id)).find( (candidate) => candidate.turnId === 'turn-1', ); assert.strictEqual(turn?.status, 'failed'); @@ -10507,7 +10517,7 @@ describe('SessionManager permission mode updates', () => { await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' })); - const [turn] = await store.listTurns(session.id); + const [turn] = await manager.listTurns(session.id); assert.strictEqual(turn?.status, 'failed'); assert.strictEqual(turn?.errorClass, 'runtime_error'); const [run] = await runStore.listSessionInvocations(session.id); @@ -10535,7 +10545,7 @@ describe('SessionManager permission mode updates', () => { await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' })); assert.strictEqual((await store.readHeader(session.id)).status, 'active'); - const [turn] = await store.listTurns(session.id); + const [turn] = await manager.listTurns(session.id); assert.strictEqual(turn?.status, 'failed'); assert.strictEqual(turn?.errorClass, 'tool_step_cap_reached'); const [run] = await runStore.listSessionInvocations(session.id); @@ -10553,6 +10563,7 @@ describe('SessionManager permission mode updates', () => { test('does not let a late complete event overwrite a prior turn error', async () => { const store = new MemorySessionStore(); + const runStore = new MemoryAgentRunStore(); const backends = new BackendRegistry(); backends.register( 'ai-sdk', @@ -10562,29 +10573,44 @@ describe('SessionManager permission mode updates', () => { { type: 'complete', stopReason: 'end_turn' }, ]), ); - const manager = new SessionManager({ store, backends, newId: nextId(), now: nextNow(10_500) }); + const manager = new SessionManager({ + store, + runStore, + runtimeEventStore: runStore, + backends, + newId: nextId(), + now: nextNow(10_500), + }); const session = await manager.createSession(makeInput()); await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' })); - const states = (await store.readMessages(session.id)).filter( + const states = (await manager.getMessages(session.id)).filter( (message) => message.type === 'turn_state' && message.turnId === 'turn-1', ); assert.deepStrictEqual( states.map((state) => (state.type === 'turn_state' ? state.status : '')), - ['running', 'failed'], + ['failed'], ); - const [turn] = await store.listTurns(session.id); + const [turn] = await manager.listTurns(session.id); assert.strictEqual(turn?.status, 'failed'); assert.strictEqual(turn?.errorClass, 'tool_failed'); }); test('stopSession records renderer abort source for diagnostics', async () => { const store = new MemorySessionStore(); + const runStore = new MemoryAgentRunStore(); const backends = new BackendRegistry(); const gate = makeGate(); backends.register('ai-sdk', (ctx) => new TestBackend(ctx, gate)); - const manager = new SessionManager({ store, backends, newId: nextId(), now: nextNow(12_500) }); + const manager = new SessionManager({ + store, + runStore, + runtimeEventStore: runStore, + backends, + newId: nextId(), + now: nextNow(12_500), + }); const session = await manager.createSession(makeInput()); const iterator = manager @@ -10593,7 +10619,7 @@ describe('SessionManager permission mode updates', () => { await iterator.next(); await manager.stopSession(session.id, { source: 'stop_button' }); - const [turn] = await store.listTurns(session.id); + const [turn] = await manager.listTurns(session.id); assert.strictEqual(turn?.status, 'aborted'); assert.strictEqual(turn?.abortSource, 'renderer.stop_button'); }); @@ -10683,7 +10709,7 @@ describe('SessionManager permission mode updates', () => { ); const [run] = await runStore.listSessionInvocations(session.id); const runtimeEvents = await runStore.readRuntimeEvents(session.id, run!.runId); - const turnStates = (await store.readMessages(session.id)).filter( + const turnStates = (await manager.getMessages(session.id)).filter( (message) => message.type === 'turn_state' && message.turnId === 'turn-1' && @@ -10736,7 +10762,7 @@ describe('SessionManager permission mode updates', () => { ); const [run] = await runStore.listSessionInvocations(session.id); const runtimeEvents = await runStore.readRuntimeEvents(session.id, run!.runId); - const turnStates = (await store.readMessages(session.id)).filter( + const turnStates = (await manager.getMessages(session.id)).filter( (message) => message.type === 'turn_state' && message.turnId === 'turn-1' && @@ -10791,7 +10817,7 @@ describe('SessionManager permission mode updates', () => { await iterator.next(); assert.strictEqual((await store.readHeader(session.id)).status, 'aborted'); - const [turn] = await store.listTurns(session.id); + const [turn] = await manager.listTurns(session.id); assert.strictEqual(turn?.status, 'aborted'); assert.strictEqual(turn?.abortSource, 'renderer.stop_button'); const [run] = await runStore.listSessionInvocations(session.id); @@ -11127,11 +11153,19 @@ describe('SessionManager permission mode updates', () => { assert.deepStrictEqual(checkpointCoverage, [10]); }); - test('startup recovery marks persisted running turns as failed instead of leaving them stuck', async () => { + test('startup recovery unsticks a legacy transcript Session and its import settles the turns', async () => { const store = new MemorySessionStore(); + const runStore = new MemoryAgentRunStore(); const backends = new BackendRegistry(); backends.register('ai-sdk', (ctx) => new TestBackend(ctx)); - const manager = new SessionManager({ store, backends, newId: nextId(), now: nextNow(12_800) }); + const manager = new SessionManager({ + store, + runStore, + runtimeEventStore: runStore, + backends, + newId: nextId(), + now: nextNow(12_800), + }); const running = await manager.createSession(makeInput({ status: 'running' })); const waiting = await manager.createSession(makeInput({ status: 'waiting_for_user' })); const activeStuck = await manager.createSession(makeInput({ status: 'active' })); @@ -11223,43 +11257,42 @@ describe('SessionManager permission mode updates', () => { }, ]); + // These Sessions predate the ledger: their transcript is all they have. + for (const seeded of [running, waiting, activeStuck, failedThenCompleted, activeDone]) { + store.markPreLedgerSession(seeded.id); + } + + // Recovery owns only what it can still decide without a ledger: a header + // left mid-turn by a crash. Nothing here re-reads the transcript to guess a + // turn's outcome — the import below is the one path that converts it. const recovered = await manager.recoverInterruptedSessions(); - assert.deepStrictEqual(recovered, [ - running.id, - waiting.id, - activeStuck.id, - failedThenCompleted.id, - ]); + assert.deepStrictEqual(recovered, [running.id, waiting.id]); assert.strictEqual((await store.readHeader(running.id)).status, 'active'); assert.strictEqual((await store.readHeader(waiting.id)).status, 'active'); assert.strictEqual((await store.readHeader(activeStuck.id)).status, 'active'); assert.strictEqual((await store.readHeader(failedThenCompleted.id)).status, 'active'); assert.strictEqual((await store.readHeader(activeDone.id)).status, 'active'); - const runningTurn = (await store.listTurns(running.id)).find( - (turn) => turn.turnId === 'running-turn', - ); - const waitingTurn = (await store.listTurns(waiting.id)).find( - (turn) => turn.turnId === 'waiting-turn', - ); - const activeStuckTurn = (await store.listTurns(activeStuck.id)).find( - (turn) => turn.turnId === 'active-stuck-turn', - ); - const failedThenCompletedTurn = (await store.listTurns(failedThenCompleted.id)).find( - (turn) => turn.turnId === 'failed-completed-turn', - ); - const activeTurn = (await store.listTurns(activeDone.id)).find( - (turn) => turn.turnId === 'active-turn', + + const turnOf = async (sessionId: string, turnId: string) => + (await manager.listTurns(sessionId)).find((turn) => turn.turnId === turnId); + // A turn the transcript never recorded an ending for converts to the + // failure it actually was, rather than to an inferred restart class. + for (const [sessionId, turnId] of [ + [running.id, 'running-turn'], + [waiting.id, 'waiting-turn'], + [activeStuck.id, 'active-stuck-turn'], + ] as const) { + const turn = await turnOf(sessionId, turnId); + assert.strictEqual(turn?.status, 'failed'); + assert.strictEqual(turn?.errorClass, 'missing_terminal_event'); + } + // A recorded ending is imported as recorded, last state wins. + assert.strictEqual( + (await turnOf(failedThenCompleted.id, 'failed-completed-turn'))?.status, + 'completed', ); - assert.strictEqual(runningTurn?.status, 'failed'); - assert.strictEqual(runningTurn?.errorClass, 'app_restarted'); - assert.strictEqual(waitingTurn?.status, 'failed'); - assert.strictEqual(waitingTurn?.errorClass, 'app_restarted'); - assert.strictEqual(activeStuckTurn?.status, 'failed'); - assert.strictEqual(activeStuckTurn?.errorClass, 'app_restarted'); - assert.strictEqual(failedThenCompletedTurn?.status, 'failed'); - assert.strictEqual(failedThenCompletedTurn?.errorClass, 'tool_failed'); - assert.strictEqual(activeTurn?.status, 'completed'); + assert.strictEqual((await turnOf(activeDone.id, 'active-turn'))?.status, 'completed'); }); test('startup recovery derives the interrupted outcome sink from the runtime store', async () => { @@ -11442,7 +11475,7 @@ describe('SessionManager permission mode updates', () => { await manager.recoverInterruptedSessions(); assert.strictEqual((await store.readHeader(session.id)).status, 'active'); - const [turn] = await store.listTurns(session.id); + const [turn] = await manager.listTurns(session.id); assert.strictEqual(turn?.status, 'failed'); // This turn owned the pending request, so its failure names the closure // rather than the bare restart. @@ -11516,7 +11549,7 @@ describe('SessionManager permission mode updates', () => { await manager.recoverInterruptedSessions(); - const [turn] = await store.listTurns(session.id); + const [turn] = await manager.listTurns(session.id); assert.strictEqual(turn?.errorClass, 'app_restarted'); }); }); @@ -11582,7 +11615,6 @@ async function steeringDeliverySession( createTestAiSdkBackend({ sessionId: ctx.sessionId, header: ctx.header, - appendMessage: async () => {}, connection: { slug: 'mock-main', providerType: 'anthropic', @@ -12667,6 +12699,15 @@ class MemorySessionStore implements SessionStore { nextReadHeaderGate: { started: Gate; release: Gate } | undefined; nextGraphOperatorProvisionGate: { started: Gate; release: Gate } | undefined; + /** A Session written before the header carried a transcript ledger version. */ + markPreLedgerSession(sessionId: string): void { + const header = this.headers.get(sessionId); + if (!header) throw new Error(`Unknown session ${sessionId}`); + const { transcriptLedgerVersion: _version, ...legacy } = header; + void _version; + this.headers.set(sessionId, legacy); + } + async createSubagent( input: CreateSessionInput, initialBoundary?: ExecutionBoundary, @@ -12766,6 +12807,7 @@ class MemorySessionStore implements SessionStore { permissionMode: input.permissionMode, collaborationMode: input.collaborationMode ?? 'agent', orchestrationMode: input.orchestrationMode ?? 'default', + transcriptLedgerVersion: 1, schemaVersion: 1, }; this.headers.set(header.id, header); diff --git a/packages/runtime/src/__tests__/session-projection-helpers.test.ts b/packages/runtime/src/__tests__/session-projection-helpers.test.ts index cda0243994..def11a42fe 100644 --- a/packages/runtime/src/__tests__/session-projection-helpers.test.ts +++ b/packages/runtime/src/__tests__/session-projection-helpers.test.ts @@ -19,15 +19,12 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; -import type { StoredMessage } from '@maka/core/session'; import { buildStatusPatch, - buildTurnStateMessage, isTerminalRunStatus, normalizeStopSessionSource, statusFromEvent, turnStatusFromEvent, - turnHasRetainedOutput, workHubDirectStopAbortSource, } from '../session-projection-helpers.js'; @@ -64,80 +61,6 @@ describe('session projection helpers', () => { }); }); - test('buildTurnStateMessage preserves lineage and terminal status fields', () => { - assert.deepStrictEqual( - buildTurnStateMessage({ - id: 'state-1', - turnId: 'turn-1', - ts: 100, - status: 'aborted', - lineage: { - parentTurnId: 'parent', - retriedFromTurnId: 'retry-source', - regeneratedFromTurnId: 'regen-source', - branchOfTurnId: 'branch-source', - parentSessionId: 'parent-session', - }, - abortSource: 'renderer.stop_button', - partialOutputRetained: true, - }), - { - type: 'turn_state', - id: 'state-1', - turnId: 'turn-1', - ts: 100, - status: 'aborted', - parentTurnId: 'parent', - retriedFromTurnId: 'retry-source', - regeneratedFromTurnId: 'regen-source', - branchOfTurnId: 'branch-source', - parentSessionId: 'parent-session', - abortedAt: 100, - abortSource: 'renderer.stop_button', - partialOutputRetained: true, - }, - ); - - assert.partialDeepStrictEqual( - buildTurnStateMessage({ - id: 'state-2', - turnId: 'turn-2', - ts: 101, - status: 'failed', - partialOutputRetained: false, - }), - { - type: 'turn_state', - id: 'state-2', - turnId: 'turn-2', - ts: 101, - status: 'failed', - errorClass: 'unknown', - partialOutputRetained: false, - }, - ); - }); - - test('turnHasRetainedOutput only treats visible assistant text and tool results as retained output', () => { - const messages: StoredMessage[] = [ - { type: 'assistant', id: 'blank', turnId: 'turn-1', ts: 1, text: ' ', modelId: 'model' }, - { type: 'assistant', id: 'other', turnId: 'turn-2', ts: 2, text: 'kept', modelId: 'model' }, - { - type: 'tool_result', - id: 'tool', - turnId: 'turn-3', - ts: 3, - toolUseId: 'call-1', - isError: false, - content: { kind: 'text', text: 'ok' }, - }, - ]; - - assert.strictEqual(turnHasRetainedOutput(messages, 'turn-1'), false); - assert.strictEqual(turnHasRetainedOutput(messages, 'turn-2'), true); - assert.strictEqual(turnHasRetainedOutput(messages, 'turn-3'), true); - }); - test('projects terminal run statuses and session terminal events', () => { assert.strictEqual(isTerminalRunStatus('completed'), true); assert.strictEqual(isTerminalRunStatus('failed'), true); diff --git a/packages/runtime/src/__tests__/shell-run-tool-result.test.ts b/packages/runtime/src/__tests__/shell-run-tool-result.test.ts index fb43b07e9a..8f01f16def 100644 --- a/packages/runtime/src/__tests__/shell-run-tool-result.test.ts +++ b/packages/runtime/src/__tests__/shell-run-tool-result.test.ts @@ -154,7 +154,7 @@ describe('shell run sandbox denial projection', () => { content, }); - const messages = await store.readMessagesForRecovery(session.id); + const messages = await store.readMessages(session.id); const result = messages.find((message) => message.id === 'tool-result-1'); assert.deepEqual(result?.type === 'tool_result' ? result.content : undefined, content); } finally { diff --git a/packages/runtime/src/__tests__/subagent-tools.test.ts b/packages/runtime/src/__tests__/subagent-tools.test.ts index 3eb0cdaa18..0a26942d44 100644 --- a/packages/runtime/src/__tests__/subagent-tools.test.ts +++ b/packages/runtime/src/__tests__/subagent-tools.test.ts @@ -1026,7 +1026,6 @@ function makeChildToolRuntime(cwd: string): ToolRuntime { header: childHeader(cwd), connection: testConnection(), modelId: 'mock-model', - appendMessage: async () => {}, newId: nextId(), now: () => 1, getPermissionPauseTarget: () => null, diff --git a/packages/runtime/src/__tests__/tool-args-violation.test.ts b/packages/runtime/src/__tests__/tool-args-violation.test.ts index efbbf44773..3b140e34a7 100644 --- a/packages/runtime/src/__tests__/tool-args-violation.test.ts +++ b/packages/runtime/src/__tests__/tool-args-violation.test.ts @@ -238,7 +238,6 @@ test('ToolRuntime validates without rewriting arguments at permission and implem header: header(), connection: connection(), modelId: 'mock-model', - appendMessage: async () => {}, newId: nextId(), now: () => 1, getPermissionPauseTarget: () => null, diff --git a/packages/runtime/src/__tests__/tool-artifacts.test.ts b/packages/runtime/src/__tests__/tool-artifacts.test.ts index 3f5056819b..4a8828f8c3 100644 --- a/packages/runtime/src/__tests__/tool-artifacts.test.ts +++ b/packages/runtime/src/__tests__/tool-artifacts.test.ts @@ -173,7 +173,6 @@ function makeToolRuntime(overrides: Partial = {}): { header: testHeader(), connection: testConnection(), modelId: 'mock-model', - appendMessage: async () => {}, newId: nextId(), now: () => 1, getPermissionPauseTarget: () => null, diff --git a/packages/runtime/src/__tests__/tool-result-archive-capability-backend.test.ts b/packages/runtime/src/__tests__/tool-result-archive-capability-backend.test.ts index 659c854215..3dd86309b0 100644 --- a/packages/runtime/src/__tests__/tool-result-archive-capability-backend.test.ts +++ b/packages/runtime/src/__tests__/tool-result-archive-capability-backend.test.ts @@ -236,7 +236,6 @@ function backendWith( return createTestAiSdkBackend({ sessionId: 'session-1', header: header(), - appendMessage: async () => {}, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', diff --git a/packages/runtime/src/__tests__/tool-runtime-argument-ownership.test.ts b/packages/runtime/src/__tests__/tool-runtime-argument-ownership.test.ts index b421bf365b..bf3ff78783 100644 --- a/packages/runtime/src/__tests__/tool-runtime-argument-ownership.test.ts +++ b/packages/runtime/src/__tests__/tool-runtime-argument-ownership.test.ts @@ -49,10 +49,6 @@ describe('ToolRuntime argument ownership', () => { header: testHeader(), connection: testConnection(), modelId: 'test-model', - appendMessage: async (message) => { - if (message.type !== 'tool_call') return; - observeAndMutate(observed, 'storage', message.args); - }, newId: nextId(), now: () => 1, getPermissionPauseTarget: () => null, @@ -90,7 +86,7 @@ describe('ToolRuntime argument ownership', () => { }); mutateArgs(providerArgs, 'provider'); - const owners = ['storage', 'event', 'implementation', 'artifact']; + const owners = ['event', 'implementation', 'artifact']; for (const owner of owners) { assert.deepEqual(observed.get(owner), initialArgs); } diff --git a/packages/runtime/src/__tests__/tool-runtime-durable-boundary.test.ts b/packages/runtime/src/__tests__/tool-runtime-durable-boundary.test.ts index 2c68e24c32..aef99fbb26 100644 --- a/packages/runtime/src/__tests__/tool-runtime-durable-boundary.test.ts +++ b/packages/runtime/src/__tests__/tool-runtime-durable-boundary.test.ts @@ -35,7 +35,6 @@ import { ToolRuntime, type MakaTool, type RuntimeManagedMutationAdmission, - type ToolRuntimeInput, } from '../tool-runtime.js'; describe('ToolRuntime durable boundary', () => { @@ -1726,7 +1725,7 @@ function makeHarness( sink: RuntimeCommitSink, order?: string[], runId: string | null = 'run-1', - overrides: Partial = {}, + overrides: Partial[0]> = {}, ) { const messages: StoredMessage[] = []; const events: SessionEvent[] = []; diff --git a/packages/runtime/src/__tests__/tool-runtime-form-interaction.test.ts b/packages/runtime/src/__tests__/tool-runtime-form-interaction.test.ts index 8959f36e04..0bb97be2fe 100644 --- a/packages/runtime/src/__tests__/tool-runtime-form-interaction.test.ts +++ b/packages/runtime/src/__tests__/tool-runtime-form-interaction.test.ts @@ -96,7 +96,6 @@ function runtime(events: SessionEvent[]) { header: header(), connection: { providerType: 'openai', slug: 'c' } as never, modelId: 'm', - appendMessage: async () => {}, newId: () => `id-${++id}`, now: () => 1, getPermissionPauseTarget: () => null, @@ -212,7 +211,6 @@ describe('ToolRuntime form Interaction', () => { header: header(), connection: { providerType: 'openai', slug: 'c' } as never, modelId: 'm', - appendMessage: async () => {}, newId: (() => { let id = 0; return () => `id-${++id}`; @@ -270,7 +268,6 @@ describe('ToolRuntime form Interaction', () => { header: header(), connection: { providerType: 'openai', slug: 'c' } as never, modelId: 'm', - appendMessage: async () => {}, newId: (() => { let id = 0; return () => `id-${++id}`; diff --git a/packages/runtime/src/__tests__/tool-runtime-progress.test.ts b/packages/runtime/src/__tests__/tool-runtime-progress.test.ts index 142e7f561d..0e77d22ecd 100644 --- a/packages/runtime/src/__tests__/tool-runtime-progress.test.ts +++ b/packages/runtime/src/__tests__/tool-runtime-progress.test.ts @@ -34,7 +34,6 @@ test('ToolRuntime emits only valid progress through the shared codec', async () header: testHeader(), connection: testConnection(), modelId: 'test-model', - appendMessage: async () => {}, newId: nextId(), now: () => 1, getPermissionPauseTarget: () => null, diff --git a/packages/runtime/src/__tests__/tool-runtime-sandbox-boundary.test.ts b/packages/runtime/src/__tests__/tool-runtime-sandbox-boundary.test.ts index e4a71162ac..4b546b7c21 100644 --- a/packages/runtime/src/__tests__/tool-runtime-sandbox-boundary.test.ts +++ b/packages/runtime/src/__tests__/tool-runtime-sandbox-boundary.test.ts @@ -55,7 +55,6 @@ describe('ToolRuntime session sandbox boundary', () => { header: header(), connection: { providerType: 'openai', slug: 'test' } as never, modelId: 'test', - appendMessage: async () => {}, newId: nextId(), now: () => 1, getPermissionPauseTarget: () => null, @@ -73,7 +72,6 @@ describe('ToolRuntime session sandbox boundary', () => { header: header(), connection: { providerType: 'openai', slug: 'test' } as never, modelId: 'test', - appendMessage: async () => {}, readExecutionBoundary: async () => ({ kind: 'managed', profile: createWorkspaceWritePermissionProfile(), @@ -117,7 +115,6 @@ describe('ToolRuntime session sandbox boundary', () => { header: header(), connection: { providerType: 'openai', slug: 'test' } as never, modelId: 'test', - appendMessage: async () => {}, readExecutionBoundary: async () => managed, createSandboxBoundaryRequest: async (input) => { created = { @@ -239,7 +236,6 @@ describe('ToolRuntime session sandbox boundary', () => { header: header(), connection: { providerType: 'openai', slug: 'test' } as never, modelId: 'test', - appendMessage: async () => {}, readExecutionBoundary: async () => managed, newId: nextId(), now: () => 1, @@ -321,7 +317,6 @@ describe('ToolRuntime session sandbox boundary', () => { header: header(), connection: { providerType: 'openai', slug: 'test' } as never, modelId: 'test', - appendMessage: async () => {}, readExecutionBoundary: async () => ({ kind: 'managed', profile: createWorkspaceWritePermissionProfile(), @@ -379,7 +374,6 @@ describe('ToolRuntime session sandbox boundary', () => { header: header(root), connection: { providerType: 'openai', slug: 'test' } as never, modelId: 'test', - appendMessage: async () => {}, readExecutionBoundary: async () => ({ kind: 'managed', profile: createWorkspaceWritePermissionProfile(), @@ -458,7 +452,6 @@ describe('ToolRuntime session sandbox boundary', () => { header: header(root), connection: { providerType: 'openai', slug: 'test' } as never, modelId: 'test', - appendMessage: async () => {}, readExecutionBoundary: async () => ({ kind: 'managed', profile: createWorkspaceWritePermissionProfile(), @@ -518,7 +511,6 @@ describe('ToolRuntime session sandbox boundary', () => { header: header(), connection: { providerType: 'openai', slug: 'test' } as never, modelId: 'test', - appendMessage: async () => {}, readExecutionBoundary: async () => managed, createSandboxBoundaryRequest: async (input) => { created = { @@ -594,7 +586,6 @@ describe('ToolRuntime session sandbox boundary', () => { header: header(), connection: { providerType: 'openai', slug: 'test' } as never, modelId: 'test', - appendMessage: async () => {}, readExecutionBoundary: async () => managed, createSandboxBoundaryRequest: async () => { markCreateStarted(); @@ -659,7 +650,6 @@ describe('ToolRuntime session sandbox boundary', () => { header: header(), connection: { providerType: 'openai', slug: 'test' } as never, modelId: 'test', - appendMessage: async () => {}, readExecutionBoundary: async () => ({ kind: 'managed', profile: createWorkspaceWritePermissionProfile(), @@ -722,7 +712,6 @@ describe('ToolRuntime session sandbox boundary', () => { header: header(), connection: { providerType: 'openai', slug: 'test' } as never, modelId: 'test', - appendMessage: async () => {}, readExecutionBoundary: async () => ({ kind: 'managed', profile: createWorkspaceWritePermissionProfile(), @@ -796,7 +785,6 @@ describe('ToolRuntime session sandbox boundary', () => { header: header(), connection: { providerType: 'openai', slug: 'test' } as never, modelId: 'test', - appendMessage: async () => {}, readExecutionBoundary: async () => ({ kind: 'managed', profile: createWorkspaceWritePermissionProfile(), @@ -877,7 +865,6 @@ describe('ToolRuntime session sandbox boundary', () => { header: header(), connection: { providerType: 'openai', slug: 'test' } as never, modelId: 'test', - appendMessage: async () => {}, readExecutionBoundary: async () => ({ kind: 'managed', profile: createWorkspaceWritePermissionProfile(), @@ -933,7 +920,6 @@ describe('ToolRuntime session sandbox boundary', () => { header: header(), connection: { providerType: 'openai', slug: 'test' } as never, modelId: 'test', - appendMessage: async () => {}, readExecutionBoundary: async () => ({ kind: 'managed', profile: createWorkspaceWritePermissionProfile(), @@ -1002,7 +988,6 @@ describe('ToolRuntime session sandbox boundary', () => { header: header(), connection: { providerType: 'openai', slug: 'test' } as never, modelId: 'test', - appendMessage: async () => {}, readExecutionBoundary: async () => ({ kind: 'managed', profile: createWorkspaceWritePermissionProfile(), diff --git a/packages/runtime/src/__tests__/tool-runtime-settlement.test.ts b/packages/runtime/src/__tests__/tool-runtime-settlement.test.ts index 2540f554b2..c6d9c4be40 100644 --- a/packages/runtime/src/__tests__/tool-runtime-settlement.test.ts +++ b/packages/runtime/src/__tests__/tool-runtime-settlement.test.ts @@ -654,7 +654,6 @@ function makeRuntime( header: header(), connection: connection(), modelId: 'model-1', - appendMessage: async () => {}, newId: nextId(), now: () => 1, getPermissionPauseTarget: () => null, diff --git a/packages/runtime/src/__tests__/tool-runtime-sqlite-boundary.test.ts b/packages/runtime/src/__tests__/tool-runtime-sqlite-boundary.test.ts index 6146bae39b..1998048d03 100644 --- a/packages/runtime/src/__tests__/tool-runtime-sqlite-boundary.test.ts +++ b/packages/runtime/src/__tests__/tool-runtime-sqlite-boundary.test.ts @@ -81,7 +81,6 @@ describe('ToolRuntime with real SQLite boundary', () => { header: header(), connection: connection(), modelId: 'model-1', - appendMessage: async () => {}, newId: nextId(), now: nextNow(), getPermissionPauseTarget: () => null, @@ -162,7 +161,6 @@ describe('ToolRuntime with real SQLite boundary', () => { header: header(), connection: connection(), modelId: 'model-1', - appendMessage: async () => {}, readExecutionBoundary: async () => createGenesisExecutionBoundary('ask'), newId: nextId(), now: nextNow(), @@ -238,7 +236,6 @@ describe('ToolRuntime with real SQLite boundary', () => { header: header(), connection: connection(), modelId: 'model-1', - appendMessage: async () => {}, newId: nextId(), now: nextNow(), getPermissionPauseTarget: () => null, @@ -337,7 +334,6 @@ describe('ToolRuntime with real SQLite boundary', () => { header: header(), connection: connection(), modelId: 'model-1', - appendMessage: async () => {}, newId: nextId(), now: nextNow(), getPermissionPauseTarget: () => null, @@ -426,7 +422,6 @@ describe('ToolRuntime with real SQLite boundary', () => { header: header(), connection: connection(), modelId: 'model-1', - appendMessage: async () => {}, newId: nextId(), now: nextNow(), getPermissionPauseTarget: () => null, @@ -634,7 +629,6 @@ describe('ToolRuntime with real SQLite boundary', () => { header: header(), connection: connection(), modelId: 'model-1', - appendMessage: async () => {}, newId: nextId(), now: nextNow(), getPermissionPauseTarget: () => null, diff --git a/packages/runtime/src/agent-run.ts b/packages/runtime/src/agent-run.ts index b63db06080..7adc8ea191 100644 --- a/packages/runtime/src/agent-run.ts +++ b/packages/runtime/src/agent-run.ts @@ -60,16 +60,15 @@ import type { SessionHeaderPatch, SessionStatus, RuntimeSystemNoteKind, - StoredMessage, - TurnRecord, UserMessage, + AssistantMessage, } from '@maka/core/session'; import type { UserMessageInput } from '@maka/core/runtime-inputs'; import { resolveEffectiveOrchestration, type EffectiveOrchestration, } from '@maka/core/orchestration'; -import { messageContentsEqual, type SessionEvent } from '@maka/core/events'; +import type { SessionEvent } from '@maka/core/events'; import type { AgentBackend, BackendSendInput } from '@maka/core/backend-types'; import type { RunTraceEvent } from './run-trace.js'; import type { StopSessionInput } from './session-manager.js'; @@ -92,8 +91,8 @@ import { type RuntimeContinuationStartAdmissionProof, } from './runtime-continuation-admission.js'; import { DEFAULT_TOOL_MODE, isToolMode, type ToolMode } from '@maka/core/tool-mode'; -import { materializeRuntimeEventTranscriptProjection } from './runtime-ledger-repair.js'; import { cloneAndFreezeRuntimeSnapshot } from './runtime-snapshot.js'; +import { projectRuntimeEventUserMessage } from './runtime-event-read-model.js'; export interface AgentRunActiveSession { sessionId: string; @@ -117,12 +116,15 @@ export interface AgentRunHooks { blockedReason?: SessionBlockedReason, ts?: number, ): Promise; - appendTurnState( + /** + * The catalog facts a durable message carries — its time, the Session list's + * preview line, and the connection lock a Session takes on its first user + * message. The transcript write used to commit these on its way to disk; the + * ledger is not that store, so the run commits them here instead. + */ + commitMessageProjection?( sessionId: string, - turnId: string, - status: TurnRecord['status'], - lineage?: AgentRunLineage, - options?: { ts?: number; errorClass?: string; abortSource?: string }, + message: UserMessage | AssistantMessage, ): Promise; } @@ -150,7 +152,6 @@ export interface AgentRunInput { runId?: string; userMessageId?: string | null; durability?: AgentRunDurability; - store: AgentRunSessionStore; runStore?: AgentRunStore; runtimeEventStore?: RuntimeEventStore; newId: () => string; @@ -173,11 +174,6 @@ export interface AgentRunInput { toolBoundaryProtocol?: ToolBoundaryProtocol; } -export interface AgentRunSessionStore { - appendMessage(sessionId: string, message: StoredMessage): Promise; - readMessages(sessionId: string): Promise; -} - export type RuntimeContinuationFailpoint = | 'after_continuation_claim_committed' | 'after_continuation_start_committed' @@ -237,6 +233,7 @@ export class AgentRun { private runStoreAvailable = true; private runtimeEventStoreAvailable = true; private runtimeEventStoreFailure: unknown; + private lastAssistantPreview: AssistantMessage | undefined; private runtimePartialStreamKey: string | undefined; private runtimePartialBuffer: RuntimeEvent[] = []; private runtimePartialBufferBytes = 0; @@ -617,8 +614,10 @@ export class AgentRun { requireTerminalWrite: options.requireTerminalWrite ?? Boolean(this.input.runtimeEventStore), }); await this.recordSessionEvent(sessionEvent, options); + await this.commitMessageProjection(this.lastAssistantPreview); return; } + this.rememberAssistantPreview(runtimeEvent); if (this.requiresDurablePersistence() && isInteractionResumeAck(sessionEvent)) { // A hosted continuation may resume execution only after its identity-only // settlement fact is durable. Session status advances next, and the queue @@ -645,50 +644,59 @@ export class AgentRun { const steering = runtimeEvent.content?.kind === 'text' && runtimeEvent.content.steering === true; await this.recordRuntimeEvents([runtimeEvent], steering ? { requireDurableWrite: true } : {}); - - await materializeRuntimeEventTranscriptProjection( - this.input.store, - this.sessionId, - runtimeEvent, - ); + if (steering) { + await this.commitMessageProjection( + projectRuntimeEventUserMessage(runtimeEvent, runtimeEvent.id), + ); + } } } + /** + * A user message is fail-CLOSED: it also takes the Session's connection lock, + * and no other path re-derives that latch now that the transcript is not a + * second authority. An assistant preview is fail-open — losing it costs a + * stale sidebar entry, never the turn. + */ + private async commitMessageProjection( + message: UserMessage | AssistantMessage | undefined, + ): Promise { + const commit = this.input.hooks.commitMessageProjection; + if (!commit || !message) return; + const committed = commit.call(this.input.hooks, this.sessionId, message); + if (message.type === 'user') return committed; + await committed.catch(() => {}); + } + + /** + * The assistant text the Session list shows once the Turn ends. + * + * Kept as the run goes so the catalog costs one write per Turn rather than + * one per streamed step, and read only after the terminal fact is durable — + * a Turn that never spoke leaves the previous preview standing. + */ + private rememberAssistantPreview(event: RuntimeEvent): void { + if (event.role !== 'model' || event.content?.kind !== 'text') return; + if (!event.content.text?.trim()) return; + this.lastAssistantPreview = { + type: 'assistant', + id: event.id, + turnId: event.turnId, + ts: event.ts, + text: event.content.text, + modelId: this.header.model, + }; + } + async begin(): Promise { await this.openInvocation(); let initialRuntimeEventId: string; const userMessageTs = this.input.now(); - if (this.input.userMessageId === null) { - initialRuntimeEventId = this.input.newId(); - } else { - const userMessageId = this.input.userMessageId ?? this.input.newId(); - initialRuntimeEventId = userMessageId; - const userMsg = cloneAndFreezeRuntimeSnapshot({ - type: 'user', - id: userMessageId, - turnId: this.turnId, - ts: userMessageTs, - text: this.input.userInput.text, - ...(this.input.userInput.displayText !== undefined - ? { displayText: this.input.userInput.displayText } - : {}), - ...(this.input.userInput.attachments - ? { attachments: this.input.userInput.attachments } - : {}), - ...(this.input.userInput.directoryReferences - ? { directoryReferences: this.input.userInput.directoryReferences } - : {}), - ...(this.input.userInput.quotes ? { quotes: this.input.userInput.quotes } : {}), - ...(this.input.userInput.inlineReferences - ? { inlineReferences: this.input.userInput.inlineReferences } - : {}), - ...(this.input.userInput.origin ? { origin: this.input.userInput.origin } : {}), - }); - await appendUserMessageOnce(this.input.store, this.sessionId, userMsg); - } - await this.input.hooks.appendTurnState(this.sessionId, this.turnId, 'running', this.lineage); + // The caller's durable message id becomes the initial event's id, so a + // same-id append with different content is refused by the store. + initialRuntimeEventId = this.input.userMessageId ?? this.input.newId(); this.lastTs = userMessageTs; const initialRuntimeEvent = cloneAndFreezeRuntimeSnapshot( @@ -697,6 +705,9 @@ export class AgentRun { await this.recordRuntimeEvents([initialRuntimeEvent], { requireDurableWrite: this.requiresDurablePersistence(), }); + await this.commitMessageProjection( + projectRuntimeEventUserMessage(initialRuntimeEvent, initialRuntimeEvent.id), + ); this.active = await this.input.hooks.reserveRun(this.sessionId, this.header, this); @@ -738,10 +749,6 @@ export class AgentRun { const startedAt = this.input.now(); this.lastTs = startedAt; - await this.input.hooks.appendTurnState(this.sessionId, this.turnId, 'running', this.lineage, { - ts: startedAt, - }); - this.active = await this.input.hooks.reserveRun(this.sessionId, this.header, this); await this.input.hooks.updateStatus(this.sessionId, 'running', undefined, startedAt); @@ -781,10 +788,6 @@ export class AgentRun { } await this.input.continuationFailpoint?.('after_continuation_start_committed'); - await this.input.hooks.appendTurnState(this.sessionId, this.turnId, 'running', this.lineage, { - ts: startedAt, - }); - this.active = await this.input.hooks.reserveRun(this.sessionId, this.header, this); await this.input.hooks.updateStatus(this.sessionId, 'running', undefined, startedAt); @@ -916,42 +919,12 @@ export class AgentRun { }; await updateSessionStatus(); } - if (turnStatus && !this.stopped) { - const appendTurnState = this.input.hooks.appendTurnState( - this.sessionId, - this.turnId, - turnStatus.status, - this.lineage, - { - ts: ev.ts, - errorClass: turnStatus.errorClass, - ...(turnStatus.status === 'aborted' && this.abortSource - ? { abortSource: this.abortSource } - : {}), - }, - ); - if (terminalSessionEvent || ev.type === 'error') { - await appendTurnState.catch((error) => - this.enqueueTraceWriteFailure(error, 'terminal session projection'), - ); - } else { - await appendTurnState; - } - } if (ev.type === 'error') { if (this.stopped) { this.finalStatus = { status: 'aborted' }; } else { this.turnFailed = true; this.finalStatus = transition ?? { status: 'blocked', blockedReason: 'unknown' }; - - await this.input.hooks - .appendTurnState(this.sessionId, this.turnId, 'failed', this.lineage, { - ts: ev.ts, - errorClass: ev.reason ?? ev.code ?? 'unknown', - }) - .catch((error) => this.enqueueTraceWriteFailure(error, 'terminal session projection')); - this.markRunFailed(ev.reason ?? ev.code ?? 'unknown', ev.message); } } @@ -1060,13 +1033,6 @@ export class AgentRun { return; } this.finalStatus = { status: 'blocked', blockedReason: 'unknown' }; - - await this.input.hooks - .appendTurnState(this.sessionId, this.turnId, 'failed', this.lineage, { - errorClass: error instanceof Error ? error.name : 'unknown', - }) - .catch(() => {}); - this.markRunFailed(error instanceof Error ? error.name : 'unknown', errorMessage(error)); } @@ -1723,27 +1689,6 @@ function errorMessage(error: unknown): string { return redactTraceString(error instanceof Error ? error.message : String(error)); } -async function appendUserMessageOnce( - store: AgentRunSessionStore, - sessionId: string, - message: UserMessage, -): Promise { - const existing = (await store.readMessages(sessionId)).find( - (candidate) => candidate.id === message.id, - ); - if (!existing) { - await store.appendMessage(sessionId, message); - return; - } - if ( - existing.type !== 'user' || - existing.turnId !== message.turnId || - !messageContentsEqual(existing, message) - ) { - throw new Error(`Durable UserMessage identity ${message.id} has conflicting content`); - } -} - function isInteractionResumeAck(event: SessionEvent): boolean { return ( event.type === 'sandbox_boundary_decision_ack' || diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index da0cff59f7..6cb58ad38d 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -104,7 +104,6 @@ export type { } from '@maka/core/backend-types'; export { INVALID_TOOL_NAME, repairMakaToolCall } from './ai-sdk-tool-repair.js'; -export type AppendMessageFn = (m: StoredMessage) => Promise; export type ToolTelemetryRecorder = (record: ToolInvocationRecord) => void; export type { HistoryCompactCheckpointLoader, @@ -119,8 +118,6 @@ export interface AiSdkBackendInput extends AiSdkCompactionCapabilities { header: SessionHeader; /** Host-frozen provider endpoint and credential ownership for this backend generation. */ providerStateIdentity?: `sha256:${string}`; - /** Append-message function bound to this session (e.g. SessionStore wrapper). */ - appendMessage: AppendMessageFn; /** Reads the authoritative session boundary immediately before every local tool invocation. */ readExecutionBoundary: ToolRuntimeInput['readExecutionBoundary']; createSandboxBoundaryRequest?: ToolRuntimeInput['createSandboxBoundaryRequest']; @@ -454,7 +451,6 @@ export class AiSdkBackend implements AgentBackend { header: input.header, connection: input.connection, modelId: input.modelId, - appendMessage: input.appendMessage, readExecutionBoundary: input.readExecutionBoundary, createSandboxBoundaryRequest: input.createSandboxBoundaryRequest, settleSandboxBoundaryRequest: input.settleSandboxBoundaryRequest, diff --git a/packages/runtime/src/ai-sdk-turn.ts b/packages/runtime/src/ai-sdk-turn.ts index 57ddbe3d65..f745e3f7e1 100644 --- a/packages/runtime/src/ai-sdk-turn.ts +++ b/packages/runtime/src/ai-sdk-turn.ts @@ -45,7 +45,6 @@ import type { AssistantThinkingPart, RuntimeSystemNoteKind, SessionHeader, - TokenUsageMessage, } from '@maka/core/session'; import type { BackendSendInput } from '@maka/core/backend-types'; import type { RuntimeEvent } from '@maka/core/runtime-event'; @@ -103,7 +102,6 @@ import { type RepairableAiSdkToolCall, } from './model-adapter.js'; import { persistedOpenAiResponsesStepMessages } from './openai-responses-continuation.js'; -import { nonCanonicalContentOrder } from './runtime-event-read-model.js'; import { composeRequestProjection, type DispatchRequestShape, @@ -923,36 +921,6 @@ export class AiSdkTurn { return; } const stepId = currentStepMessageId; - const thinkingText = stepThinkingParts.map((part) => part.text).join(''); - const contentOrder = nonCanonicalContentOrder(stepContentOrder); - const msg: AssistantMessage = { - type: 'assistant', - id: stepId, - turnId, - ts: this.deps.now(), - text: stepText, - ...(stepTextProviderOptions !== undefined - ? { providerOptions: stepTextProviderOptions } - : {}), - ...(contentOrder ? { contentOrder } : {}), - modelId: this.deps.backend.modelId, - ...(hasThinking - ? { - thinking: { - text: thinkingText, - ...(stepThinkingParts.length === 1 && stepThinkingParts[0]!.signature !== undefined - ? { signature: stepThinkingParts[0]!.signature } - : {}), - ...(stepThinkingParts.length === 1 && - stepThinkingParts[0]!.providerOptions !== undefined - ? { providerOptions: stepThinkingParts[0]!.providerOptions } - : {}), - ...(stepThinkingParts.length > 1 ? { parts: stepThinkingParts } : {}), - }, - } - : {}), - }; - await this.deps.backend.appendMessage(msg); if (hasThinking) { for (const part of stepThinkingParts) { queue.push({ @@ -2503,14 +2471,6 @@ export class AiSdkTurn { } : {}), }; - const tu: TokenUsageMessage = { - type: 'token_usage', - id: this.deps.newId(), - turnId, - ts: this.deps.now(), - ...usageFields, - }; - await this.deps.backend.appendMessage(tu).catch(() => {}); // Settlement fallback: a mid-turn or request-hook fold is only // known here. Notes already written at decision time are skipped // by the flags inside. diff --git a/packages/runtime/src/runtime-kernel.ts b/packages/runtime/src/runtime-kernel.ts index 358b314dbd..672b4ff1cb 100644 --- a/packages/runtime/src/runtime-kernel.ts +++ b/packages/runtime/src/runtime-kernel.ts @@ -45,9 +45,6 @@ import type { SessionHeader, SessionHeaderPatch, SessionStatus, - StoredMessage, - TurnRecord, - TurnStateMessage, } from '@maka/core/session'; import { isDeepStrictEqual } from 'node:util'; import type { UserMessageInput } from '@maka/core/runtime-inputs'; @@ -64,6 +61,7 @@ import { type AgentRunActiveSession, type AgentRunBeginResult, type AgentRunDurability, + type AgentRunHooks, type AgentRunLineage, type RuntimeContinuationFailpoint, } from './agent-run.js'; @@ -93,12 +91,7 @@ import type { } from './session-manager.js'; import type { TurnShellPlan } from './shell-detect.js'; import type { ShellRunProcessManager } from './shell-run-manager.js'; -import { - buildStatusPatch, - buildTurnStateMessage, - normalizeStopSessionSource, - turnHasRetainedOutput as messagesHaveRetainedOutput, -} from './session-projection-helpers.js'; +import { buildStatusPatch, normalizeStopSessionSource } from './session-projection-helpers.js'; import { buildToolsForAgentDefinition } from './agent-catalog.js'; import { loadLatestHistoryCompactCheckpointFromRunLedger } from './history-compact-ledger.js'; import { loadModelProjectionTransitionsFromRunLedger } from './model-projection-transition-ledger.js'; @@ -321,16 +314,6 @@ interface StopOperation { abortSource: string | undefined; ts: number; statusProjected: boolean; - turnProjections: Map< - string, - { - id: string; - turnId: string; - lineage: AgentRunLineage; - message?: TurnStateMessage; - projected: boolean; - } - >; targets: Map; queue: Promise; } @@ -657,7 +640,6 @@ export class RuntimeKernel implements RuntimeKernelLike { runId: options.runId, userMessageId: options.userMessageId, durability: options.durability, - store: this.deps.store, runStore: this.deps.runStore, runtimeEventStore: this.deps.runtimeEventStore, ...(this.deps.toolBoundaryProtocol @@ -681,8 +663,7 @@ export class RuntimeKernel implements RuntimeKernelLike { updateHeader: (targetSessionId, patch) => this.updateHeader(targetSessionId, patch), updateStatus: (targetSessionId, status, blockedReason, ts) => this.updateStatus(targetSessionId, status, blockedReason, ts), - appendTurnState: (targetSessionId, turnId, status, lineage, options) => - this.appendTurnState(targetSessionId, turnId, status, lineage, options), + ...this.messageProjectionHook(), }, }); if (options.admitTurn && (await options.admitTurn()) === 'cancelled') { @@ -833,7 +814,6 @@ export class RuntimeKernel implements RuntimeKernelLike { runLineage: { parentRunId: continuation.sourceRunId }, runId: continuation.runId, invocationId: continuation.invocationId, - store: this.deps.store, runStore: this.deps.runStore, runtimeEventStore: this.deps.runtimeEventStore, ...(continuationToolBoundaryProtocol @@ -916,8 +896,7 @@ export class RuntimeKernel implements RuntimeKernelLike { updateHeader: (targetSessionId, patch) => this.updateHeader(targetSessionId, patch), updateStatus: (targetSessionId, status, blockedReason, ts) => this.updateStatus(targetSessionId, status, blockedReason, ts), - appendTurnState: (targetSessionId, turnId, status, lineage, options) => - this.appendTurnState(targetSessionId, turnId, status, lineage, options), + ...this.messageProjectionHook(), }, }); @@ -993,7 +972,6 @@ export class RuntimeKernel implements RuntimeKernelLike { userInput: { turnId, text: '' }, rootExecutionKind: 'context_compact', ...(input.hostedRoot ? { runId: input.hostedRoot.runId } : {}), - store: this.deps.store, runStore: this.deps.runStore, runtimeEventStore: this.deps.runtimeEventStore, ...(this.deps.toolBoundaryProtocol @@ -1017,8 +995,7 @@ export class RuntimeKernel implements RuntimeKernelLike { updateHeader: (targetSessionId, patch) => this.updateHeader(targetSessionId, patch), updateStatus: (targetSessionId, status, blockedReason, ts) => this.updateStatus(targetSessionId, status, blockedReason, ts), - appendTurnState: (targetSessionId, nextTurnId, status, lineage, options) => - this.appendTurnState(targetSessionId, nextTurnId, status, lineage, options), + ...this.messageProjectionHook(), }, }); @@ -1802,15 +1779,6 @@ export class RuntimeKernel implements RuntimeKernelLike { delivery: { kind: 'pending' }, } satisfies StopTarget); const needsRun = !target.runs.has(run.runId); - const projection = - needsRun && run.isSessionInline() && !operation.turnProjections.has(run.runId) - ? { - id: this.deps.newId(), - turnId: run.turnId, - lineage: run.lineage, - projected: false, - } - : undefined; if (!existingOperation) this.stopOperations.set(sessionId, operation); if (!existingTarget) { @@ -1825,7 +1793,6 @@ export class RuntimeKernel implements RuntimeKernelLike { sessionInline: run.isSessionInline(), stopCompleted: false, }); - if (projection) operation.turnProjections.set(run.runId, projection); } return operation; } @@ -1837,7 +1804,6 @@ export class RuntimeKernel implements RuntimeKernelLike { abortSource, ts, statusProjected: false, - turnProjections: new Map(), targets: new Map(), queue: Promise.resolve(), }; @@ -1917,24 +1883,11 @@ export class RuntimeKernel implements RuntimeKernelLike { await this.updateStatus(sessionId, 'aborted', undefined, operation.ts); operation.statusProjected = true; } - for (const projection of operation.turnProjections.values()) { - if (projection.projected) continue; - projection.message ??= buildTurnStateMessage({ - id: projection.id, - turnId: projection.turnId, - ts: operation.ts, - status: 'aborted', - lineage: projection.lineage, - ...(operation.abortSource ? { abortSource: operation.abortSource } : {}), - partialOutputRetained: await this.turnHasRetainedOutput(sessionId, projection.turnId), - }); - await this.appendStopProjection(sessionId, projection.message); - projection.projected = true; - } - // The Session projection above now reads as aborted. The ledger has to say - // the same thing before this stop reports success: a Run left non-terminal - // here stays that way, because the stream that would have finalized it is - // exactly the one the stop could not wake. + // The ledger has to say this turn was aborted before the stop reports + // success: a Run left non-terminal here stays that way, because the stream + // that would have finalized it is exactly the one the stop could not wake. + // Nothing else records the abort — the transcript reads it back off this + // terminal fact. // // Without a Host interaction authority, Runtime owns terminal settlement. // A Hosted Run's terminal fact belongs to the Host, which also parks @@ -1956,7 +1909,6 @@ export class RuntimeKernel implements RuntimeKernelLike { } const completed = operation.statusProjected && - [...operation.turnProjections.values()].every((projection) => projection.projected) && [...operation.targets.values()].every( (target) => target.delivery.kind !== 'pending' && @@ -1976,19 +1928,6 @@ export class RuntimeKernel implements RuntimeKernelLike { failures.throwIfAny(`Stop cleanup failed for session ${sessionId}`); } - private async appendStopProjection(sessionId: string, message: StoredMessage): Promise { - const existing = (await this.deps.store.readMessages(sessionId)).find( - (candidate) => candidate.id === message.id, - ); - if (existing) { - if (!isDeepStrictEqual(existing, message)) { - throw new Error(`stop projection ${message.id} conflicts with an existing message`); - } - return; - } - await this.deps.store.appendMessage(sessionId, message); - } - async respondToSandboxBoundary( sessionId: string, response: SandboxBoundaryResponse, @@ -2738,32 +2677,16 @@ export class RuntimeKernel implements RuntimeKernelLike { return next; } - private async appendTurnState( - sessionId: string, - turnId: string, - status: TurnRecord['status'], - lineage: AgentRunLineage = {}, - options: { id?: string; ts?: number; errorClass?: string; abortSource?: string } = {}, - ): Promise { - const ts = options.ts ?? this.deps.now(); - await this.deps.store.appendMessage( - sessionId, - buildTurnStateMessage({ - id: options.id ?? this.deps.newId(), - turnId, - ts, - status, - lineage, - ...(options.abortSource ? { abortSource: options.abortSource } : {}), - ...(options.errorClass !== undefined ? { errorClass: options.errorClass } : {}), - partialOutputRetained: await this.turnHasRetainedOutput(sessionId, turnId), - }), - ); - } - - private async turnHasRetainedOutput(sessionId: string, turnId: string): Promise { - const messages = await this.deps.store.readMessages(sessionId).catch(() => []); - return messagesHaveRetainedOutput(messages, turnId); + /** Present only when the store keeps a Session catalog to project into. */ + private messageProjectionHook(): Pick { + const commit = this.deps.store.commitMessageCatalogProjection; + if (!commit) return {}; + return { + commitMessageProjection: async (sessionId, message) => { + await commit.call(this.deps.store, sessionId, message); + this.updateCachedHeader(sessionId, await this.deps.store.readHeader(sessionId)); + }, + }; } } diff --git a/packages/runtime/src/runtime-ledger-repair.ts b/packages/runtime/src/runtime-ledger-repair.ts index e736ee25f5..061147b5e8 100644 --- a/packages/runtime/src/runtime-ledger-repair.ts +++ b/packages/runtime/src/runtime-ledger-repair.ts @@ -20,7 +20,6 @@ import { createHash } from 'node:crypto'; import { deriveTurnRecords } from '@maka/core/session'; import { DEFAULT_TOOL_MODE } from '@maka/core/tool-mode'; -import { isTerminalRuntimeEvent } from '@maka/core/runtime-event'; import type { RuntimeEvent, RuntimeEventInvocationOpenedContent } from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; import { @@ -35,37 +34,14 @@ import type { SessionHeader } from '@maka/core/session'; import type { StoredMessage, TurnRecord } from '@maka/core/session'; import { backfillRuntimeEventsFromStoredMessages } from './runtime-event-backfill.js'; import type { RuntimeEventBackfillOutcome } from './runtime-event-backfill.js'; -import { projectRuntimeEventUserMessage } from './runtime-event-read-model.js'; export interface RuntimeLedgerRepairDeps { runtimeEventStore: RuntimeEventStore; + /** The legacy transcript this converter reads; nothing writes back to it. */ readMessages(sessionId: string): Promise; - appendMessage(sessionId: string, message: StoredMessage): Promise; - newId: () => string; now: () => number; } -interface RuntimeEventTranscriptProjectionDeps { - readMessages(sessionId: string): Promise; - appendMessage(sessionId: string, message: StoredMessage): Promise; -} - -export async function materializeRuntimeEventTranscriptProjection( - deps: RuntimeEventTranscriptProjectionDeps, - sessionId: string, - event: RuntimeEvent, - knownMessageIds?: Set, -): Promise { - const message = steeringMessageFromRuntimeEvent(event); - if (!message) return false; - const messageIds = - knownMessageIds ?? new Set((await deps.readMessages(sessionId)).map((item) => item.id)); - if (messageIds.has(message.id)) return false; - await deps.appendMessage(sessionId, message); - messageIds.add(message.id); - return true; -} - export class RuntimeLedgerRepair { private readonly queues = new Map>(); @@ -82,14 +58,24 @@ export class RuntimeLedgerRepair { */ async materializeTranscriptLedger(header: SessionHeader): Promise { const sessionId = header.id; - return this.withRepairQueue(sessionId, 'transcript-runs', async () => { + return this.withRepairQueue(sessionId, async () => { const messages = await this.deps.readMessages(sessionId); const ledgerMessages = messages.filter( (message) => message.type !== 'user' || message.steeringEventId === undefined, ); - const endedTurnIds = new Set( + // A turn the ledger already owns is not converted again. Its own run is + // the authority even when it never ended — a crashed turn is settled by + // recovery on that run, and a second, transcript-derived invocation for + // the same turn would make the Session read as two. The one exception is + // this converter's own run: an interrupted import re-derives it, and the + // deterministic ids let the store dedupe what already landed. + const ownedTurnIds = new Set( (await this.listInlineInvocations(sessionId)) - .filter((invocation) => invocation.terminalEvent) + .filter( + (invocation) => + invocation.terminalEvent || + invocation.runId !== transcriptRunId(sessionId, invocation.turnId), + ) .map((invocation) => invocation.turnId), ); const messagesByTurn = groupMessagesByTurn(ledgerMessages); @@ -101,7 +87,7 @@ export class RuntimeLedgerRepair { const firstOpenedAt = Math.max(0, header.createdAt - turns.length); for (const [index, turn] of turns.entries()) { - if (endedTurnIds.has(turn.turnId)) continue; + if (ownedTurnIds.has(turn.turnId)) continue; const turnMessages = messagesByTurn.get(turn.turnId) ?? []; const runId = transcriptRunId(sessionId, turn.turnId); const openedAt = firstOpenedAt + index; @@ -128,38 +114,13 @@ export class RuntimeLedgerRepair { }); } - async repairSteeringMessagesOnce(sessionId: string): Promise { - return this.withRepairQueue(sessionId, 'steering-transcript', async () => { - const messages = await this.deps.readMessages(sessionId); - const messageIds = new Set(messages.map((message) => message.id)); - const inlineRunIds = new Set( - (await this.listInlineInvocations(sessionId)).map((invocation) => invocation.runId), - ); - let repaired = 0; - for (const event of await this.deps.runtimeEventStore.readSessionRuntimeEvents(sessionId)) { - if (!inlineRunIds.has(event.runId)) continue; - if ( - await materializeRuntimeEventTranscriptProjection(this.deps, sessionId, event, messageIds) - ) { - repaired += 1; - } - } - return repaired; - }); - } - private async listInlineInvocations(sessionId: string): Promise { return (await this.deps.runtimeEventStore.listSessionInvocations(sessionId)).filter( (invocation) => isSessionInlineInvocation(invocation.opening), ); } - private async withRepairQueue( - sessionId: string, - runId: string, - operation: () => Promise, - ): Promise { - const key = `${sessionId}:${runId}`; + private async withRepairQueue(key: string, operation: () => Promise): Promise { const previous = this.queues.get(key) ?? Promise.resolve(); const current = previous.then(operation, operation); const cleanup = current.then( @@ -279,17 +240,3 @@ function groupMessagesByTurn(messages: readonly StoredMessage[]): Map; list(filter?: SessionListFilter): Promise; readHeader(sessionId: string): Promise; + /** The legacy transcript, read only to convert it onto the ledger. */ readMessages(sessionId: string): Promise; - readMessagesSnapshot?(sessionId: string): Promise; - listTurns(sessionId: string): Promise; - appendMessage(sessionId: string, m: StoredMessage): Promise; - appendMessages(sessionId: string, ms: StoredMessage[]): Promise; + /** Commit the Session-list facts a durable message carries. */ + commitMessageCatalogProjection?( + sessionId: string, + message: UserMessage | AssistantMessage, + ): Promise; updateHeader(sessionId: string, patch: SessionHeaderPatch): Promise; updateHeaderVersioned?( sessionId: string, @@ -641,7 +638,6 @@ export interface SessionStore { export interface StrictRecoverySessionStore extends SessionStore { listForRecovery(): Promise; - readMessagesForRecovery(sessionId: string): Promise; } export interface StrictRecoveryAgentRunStore extends AgentRunStore { @@ -664,7 +660,6 @@ export interface BackendFactoryContext { store: SessionStore; /** Process-local cancellation for the execution that owns this activation. */ abortSignal?: AbortSignal; - appendMessage?: (message: StoredMessage) => Promise; /** * Child-agent instruction channel. Linked child sessions populate this; an * ordinary main-session activation leaves it undefined. A @@ -908,8 +903,6 @@ export class SessionManager { this.runtimeLedgerRepair = new RuntimeLedgerRepair({ runtimeEventStore: deps.runtimeEventStore, readMessages: (sessionId) => deps.store.readMessages(sessionId), - appendMessage: (sessionId, message) => deps.store.appendMessage(sessionId, message), - newId: deps.newId, now: deps.now, }); } @@ -1419,13 +1412,6 @@ export class SessionManager { ); const recovered = new Set(); for (const session of interrupted) { - if (this.runtimeLedgerRepair) { - await recoverOr( - policy, - () => this.runtimeLedgerRepair!.repairSteeringMessagesOnce(session.id), - 0, - ); - } if (this.runtimeKernel.hasActiveRuns(session.id)) continue; // Fail-closed: a request whose live owner died can never be answered, so // it settles as `deny` with a durable `host_restarted` reason. The run @@ -1471,18 +1457,6 @@ export class SessionManager { ); if (recoveredShellRuns > 0) recovered.add(session.id); } - let messages: StoredMessage[] = []; - let messagesReadable = true; - try { - messages = - policy.kind === 'strict' - ? await policy.stores.sessionStore.readMessagesForRecovery(session.id) - : await this.deps.store.readMessages(session.id); - } catch (error) { - if (policy.kind === 'strict') throw error; - messagesReadable = false; - } - // A revision copy still `preparing` is settled by the Host's revision // coordinator, which reads the admission ledger and runs before this // recovery. Deciding it a second time here — off a transcript scan, and @@ -1518,51 +1492,24 @@ export class SessionManager { if (runRecovery.recovered || continuationClaimRecovered) { await recoverOr(policy, () => this.updateStatus(session.id, 'active'), undefined); recovered.add(session.id); - } else if ( - !messagesReadable && - (session.status === 'running' || session.status === 'waiting_for_user') - ) { - await recoverOr(policy, () => this.updateStatus(session.id, 'active'), undefined); - recovered.add(session.id); } continue; } } - if (!messagesReadable) { - if (session.status === 'running' || session.status === 'waiting_for_user') { - // Recovery may run in BACKGROUND startup (#456): re-check for a - // run the user started while this session's recovery was in - // flight, so we never stomp a live run's status. - if (this.runtimeKernel.hasActiveRuns(session.id)) continue; - await recoverOr(policy, () => this.updateStatus(session.id, 'active'), undefined); - recovered.add(session.id); - } - continue; - } - - const recoveries = interruptedTurnRecoveries(messages); - if (recoveries.length === 0) continue; - for (const recovery of recoveries) { - await recoverOr( - policy, - () => - this.appendTurnState(session.id, recovery.turnId, 'failed', recovery.lineage, { - errorClass: recovery.errorClass, - }), - undefined, - ); - } + // No ledger and nothing to recover from it. A Session whose turns were + // interrupted before this process started is settled by the transcript + // importer, which converts a turn that never recorded how it ended into + // the failed terminal fact it actually was — this recovery has no second + // transcript to read that from. if (session.status === 'running' || session.status === 'waiting_for_user') { - // Same double-check as above: a message sent mid-recovery owns - // the session status now (its own transitions will settle it). - if (this.runtimeKernel.hasActiveRuns(session.id)) { - recovered.add(session.id); - continue; - } + // Recovery may run in BACKGROUND startup (#456): re-check for a run the + // user started while this session's recovery was in flight, so we never + // stomp a live run's status. + if (this.runtimeKernel.hasActiveRuns(session.id)) continue; await recoverOr(policy, () => this.updateStatus(session.id, 'active'), undefined); + recovered.add(session.id); } - recovered.add(session.id); } return [...recovered]; } @@ -2789,19 +2736,16 @@ export class SessionManager { await this.finalizeChildWorkspacePatches(child.id); - const [runs, messages] = await Promise.all([ - this.listInvocations(child.id), - this.deps.store.readMessages(child.id), - ]); - const turnOwner = runs.find((candidate) => candidate.turnId === claim.targetTurnId); + // An invocation is what makes a Turn exist on the ledger, so the run listing + // is the whole occupancy check: a Turn with durable content has one. + const turnOwner = (await this.listInvocations(child.id)).find( + (candidate) => candidate.turnId === claim.targetTurnId, + ); if (turnOwner) { throw new Error( `Claimed graph turn ${claim.targetTurnId} is already owned by run ${turnOwner.runId}`, ); } - if (messages.some((message) => 'turnId' in message && message.turnId === claim.targetTurnId)) { - throw new Error(`Claimed graph turn ${claim.targetTurnId} already has durable messages`); - } if (child.isArchived || child.status === 'aborted') { throw new Error('Claimed graph execution target child session is terminated'); } @@ -2981,7 +2925,7 @@ export class SessionManager { prompt: string, expectedUserMessageId?: string, ): Promise { - const messages = await this.deps.store.readMessages(sessionId); + const { messages } = await this.getSessionView(sessionId); const userMessages = messages.filter( (message): message is UserMessage => message.type === 'user' && message.turnId === turnId, ); @@ -3289,18 +3233,10 @@ export class SessionManager { const snapshot = child.subagentRuntime; if (!snapshot) throw new Error('Stored child session is missing its durable runtime snapshot'); const facts = invocationListingFacts(run); - const [messages, runtimeEvents, artifacts] = await Promise.all([ - this.deps.store.readMessages(child.id), + const [runtimeEvents, artifacts] = await Promise.all([ this.deps.runtimeEventStore.readRuntimeEvents(child.id, run.runId), this.finalizeAndListChildTurnArtifacts(child.id, run.turnId, facts.status), ]); - const storedSummary = - messages - .filter( - (message): message is Extract => - message.type === 'assistant' && message.turnId === run.turnId, - ) - .at(-1)?.text ?? ''; const runtimeText = runtimeEvents.filter( ( event, @@ -3323,7 +3259,7 @@ export class SessionManager { runId: run.runId, status: agentRunStatusForSpawnResult(facts.status), permissionMode: child.permissionMode, - summary: trimSummary(durableRuntimeSummary ?? (storedSummary || partialRuntimeSummary)), + summary: trimSummary(durableRuntimeSummary ?? partialRuntimeSummary), artifactIds: artifacts.map((artifact) => artifact.id), startedAt: facts.createdAt, completedAt: facts.updatedAt, @@ -3632,6 +3568,12 @@ export class SessionManager { turnId: string; runId: string; admittedAt: number; + /** + * The message this admission owns, when the crash beat the Run that would + * have recorded it. Recovery writes it into the invocation it opens below, + * so the Turn the user sees still carries what they asked for. + */ + userMessage?: { id: string; content: MessageContent; origin?: UserMessage['origin'] }; execution: Exclude< RootExecutionDescriptor, | { kind: 'regenerate' } @@ -3813,6 +3755,34 @@ export class SessionManager { }), ); + if (input.userMessage) { + await this.deps.runtimeEventStore.appendRuntimeEvent(input.sessionId, input.runId, { + id: input.userMessage.id, + ...run, + ts: input.admittedAt, + partial: false, + role: 'user', + author: input.userMessage.origin ? 'host' : 'user', + content: { + kind: 'text', + text: input.userMessage.content.text, + ...(input.userMessage.content.displayText !== undefined + ? { displayText: input.userMessage.content.displayText } + : {}), + ...(input.userMessage.content.attachments?.length + ? { attachments: input.userMessage.content.attachments } + : {}), + ...(input.userMessage.content.directoryReferences?.length + ? { directoryReferences: input.userMessage.content.directoryReferences } + : {}), + ...(input.userMessage.content.quotes?.length + ? { quotes: input.userMessage.content.quotes } + : {}), + ...(input.userMessage.origin ? { origin: input.userMessage.origin } : {}), + }, + }); + } + const ts = this.deps.now(); const terminalEvent = buildRecoveredTerminalRuntimeEvent({ id: this.deps.newId(), @@ -3960,20 +3930,7 @@ export class SessionManager { /** Canonical, repaired source view for a Host-owned cross-Session copy. */ async readConversationCopySnapshot(sessionId: string): Promise { - const readMessagesSnapshot = this.deps.store.readMessagesSnapshot; - if (!readMessagesSnapshot) { - throw new Error('Conversation copy requires a side-effect-free message snapshot'); - } - const readMessages = readMessagesSnapshot.bind(this.deps.store); - const view = await this.getSessionView(sessionId); - if (view.invocations.length > 0 || view.messages.length > 0) return view; - const messages = await readMessages(sessionId); - if (messages.length === 0) return view; - return { - ...view, - messages, - turns: deriveTurnRecords(messages), - }; + return this.getSessionView(sessionId); } async respondToSandboxBoundary( @@ -4061,13 +4018,10 @@ export class SessionManager { return await this.getMessages(sessionId); } catch (error) { if (!(error instanceof RuntimeReadModelError)) throw error; - // ShellRun hydration is a best-effort UI projection. A legacy RuntimeEvent - // incompatibility must not turn its retry loop into a permanent IPC error. - try { - return await this.deps.store.readMessages(sessionId); - } catch { - return null; - } + // ShellRun hydration is a best-effort UI projection. A ledger the read + // model cannot project yet must not turn its retry loop into a permanent + // IPC error; there is no second transcript to fall back to. + return null; } } @@ -4245,34 +4199,6 @@ export class SessionManager { } } - private async appendTurnState( - sessionId: string, - turnId: string, - status: TurnRecord['status'], - lineage: AgentRunLineage = {}, - options: { ts?: number; errorClass?: string; abortSource?: string } = {}, - ): Promise { - const ts = options.ts ?? this.deps.now(); - await this.deps.store.appendMessage( - sessionId, - buildTurnStateMessage({ - id: this.deps.newId(), - turnId, - ts, - status, - lineage, - ...(options.abortSource ? { abortSource: options.abortSource } : {}), - ...(options.errorClass !== undefined ? { errorClass: options.errorClass } : {}), - partialOutputRetained: await this.turnHasRetainedOutput(sessionId, turnId), - }), - ); - } - - private async turnHasRetainedOutput(sessionId: string, turnId: string): Promise { - const messages = await this.deps.store.readMessages(sessionId).catch(() => []); - return messagesHaveRetainedOutput(messages, turnId); - } - private async requireTurnForAction( sessionId: string, turnId: string, @@ -4289,7 +4215,16 @@ export class SessionManager { return turn; } + /** + * The Session as its ledger tells it. + * + * A transcript written before the ledger owned execution facts is converted + * here, on the first read, because there is no second transcript left to read + * it from: the importer is what gives those turns an invocation to be + * projected from, so a read that skipped it would report the Session empty. + */ private async getSessionView(sessionId: string): Promise { + await this.ensureTranscriptLedgerForRead(sessionId); return this.readModel().getSessionView(sessionId); } @@ -4305,6 +4240,18 @@ export class SessionManager { }); } + /** + * Convert a transcript written before the ledger owned execution facts, so a + * reader that goes straight to the ledger still sees the whole Session. + * + * Idempotent and cheap after the first call: the conversion is remembered per + * Session, and a Session born on the ledger has nothing to convert. + */ + async ensureTranscriptLedgerForRead(sessionId: string): Promise { + const repair = this.runtimeLedgerRepair; + if (repair) await this.ensureTranscriptLedger(sessionId, repair, 'compatibility'); + } + async prepareImportedSessionHistory(sessionId: string): Promise { const repair = this.runtimeLedgerRepair; if (!repair) throw new Error('Imported Session history requires canonical Runtime stores'); @@ -4616,47 +4563,10 @@ export class SessionManager { return false; } - const appendedTurnState = await recoverOr( - policy, - () => - this.appendTerminalTurnStateIfNeeded( - sessionId, - inspected.invocation, - decision, - terminalTurnStatus(status), - { - ts, - ...(failureClass ? { errorClass: failureClass } : {}), - ...(abortSource ? { abortSource } : {}), - }, - policy, - ), - false, - ); - // A run that already carried a complete terminal fact and a terminal Turn - // state had nothing to recover. Saying otherwise makes recovery rewrite the - // Session status of every healthy run it walks past. - return inspected.terminalRuntimeFact === undefined || appendedTurnState; - } - - private async appendTerminalTurnStateIfNeeded( - sessionId: string, - run: RuntimeInvocationRecord, - decision: AgentRunRecoveryDecision, - status: TurnRecord['status'], - options: { ts: number; errorClass?: string; abortSource?: string }, - policy: RecoveryPolicy = { kind: 'best_effort' }, - ): Promise { - if (!isSessionInlineInvocation(run.opening)) return false; - const messages = await recoverOr( - policy, - () => this.deps.store.readMessages(sessionId), - [] as StoredMessage[], - ); - const latest = latestTurnState(messages, decision.turnId); - if (latest && isTerminalTurnStatus(latest.status) && latest.status === status) return false; - await this.appendTurnState(sessionId, decision.turnId, status, decision.lineage, options); - return true; + // A run that already carried a complete terminal fact had nothing to + // recover. Saying otherwise makes recovery rewrite the Session status of + // every healthy run it walks past. + return inspected.terminalRuntimeFact === undefined; } } @@ -5118,96 +5028,10 @@ class ChildAgentSummaryAccumulator { } } -interface InterruptedTurnRecovery { - turnId: string; - errorClass: string; - lineage: Partial< - Pick< - UserMessageInput, - | 'parentTurnId' - | 'retriedFromTurnId' - | 'regeneratedFromTurnId' - | 'branchOfTurnId' - | 'parentSessionId' - > - >; -} - -function interruptedTurnRecoveries(messages: readonly StoredMessage[]): InterruptedTurnRecovery[] { - const byTurn = new Map< - string, - { - hasAssistant: boolean; - states: Array>; - } - >(); - for (const message of messages) { - const turnId = (message as { turnId?: string }).turnId; - if (!turnId) continue; - const bucket = byTurn.get(turnId) ?? { hasAssistant: false, states: [] }; - if (message.type === 'assistant') bucket.hasAssistant = true; - if (message.type === 'turn_state') bucket.states.push(message); - byTurn.set(turnId, bucket); - } - - const recoveries: InterruptedTurnRecovery[] = []; - for (const [turnId, bucket] of byTurn) { - const latest = bucket.states.at(-1); - if (!latest) continue; - if (latest.status === 'running') { - recoveries.push({ - turnId, - errorClass: 'app_restarted', - lineage: turnStateLineage(latest), - }); - continue; - } - const failed = [...bucket.states].reverse().find((state) => state.status === 'failed'); - if (latest.status === 'completed' && !bucket.hasAssistant && failed) { - recoveries.push({ - turnId, - errorClass: failed.errorClass ?? 'unknown', - lineage: turnStateLineage(failed), - }); - } - } - return recoveries; -} - -function turnStateLineage( - state: Extract, -): Partial< - Pick< - UserMessageInput, - | 'parentTurnId' - | 'retriedFromTurnId' - | 'regeneratedFromTurnId' - | 'branchOfTurnId' - | 'parentSessionId' - > -> { - return { - ...(state.parentTurnId ? { parentTurnId: state.parentTurnId } : {}), - ...(state.retriedFromTurnId ? { retriedFromTurnId: state.retriedFromTurnId } : {}), - ...(state.regeneratedFromTurnId ? { regeneratedFromTurnId: state.regeneratedFromTurnId } : {}), - ...(state.branchOfTurnId ? { branchOfTurnId: state.branchOfTurnId } : {}), - ...(state.parentSessionId ? { parentSessionId: state.parentSessionId } : {}), - }; -} - function isTerminalRunStatus(status: RunLifecycleStatus): boolean { return status === 'completed' || status === 'failed' || status === 'cancelled'; } -function isTerminalTurnStatus(status: TurnRecord['status']): boolean { - return status === 'completed' || status === 'failed' || status === 'aborted'; -} - -function terminalTurnStatus(status: AgentRunRecoveryDecision['status']): TurnRecord['status'] { - if (status === 'cancelled') return 'aborted'; - return status; -} - function diagnosticRecoveryReason(diagnostic: Record | undefined): string { const recoveryReason = diagnostic?.recoveryReason; return typeof recoveryReason === 'string' && recoveryReason.length > 0 @@ -5215,17 +5039,6 @@ function diagnosticRecoveryReason(diagnostic: Record | undefine : 'agent_run_recovery'; } -function latestTurnState( - messages: readonly StoredMessage[], - turnId: string, -): Extract | undefined { - for (let index = messages.length - 1; index >= 0; index -= 1) { - const message = messages[index]; - if (message?.type === 'turn_state' && message.turnId === turnId) return message; - } - return undefined; -} - function runtimeTerminalFactToRecoveryDecision( invocation: RuntimeInvocationRecord, fact: RuntimeEventTerminalFact, diff --git a/packages/runtime/src/session-projection-helpers.ts b/packages/runtime/src/session-projection-helpers.ts index 456f0ddd7a..ff12795b25 100644 --- a/packages/runtime/src/session-projection-helpers.ts +++ b/packages/runtime/src/session-projection-helpers.ts @@ -24,33 +24,9 @@ import type { SessionBlockedReason, SessionHeader, SessionStatus, - StoredMessage, TurnRecord, - TurnStateMessage, } from '@maka/core/session'; -export type TurnStateLineage = Partial< - Pick< - TurnStateMessage, - | 'parentTurnId' - | 'retriedFromTurnId' - | 'regeneratedFromTurnId' - | 'branchOfTurnId' - | 'parentSessionId' - > ->; - -export interface BuildTurnStateMessageInput { - id: string; - turnId: string; - ts: number; - status: TurnRecord['status']; - lineage?: TurnStateLineage; - errorClass?: string; - abortSource?: string; - partialOutputRetained: boolean; -} - export function buildStatusPatch( status: SessionStatus, ts: number, @@ -63,38 +39,6 @@ export function buildStatusPatch( }; } -export function buildTurnStateMessage(input: BuildTurnStateMessageInput): TurnStateMessage { - const lineage = input.lineage ?? {}; - return { - type: 'turn_state', - id: input.id, - turnId: input.turnId, - ts: input.ts, - status: input.status, - ...(lineage.parentTurnId ? { parentTurnId: lineage.parentTurnId } : {}), - ...(lineage.retriedFromTurnId ? { retriedFromTurnId: lineage.retriedFromTurnId } : {}), - ...(lineage.regeneratedFromTurnId - ? { regeneratedFromTurnId: lineage.regeneratedFromTurnId } - : {}), - ...(lineage.branchOfTurnId ? { branchOfTurnId: lineage.branchOfTurnId } : {}), - ...(lineage.parentSessionId ? { parentSessionId: lineage.parentSessionId } : {}), - ...(input.status === 'aborted' ? { abortedAt: input.ts } : {}), - ...(input.status === 'aborted' && input.abortSource ? { abortSource: input.abortSource } : {}), - ...(input.status === 'failed' ? { errorClass: input.errorClass ?? 'unknown' } : {}), - partialOutputRetained: input.partialOutputRetained, - }; -} - -export function turnHasRetainedOutput(messages: readonly StoredMessage[], turnId: string): boolean { - return messages.some( - (message) => - (message.type === 'assistant' && - message.turnId === turnId && - message.text.trim().length > 0) || - (message.type === 'tool_result' && message.turnId === turnId), - ); -} - export function normalizeStopSessionSource( source: 'stop_button' | 'graph_supervisor' | 'workhub_direct_stop' | undefined, workHubActionId?: string, diff --git a/packages/runtime/src/test-only/fake-backend.ts b/packages/runtime/src/test-only/fake-backend.ts index 213089f261..8c41be3681 100644 --- a/packages/runtime/src/test-only/fake-backend.ts +++ b/packages/runtime/src/test-only/fake-backend.ts @@ -18,7 +18,7 @@ */ import { randomUUID } from 'node:crypto'; -import type { PersistedBackendKind, SessionHeader, StoredMessage } from '@maka/core/session'; +import type { PersistedBackendKind } from '@maka/core/session'; import type { SessionEvent } from '@maka/core/events'; import type { AgentBackend, @@ -36,7 +36,6 @@ import { RuntimeInteractionInvariantError, type RuntimeUserQuestionClosureReason, } from '../interaction-authority.js'; -import type { SessionStore } from '../session-manager.js'; const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); export const FAKE_ASK_USER_QUESTION_PROMPT = '__e2e_ask_user_question__'; @@ -76,14 +75,7 @@ export class FakeBackend implements AgentBackend { private readonly stopWaiters: Array<() => void> = []; private questionAdmissionWaiting = false; - constructor( - private readonly ctx: { - sessionId: string; - header: SessionHeader; - store: SessionStore; - appendMessage?: (message: StoredMessage) => Promise; - }, - ) { + constructor(ctx: { sessionId: string }) { this.sessionId = ctx.sessionId; } @@ -314,17 +306,6 @@ export class FakeBackend implements AgentBackend { } const ts = Date.now(); - const appendMessage = - this.ctx.appendMessage ?? - ((message: StoredMessage) => this.ctx.store.appendMessage(this.sessionId, message)); - await appendMessage({ - type: 'assistant', - id: messageId, - turnId, - ts, - text, - modelId: this.ctx.header.model, - }); yield { type: 'text_complete', id: randomUUID(), turnId, ts, messageId, text }; yield { type: 'complete', id: randomUUID(), turnId, ts: Date.now(), stopReason: 'end_turn' }; } finally { @@ -435,19 +416,7 @@ export class FakeBackend implements AgentBackend { options: [{ label: '是' }, { label: '否' }], }, ]; - const appendMessage = - this.ctx.appendMessage ?? - ((message: StoredMessage) => this.ctx.store.appendMessage(this.sessionId, message)); const startedAt = Date.now(); - await appendMessage({ - type: 'tool_call', - id: toolUseId, - turnId, - stepId, - ts: startedAt, - toolName: 'AskUserQuestion', - args: { questions }, - }); yield { type: 'tool_start', id: randomUUID(), @@ -519,15 +488,6 @@ export class FakeBackend implements AgentBackend { }; const resultContent = { kind: 'json' as const, value: result }; const resultTs = Date.now(); - await appendMessage({ - type: 'tool_result', - id: randomUUID(), - turnId, - ts: resultTs, - toolUseId, - isError: false, - content: resultContent, - }); yield { type: 'tool_result', id: randomUUID(), @@ -551,14 +511,6 @@ export class FakeBackend implements AgentBackend { }; } const completedAt = Date.now(); - await appendMessage({ - type: 'assistant', - id: messageId, - turnId, - ts: completedAt, - text, - modelId: this.ctx.header.model, - }); yield { type: 'text_complete', id: randomUUID(), turnId, ts: completedAt, messageId, text }; yield { type: 'complete', id: randomUUID(), turnId, ts: Date.now(), stopReason: 'end_turn' }; } @@ -598,19 +550,7 @@ export class FakeBackend implements AgentBackend { const stepId = randomUUID(); const expansion = { network: { enabled: true as const } }; const justification = 'Connect to the deterministic fake test endpoint.'; - const appendMessage = - this.ctx.appendMessage ?? - ((message: StoredMessage) => this.ctx.store.appendMessage(this.sessionId, message)); const startedAt = Date.now(); - await appendMessage({ - type: 'tool_call', - id: toolUseId, - turnId, - stepId, - ts: startedAt, - toolName: 'RequestSandboxBoundary', - args: { expansion, justification }, - }); yield { type: 'tool_start', id: randomUUID(), @@ -685,15 +625,6 @@ export class FakeBackend implements AgentBackend { value: { decision, status: settlement.request.status }, }; const resultTs = Date.now(); - await appendMessage({ - type: 'tool_result', - id: randomUUID(), - turnId, - ts: resultTs, - toolUseId, - isError: decision === 'deny', - content: resultContent, - }); yield { type: 'tool_result', id: randomUUID(), @@ -715,14 +646,6 @@ export class FakeBackend implements AgentBackend { text, }; const completedAt = Date.now(); - await appendMessage({ - type: 'assistant', - id: messageId, - turnId, - ts: completedAt, - text, - modelId: this.ctx.header.model, - }); yield { type: 'text_complete', id: randomUUID(), turnId, ts: completedAt, messageId, text }; yield { type: 'complete', id: randomUUID(), turnId, ts: Date.now(), stopReason: 'end_turn' }; } diff --git a/packages/runtime/src/tool-runtime.ts b/packages/runtime/src/tool-runtime.ts index a12fdce73e..06adbaa5ae 100644 --- a/packages/runtime/src/tool-runtime.ts +++ b/packages/runtime/src/tool-runtime.ts @@ -47,7 +47,6 @@ import type { ToolUncertainOutcomeSignal, UserQuestionRequestEvent, } from '@maka/core/events'; -import type { ToolCallMessage, ToolResultMessage } from '@maka/core/session'; import type { HostedFormSettlement, HostedInteractionBridge, @@ -320,7 +319,6 @@ export interface MakaToolContext { ) => Promise; } -export type AppendMessageFn = (m: ToolCallMessage | ToolResultMessage) => Promise; export type ToolTelemetryRecorder = (record: ToolInvocationRecord) => void; /** @@ -373,7 +371,6 @@ export interface ToolRuntimeInput { header: SessionHeader; connection: RuntimeExecutionConnection; modelId: string; - appendMessage: AppendMessageFn; readExecutionBoundary: () => Promise; createSandboxBoundaryRequest?: ( input: CreateSandboxBoundaryRequest, @@ -1077,17 +1074,6 @@ export class ToolRuntime { this.input.sessionId, ) ?? DURABLE_TOOL_RESULT_PROJECTION_FAILURE; const durableOutcome = await durableAttempt?.commitOutcome(content, true, modelProjection); - const msg: ToolResultMessage = { - type: 'tool_result', - id: this.input.newId(), - turnId, - ts: this.input.now(), - toolUseId, - isError: true, - content, - ...activityIdentity, - }; - await this.input.appendMessage(msg); queue.push({ type: 'tool_result', id: durableOutcome?.id ?? this.input.newId(), @@ -1281,29 +1267,6 @@ export class ToolRuntime { queue.push(event); callEventPublished = true; }; - const callMsg: ToolCallMessage = { - type: 'tool_call', - id: toolUseId, - turnId, - ts: now, - toolName: tool.name, - ...activityIdentity, - ...(tool.activityKind ? { activityKind: tool.activityKind } : {}), - ...(tool.displayName ? { displayName: tool.displayName } : {}), - args: structuredClone(persistedArgs), - ...(ctx.providerOptions !== undefined - ? { providerOptions: structuredClone(ctx.providerOptions) } - : {}), - // Persist the same step id the tool_start event carries so the UI - // timeline and post-restart backfill can pair this call with its step. - ...(stepId !== undefined ? { stepId } : {}), - }; - let callMessageAppended = false; - const appendCallMessage = async (): Promise => { - if (callMessageAppended) return; - await this.input.appendMessage(callMsg); - callMessageAppended = true; - }; const emitToolStartedTrace = (): void => { trace?.emit('tool', 'tool_started', 'Tool execution started', { toolUseId, @@ -1320,7 +1283,6 @@ export class ToolRuntime { text: string, sandboxFailure?: Extract['sandboxFailure'], ): Promise => { - await appendCallMessage(); publishCallEvent(buildCallEvent('preflight')); emitToolStartedTrace(); await this.writeSyntheticToolResult( @@ -1623,7 +1585,6 @@ export class ToolRuntime { await disposeManagedMutationAdmission(managedMutationAdmission); throw error; } - await appendCallMessage(); publishCallEvent(buildCallEvent('dispatch')); emitToolStartedTrace(); if (durableAttempt) { @@ -1935,18 +1896,6 @@ export class ToolRuntime { }, ); } - const resultMsg: ToolResultMessage = { - type: 'tool_result', - id: this.input.newId(), - turnId, - ts: this.input.now(), - toolUseId, - isError: toolResultStatus !== 'success', - content, - durationMs, - ...activityIdentity, - }; - await this.input.appendMessage(resultMsg); queue.push({ type: 'tool_result', id: durableOutcome?.id ?? this.input.newId(), @@ -2086,18 +2035,6 @@ export class ToolRuntime { modelProjection, durationMs, ); - const resultMsg: ToolResultMessage = { - type: 'tool_result', - id: this.input.newId(), - turnId, - ts: this.input.now(), - toolUseId, - isError: true, - content: terminalFailure.content, - durationMs, - ...activityIdentity, - }; - await this.input.appendMessage(resultMsg); queue.push({ type: 'tool_result', id: durableOutcome?.id ?? this.input.newId(), diff --git a/packages/storage/package.json b/packages/storage/package.json index dc28e6f7aa..92872ea09b 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -46,6 +46,7 @@ "./session-bundle-policy": "./dist/session-bundle-policy.js", "./session-copy-cleanup": "./dist/session-copy-cleanup.js", "./session-todo-authority": "./dist/session-todo-authority.js", + "./session-message-projection": "./dist/session-message-projection.js", "./session-store": "./dist/session-store.js", "./settings-store": "./dist/settings-store.js", "./shell-run-authority": "./dist/shell-run-authority.js", diff --git a/packages/storage/src/__tests__/session-store.test.ts b/packages/storage/src/__tests__/session-store.test.ts index 39b91a7a3e..ba594bcd34 100644 --- a/packages/storage/src/__tests__/session-store.test.ts +++ b/packages/storage/src/__tests__/session-store.test.ts @@ -18,7 +18,6 @@ */ import assert from 'node:assert/strict'; -import { createHash } from 'node:crypto'; import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -637,7 +636,7 @@ describe('SQLite SessionStore', () => { } }); - test('pages the durable transcript by sequence, bytes, and a fixed watermark', async () => { + test('bounds durable message lookups by a fixed transcript watermark', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-session-transcript-pages-')); const store = createSessionStore(root); try { @@ -651,27 +650,6 @@ describe('SQLite SessionStore', () => { })); await store.appendMessages(session.id, messages); - const tail = await store.readTranscriptPageSnapshot(session.id, { - direction: 'older', - maxBytes: 64 * 1024, - maxMessages: 2, - }); - assert.equal(tail.throughSequence, 3); - assert.deepEqual( - tail.fragments.map(({ sequence }) => sequence), - [3, 2], - ); - assert.deepEqual(tail.next, { position: 1, byteOffset: null }); - - const decodedTail = await store.readTranscriptRecordsSnapshot(session.id, { - direction: 'older', - maxStoredBytes: 1, - maxMessages: 2, - }); - assert.equal(decodedTail.throughSequence, 3); - assert.deepEqual(decodedTail.records, [{ sequence: 3, message: messages[3] }]); - assert.equal(decodedTail.nextPosition, 2); - await store.appendMessage(session.id, { type: 'user', id: 'message-4', @@ -714,89 +692,23 @@ describe('SQLite SessionStore', () => { }, ], ); - assert.deepEqual( - await store.readTranscriptPageSnapshot(session.id, { - direction: 'older', - throughSequence: null, - maxBytes: 64 * 1024, - maxMessages: 2, - }), - { throughSequence: null, fragments: [], rawBytes: 0, next: null }, - ); - const older = await store.readTranscriptPageSnapshot(session.id, { - direction: 'older', - throughSequence: tail.throughSequence ?? undefined, - position: 1, - maxBytes: 64 * 1024, - maxMessages: 2, - }); - assert.equal(older.throughSequence, 3); - assert.deepEqual( - older.fragments.map(({ sequence }) => sequence), - [1, 0], - ); - assert.equal(older.next, null); - - const newer = await store.readTranscriptPageSnapshot(session.id, { - direction: 'newer', - throughSequence: 3, - position: 2, - maxBytes: 64 * 1024, - maxMessages: 10, - }); - assert.deepEqual( - newer.fragments.map(({ sequence }) => sequence), - [2, 3], - ); - assert.equal(newer.next, null); - - const oversized = await store.readTranscriptPageSnapshot(session.id, { - direction: 'older', - throughSequence: 3, - maxBytes: 1, - maxMessages: 10, - }); - assert.deepEqual( - oversized.fragments.map(({ sequence }) => sequence), - [3], - ); - assert.equal(oversized.fragments[0]!.data.byteLength, 1); - assert.ok(oversized.fragments[0]!.totalBytes > 1); - assert.deepEqual(oversized.next, { - position: 3, - byteOffset: oversized.fragments[0]!.byteOffset, - }); - const fragments = [...oversized.fragments]; - let continuation: { - readonly position: number; - readonly byteOffset: number | null; - } | null = oversized.next; - while (continuation?.position === 3 && continuation.byteOffset !== null) { - const page = await store.readTranscriptPageSnapshot(session.id, { - direction: 'older', - throughSequence: 3, - position: continuation.position, - byteOffset: continuation.byteOffset, - maxBytes: 7, - maxMessages: 10, - }); - fragments.push(...page.fragments); - continuation = page.next; - } - const reconstructed = Buffer.concat( - fragments - .filter((fragment) => fragment.sequence === 3) - .sort((left, right) => left.byteOffset - right.byteOffset) - .map((fragment) => fragment.data), - ); - assert.deepEqual(JSON.parse(reconstructed.toString('utf8')), messages[3]); + assert.deepEqual(await store.readMessages(session.id), [ + ...messages, + { + type: 'user', + id: 'message-4', + turnId: 'turn-4', + ts: 5, + text: 'appended after the watermark', + }, + ]); } finally { await store.close?.(); await rm(root, { recursive: true, force: true }); } }); - test('pages both new chunked messages and legacy inline v22 records', async () => { + test('reads both new chunked messages and legacy inline v22 records', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-session-transcript-chunks-')); const message = { type: 'user' as const, @@ -818,28 +730,7 @@ describe('SQLite SessionStore', () => { const session = await store.create(makeInput()); sessionId = session.id; await store.appendMessages(session.id, [message, smallMessage]); - const fragments = []; - let position = 0; - let byteOffset: number | undefined; - do { - const page = await store.readTranscriptPageSnapshot(session.id, { - direction: 'newer', - throughSequence: 0, - position, - ...(byteOffset === undefined ? {} : { byteOffset }), - maxBytes: 50_000, - maxMessages: 1, - }); - fragments.push(...page.fragments); - position = page.next?.position ?? 1; - byteOffset = page.next?.byteOffset ?? undefined; - } while (position === 0); - assert.deepEqual( - JSON.parse(Buffer.concat(fragments.map(({ data }) => data)).toString('utf8')), - message, - ); - assert.equal(new Set(fragments.map(({ payloadDigest }) => payloadDigest)).size, 1); - assert.match(fragments[0]?.payloadDigest ?? '', /^sha256:[0-9a-f]{64}$/); + assert.deepEqual(await store.readMessages(session.id), [message, smallMessage]); } finally { await store.close?.(); } @@ -873,17 +764,6 @@ describe('SQLite SessionStore', () => { const migrated = createSessionStore(root); try { - const page = await migrated.readTranscriptPageSnapshot(sessionId, { - direction: 'older', - throughSequence: 0, - maxBytes: 50_000, - maxMessages: 1, - }); - assert.equal(page.fragments[0]?.data.byteLength, 50_000); - assert.equal( - page.fragments[0]?.totalBytes, - Buffer.byteLength(JSON.stringify(message), 'utf8'), - ); assert.deepEqual(await migrated.readMessages(sessionId), [message, smallMessage]); } finally { await migrated.close?.(); @@ -892,7 +772,7 @@ describe('SQLite SessionStore', () => { await rm(root, { recursive: true, force: true }); }); - test('rejects corrupt chunked messages on paged and ordinary reads', async () => { + test('rejects corrupt chunked messages on ordinary reads', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-session-transcript-corruption-')); const store = createSessionStore(root); let sessionId = ''; @@ -928,47 +808,6 @@ describe('SQLite SessionStore', () => { const corrupted = createSessionStore(root); try { - await assert.rejects( - corrupted.readTranscriptPageSnapshot(sessionId, { - direction: 'newer', - throughSequence: 0, - position: 0, - byteOffset: 64 * 1024, - maxBytes: 1_000, - maxMessages: 1, - }), - /incompatible/i, - ); - const rewritten = new DatabaseSync(path); - try { - const chunk = rewritten - .prepare( - ` - SELECT data FROM session_message_chunks - WHERE session_id = ? AND sequence = 0 AND chunk_index = 1 - `, - ) - .get(sessionId) as { data: Uint8Array }; - rewritten - .prepare( - ` - UPDATE session_message_chunks SET sha256 = ? - WHERE session_id = ? AND sequence = 0 AND chunk_index = 1 - `, - ) - .run(createHash('sha256').update(chunk.data).digest('hex'), sessionId); - } finally { - rewritten.close(); - } - const page = await corrupted.readTranscriptPageSnapshot(sessionId, { - direction: 'newer', - throughSequence: 0, - position: 0, - byteOffset: 64 * 1024, - maxBytes: 1_000, - maxMessages: 1, - }); - assert.match(page.fragments[0]?.payloadDigest ?? '', /^sha256:[0-9a-f]{64}$/); await assert.rejects(corrupted.readMessages(sessionId), /incompatible/i); } finally { await corrupted.close?.(); @@ -1060,319 +899,6 @@ describe('SQLite SessionStore', () => { } }); - test('pages turn contributions at a fixed transcript watermark', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-turn-contributions-')); - const store = createSessionStore(root); - try { - const session = await store.create(makeInput()); - await store.appendMessages(session.id, [ - { type: 'user', id: 'user-1', turnId: 'turn-1', ts: 1, text: 'one' }, - { - type: 'assistant', - id: 'assistant-1', - turnId: 'turn-1', - ts: 2, - text: 'answer', - modelId: 'model-1', - }, - { - type: 'turn_state', - id: 'state-1', - turnId: 'turn-1', - ts: 3, - status: 'completed', - partialOutputRetained: true, - }, - { type: 'user', id: 'user-2', turnId: 'turn-2', ts: 4, text: 'two' }, - ]); - - const first = await store.readTurnContributionsSnapshot(session.id, null, 0, 1); - assert.equal(first.throughSequence, 3); - assert.equal(first.nextPosition, 3); - assert.deepEqual(first.contributions, [ - { - turnId: 'turn-1', - firstSequence: 0, - latestState: { - sequence: 2, - message: { - type: 'turn_state', - id: 'state-1', - turnId: 'turn-1', - ts: 3, - status: 'completed', - partialOutputRetained: true, - }, - }, - userPromptPreview: 'one', - hasAssistantMessage: true, - hasAssistantOutput: true, - hasToolResult: false, - hasFailedToolResult: false, - hasAbortNote: false, - }, - ]); - - await store.appendMessage(session.id, { - type: 'assistant', - id: 'assistant-2', - turnId: 'turn-2', - ts: 5, - text: 'later', - modelId: 'model-1', - }); - const second = await store.readTurnContributionsSnapshot( - session.id, - first.throughSequence, - first.nextPosition!, - 1, - ); - assert.equal(second.throughSequence, 3); - assert.deepEqual( - second.contributions.map((entry) => entry.turnId), - ['turn-2'], - ); - assert.equal(second.nextPosition, null); - - await store.appendMessage(session.id, { - type: 'assistant', - id: 'assistant-large', - turnId: 'turn-large', - ts: 6, - text: 'x'.repeat(70 * 1024), - modelId: 'model-1', - }); - const chunked = await store.readTurnContributionsSnapshot(session.id, null, 0, 128); - assert.deepEqual( - chunked.contributions.find((entry) => entry.turnId === 'turn-large'), - { - turnId: 'turn-large', - firstSequence: 5, - latestState: null, - userPromptPreview: null, - hasAssistantMessage: true, - hasAssistantOutput: true, - hasToolResult: false, - hasFailedToolResult: false, - hasAbortNote: false, - }, - ); - } finally { - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('bounds turn contribution source scanning independently of turn count', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-turn-source-bound-')); - const store = createSessionStore(root); - try { - const session = await store.create(makeInput()); - await store.appendMessages( - session.id, - Array.from({ length: 1_025 }, (_, index) => ({ - type: 'assistant' as const, - id: `assistant-${index}`, - turnId: 'turn-1', - ts: index, - text: 'x', - modelId: 'model-1', - })), - ); - - const first = await store.readTurnContributionsSnapshot(session.id, null, 0, 128); - assert.equal(first.nextPosition, 1_024); - assert.equal(first.contributions.length, 1); - const second = await store.readTurnContributionsSnapshot( - session.id, - first.throughSequence, - first.nextPosition!, - 128, - ); - assert.equal(second.nextPosition, null); - assert.equal(second.contributions.length, 1); - } finally { - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('samples a bounded prompt landmark index across the durable transcript', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-turn-landmarks-')); - const store = createSessionStore(root); - try { - const session = await store.create(makeInput()); - await store.appendMessages( - session.id, - Array.from({ length: 40 }, (_, index) => [ - { - type: 'user' as const, - id: `user-${index}`, - turnId: `turn-${index}`, - ts: index * 2, - text: index === 20 ? 'x'.repeat(70 * 1024) : `prompt ${index}`, - }, - { - type: 'assistant' as const, - id: `assistant-${index}`, - turnId: `turn-${index}`, - ts: index * 2 + 1, - text: 'answer', - modelId: 'model-1', - }, - ]).flat(), - ); - - const snapshot = await store.readTurnLandmarksSnapshot(session.id, 8); - - assert.equal(snapshot.throughSequence, 79); - assert.ok(snapshot.landmarks.length <= 8); - assert.ok(snapshot.landmarks.length > 1); - assert.equal( - snapshot.landmarks.some((landmark) => landmark.turnId === 'turn-20'), - false, - ); - assert.deepEqual( - [...snapshot.landmarks].sort((left, right) => left.sequence - right.sequence), - snapshot.landmarks, - ); - } finally { - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('keeps every prompt landmark when long turns fit within the landmark limit', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-turn-landmarks-long-turns-')); - const store = createSessionStore(root); - try { - const session = await store.create(makeInput()); - await store.appendMessages( - session.id, - Array.from({ length: 3 }, (_, turnIndex) => [ - { - type: 'user' as const, - id: `user-${turnIndex}`, - turnId: `turn-${turnIndex}`, - ts: turnIndex * 10_000, - text: `prompt ${turnIndex}`, - }, - ...Array.from({ length: turnIndex === 0 ? 1_000 : 4_000 }, (_, messageIndex) => ({ - type: 'assistant' as const, - id: `assistant-${turnIndex}-${messageIndex}`, - turnId: `turn-${turnIndex}`, - ts: turnIndex * 10_000 + messageIndex + 1, - text: 'x', - modelId: 'model-1', - })), - ]).flat(), - ); - const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); - try { - const insert = database.prepare(` - INSERT INTO core_root_turn_admissions(session_id, turn_id, admitted_at, record_json) - VALUES (?, ?, ?, ?) - `); - for (let turnIndex = 0; turnIndex < 3; turnIndex += 1) { - insert.run( - session.id, - `turn-${turnIndex}`, - turnIndex, - JSON.stringify({ userMessageId: `user-${turnIndex}` }), - ); - } - } finally { - database.close(); - } - - const snapshot = await store.readTurnLandmarksSnapshot(session.id, 64); - - assert.deepEqual( - snapshot.landmarks.map((landmark) => landmark.turnId), - ['turn-0', 'turn-1', 'turn-2'], - ); - } finally { - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('keeps legacy prompts when newer turns have indexed admissions', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-turn-landmarks-mixed-')); - const store = createSessionStore(root); - try { - const session = await store.create(makeInput()); - await store.appendMessages( - session.id, - Array.from({ length: 10 }, (_, index) => [ - { - type: 'user' as const, - id: `user-${index}`, - turnId: `turn-${index}`, - ts: index * 2, - text: `prompt ${index}`, - }, - { - type: 'assistant' as const, - id: `assistant-${index}`, - turnId: `turn-${index}`, - ts: index * 2 + 1, - text: 'answer', - modelId: 'model-1', - }, - ]).flat(), - ); - const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); - try { - database - .prepare(` - INSERT INTO core_root_turn_admissions(session_id, turn_id, admitted_at, record_json) - VALUES (?, ?, ?, ?) - `) - .run(session.id, 'turn-9', 9, JSON.stringify({ userMessageId: 'user-9' })); - } finally { - database.close(); - } - - const snapshot = await store.readTurnLandmarksSnapshot(session.id, 8); - - assert.equal(snapshot.landmarks.length, 8); - assert.equal(snapshot.landmarks[0]?.turnId, 'turn-0'); - assert.equal(snapshot.landmarks.at(-1)?.turnId, 'turn-9'); - } finally { - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('clears unread when the current read marker is already the latest visible message', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-read-marker-')); - const store = createSessionStore(root); - try { - const session = await store.create(makeInput()); - await store.appendMessage(session.id, { - type: 'assistant', - id: 'message-1', - turnId: 'turn-1', - ts: 20, - text: 'already read', - modelId: 'fake-model', - }); - await store.updateHeader(session.id, { - lastReadMessageId: 'message-1', - hasUnread: true, - }); - - const updated = await store.markSessionReadThroughMessage(session.id, 'message-1'); - - assert.equal(updated.header.lastReadMessageId, 'message-1'); - assert.equal(updated.header.hasUnread, false); - } finally { - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - test('reads back a legacy fake-backend session instead of migrating or rejecting it', async () => { // #3211: `'fake'` was retired as a live backend but never migrated out of // storage. Narrowing the header validator would make these rows decode as diff --git a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts index 906c5d6ace..3d5df5370d 100644 --- a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts @@ -281,7 +281,7 @@ describe('SqliteSessionMetadataStore', () => { const store = createSqliteSessionMetadataStore(path); try { await assert.rejects( - () => store.readMessagesForRecovery('session-1'), + () => store.readMessages('session-1'), (error: unknown) => error instanceof StoredSessionMessageIncompatibleError && error.code === 'stored_session_message_incompatible' && @@ -365,7 +365,7 @@ describe('SqliteSessionMetadataStore', () => { } }); - test('materializes an accepted steering draft when it is handed off', async () => { + test('retires an accepted steering draft when it is handed off', async () => { const store = createSqliteSessionMetadataStore(':memory:'); try { await store.create(fullHeader({ id: 'session-1', connectionLocked: false })); @@ -424,27 +424,12 @@ describe('SqliteSessionMetadataStore', () => { messageIds: ['message-1'], turnId: 'turn-1', }); - assert.deepEqual( - (await store.readMessages('session-1')).map((message) => ({ - id: message.id, - type: message.type, - turnId: message.turnId, - text: message.type === 'user' ? message.text : undefined, - steeringEventId: message.type === 'user' ? message.steeringEventId : undefined, - })), - [ - { - id: 'message-1', - type: 'user', - turnId: 'turn-1', - text: 'submitted', - steeringEventId: 'message-1', - }, - ], - ); - assert.equal((await store.read('session-1')).header.lastMessageAt, 10); - assert.equal((await store.readCatalogRecord('session-1')).lastMessagePreview, 'submitted'); - assert.equal((await store.read('session-1')).header.connectionLocked, true); + // Handoff retires the admission and nothing else: the message itself is a + // RuntimeEvent, and its catalog facts come from the run that wrote it. + assert.deepEqual(await store.readMessages('session-1'), []); + assert.equal((await store.read('session-1')).header.lastMessageAt, 3); + assert.equal((await store.readCatalogRecord('session-1')).lastMessagePreview, undefined); + assert.equal((await store.read('session-1')).header.connectionLocked, false); assert.deepEqual(await store.listMessageAdmissions('session-1'), []); } finally { store.close(); @@ -560,229 +545,6 @@ describe('SqliteSessionMetadataStore', () => { } }); - test('materializes a proven Root message when its admission is absent', async () => { - const store = createSqliteSessionMetadataStore(':memory:'); - try { - await store.create(fullHeader({ id: 'session-legacy-root' })); - - await markMessagesHandedOffWithProvenRoots(store, { - sessionId: 'session-legacy-root', - messageIds: ['message-legacy-root'], - turnId: 'turn-legacy-root', - provenRootMessages: [ - { - messageId: 'message-legacy-root', - content: { text: 'retained by the legacy Root', displayText: 'legacy display' }, - admittedAt: 17, - }, - ], - }); - - assert.deepEqual(await store.readMessages('session-legacy-root'), [ - { - type: 'user', - id: 'message-legacy-root', - turnId: 'turn-legacy-root', - ts: 17, - text: 'retained by the legacy Root', - displayText: 'legacy display', - steeringEventId: 'message-legacy-root', - }, - ]); - assert.deepEqual(await store.listMessageAdmissions('session-legacy-root'), []); - } finally { - store.close(); - } - }); - - test('inserts proven Root messages before existing output from their Turn', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-legacy-root-order-')); - const path = join(root, 'state.sqlite'); - const store = createSqliteSessionMetadataStore(path); - try { - await store.create(fullHeader({ id: 'session-legacy-order' })); - const legacyOutput = 'existing chunked output '.repeat(4_096); - await store.appendMessages( - 'session-legacy-order', - [ - { - type: 'assistant', - id: 'message-prior-output', - turnId: 'turn-prior', - ts: 10, - text: 'prior output', - modelId: 'fake-model', - }, - { - type: 'assistant', - id: 'message-legacy-output', - turnId: 'turn-legacy-order', - ts: 18, - text: legacyOutput, - modelId: 'fake-model', - }, - { - type: 'user', - id: 'message-newer-user', - turnId: 'turn-newer', - ts: 30, - text: 'newest preview', - }, - ], - { lastMessageAt: 30, lastMessagePreview: 'newest preview' }, - ); - - await markMessagesHandedOffWithProvenRoots(store, { - sessionId: 'session-legacy-order', - messageIds: ['message-legacy-followup', 'message-legacy-steering'], - turnId: 'turn-legacy-order', - provenRootMessages: [ - { - messageId: 'message-legacy-followup', - content: { text: 'legacy follow-up' }, - admittedAt: 17, - }, - { - messageId: 'message-legacy-steering', - content: { text: 'legacy steering' }, - admittedAt: 17, - }, - ], - }); - - assert.deepEqual( - (await store.readMessages('session-legacy-order')).map((message) => message.id), - [ - 'message-prior-output', - 'message-legacy-followup', - 'message-legacy-steering', - 'message-legacy-output', - 'message-newer-user', - ], - ); - assert.equal((await store.read('session-legacy-order')).header.lastMessageAt, 30); - const shiftedOutput = (await store.readMessages('session-legacy-order')).find( - (message) => message.id === 'message-legacy-output', - ); - assert.equal(shiftedOutput?.type, 'assistant'); - assert.equal( - shiftedOutput?.type === 'assistant' ? shiftedOutput.text : undefined, - legacyOutput, - ); - assert.equal( - (await store.readCatalogRecord('session-legacy-order')).lastMessagePreview, - 'newest preview', - ); - const audit = new DatabaseSync(path, { readOnly: true }); - try { - assert.deepEqual(audit.prepare('PRAGMA foreign_key_check').all(), []); - } finally { - audit.close(); - } - } finally { - store.close(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('places a proven Root message before an equally-timed newer transcript row', async () => { - const store = createSqliteSessionMetadataStore(':memory:'); - try { - await store.create(fullHeader({ id: 'session-legacy-time-tie' })); - await store.appendMessages( - 'session-legacy-time-tie', - [ - { - type: 'user', - id: 'message-newer-time-tie', - turnId: 'turn-newer-time-tie', - ts: 17, - text: 'newer same-millisecond preview', - }, - ], - { lastMessageAt: 17, lastMessagePreview: 'newer same-millisecond preview' }, - ); - - await markMessagesHandedOffWithProvenRoots(store, { - sessionId: 'session-legacy-time-tie', - messageIds: ['message-legacy-time-tie'], - turnId: 'turn-legacy-time-tie', - provenRootMessages: [ - { - messageId: 'message-legacy-time-tie', - content: { text: 'legacy same-millisecond source' }, - admittedAt: 17, - }, - ], - }); - - assert.deepEqual( - (await store.readMessages('session-legacy-time-tie')).map((message) => message.id), - ['message-legacy-time-tie', 'message-newer-time-tie'], - ); - assert.equal( - (await store.readCatalogRecord('session-legacy-time-tie')).lastMessagePreview, - 'newer same-millisecond preview', - ); - } finally { - store.close(); - } - }); - - test('keeps ordinary admission handoff append semantics when Root proof is also supplied', async () => { - const store = createSqliteSessionMetadataStore(':memory:'); - try { - await store.create(fullHeader({ id: 'session-ordinary-handoff-order' })); - await store.appendMessages( - 'session-ordinary-handoff-order', - [ - { - type: 'assistant', - id: 'message-existing-ordinary-output', - turnId: 'turn-ordinary-handoff-order', - ts: 20, - text: 'existing output', - modelId: 'fake-model', - }, - ], - { lastMessageAt: 20, lastMessagePreview: 'existing output' }, - ); - await store.commitMessageAdmission({ - sessionId: 'session-ordinary-handoff-order', - turnId: 'turn-ordinary-handoff-order', - runId: 'run-ordinary-handoff-order', - messageId: 'message-ordinary-admission', - content: { text: 'ordinary admission' }, - submittedContentDigest: messageContentDigest({ text: 'ordinary admission' }), - submittedPlacement: 'current_turn', - placement: 'current_turn', - disposition: 'steering', - skillInvocation: { loaded: [], failed: [], receipts: [] }, - admittedAt: 10, - }); - - await markMessagesHandedOffWithProvenRoots(store, { - sessionId: 'session-ordinary-handoff-order', - messageIds: ['message-ordinary-admission'], - turnId: 'turn-ordinary-handoff-order', - provenRootMessages: [ - { - messageId: 'message-ordinary-admission', - content: { text: 'ordinary admission' }, - admittedAt: 10, - }, - ], - }); - - assert.deepEqual( - (await store.readMessages('session-ordinary-handoff-order')).map((message) => message.id), - ['message-existing-ordinary-output', 'message-ordinary-admission'], - ); - } finally { - store.close(); - } - }); - test('rejects an admission handed off to a different Turn', async () => { const store = createSqliteSessionMetadataStore(':memory:'); try { @@ -819,181 +581,23 @@ describe('SqliteSessionMetadataStore', () => { } }); - test('rejects fully materialized proven Root sources in a conflicting order', async () => { - const store = createSqliteSessionMetadataStore(':memory:'); - try { - await store.create(fullHeader({ id: 'session-existing-source-order' })); - await store.appendMessages( - 'session-existing-source-order', - [ - { - type: 'user', - id: 'message-existing-source-b', - turnId: 'turn-existing-source-order', - ts: 25, - text: 'source b', - }, - { - type: 'user', - id: 'message-existing-source-a', - turnId: 'turn-existing-source-order', - ts: 25, - text: 'source a', - }, - ], - { lastMessageAt: 25, lastMessagePreview: 'source a' }, - ); - - await assert.rejects( - markMessagesHandedOffWithProvenRoots(store, { - sessionId: 'session-existing-source-order', - messageIds: ['message-existing-source-a', 'message-existing-source-b'], - turnId: 'turn-existing-source-order', - provenRootMessages: [ - { - messageId: 'message-existing-source-a', - content: { text: 'source a' }, - admittedAt: 25, - }, - { - messageId: 'message-existing-source-b', - content: { text: 'source b' }, - admittedAt: 25, - }, - ], - }), - /source order conflict/, - ); - } finally { - store.close(); - } - }); - - test('rejects a partial proven Root group that already crosses newer history', async () => { - const store = createSqliteSessionMetadataStore(':memory:'); - try { - await store.create(fullHeader({ id: 'session-partial-source-order' })); - await store.appendMessages( - 'session-partial-source-order', - [ - { - type: 'user', - id: 'message-partial-source-a', - turnId: 'turn-partial-source-order', - ts: 15, - text: 'source a', - }, - { - type: 'user', - id: 'message-partial-newer-tail', - turnId: 'turn-partial-newer', - ts: 30, - text: 'newer tail', - }, - { - type: 'user', - id: 'message-partial-source-c', - turnId: 'turn-partial-source-order', - ts: 15, - text: 'source c', - }, - ], - { lastMessageAt: 30, lastMessagePreview: 'newer tail' }, - ); - - await assert.rejects( - markMessagesHandedOffWithProvenRoots(store, { - sessionId: 'session-partial-source-order', - messageIds: [ - 'message-partial-source-a', - 'message-partial-source-b', - 'message-partial-source-c', - ], - turnId: 'turn-partial-source-order', - provenRootMessages: [ - { - messageId: 'message-partial-source-a', - content: { text: 'source a' }, - admittedAt: 15, - }, - { - messageId: 'message-partial-source-b', - content: { text: 'source b' }, - admittedAt: 15, - }, - { - messageId: 'message-partial-source-c', - content: { text: 'source c' }, - admittedAt: 15, - }, - ], - }), - /source order conflict/, - ); - assert.deepEqual( - (await store.readMessages('session-partial-source-order')).map((message) => message.id), - ['message-partial-source-a', 'message-partial-newer-tail', 'message-partial-source-c'], - ); - } finally { - store.close(); - } - }); - - test('rejects an unsafe proven Root tail insertion range', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-legacy-root-overflow-')); - const path = join(root, 'state.sqlite'); - const store = createSqliteSessionMetadataStore(path); - try { - await store.create(fullHeader({ id: 'session-legacy-overflow' })); - await store.appendMessages( - 'session-legacy-overflow', - [ - { - type: 'assistant', - id: 'message-overflow-anchor', - turnId: 'turn-overflow-anchor', - ts: 1, - text: 'anchor', - modelId: 'fake-model', - }, - ], - { lastMessageAt: 1, lastMessagePreview: 'anchor' }, - ); - const database = new DatabaseSync(path); - try { - database - .prepare('UPDATE session_messages SET sequence = ? WHERE session_id = ? AND sequence = 0') - .run(Number.MAX_SAFE_INTEGER - 1, 'session-legacy-overflow'); - } finally { - database.close(); - } - - await assert.rejects( - markMessagesHandedOffWithProvenRoots(store, { - sessionId: 'session-legacy-overflow', - messageIds: ['message-overflow-a', 'message-overflow-b'], - turnId: 'turn-legacy-overflow', - provenRootMessages: [ - { messageId: 'message-overflow-a', content: { text: 'a' }, admittedAt: 2 }, - { messageId: 'message-overflow-b', content: { text: 'b' }, admittedAt: 2 }, - ], - }), - /sequence overflow/, - ); - assert.deepEqual( - (await store.readMessages('session-legacy-overflow')).map((message) => message.id), - ['message-overflow-anchor'], - ); - } finally { - store.close(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('repeats a proven Root message handoff without duplicating its transcript', async () => { + test('repeats a proven Root message handoff after its admission is gone', async () => { const store = createSqliteSessionMetadataStore(':memory:'); try { await store.create(fullHeader({ id: 'session-legacy-repeat' })); + await store.commitMessageAdmission({ + sessionId: 'session-legacy-repeat', + turnId: 'turn-legacy-repeat', + runId: 'run-legacy-repeat', + messageId: 'message-legacy-repeat', + content: { text: 'a single durable message' }, + submittedContentDigest: messageContentDigest({ text: 'a single durable message' }), + submittedPlacement: 'current_turn', + placement: 'current_turn', + disposition: 'steering', + skillInvocation: { loaded: [], failed: [], receipts: [] }, + admittedAt: 18, + }); const input = { sessionId: 'session-legacy-repeat', messageIds: ['message-legacy-repeat'], @@ -1001,7 +605,7 @@ describe('SqliteSessionMetadataStore', () => { provenRootMessages: [ { messageId: 'message-legacy-repeat', - content: { text: 'a single durable transcript message' }, + content: { text: 'a single durable message' }, admittedAt: 18, }, ], @@ -1010,14 +614,8 @@ describe('SqliteSessionMetadataStore', () => { await markMessagesHandedOffWithProvenRoots(store, input); await markMessagesHandedOffWithProvenRoots(store, input); - assert.deepEqual( - (await store.readMessages('session-legacy-repeat')).map((message) => ({ - id: message.id, - turnId: message.turnId, - ts: message.ts, - })), - [{ id: 'message-legacy-repeat', turnId: 'turn-legacy-repeat', ts: 18 }], - ); + assert.deepEqual(await store.listMessageAdmissions('session-legacy-repeat'), []); + assert.deepEqual(await store.readMessages('session-legacy-repeat'), []); } finally { store.close(); } @@ -1080,102 +678,6 @@ describe('SqliteSessionMetadataStore', () => { } }); - test('rejects proven Root repeats with an existing transcript content or Turn conflict', async () => { - const store = createSqliteSessionMetadataStore(':memory:'); - try { - await store.create(fullHeader({ id: 'session-legacy-conflict' })); - await markMessagesHandedOffWithProvenRoots(store, { - sessionId: 'session-legacy-conflict', - messageIds: ['message-legacy-conflict'], - turnId: 'turn-legacy-conflict', - provenRootMessages: [ - { - messageId: 'message-legacy-conflict', - content: { text: 'canonical text' }, - admittedAt: 20, - }, - ], - }); - - await assert.rejects( - markMessagesHandedOffWithProvenRoots(store, { - sessionId: 'session-legacy-conflict', - messageIds: ['message-legacy-conflict'], - turnId: 'turn-legacy-conflict', - provenRootMessages: [ - { - messageId: 'message-legacy-conflict', - content: { text: 'different text' }, - admittedAt: 20, - }, - ], - }), - /transcript identity conflict/, - ); - await assert.rejects( - markMessagesHandedOffWithProvenRoots(store, { - sessionId: 'session-legacy-conflict', - messageIds: ['message-legacy-conflict'], - turnId: 'turn-legacy-conflict-different', - provenRootMessages: [ - { - messageId: 'message-legacy-conflict', - content: { text: 'canonical text' }, - admittedAt: 20, - }, - ], - }), - /transcript Turn conflict/, - ); - } finally { - store.close(); - } - }); - - test('keeps an admission as the content and timestamp authority during handoff', async () => { - const store = createSqliteSessionMetadataStore(':memory:'); - try { - await store.create(fullHeader({ id: 'session-admission-authority' })); - await store.commitMessageAdmission({ - sessionId: 'session-admission-authority', - turnId: 'turn-admission-authority', - runId: 'run-admission-authority', - messageId: 'message-admission-authority', - content: { text: 'admission authority', displayText: 'submitted display' }, - submittedContentDigest: messageContentDigest({ text: 'admission authority' }), - submittedPlacement: 'current_turn', - placement: 'current_turn', - disposition: 'steering', - skillInvocation: { loaded: [], failed: [], receipts: [] }, - admittedAt: 21, - }); - - await markMessagesHandedOffWithProvenRoots(store, { - sessionId: 'session-admission-authority', - messageIds: ['message-admission-authority'], - turnId: 'turn-admission-authority', - provenRootMessages: [ - { - messageId: 'message-admission-authority', - content: { text: 'admission authority', displayText: 'submitted display' }, - admittedAt: 99, - }, - ], - }); - - assert.deepEqual( - (await store.readMessages('session-admission-authority')).map((message) => ({ - text: message.type === 'user' ? message.text : undefined, - ts: message.ts, - })), - [{ text: 'admission authority', ts: 21 }], - ); - assert.deepEqual(await store.listMessageAdmissions('session-admission-authority'), []); - } finally { - store.close(); - } - }); - test('rejects proven Root fallback content that drifts from an admission', async () => { const store = createSqliteSessionMetadataStore(':memory:'); try { @@ -1288,7 +790,7 @@ describe('SqliteSessionMetadataStore', () => { } }); - test('removes the accepted payload after transcript handoff', async () => { + test('removes the accepted payload without writing a transcript row', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-message-handoff-')); const path = join(root, 'state.sqlite'); const store = createSqliteSessionMetadataStore(path); @@ -1332,7 +834,7 @@ describe('SqliteSessionMetadataStore', () => { 'SELECT COUNT(*) AS count FROM session_messages WHERE session_id = ? AND message_id = ?', ) .get('session-1', 'message-1')?.count, - 1, + 0, ); } finally { persisted.close(); @@ -1425,14 +927,7 @@ describe('SqliteSessionMetadataStore', () => { }); assert.equal(await store.readMessageAdmission('session-1', 'message-1'), undefined); - assert.deepEqual( - (await store.readMessages('session-1')).map((message) => ({ - id: message.id, - turnId: message.turnId, - text: message.type === 'user' ? message.text : undefined, - })), - [{ id: 'message-1', turnId: 'turn-2', text: content.text }], - ); + assert.deepEqual(await store.readMessages('session-1'), []); } finally { store.close(); await rm(root, { recursive: true, force: true }); @@ -1605,7 +1100,7 @@ describe('SqliteSessionMetadataStore', () => { } }); - test('materializes an accepted follow-up under its successor root', async () => { + test('retires an accepted follow-up under its successor root', async () => { const store = createSqliteSessionMetadataStore(':memory:'); try { await store.create(fullHeader({ id: 'session-followup-admission' })); @@ -1626,23 +1121,22 @@ describe('SqliteSessionMetadataStore', () => { }); assert.equal(admission.disposition, 'followup'); assert.deepEqual(await store.readMessages('session-followup-admission'), []); - await store.markMessagesHandedOff({ + const handoff = { sessionId: 'session-followup-admission', messageIds: ['message-followup'], turnId: 'turn-successor', - }); - await store.markMessagesHandedOff({ - sessionId: 'session-followup-admission', - messageIds: ['message-followup'], - turnId: 'turn-successor', - }); - assert.deepEqual( - (await store.readMessages('session-followup-admission')).map((message) => ({ - id: message.id, - turnId: message.turnId, - })), - [{ id: 'message-followup', turnId: 'turn-successor' }], - ); + provenRootMessages: [ + { + messageId: 'message-followup', + content: { text: 'queued before the successor root' }, + admittedAt: 11, + }, + ], + }; + await markMessagesHandedOffWithProvenRoots(store, handoff); + await markMessagesHandedOffWithProvenRoots(store, handoff); + assert.deepEqual(await store.listMessageAdmissions('session-followup-admission'), []); + assert.deepEqual(await store.readMessages('session-followup-admission'), []); } finally { store.close(); } diff --git a/packages/storage/src/__tests__/workhub-message-assignment.test.ts b/packages/storage/src/__tests__/workhub-message-assignment.test.ts index a3e69d8160..1faa540c70 100644 --- a/packages/storage/src/__tests__/workhub-message-assignment.test.ts +++ b/packages/storage/src/__tests__/workhub-message-assignment.test.ts @@ -33,6 +33,7 @@ import { type WorkHubDelegationStopResolvedMessage, type WorkHubDelegationSupersededMessage, } from '@maka/core/session'; +import { createSqliteAgentRunStore } from '../agent-run-store.js'; import { createSessionStore, isSessionNotFoundError } from '../session-store.js'; test('atomically commits one WorkHub assignment and target admission', async () => { @@ -82,11 +83,7 @@ test('atomically commits one WorkHub assignment and target admission', async () ]); const coordination = await store.readHeaderSnapshot(WORKHUB_COORDINATION_SESSION_ID); assert.equal(coordination.lastMessageAt, request.assignment.ts); - await store.markMessagesHandedOff({ - sessionId: target.id, - messageIds: [request.admission.messageId], - turnId: request.admission.turnId, - }); + await handOffToRootTurn(store, root, request); const replayAfterConsumption = await store.assignWorkHubMessage(request); assert.equal(replayAfterConsumption.kind, 'existing'); assert.deepEqual(replayAfterConsumption.assignment, request.assignment); @@ -126,11 +123,7 @@ test('scans every target Message lifecycle once and preserves Coordination order assignmentRequest('unrelated-action', unrelated.id, 'Login', 'unrelated-turn'), ); await store.assignWorkHubMessage(middle); - await store.markMessagesHandedOff({ - sessionId: target.id, - messageIds: [middle.admission.messageId], - turnId: middle.admission.turnId, - }); + await handOffToRootTurn(store, root, middle); await store.assignWorkHubMessage(newest); assert.equal( await store.claimMessageAdmissionCancellation( @@ -169,11 +162,7 @@ test('keeps target assignments reachable when their Message lifecycle changes', .sort((left, right) => left.admission.messageId.localeCompare(right.admission.messageId)); for (const request of requests) await store.assignWorkHubMessage(request); - await store.markMessagesHandedOff({ - sessionId: target.id, - messageIds: [requests[1]!.admission.messageId], - turnId: requests[1]!.admission.turnId, - }); + await handOffToRootTurn(store, root, requests[1]!); assert.equal( await store.claimMessageAdmissionCancellation( target.id, @@ -722,6 +711,49 @@ function terminalSuffix(delegationId: string): string { return createHash('sha256').update(delegationId, 'utf8').digest('hex').slice(0, 48); } +/** + * Hand a Message off the way a Turn does: the Root admission that consumed it + * is what keeps its identity durable once the pending admission is retired. + */ +async function handOffToRootTurn( + store: ReturnType, + root: string, + request: AssignmentRequest, +): Promise { + const runStore = createSqliteAgentRunStore(root); + try { + await runStore.admitRootTurn({ + sessionId: request.admission.sessionId, + turnId: request.admission.turnId, + proposedRunId: request.admission.runId, + proposedUserMessageId: request.admission.messageId, + execution: { + kind: 'external_message', + inputDigest: request.admission.submittedContentDigest, + }, + previousRootTurnId: null, + normalizedInput: request.admission.content, + sourceMessages: [ + { + messageId: request.admission.messageId, + content: request.admission.content, + submittedContentDigest: request.admission.submittedContentDigest, + placement: request.admission.placement, + disposition: request.admission.disposition, + }, + ], + admittedAt: request.admission.admittedAt, + }); + } finally { + runStore.close?.(); + } + await store.markMessagesHandedOff({ + sessionId: request.admission.sessionId, + messageIds: [request.admission.messageId], + turnId: request.admission.turnId, + }); +} + type AssignmentRequest = ReturnType; function assignmentRequest( diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index 5b066c5d4a..0343df2cc1 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -83,10 +83,7 @@ export { normalizeRootTurnAdmissionPayload, rootTurnAdmissionRecordFits, } from './agent-run-store.js'; -export { - isSessionNotFoundError, - SessionReadMarkerMessageNotFoundError, -} from './session-store.js'; +export { isSessionNotFoundError } from './session-store.js'; export { SessionMetadataConflictError, SessionMetadataVersionConflictError, @@ -131,6 +128,10 @@ export type { SessionTranscriptRecordScanRequest, SessionTranscriptStoragePage, SessionTranscriptStorageFragment, + SessionTurnContribution, + SessionTurnContributionPage, + SessionTurnLandmark, + SessionTurnLandmarkSnapshot, } from './session-store.js'; export type ExecutionSessionWriter = SessionAuthorityStore; @@ -229,6 +230,10 @@ export interface ExecutionRuntimeEventReader { ): Promise>; readImmutableRuntimeEvents(sessionId: string, runId: string): Promise; readSessionRuntimeEvents(sessionId: string): Promise; + /** Session-wide events with the ordinal that fixes their transcript order. */ + readSessionRuntimeEventEntries( + sessionId: string, + ): Promise>; } interface ExecutionStoresReaderBase { @@ -428,27 +433,10 @@ async function createExecutionStoresForWrite run(() => sessionStore.readCatalogRecord(sessionId)), probeSessionRemoval: (sessionId) => run(() => sessionStore.probeSessionRemoval(sessionId)), readMessagesSnapshot: (sessionId) => run(() => sessionStore.readMessagesSnapshot(sessionId)), - readTranscriptPageSnapshot: (sessionId, request) => - run(() => sessionStore.readTranscriptPageSnapshot(sessionId, request)), - readTranscriptRecordsSnapshot: (sessionId, request) => - run(() => sessionStore.readTranscriptRecordsSnapshot(sessionId, request)), readTranscriptMessagesSnapshot: (sessionId, request) => run(() => sessionStore.readTranscriptMessagesSnapshot(sessionId, request)), readTranscriptHighWaterSnapshot: (sessionId) => run(() => sessionStore.readTranscriptHighWaterSnapshot(sessionId)), - readTurnContributionsSnapshot: (sessionId, throughSequence, position, maxContributions) => - run(() => - sessionStore.readTurnContributionsSnapshot( - sessionId, - throughSequence, - position, - maxContributions, - ), - ), - readTurnLandmarksSnapshot: (sessionId, maxLandmarks) => - run(() => sessionStore.readTurnLandmarksSnapshot(sessionId, maxLandmarks)), - readMessagesForRecovery: (sessionId) => - run(() => sessionStore.readMessagesForRecovery(sessionId)), listTurnsSnapshot: (sessionId) => run(() => sessionStore.listTurnsSnapshot(sessionId)), readHeader: (sessionId) => run(() => sessionStore.readHeader(sessionId)), readMessages: (sessionId) => run(() => sessionStore.readMessages(sessionId)), @@ -457,6 +445,8 @@ async function createExecutionStoresForWrite sessionStore.appendMessage(sessionId, message)), appendMessages: (sessionId, messages) => run(() => sessionStore.appendMessages(sessionId, messages)), + commitMessageCatalogProjection: (sessionId, message) => + run(() => sessionStore.commitMessageCatalogProjection(sessionId, message)), commitMessageAdmission: (admission) => run(() => sessionStore.commitMessageAdmission(admission)), readMessageAdmission: (sessionId, messageId) => @@ -480,8 +470,6 @@ async function createExecutionStoresForWrite sessionStore.updateHeaderVersioned(sessionId, patch, expectedRevision)), updateSessionConfiguration: (sessionId, input) => run(() => sessionStore.updateSessionConfiguration(sessionId, input)), - markSessionReadThroughMessage: (sessionId, messageId) => - run(() => sessionStore.markSessionReadThroughMessage(sessionId, messageId)), setFlagged: (sessionId, isFlagged) => run(() => sessionStore.setFlagged(sessionId, isFlagged)), rename: (sessionId, name) => run(() => sessionStore.rename(sessionId, name)), @@ -698,6 +686,8 @@ async function openExecutionStoresForRead runtimeEventStore.readInvocation(sessionId, invocationId)), readSessionRuntimeEvents: (sessionId) => run(() => runtimeEventStore.readSessionRuntimeEvents(sessionId)), + readSessionRuntimeEventEntries: (sessionId) => + run(() => runtimeEventStore.readSessionRuntimeEventEntries(sessionId)), }, }; freezeExecutionStoresFacade(stores); diff --git a/packages/storage/src/runtime-event-persistence.ts b/packages/storage/src/runtime-event-persistence.ts index 893e734e50..9141585e98 100644 --- a/packages/storage/src/runtime-event-persistence.ts +++ b/packages/storage/src/runtime-event-persistence.ts @@ -65,6 +65,10 @@ export interface RuntimeEventReadStore { ): Promise>; readImmutableRuntimeEvents(sessionId: string, runId: string): Promise; readSessionRuntimeEvents(sessionId: string): Promise; + /** Session-wide events with the ordinal that fixes their transcript order. */ + readSessionRuntimeEventEntries( + sessionId: string, + ): Promise>; } export async function openRuntimeEventPersistence(input: { @@ -112,6 +116,8 @@ export async function openRuntimeEventReadPersistence(input: { readImmutableRuntimeEvents: (sessionId: string, runId: string) => store.readImmutableRuntimeEvents(sessionId, runId), readSessionRuntimeEvents: (sessionId: string) => store.readSessionRuntimeEvents(sessionId), + readSessionRuntimeEventEntries: (sessionId: string) => + store.readSessionRuntimeEventEntries(sessionId), }), close: () => store.close(), }; diff --git a/packages/storage/src/session-message-projection.ts b/packages/storage/src/session-message-projection.ts index 5284e5720a..94fc0fb3a2 100644 --- a/packages/storage/src/session-message-projection.ts +++ b/packages/storage/src/session-message-projection.ts @@ -18,6 +18,7 @@ */ import type { StoredMessage, UserMessage } from '@maka/core/session'; +import type { SessionTurnContribution } from './session-store.js'; export function projectSessionCatalogMessages(messages: readonly StoredMessage[]): { readonly lastMessageAt?: number; @@ -87,3 +88,43 @@ function truncatePreview(text: string, maxLength = 96): string { if (chars.length <= maxLength) return text; return `${chars.slice(0, maxLength - 1).join('')}…`; } + +/** + * One Turn's summary, folded message by message in transcript order. + * + * Both transcript authorities fold the same way: the sqlite catalog over its + * own rows, and the ledger reader over the messages a run projects. + */ +export function foldTurnContribution( + current: SessionTurnContribution | undefined, + turnId: string, + sequence: number, + message: StoredMessage, +): SessionTurnContribution { + const contribution = current ?? { + turnId, + firstSequence: sequence, + latestState: null, + userPromptPreview: null, + hasAssistantMessage: false, + hasAssistantOutput: false, + hasToolResult: false, + hasFailedToolResult: false, + hasAbortNote: false, + }; + const userPrompt = message.type === 'user' ? (message.displayText ?? message.text).trim() : ''; + return { + ...contribution, + latestState: message.type === 'turn_state' ? { sequence, message } : contribution.latestState, + userPromptPreview: contribution.userPromptPreview ?? (userPrompt || null), + hasAssistantMessage: contribution.hasAssistantMessage || message.type === 'assistant', + hasAssistantOutput: + contribution.hasAssistantOutput || + (message.type === 'assistant' && message.text.trim().length > 0), + hasToolResult: contribution.hasToolResult || message.type === 'tool_result', + hasFailedToolResult: + contribution.hasFailedToolResult || (message.type === 'tool_result' && message.isError), + hasAbortNote: + contribution.hasAbortNote || (message.type === 'system_note' && message.kind === 'abort'), + }; +} diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index eb05f3fb1f..992cb8322f 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -82,6 +82,7 @@ import { type StoredMessage, type TurnRecord, type TurnStateMessage, + type AssistantMessage, type UserMessage, type WorkHubDelegationAssignedMessage, type WorkHubDelegationReplacementAbortedMessage, @@ -97,12 +98,7 @@ import type { MessageAdmissionStore, PendingMessageAdmission, } from './message-admission-store.js'; -import { - isVisibleSessionMessage, - lastMessagePreviewForMessages, - latestVisibleMessageAt, - projectSessionCatalogMessages, -} from './session-message-projection.js'; +import { projectSessionCatalogMessages } from './session-message-projection.js'; export { projectSessionCatalogMessages }; const SESSION_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; @@ -127,18 +123,6 @@ export function isSessionNotFoundError(error: unknown): error is SessionNotFound return error instanceof SessionNotFoundError; } -export class SessionReadMarkerMessageNotFoundError extends Error { - readonly name = 'SessionReadMarkerMessageNotFoundError'; - readonly code = 'session_read_marker_message_not_found'; - - constructor( - readonly sessionId: string, - readonly messageId: string, - ) { - super(`Session read marker message does not exist: ${messageId}`); - } -} - export interface SessionHeaderSnapshot { readonly header: SessionHeader; readonly revision: number; @@ -318,33 +302,19 @@ export interface SessionStore { listForRecovery(): Promise; /** Read only the durable header without triggering connection-lock self-healing. */ readHeaderSnapshot(sessionId: string): Promise; - /** Read durable messages without triggering connection-lock self-healing. */ readMessagesSnapshot(sessionId: string): Promise; - /** Read one byte-bounded page directly from the durable append-only ledger. */ - readTranscriptPageSnapshot( - sessionId: string, - request: SessionTranscriptPageRequest, - ): Promise; readTranscriptHighWaterSnapshot(sessionId: string): Promise; - readTurnContributionsSnapshot( - sessionId: string, - throughSequence: number | null, - position: number, - maxContributions: number, - ): Promise; - readTurnLandmarksSnapshot( - sessionId: string, - maxLandmarks: number, - ): Promise; - /** Read durable messages for startup recovery. */ - readMessagesForRecovery(sessionId: string): Promise; - /** Derive durable turns without triggering connection-lock self-healing. */ listTurnsSnapshot(sessionId: string): Promise; readHeader(sessionId: string): Promise; readMessages(sessionId: string): Promise; listTurns(sessionId: string): Promise; appendMessage(sessionId: string, message: StoredMessage): Promise; appendMessages(sessionId: string, messages: StoredMessage[]): Promise; + /** Commit the Session-list facts a durable message carries. */ + commitMessageCatalogProjection( + sessionId: string, + message: UserMessage | AssistantMessage, + ): Promise; updateHeader(sessionId: string, patch: SessionHeaderPatch): Promise; setFlagged(sessionId: string, isFlagged: boolean): Promise; rename(sessionId: string, name: string): Promise; @@ -354,11 +324,6 @@ export interface SessionStore { } export interface SessionAuthorityStore extends SessionStore, MessageAdmissionStore { - /** Decode a bounded ledger range for an authority-owned wire projection. */ - readTranscriptRecordsSnapshot( - sessionId: string, - request: SessionTranscriptRecordScanRequest, - ): Promise; /** Read a bounded set of durable messages at an inclusive transcript watermark. */ readTranscriptMessagesSnapshot( sessionId: string, @@ -470,10 +435,6 @@ export interface SessionAuthorityStore extends SessionStore, MessageAdmissionSto sessionId: string, input: UpdateSessionConfigurationRequest, ): Promise; - markSessionReadThroughMessage( - sessionId: string, - messageId: string, - ): Promise; probeSessionRemoval(sessionId: string): Promise; setSessionsArchivedVersioned( sessions: readonly VersionedSessionIdentity[], @@ -898,41 +859,9 @@ class SqliteSessionStore implements SessionAuthorityStore { async list(filter?: SessionListFilter): Promise { await this.ensureReady(); - const records = (await this.metadata.list(filter, 'ordinary')).filter( - (record) => record.header.conversationCopy?.state !== 'preparing', - ); - const withPreviews: Array<{ - record: SessionMetadataRecord; - previewMessages: StoredMessage[]; - }> = []; - for (const record of records) { - const previewMessages = await this.metadata.readPreviewMessages(record.header.id); - withPreviews.push({ record, previewMessages }); - } - withPreviews.sort((a, b) => { - const aLastMessageAt = maxTimestamp( - a.record.header.lastMessageAt, - latestVisibleMessageAt(a.previewMessages), - ); - const bLastMessageAt = maxTimestamp( - b.record.header.lastMessageAt, - latestVisibleMessageAt(b.previewMessages), - ); - const tsDelta = (bLastMessageAt ?? 0) - (aLastMessageAt ?? 0); - return tsDelta !== 0 ? tsDelta : a.record.header.id.localeCompare(b.record.header.id); - }); - - const summaries: SessionSummary[] = []; - for (let index = 0; index < withPreviews.length; index += 1) { - const { record, previewMessages } = withPreviews[index]!; - const { header } = record; - let messages = previewMessages.slice(-10); - if (index < 3) { - messages = (await this.metadata.readMessages(header.id)).slice(-10); - } - summaries.push(toSummary(header, messages)); - } - return summaries; + return (await this.metadata.list(filter, 'ordinary')) + .filter((record) => record.header.conversationCopy?.state !== 'preparing') + .map((record) => toCatalogSummary(record.header, record.lastMessagePreview)); } async listCatalogPage( @@ -965,11 +894,7 @@ class SqliteSessionStore implements SessionAuthorityStore { } async listForRecovery(): Promise { - const headers = await this.listHeaders(); - for (const header of headers) { - await this.metadata.readMessagesForRecovery(header.id); - } - return headers; + return this.listHeaders(); } async listHeaders(): Promise { @@ -1006,14 +931,6 @@ class SqliteSessionStore implements SessionAuthorityStore { return this.metadata.readMessages(sessionId); } - async readTranscriptPageSnapshot( - sessionId: string, - request: SessionTranscriptPageRequest, - ): Promise { - await this.ensureReady(); - return this.metadata.readTranscriptPage(sessionId, request); - } - async readTranscriptMessagesSnapshot( sessionId: string, request: SessionTranscriptMessageLookupRequest, @@ -1022,47 +939,11 @@ class SqliteSessionStore implements SessionAuthorityStore { return this.metadata.readTranscriptMessages(sessionId, request); } - async readTranscriptRecordsSnapshot( - sessionId: string, - request: SessionTranscriptRecordScanRequest, - ): Promise { - await this.ensureReady(); - return this.metadata.readTranscriptRecords(sessionId, request); - } - async readTranscriptHighWaterSnapshot(sessionId: string): Promise { await this.ensureReady(); return this.metadata.readTranscriptHighWater(sessionId); } - async readTurnContributionsSnapshot( - sessionId: string, - throughSequence: number | null, - position: number, - maxContributions: number, - ): Promise { - await this.ensureReady(); - return this.metadata.readTurnContributions( - sessionId, - throughSequence, - position, - maxContributions, - ); - } - - async readTurnLandmarksSnapshot( - sessionId: string, - maxLandmarks: number, - ): Promise { - await this.ensureReady(); - return this.metadata.readTurnLandmarks(sessionId, maxLandmarks); - } - - async readMessagesForRecovery(sessionId: string): Promise { - await this.ensureReady(); - return this.metadata.readMessagesForRecovery(sessionId); - } - async listTurnsSnapshot(sessionId: string): Promise { return deriveTurnRecords(await this.readMessagesSnapshot(sessionId)); } @@ -1094,6 +975,15 @@ class SqliteSessionStore implements SessionAuthorityStore { for (const listener of this.transcriptChangeListeners) listener(sessionId); } + /** @see SqliteSessionMetadataStore.commitMessageCatalogProjection */ + async commitMessageCatalogProjection( + sessionId: string, + message: UserMessage | AssistantMessage, + ): Promise { + await this.ensureReady(); + await this.metadata.commitMessageCatalogProjection(sessionId, message); + } + async commitMessageAdmission( admission: PendingMessageAdmission, ): Promise { @@ -1177,42 +1067,6 @@ class SqliteSessionStore implements SessionAuthorityStore { return projectHeaderSnapshot(await this.metadata.updateSessionConfiguration(sessionId, input)); } - async markSessionReadThroughMessage( - sessionId: string, - messageId: string, - ): Promise { - for (let attempt = 0; attempt < 3; attempt += 1) { - const record = await this.readHeaderRecordSnapshot(sessionId); - const messages = await this.readMessagesSnapshot(sessionId); - const visibleMessages = messages.filter(isVisibleSessionMessage); - const targetIndex = visibleMessages.findIndex((message) => message.id === messageId); - if (targetIndex < 0) { - throw new SessionReadMarkerMessageNotFoundError(sessionId, messageId); - } - const currentIndex = - record.header.lastReadMessageId === undefined - ? -1 - : visibleMessages.findIndex((message) => message.id === record.header.lastReadMessageId); - const hasUnread = targetIndex < visibleMessages.length - 1; - if ( - targetIndex < currentIndex || - (targetIndex === currentIndex && record.header.hasUnread === hasUnread) - ) { - return record; - } - try { - return await this.updateHeaderVersioned( - sessionId, - { lastReadMessageId: messageId, hasUnread }, - record.revision, - ); - } catch (error) { - if (!(error instanceof SessionMetadataVersionConflictError) || attempt === 2) throw error; - } - } - throw new Error('Session read marker retry loop did not terminate'); - } - async probeSessionRemoval(sessionId: string): Promise { await this.ensureReady(); return projectRemovalProbe(await this.metadata.probeRemoval(sessionId)); @@ -1392,6 +1246,11 @@ function buildSessionHeader( collaborationMode: input.collaborationMode ?? 'agent', orchestrationMode: input.orchestrationMode ?? 'default', ...(input.thinkingLevel !== undefined ? { thinkingLevel: input.thinkingLevel } : {}), + // Born on the ledger: a Session created here records its execution facts as + // RuntimeEvents from its first turn, so there is no transcript to convert. + // Only an imported transcript (staged at 0) and a Session written before + // this field existed have anything for the converter to do. + transcriptLedgerVersion: 1, schemaVersion: 1, }; assertValidSessionLineage(header); @@ -1639,10 +1498,8 @@ function projectStableSessionCreateProbe( : probe; } -function toSummary(header: SessionHeader, messages: StoredMessage[] = []): SessionSummary { - const preview = lastMessagePreviewForMessages(messages); - const derivedLastMessageAt = latestVisibleMessageAt(messages); - const lastMessageAt = maxTimestamp(header.lastMessageAt, derivedLastMessageAt); +function toSummary(header: SessionHeader): SessionSummary { + const lastMessageAt = header.lastMessageAt; return { id: header.id, cwd: header.cwd, @@ -1653,7 +1510,6 @@ function toSummary(header: SessionHeader, messages: StoredMessage[] = []): Sessi labels: header.labels, hasUnread: header.hasUnread, lastMessageAt, - ...(preview ? { lastMessagePreview: preview } : {}), status: header.status, ...(header.blockedReason ? { blockedReason: header.blockedReason } : {}), ...(header.statusUpdatedAt !== undefined ? { statusUpdatedAt: header.statusUpdatedAt } : {}), @@ -1697,12 +1553,6 @@ function toCatalogSummary( }; } -function maxTimestamp(left: number | undefined, right: number | undefined): number | undefined { - if (left === undefined) return right; - if (right === undefined) return left; - return Math.max(left, right); -} - function normalizeSessionName(name: string): string { return name === 'New Session' ? DEFAULT_SESSION_NAME : name; } diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index a9bbe0e089..159d7547f8 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -91,6 +91,8 @@ import { type SessionHeader, type SessionHeaderPatch, type StoredMessage, + type AssistantMessage, + type UserMessage, type SubagentSessionParent, type WorkHubActionClaim, type WorkHubActionClaimOutcome, @@ -141,19 +143,12 @@ import { SessionNotFoundError, type ExternalSessionImportLookupResult, type SessionTranscriptMessageLookupRequest, - type SessionTranscriptPageRequest, - type SessionTranscriptRecordScanPage, - type SessionTranscriptRecordScanRequest, - type SessionTranscriptStoragePage, - type SessionTurnContribution, - type SessionTurnContributionPage, - type SessionTurnLandmarkSnapshot, } from './session-store.js'; import { isDiscardableConversationCopy, isValidConversationCopyTransition, } from './session-conversation-copy.js'; -import { catalogPreviewForUserMessage } from './session-message-projection.js'; +import { projectSessionCatalogMessages } from './session-message-projection.js'; import { configureSqliteSessionMetadataDatabase, migrateSqliteSessionMetadataDatabase, @@ -175,9 +170,6 @@ import { export { SQLITE_SESSION_METADATA_SCHEMA_VERSION } from './sqlite-session-metadata-schema.js'; const SQLITE_TRANSCRIPT_MESSAGE_LOOKUP_BATCH_SIZE = 256; -const SQLITE_TURN_CONTRIBUTION_MAX_SOURCE_MESSAGES = 1_024; -const SQLITE_TURN_CONTRIBUTION_MAX_SOURCE_BYTES = 4 * 1024 * 1024; -const SQLITE_TURN_LANDMARK_LEGACY_NEIGHBOR_MESSAGES = 32; // Each target Session binds three parameters in the linkage query. Stay well // inside SQLite's bound-parameter limit. const WORKHUB_TARGET_LINKAGE_MAX_SESSIONS = 256; @@ -1386,10 +1378,14 @@ export class SqliteSessionMetadataStore { }); } + /** + * Session records in catalog order, each carrying the projection the Session + * list shows. + */ async list( filter: SessionListFilter | undefined, roleScope: SessionMetadataRoleScope, - ): Promise { + ): Promise { this.assertOpen(); const { where, parameters } = buildSessionListPredicate(filter ?? {}); if (roleScope === 'ordinary') { @@ -1404,18 +1400,22 @@ export class SqliteSessionMetadataStore { const rows = this.db .prepare( ` - SELECT session_id, payload_json, metadata_version, committed_at + SELECT + metadata.session_id, + metadata.payload_json, + metadata.metadata_version, + metadata.committed_at, + COALESCE(projection.activity_at, 0) AS activity_at, + projection.last_message_preview FROM session_metadata metadata + LEFT JOIN session_catalog_projection projection + ON projection.session_id = metadata.session_id ${where.length > 0 ? `WHERE ${where.join(' AND ')}` : ''} - ORDER BY ( - SELECT activity_at - FROM session_catalog_projection projection - WHERE projection.session_id = metadata.session_id - ) DESC, session_id ASC + ORDER BY activity_at DESC, metadata.session_id ASC `, ) - .all(...parameters) as unknown as SessionMetadataRow[]; - return rows.map(decodeRecord); + .all(...parameters) as unknown as SessionMetadataCatalogRow[]; + return rows.map(decodeCatalogRecord); } async listCatalogPage( @@ -2152,11 +2152,12 @@ export class SqliteSessionMetadataStore { return this.readTransaction(() => { type Row = { session_id?: unknown; message_id?: unknown }; const list = targets.map(() => '?').join(', '); - // One Message moves between these lifecycle tables. Combine every target's - // identities once, then resolve activity from the canonical Coordination - // ledger in this same read transaction. That avoids rebuilding the target - // set once per page or once per candidate, without introducing another - // durable representation. + // One Message moves between these lifecycle tables — pending, admitted + // into a Turn, cancelled. Combine every target's identities once, then + // resolve activity from the canonical Coordination ledger in this same + // read transaction. That avoids rebuilding the target set once per page + // or once per candidate, without introducing another durable + // representation. const rows = this.db .prepare( ` @@ -2168,7 +2169,7 @@ export class SqliteSessionMetadataStore { AND length(message_id) = 52 UNION SELECT session_id, message_id - FROM session_messages + FROM core_root_source_message_proofs WHERE session_id IN (${list}) AND message_id GLOB 'whm_*' AND length(message_id) = 52 @@ -2306,29 +2307,6 @@ export class SqliteSessionMetadataStore { provenSteeringMessages.set(normalized.messageId, normalized); } this.transaction(() => { - const lastSequenceRow = this.db - .prepare( - 'SELECT COALESCE(MAX(sequence), -1) AS last_sequence FROM session_messages WHERE session_id = ?', - ) - .get(input.sessionId) as { last_sequence?: unknown }; - if ( - typeof lastSequenceRow.last_sequence !== 'number' || - !Number.isSafeInteger(lastSequenceRow.last_sequence) || - lastSequenceRow.last_sequence < -1 - ) { - throw new SessionMetadataConflictError('Invalid Session message sequence'); - } - const lastSequence = lastSequenceRow.last_sequence; - const historicalMissingMessages = new Map< - string, - { readonly message: StoredMessage; readonly json: string } - >(); - const ordinaryMissingMessages = new Map< - string, - { readonly message: StoredMessage; readonly json: string } - >(); - const historicalMessageIdSet = new Set(); - const existingSequences = new Map(); for (const messageId of unique) { const fallback = provenRootMessages.get(messageId); const steeringProof = provenSteeringMessages.get(messageId); @@ -2372,9 +2350,6 @@ export class SqliteSessionMetadataStore { ) { throw new SessionMetadataConflictError('Message admission fallback content conflict'); } - if (admission === undefined && fallback !== undefined) { - historicalMessageIdSet.add(messageId); - } if ( !admission && this.db @@ -2385,74 +2360,8 @@ export class SqliteSessionMetadataStore { ) { throw new SessionMetadataConflictError('Message admission is already cancelled'); } - const rows = this.db - .prepare( - ` - SELECT message.sequence, message.record_json, payload.record_bytes, payload.sha256 - FROM session_messages AS message - LEFT JOIN session_message_payloads AS payload - ON payload.session_id = message.session_id AND payload.sequence = message.sequence - WHERE message.session_id = ? AND message.message_id = ? - `, - ) - .all(input.sessionId, messageId) as Array<{ - sequence?: unknown; - record_json?: unknown; - record_bytes?: unknown; - sha256?: unknown; - }>; - if (rows.length > 1) { - throw new SessionMetadataConflictError( - 'Message admission transcript identity is ambiguous', - ); - } - if (rows.length === 0) { - const source = admission ?? fallback; - if (!source) throw new SessionMetadataConflictError('Message admission does not exist'); - const message = decodeCanonicalMessage({ - type: 'user', - id: messageId, - turnId: input.turnId, - ts: steeringProof?.eventTs ?? source.admittedAt, - ...source.content, - steeringEventId: steeringProof?.eventId ?? messageId, - }); - const json = JSON.stringify(message); - (historicalMessageIdSet.has(messageId) - ? historicalMissingMessages - : ordinaryMissingMessages - ).set(messageId, { message, json }); - } else { - const sequence = rows[0]?.sequence; - if (typeof sequence !== 'number' || !Number.isSafeInteger(sequence)) { - throw new SessionMetadataConflictError('Invalid Message transcript sequence'); - } - const row = rows[0]!; - const recordJson = readStoredMessageRecordJson(this.db, input.sessionId, sequence, row); - const message = decodeStoredMessage(JSON.parse(recordJson) as unknown); - const expectedSource = admission ?? fallback ?? steeringProof; - if ( - message.type !== 'user' || - message.id !== messageId || - (expectedSource !== undefined && - !messageContentsEqual(normalizeMessageContent(message), expectedSource.content)) - ) { - throw new SessionMetadataConflictError( - 'Message admission transcript identity conflict', - ); - } - if (message.turnId !== input.turnId) { - throw new SessionMetadataConflictError('Message admission transcript Turn conflict'); - } - if ( - steeringProof !== undefined && - (message.ts !== steeringProof.eventTs || - message.turnId !== steeringProof.executionTurnId || - message.steeringEventId !== steeringProof.eventId) - ) { - throw new SessionMetadataConflictError('Proven steering transcript identity conflict'); - } - existingSequences.set(messageId, sequence); + if (admission === undefined && fallback === undefined && steeringProof === undefined) { + throw new SessionMetadataConflictError('Message admission does not exist'); } if (admission) { const deleted = this.db @@ -2463,120 +2372,29 @@ export class SqliteSessionMetadataStore { } } } - let tailLatest: StoredMessage | undefined; - if (historicalMessageIdSet.size > 0) { - const transcript = this.readSessionMessageOrderingSync(input.sessionId); - let previousExistingSequence = -1; - const historicalMessageIds = unique.filter((messageId) => - historicalMessageIdSet.has(messageId), - ); - for (const messageId of historicalMessageIds) { - const sequence = existingSequences.get(messageId); - if (sequence === undefined) continue; - const admittedAt = provenRootMessages.get(messageId)!.admittedAt; - const blockingRow = transcript.find( - ({ sequence: candidateSequence, message }) => - candidateSequence > previousExistingSequence && - candidateSequence < sequence && - !historicalMessageIdSet.has(message.id) && - (message.turnId === input.turnId || message.ts >= admittedAt), - ); - if (sequence <= previousExistingSequence || blockingRow !== undefined) { - throw new SessionMetadataConflictError( - 'Message admission transcript source order conflict', - ); - } - previousExistingSequence = sequence; - } - - const insertionGroups: Array<{ - readonly boundary: number; - readonly entries: Array<{ readonly message: StoredMessage; readonly json: string }>; - }> = []; - let pending: Array<{ readonly message: StoredMessage; readonly json: string }> = []; - let previousAnchor = -1; - for (const messageId of historicalMessageIds) { - const missing = historicalMissingMessages.get(messageId); - if (missing) { - pending.push(missing); - continue; - } - const boundary = existingSequences.get(messageId); - if (pending.length > 0 && boundary !== undefined) { - const admittedAt = Math.min(...pending.map(({ message }) => message.ts)); - const earlierBoundary = transcript.find( - ({ sequence, message }) => - sequence > previousAnchor && - sequence < boundary && - !historicalMessageIdSet.has(message.id) && - (message.turnId === input.turnId || message.ts >= admittedAt), - ); - if (earlierBoundary) { - throw new SessionMetadataConflictError( - 'Message admission transcript source order conflict', - ); - } - insertionGroups.push({ boundary, entries: pending }); - pending = []; - } - if (boundary !== undefined) previousAnchor = boundary; - } - if (pending.length > 0) { - const admittedAt = Math.min(...pending.map(({ message }) => message.ts)); - const repairBoundary = transcript.find( - ({ sequence, message }) => - sequence > previousAnchor && - !historicalMessageIdSet.has(message.id) && - (message.turnId === input.turnId || message.ts >= admittedAt), - )?.sequence; - insertionGroups.push({ - boundary: repairBoundary ?? lastSequence + 1, - entries: pending, - }); - } + }); + } - for (const group of insertionGroups.reverse()) { - this.shiftSessionMessageSuffixSync(input.sessionId, group.boundary, group.entries.length); - this.insertSessionMessagesSync(input.sessionId, group.boundary, group.entries); - if (group.boundary === lastSequence + 1) { - tailLatest = group.entries.at(-1)?.message; - } - } - } - if (ordinaryMissingMessages.size > 0) { - const ordinaryEntries = unique.flatMap((messageId) => { - const entry = ordinaryMissingMessages.get(messageId); - return entry ? [entry] : []; - }); - const currentLastSequenceRow = this.db - .prepare( - 'SELECT COALESCE(MAX(sequence), -1) AS last_sequence FROM session_messages WHERE session_id = ?', - ) - .get(input.sessionId) as { last_sequence?: unknown }; - const currentLastSequence = currentLastSequenceRow.last_sequence; - if ( - typeof currentLastSequence !== 'number' || - !Number.isSafeInteger(currentLastSequence) || - currentLastSequence < -1 - ) { - throw new SessionMetadataConflictError('Invalid Session message sequence'); - } - this.insertSessionMessagesSync(input.sessionId, currentLastSequence + 1, ordinaryEntries); - tailLatest = ordinaryEntries.at(-1)?.message; - } - if (tailLatest?.type === 'user') { - this.updateCatalogProjectionSync( - input.sessionId, - { - lastMessageAt: tailLatest.ts, - lastMessagePreview: catalogPreviewForUserMessage(tailLatest), - }, - false, - true, - ); - } else if (historicalMissingMessages.size > 0 || ordinaryMissingMessages.size > 0) { - this.updateCatalogProjectionSync(input.sessionId, {}, false, true); - } + /** + * The catalog facts a durable message carries, committed without a transcript + * row to carry them: the Session list's preview line, its time, and the + * connection lock a Session takes on its first user message. + */ + async commitMessageCatalogProjection( + sessionId: string, + message: UserMessage | AssistantMessage, + ): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + this.transaction(() => { + const record = this.readRecordSync(sessionId); + if (!record) throw new SessionNotFoundError(sessionId); + this.updateCatalogProjectionSync( + sessionId, + projectSessionCatalogMessages([message]), + false, + message.type === 'user' && !record.header.connectionLocked, + ); }); } @@ -2734,130 +2552,6 @@ export class SqliteSessionMetadataStore { return this.readMessagesWith(sessionId, decodeStoredMessage); } - async readTranscriptPage( - sessionId: string, - request: SessionTranscriptPageRequest, - ): Promise { - this.assertOpen(); - assertSafeSessionId(sessionId); - assertTranscriptPageRequest(request); - return this.readTransaction(() => { - if (!this.readRecordSync(sessionId)) throw new SessionNotFoundError(sessionId); - const highWaterRow = this.db - .prepare('SELECT MAX(sequence) AS high_water FROM session_messages WHERE session_id = ?') - .get(sessionId) as { high_water?: unknown }; - const actualHighWater = nullableStoredMessageSequence(highWaterRow.high_water, sessionId); - const throughSequence = - request.throughSequence === undefined ? actualHighWater : request.throughSequence; - if (throughSequence === null) { - return { - throughSequence: null, - fragments: [], - rawBytes: 0, - next: null, - }; - } - if (actualHighWater === null || throughSequence > actualHighWater) { - throw new Error(`Session transcript watermark is ahead of durable storage: ${sessionId}`); - } - const position = request.position ?? (request.direction === 'older' ? throughSequence : 0); - const comparison = request.direction === 'older' ? '<=' : '>='; - const order = request.direction === 'older' ? 'DESC' : 'ASC'; - const rows = this.db - .prepare( - ` - SELECT message.sequence, - coalesce(payload.record_bytes, length(CAST(message.record_json AS BLOB))) AS total_bytes, - payload.record_bytes IS NOT NULL AS chunked, - payload.sha256 AS payload_sha256 - FROM session_messages AS message - LEFT JOIN session_message_payloads AS payload - ON payload.session_id = message.session_id AND payload.sequence = message.sequence - WHERE message.session_id = ? - AND message.sequence <= ? - AND message.sequence ${comparison} ? - ORDER BY message.sequence ${order} - LIMIT ? - `, - ) - .all(sessionId, throughSequence, position, request.maxMessages + 1) as Array<{ - sequence?: unknown; - total_bytes?: unknown; - chunked?: unknown; - payload_sha256?: unknown; - }>; - const slices: TranscriptRecordSlice[] = []; - let rawBytes = 0; - let next: { position: number; byteOffset: number | null } | null = null; - for (const row of rows) { - if (slices.length >= request.maxMessages || rawBytes >= request.maxBytes) break; - const sequence = requireStoredMessageSequence(row.sequence, sessionId); - const totalBytes = requireTranscriptRecordByteLength(row.total_bytes, sessionId, sequence); - const chunked = row.chunked === 1; - const payloadDigest = chunked - ? requireTranscriptPayloadDigest(row.payload_sha256, sessionId, sequence) - : null; - const continued = sequence === position && request.byteOffset !== undefined; - const edge = continued - ? request.byteOffset! - : request.direction === 'older' - ? totalBytes - : 0; - if ( - (request.direction === 'older' && (edge < 1 || edge > totalBytes)) || - (request.direction === 'newer' && (edge < 0 || edge >= totalBytes)) - ) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence); - } - const available = request.maxBytes - rawBytes; - const byteOffset = request.direction === 'older' ? Math.max(0, edge - available) : edge; - const byteLength = - request.direction === 'older' - ? edge - byteOffset - : Math.min(totalBytes - edge, available); - const complete = - request.direction === 'older' ? byteOffset === 0 : byteOffset + byteLength === totalBytes; - slices.push({ - sequence, - byteOffset, - totalBytes, - byteLength, - chunked, - payloadDigest, - }); - rawBytes += byteLength; - if (!complete) { - next = { - position: sequence, - byteOffset: request.direction === 'older' ? byteOffset : byteOffset + byteLength, - }; - break; - } - } - if (next === null && slices.length > 0 && slices.length < rows.length) { - const sequence = slices.at(-1)!.sequence; - next = { - position: sequence + (request.direction === 'older' ? -1 : 1), - byteOffset: null, - }; - } - const dataBySequence = readTranscriptSlices(this.db, sessionId, slices); - const fragments = slices.map( - ({ sequence, byteOffset, totalBytes, byteLength, payloadDigest }) => { - const data = dataBySequence.get(sequence); - if (!data || data.byteLength !== byteLength) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence); - } - if (byteOffset === 0 && byteLength === totalBytes) { - validateTranscriptRecord(data, sessionId, sequence); - } - return { sequence, byteOffset, totalBytes, payloadDigest, data }; - }, - ); - return { throughSequence, fragments, rawBytes, next }; - }); - } - async readTranscriptMessages( sessionId: string, request: SessionTranscriptMessageLookupRequest, @@ -2965,86 +2659,6 @@ export class SqliteSessionMetadataStore { }); } - async readTranscriptRecords( - sessionId: string, - request: SessionTranscriptRecordScanRequest, - ): Promise { - this.assertOpen(); - assertSafeSessionId(sessionId); - assertTranscriptRecordScanRequest(request); - return this.readTransaction(() => { - if (!this.readRecordSync(sessionId)) throw new SessionNotFoundError(sessionId); - const highWaterRow = this.db - .prepare('SELECT MAX(sequence) AS high_water FROM session_messages WHERE session_id = ?') - .get(sessionId) as { high_water?: unknown }; - const actualHighWater = nullableStoredMessageSequence(highWaterRow.high_water, sessionId); - const throughSequence = - request.throughSequence === undefined ? actualHighWater : request.throughSequence; - if (throughSequence === null) { - return { throughSequence: null, records: [], nextPosition: null }; - } - if (actualHighWater === null || throughSequence > actualHighWater) { - throw new Error(`Session transcript watermark is ahead of durable storage: ${sessionId}`); - } - const position = request.position ?? (request.direction === 'older' ? throughSequence : 0); - const comparison = request.direction === 'older' ? '<=' : '>='; - const order = request.direction === 'older' ? 'DESC' : 'ASC'; - const rows = this.db - .prepare( - ` - SELECT message.sequence, - coalesce(payload.record_bytes, length(CAST(message.record_json AS BLOB))) AS stored_bytes - FROM session_messages AS message - LEFT JOIN session_message_payloads AS payload - ON payload.session_id = message.session_id AND payload.sequence = message.sequence - WHERE message.session_id = ? - AND message.sequence <= ? - AND message.sequence ${comparison} ? - ORDER BY message.sequence ${order} - LIMIT ? - `, - ) - .all(sessionId, throughSequence, position, request.maxMessages + 1) as Array<{ - sequence?: unknown; - stored_bytes?: unknown; - }>; - const selected: number[] = []; - let storedBytes = 0; - for (const row of rows) { - if (selected.length >= request.maxMessages) break; - const sequence = requireStoredMessageSequence(row.sequence, sessionId); - const bytes = requireTranscriptRecordByteLength(row.stored_bytes, sessionId, sequence); - if (selected.length > 0 && storedBytes + bytes > request.maxStoredBytes) break; - selected.push(sequence); - storedBytes += bytes; - } - const decoded = new Map(); - for (const row of readStoredMessageRows(this.db, sessionId, selected)) { - try { - decoded.set(row.sequence, decodeStoredMessage(JSON.parse(row.recordJson) as unknown)); - } catch (error) { - throw new StoredSessionMessageIncompatibleError(sessionId, row.sequence, { - cause: error, - }); - } - } - const records = selected.map((sequence) => { - const message = decoded.get(sequence); - if (!message) throw new StoredSessionMessageIncompatibleError(sessionId, sequence); - return { sequence, message }; - }); - const last = selected.at(-1); - return { - throughSequence, - records, - nextPosition: - last !== undefined && rows.length > selected.length - ? last + (request.direction === 'older' ? -1 : 1) - : null, - }; - }); - } - async readTranscriptHighWater(sessionId: string): Promise { this.assertOpen(); assertSafeSessionId(sessionId); @@ -3055,349 +2669,6 @@ export class SqliteSessionMetadataStore { return nullableStoredMessageSequence(row.high_water, sessionId); } - async readTurnContributions( - sessionId: string, - throughSequence: number | null, - position: number, - maxContributions: number, - ): Promise { - this.assertOpen(); - assertSafeSessionId(sessionId); - if ( - (throughSequence !== null && - (!Number.isSafeInteger(throughSequence) || throughSequence < 0)) || - !Number.isSafeInteger(position) || - position < 0 || - !Number.isSafeInteger(maxContributions) || - maxContributions < 1 || - maxContributions > 128 - ) { - throw new Error('Invalid Session turn contribution request'); - } - return this.readTransaction(() => { - if (!this.readRecordSync(sessionId)) throw new SessionNotFoundError(sessionId); - const highWaterRow = this.db - .prepare('SELECT MAX(sequence) AS high_water FROM session_messages WHERE session_id = ?') - .get(sessionId) as { high_water?: unknown }; - const actualHighWater = nullableStoredMessageSequence(highWaterRow.high_water, sessionId); - const fixedThrough = throughSequence ?? actualHighWater; - if (fixedThrough === null) { - return { throughSequence: null, contributions: [], nextPosition: null }; - } - if (actualHighWater === null || fixedThrough > actualHighWater) { - throw new Error(`Session turn watermark is ahead of durable storage: ${sessionId}`); - } - const contributions = new Map(); - let nextPosition: number | null = position; - let sourceMessages = 0; - let sourceBytes = 0; - while (nextPosition <= fixedThrough) { - const rows = this.db - .prepare( - ` - SELECT message.sequence, message.record_json, payload.record_bytes, payload.sha256 - FROM session_messages AS message - LEFT JOIN session_message_payloads AS payload - ON payload.session_id = message.session_id AND payload.sequence = message.sequence - WHERE message.session_id = ? - AND message.sequence >= ? - AND message.sequence <= ? - ORDER BY message.sequence ASC - LIMIT 128 - `, - ) - .all(sessionId, nextPosition, fixedThrough) as StoredSessionMessagePayloadRow[]; - if (rows.length === 0) { - throw new StoredSessionMessageIncompatibleError(sessionId, nextPosition); - } - for (const row of rows) { - const sequence = requireStoredMessageSequence(row.sequence, sessionId); - const recordBytes = storedMessageRecordBytes(row, sessionId, sequence); - if ( - sourceMessages > 0 && - (sourceMessages >= SQLITE_TURN_CONTRIBUTION_MAX_SOURCE_MESSAGES || - sourceBytes + recordBytes > SQLITE_TURN_CONTRIBUTION_MAX_SOURCE_BYTES) - ) { - return { - throughSequence: fixedThrough, - contributions: [...contributions.values()], - nextPosition: sequence, - }; - } - const message = decodeStoredMessageRecordRow(this.db, sessionId, row); - sourceMessages += 1; - sourceBytes += recordBytes; - if (!('turnId' in message) || typeof message.turnId !== 'string') { - nextPosition = sequence + 1; - continue; - } - const turnId = message.turnId; - if (turnId && !contributions.has(turnId) && contributions.size >= maxContributions) { - nextPosition = sequence; - return { - throughSequence: fixedThrough, - contributions: [...contributions.values()], - nextPosition, - }; - } - contributions.set( - turnId, - foldTurnContribution(contributions.get(turnId), turnId, sequence, message), - ); - nextPosition = sequence + 1; - } - } - return { - throughSequence: fixedThrough, - contributions: [...contributions.values()], - nextPosition: null, - }; - }); - } - - async readTurnLandmarks( - sessionId: string, - maxLandmarks: number, - ): Promise { - this.assertOpen(); - assertSafeSessionId(sessionId); - if (!Number.isSafeInteger(maxLandmarks) || maxLandmarks < 1 || maxLandmarks > 64) { - throw new Error('Invalid Session turn landmark limit'); - } - return this.readTransaction(() => { - if (!this.readRecordSync(sessionId)) throw new SessionNotFoundError(sessionId); - const throughRow = this.db - .prepare( - ` - SELECT sequence AS through_sequence - FROM session_messages - WHERE session_id = ? - ORDER BY sequence DESC - LIMIT 1 - `, - ) - .get(sessionId) as { through_sequence?: unknown } | undefined; - const throughSequence = nullableStoredMessageSequence( - throughRow?.through_sequence, - sessionId, - ); - if (throughSequence === null) { - return { throughSequence: null, landmarks: [] }; - } - - const promptRows = this.db - .prepare( - ` - SELECT admission.admitted_at, message.sequence - FROM core_root_turn_admissions AS admission - JOIN session_messages AS message - ON message.session_id = admission.session_id - AND message.message_id = json_extract(admission.record_json, '$.userMessageId') - LEFT JOIN session_message_payloads AS payload - ON payload.session_id = message.session_id AND payload.sequence = message.sequence - WHERE admission.session_id = ? AND payload.sequence IS NULL - ORDER BY admission.admitted_at ASC, admission.turn_id ASC - LIMIT ? - `, - ) - .all(sessionId, maxLandmarks + 1) as TurnLandmarkCandidateRow[]; - const selected = new Set(); - if (promptRows.length <= maxLandmarks) { - for (const row of promptRows) { - selected.add(requireStoredMessageSequence(row.sequence, sessionId)); - } - } - - const forward = this.db.prepare(` - SELECT admission.admitted_at, message.sequence - FROM core_root_turn_admissions AS admission - JOIN session_messages AS message - ON message.session_id = admission.session_id - AND message.message_id = json_extract(admission.record_json, '$.userMessageId') - LEFT JOIN session_message_payloads AS payload - ON payload.session_id = message.session_id AND payload.sequence = message.sequence - WHERE admission.session_id = ? AND admission.admitted_at >= ? AND payload.sequence IS NULL - ORDER BY admission.admitted_at ASC, admission.turn_id ASC - LIMIT 1 - `); - const backward = this.db.prepare(` - SELECT admission.admitted_at, message.sequence - FROM core_root_turn_admissions AS admission - JOIN session_messages AS message - ON message.session_id = admission.session_id - AND message.message_id = json_extract(admission.record_json, '$.userMessageId') - LEFT JOIN session_message_payloads AS payload - ON payload.session_id = message.session_id AND payload.sequence = message.sequence - WHERE admission.session_id = ? AND admission.admitted_at < ? AND payload.sequence IS NULL - ORDER BY admission.admitted_at DESC, admission.turn_id DESC - LIMIT 1 - `); - if (promptRows.length > maxLandmarks) { - const firstAdmittedAt = requireTurnLandmarkAdmittedAt(promptRows[0]?.admitted_at); - const lastRow = this.db - .prepare( - ` - SELECT admitted_at - FROM core_root_turn_admissions - WHERE session_id = ? - ORDER BY admitted_at DESC, turn_id DESC - LIMIT 1 - `, - ) - .get(sessionId) as TurnLandmarkCandidateRow | undefined; - const lastAdmittedAt = requireTurnLandmarkAdmittedAt(lastRow?.admitted_at); - for (let index = 0; index < maxLandmarks; index += 1) { - const target = - maxLandmarks === 1 - ? lastAdmittedAt - : firstAdmittedAt + - Math.floor(((lastAdmittedAt - firstAdmittedAt) * index) / (maxLandmarks - 1)); - const candidates = [ - ...(forward.all(sessionId, target) as TurnLandmarkCandidateRow[]), - ...(backward.all(sessionId, target) as TurnLandmarkCandidateRow[]), - ]; - let nearest: TurnLandmarkCandidateRow | undefined; - for (const candidate of candidates) { - const admittedAt = requireTurnLandmarkAdmittedAt(candidate.admitted_at); - if ( - nearest === undefined || - Math.abs(admittedAt - target) < - Math.abs(requireTurnLandmarkAdmittedAt(nearest.admitted_at) - target) - ) { - nearest = candidate; - } - } - if (nearest) selected.add(requireStoredMessageSequence(nearest.sequence, sessionId)); - } - } - const firstIndexedSequence = - promptRows.length > 0 - ? requireStoredMessageSequence(promptRows[0]?.sequence, sessionId) - : null; - const legacyThrough = - firstIndexedSequence === null ? throughSequence : firstIndexedSequence - 1; - if (legacyThrough >= 0) { - const firstRow = this.db - .prepare( - ` - SELECT sequence AS first_sequence - FROM session_messages - WHERE session_id = ? - ORDER BY sequence ASC - LIMIT 1 - `, - ) - .get(sessionId) as { first_sequence?: unknown } | undefined; - const firstSequence = nullableStoredMessageSequence(firstRow?.first_sequence, sessionId); - if (firstSequence === null || firstSequence > legacyThrough) { - throw new StoredSessionMessageIncompatibleError(sessionId, legacyThrough); - } - const forwardLegacy = this.db.prepare(` - SELECT message.sequence, message.message_type, payload.sequence AS payload_sequence - FROM session_messages AS message - LEFT JOIN session_message_payloads AS payload - ON payload.session_id = message.session_id AND payload.sequence = message.sequence - WHERE message.session_id = ? AND message.sequence >= ? AND message.sequence <= ? - ORDER BY message.sequence ASC - LIMIT ${SQLITE_TURN_LANDMARK_LEGACY_NEIGHBOR_MESSAGES} - `); - const backwardLegacy = this.db.prepare(` - SELECT message.sequence, message.message_type, payload.sequence AS payload_sequence - FROM session_messages AS message - LEFT JOIN session_message_payloads AS payload - ON payload.session_id = message.session_id AND payload.sequence = message.sequence - WHERE message.session_id = ? AND message.sequence < ? AND message.sequence >= ? - ORDER BY message.sequence DESC - LIMIT ${SQLITE_TURN_LANDMARK_LEGACY_NEIGHBOR_MESSAGES} - `); - const targetCount = Math.min(maxLandmarks, legacyThrough - firstSequence + 1); - for (let index = 0; index < targetCount; index += 1) { - const target = - targetCount === 1 - ? legacyThrough - : firstSequence + - Math.floor(((legacyThrough - firstSequence) * index) / (targetCount - 1)); - const candidates = [ - ...(forwardLegacy.all(sessionId, target, legacyThrough) as LegacyTurnLandmarkRow[]), - ...(backwardLegacy.all(sessionId, target, firstSequence) as LegacyTurnLandmarkRow[]), - ]; - let nearest: number | undefined; - for (const candidate of candidates) { - const sequence = requireStoredMessageSequence(candidate.sequence, sessionId); - if (candidate.message_type !== 'user' || candidate.payload_sequence !== null) continue; - if (nearest === undefined || Math.abs(sequence - target) < Math.abs(nearest - target)) { - nearest = sequence; - } - } - if (nearest !== undefined) selected.add(nearest); - } - } - - const selectedSequences = [...selected].sort((left, right) => left - right); - const sampledSequences = - selectedSequences.length <= maxLandmarks - ? selectedSequences - : Array.from( - { length: maxLandmarks }, - (_, index) => - selectedSequences[ - maxLandmarks === 1 - ? selectedSequences.length - 1 - : Math.floor(((selectedSequences.length - 1) * index) / (maxLandmarks - 1)) - ]!, - ); - const landmarks = readStoredMessageRows(this.db, sessionId, sampledSequences).flatMap( - ({ sequence, recordJson }) => { - let message: StoredMessage; - try { - message = decodeStoredMessage(JSON.parse(recordJson) as unknown); - } catch (error) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence, { cause: error }); - } - if (message.type !== 'user') { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence); - } - const label = (message.displayText ?? message.text).trim(); - return label ? [{ turnId: message.turnId, sequence, label }] : []; - }, - ); - return { throughSequence, landmarks }; - }); - } - - async readMessagesForRecovery(sessionId: string): Promise { - return this.readMessagesWith(sessionId, decodeStoredMessage); - } - - async readPreviewMessages(sessionId: string, limit = 10): Promise { - this.assertOpen(); - assertSafeSessionId(sessionId); - if (!Number.isSafeInteger(limit) || limit < 1 || limit > 128) { - throw new Error('Session message preview limit must be between 1 and 128'); - } - if (!this.readRecordSync(sessionId)) throw new SessionNotFoundError(sessionId); - const sequences = ( - this.db - .prepare( - ` - SELECT sequence FROM session_messages - WHERE session_id = ? ORDER BY sequence DESC LIMIT ? - `, - ) - .all(sessionId, limit) as Array<{ sequence?: unknown }> - ) - .map((row) => requireStoredMessageSequence(row.sequence, sessionId)) - .reverse(); - return readStoredMessageRows( - this.db, - sessionId, - sequences, - sequences.map(() => '?').join(', '), - ).map((row) => - decodeStoredMessageRow({ sequence: row.sequence, record_json: row.recordJson }, sessionId), - ); - } - async beginCatalogProjectionWrite(): Promise { this.assertOpen(); this.transaction(() => { @@ -5481,84 +4752,6 @@ export class SqliteSessionMetadataStore { return row ? decodeStoredMessageRecordRow(this.db, sessionId, row) : undefined; } - private readSessionMessageOrderingSync( - sessionId: string, - ): Array<{ readonly sequence: number; readonly message: StoredMessage }> { - const rows = this.db - .prepare( - ` - SELECT message.sequence, message.record_json, payload.record_bytes, payload.sha256 - FROM session_messages AS message - LEFT JOIN session_message_payloads AS payload - ON payload.session_id = message.session_id AND payload.sequence = message.sequence - WHERE message.session_id = ? - ORDER BY message.sequence - `, - ) - .all(sessionId) as StoredSessionMessagePayloadRow[]; - return rows.map((row) => { - const sequence = requireStoredMessageSequence(row.sequence, sessionId); - const recordJson = readStoredMessageRecordJson(this.db, sessionId, sequence, row); - return { - sequence, - message: decodeStoredMessage(JSON.parse(recordJson) as unknown), - }; - }); - } - - private shiftSessionMessageSuffixSync( - sessionId: string, - firstSequence: number, - amount: number, - ): void { - if (!Number.isSafeInteger(firstSequence) || firstSequence < 0) { - throw new SessionMetadataConflictError('Invalid transcript insertion sequence'); - } - if (!Number.isSafeInteger(amount) || amount < 1) { - throw new SessionMetadataConflictError('Invalid transcript insertion size'); - } - const sequences = ( - this.db - .prepare( - ` - SELECT sequence - FROM session_messages - WHERE session_id = ? AND sequence >= ? - ORDER BY sequence DESC - `, - ) - .all(sessionId, firstSequence) as Array<{ sequence?: unknown }> - ).map((row) => requireStoredMessageSequence(row.sequence, sessionId)); - const highest = sequences[0]; - if (highest !== undefined && highest > Number.MAX_SAFE_INTEGER - amount) { - throw new SessionMetadataConflictError('Session message sequence overflow'); - } - if (sequences.length === 0) return; - - this.db.exec('PRAGMA defer_foreign_keys = ON'); - const moveChunks = this.db.prepare( - 'UPDATE session_message_chunks SET sequence = ? WHERE session_id = ? AND sequence = ?', - ); - const movePayload = this.db.prepare( - 'UPDATE session_message_payloads SET sequence = ? WHERE session_id = ? AND sequence = ?', - ); - const moveMessage = this.db.prepare( - 'UPDATE session_messages SET sequence = ? WHERE session_id = ? AND sequence = ?', - ); - for (const sequence of sequences) { - const shifted = sequence + amount; - moveChunks.run(shifted, sessionId, sequence); - const payload = movePayload.run(shifted, sessionId, sequence); - if (payload.changes !== 0 && payload.changes !== 1) { - throw new SessionMetadataConflictError('Message payload sequence is ambiguous'); - } - const message = moveMessage.run(shifted, sessionId, sequence); - if (message.changes !== 1) { - throw new SessionMetadataConflictError('Message transcript sequence changed during repair'); - } - } - } - private insertSessionMessagesSync( sessionId: string, firstSequence: number, @@ -7109,41 +6302,6 @@ interface StoredSessionMessagePayloadRow { readonly sha256?: unknown; } -interface TurnLandmarkCandidateRow { - readonly sequence?: unknown; - readonly admitted_at?: unknown; -} - -interface LegacyTurnLandmarkRow { - readonly sequence?: unknown; - readonly message_type?: unknown; - readonly payload_sequence?: unknown; -} - -function requireTurnLandmarkAdmittedAt(value: unknown): number { - if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { - throw new Error('Invalid root Turn admission timestamp'); - } - return value; -} - -function storedMessageRecordBytes( - row: StoredSessionMessagePayloadRow, - sessionId: string, - sequence: number, -): number { - if (row.record_bytes !== null) { - return requireTranscriptRecordByteLength(row.record_bytes, sessionId, sequence); - } - if ( - typeof row.record_json !== 'string' || - row.record_json === SQLITE_SESSION_MESSAGE_CHUNK_MARKER - ) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence); - } - return Buffer.byteLength(row.record_json, 'utf8'); -} - function decodeStoredMessageRecordRow( db: DatabaseSync, sessionId: string, @@ -7233,40 +6391,6 @@ function sameWorkHubAssignmentRequest( ); } -function foldTurnContribution( - current: SessionTurnContribution | undefined, - turnId: string, - sequence: number, - message: StoredMessage, -): SessionTurnContribution { - const contribution = current ?? { - turnId, - firstSequence: sequence, - latestState: null, - userPromptPreview: null, - hasAssistantMessage: false, - hasAssistantOutput: false, - hasToolResult: false, - hasFailedToolResult: false, - hasAbortNote: false, - }; - const userPrompt = message.type === 'user' ? (message.displayText ?? message.text).trim() : ''; - return { - ...contribution, - latestState: message.type === 'turn_state' ? { sequence, message } : contribution.latestState, - userPromptPreview: contribution.userPromptPreview ?? (userPrompt || null), - hasAssistantMessage: contribution.hasAssistantMessage || message.type === 'assistant', - hasAssistantOutput: - contribution.hasAssistantOutput || - (message.type === 'assistant' && message.text.trim().length > 0), - hasToolResult: contribution.hasToolResult || message.type === 'tool_result', - hasFailedToolResult: - contribution.hasFailedToolResult || (message.type === 'tool_result' && message.isError), - hasAbortNote: - contribution.hasAbortNote || (message.type === 'system_note' && message.kind === 'abort'), - }; -} - function readStoredMessageRows( db: DatabaseSync, sessionId: string, @@ -7355,26 +6479,6 @@ function nullableStoredMessageSequence(value: unknown, sessionId: string): numbe return requireStoredMessageSequence(value, sessionId); } -interface TranscriptRecordSlice { - readonly sequence: number; - readonly byteOffset: number; - readonly totalBytes: number; - readonly byteLength: number; - readonly chunked: boolean; - readonly payloadDigest: `sha256:${string}` | null; -} - -function requireTranscriptPayloadDigest( - value: unknown, - sessionId: string, - sequence: number, -): `sha256:${string}` { - if (typeof value !== 'string' || !/^[0-9a-f]{64}$/.test(value)) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence); - } - return `sha256:${value}`; -} - function requireTranscriptRecordByteLength( value: unknown, sessionId: string, @@ -7385,207 +6489,3 @@ function requireTranscriptRecordByteLength( } return value as number; } - -function readTranscriptSlices( - db: DatabaseSync, - sessionId: string, - slices: readonly TranscriptRecordSlice[], -): Map { - if (slices.length === 0) return new Map(); - const chunkedSlices = slices.filter((slice) => slice.chunked); - const values = chunkedSlices.map(() => '(?, ?, ?)').join(', '); - const parameters = chunkedSlices.flatMap((slice) => [ - slice.sequence, - Math.floor(slice.byteOffset / SQLITE_SESSION_MESSAGE_CHUNK_BYTES), - Math.floor((slice.byteOffset + slice.byteLength - 1) / SQLITE_SESSION_MESSAGE_CHUNK_BYTES), - ]); - const rows = - chunkedSlices.length === 0 - ? [] - : (db - .prepare( - ` - WITH requested(sequence, first_chunk, last_chunk) AS (VALUES ${values}) - SELECT requested.sequence, chunk.chunk_index, chunk.data, chunk.sha256 - FROM requested - INNER JOIN session_message_chunks AS chunk - ON chunk.session_id = ? - AND chunk.sequence = requested.sequence - AND chunk.chunk_index BETWEEN requested.first_chunk AND requested.last_chunk - ORDER BY requested.sequence, chunk.chunk_index - `, - ) - .all(...parameters, sessionId) as Array<{ - sequence?: unknown; - chunk_index?: unknown; - data?: unknown; - sha256?: unknown; - }>); - const rowsBySequence = new Map(); - for (const row of rows) { - const sequence = requireStoredMessageSequence(row.sequence, sessionId); - const grouped = rowsBySequence.get(sequence); - if (grouped) grouped.push(row); - else rowsBySequence.set(sequence, [row]); - } - const result = new Map(); - for (const slice of slices) { - if (!slice.chunked) continue; - const selected = rowsBySequence.get(slice.sequence) ?? []; - const firstChunk = Math.floor(slice.byteOffset / SQLITE_SESSION_MESSAGE_CHUNK_BYTES); - const lastChunk = Math.floor( - (slice.byteOffset + slice.byteLength - 1) / SQLITE_SESSION_MESSAGE_CHUNK_BYTES, - ); - if (selected.length !== lastChunk - firstChunk + 1) { - throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); - } - const chunks: Buffer[] = []; - for (let index = 0; index < selected.length; index += 1) { - const row = selected[index]!; - if ( - row.chunk_index !== firstChunk + index || - !(row.data instanceof Uint8Array) || - typeof row.sha256 !== 'string' - ) { - throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); - } - const chunk = Buffer.from(row.data); - if (createHash('sha256').update(chunk).digest('hex') !== row.sha256) { - throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); - } - chunks.push(chunk); - } - const joined = Buffer.concat(chunks); - const start = slice.byteOffset - firstChunk * SQLITE_SESSION_MESSAGE_CHUNK_BYTES; - const data = joined.subarray(start, start + slice.byteLength); - if (data.byteLength !== slice.byteLength) { - throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); - } - result.set(slice.sequence, data); - } - const inlineSlices = slices.filter((slice) => !slice.chunked); - if (inlineSlices.length > 0) { - const inlineValues = inlineSlices.map(() => '(?, ?, ?)').join(', '); - const inlineParameters = inlineSlices.flatMap((slice) => [ - slice.sequence, - slice.byteOffset + 1, - slice.byteLength, - ]); - const inlineRows = db - .prepare( - ` - WITH requested(sequence, byte_start, byte_length) AS (VALUES ${inlineValues}) - SELECT requested.sequence, - substr(CAST(message.record_json AS BLOB), requested.byte_start, requested.byte_length) - AS data - FROM requested - INNER JOIN session_messages AS message - ON message.session_id = ? AND message.sequence = requested.sequence - `, - ) - .all(...inlineParameters, sessionId) as Array<{ - sequence?: unknown; - data?: unknown; - }>; - for (const row of inlineRows) { - const sequence = requireStoredMessageSequence(row.sequence, sessionId); - if (!(row.data instanceof Uint8Array)) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence); - } - result.set(sequence, Buffer.from(row.data)); - } - } - return result; -} - -function validateTranscriptRecord( - data: string | Buffer, - sessionId: string, - sequence: number, -): void { - try { - decodeStoredMessage( - markPersisted( - JSON.parse(typeof data === 'string' ? data : data.toString('utf8')), - ), - ); - } catch (error) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence, { - cause: error, - }); - } -} - -function assertTranscriptPageRequest(request: SessionTranscriptPageRequest): void { - if (request.direction !== 'older' && request.direction !== 'newer') { - throw new Error('Invalid Session transcript page direction'); - } - if ( - request.throughSequence !== undefined && - request.throughSequence !== null && - (!Number.isSafeInteger(request.throughSequence) || request.throughSequence < 0) - ) { - throw new Error('Invalid Session transcript watermark'); - } - if ( - request.position !== undefined && - (!Number.isSafeInteger(request.position) || request.position < 0) - ) { - throw new Error('Invalid Session transcript position'); - } - if ( - request.byteOffset !== undefined && - (request.position === undefined || - !Number.isSafeInteger(request.byteOffset) || - request.byteOffset < 0) - ) { - throw new Error('Invalid Session transcript byte offset'); - } - if ( - !Number.isSafeInteger(request.maxBytes) || - request.maxBytes < 1 || - request.maxBytes > 1024 * 1024 - ) { - throw new Error('Session transcript page byte limit must be between 1 and 1048576'); - } - if ( - !Number.isSafeInteger(request.maxMessages) || - request.maxMessages < 1 || - request.maxMessages > 256 - ) { - throw new Error('Session transcript page message limit must be between 1 and 256'); - } -} - -function assertTranscriptRecordScanRequest(request: SessionTranscriptRecordScanRequest): void { - if (request.direction !== 'older' && request.direction !== 'newer') { - throw new Error('Invalid Session transcript record direction'); - } - if ( - request.throughSequence !== undefined && - request.throughSequence !== null && - (!Number.isSafeInteger(request.throughSequence) || request.throughSequence < 0) - ) { - throw new Error('Invalid Session transcript watermark'); - } - if ( - request.position !== undefined && - (!Number.isSafeInteger(request.position) || request.position < 0) - ) { - throw new Error('Invalid Session transcript position'); - } - if ( - !Number.isSafeInteger(request.maxStoredBytes) || - request.maxStoredBytes < 1 || - request.maxStoredBytes > 16 * 1024 * 1024 - ) { - throw new Error('Invalid Session transcript record byte limit'); - } - if ( - !Number.isSafeInteger(request.maxMessages) || - request.maxMessages < 1 || - request.maxMessages > 256 - ) { - throw new Error('Invalid Session transcript record count limit'); - } -} From aadef6eb4d99ae4ff8ae5abf154c67599be52fe4 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 6 Sep 2026 17:33:54 +0800 Subject: [PATCH 07/32] fix(runtime): derive an admitted prompt's id once, and never seal a run without it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of #4879 found the failure the PR exists to remove, reintroduced on the recovery path. `begin()` derived the prompt event's id as `userMessageId ?? newId()`; recovery derived it as `userMessageId ?? ${runId}-admitted-prompt`. The two rules agree only for a single-source Root. A Root folded from several queued Messages has no Message identity, so a Host that died after the prompt landed and before the terminal came back to a ledger whose prompt it could not see, and recorded the same executed prompt a second time. `admittedPromptEventId` is now the one derivation, and recovery asks whether the Turn has a prompt rather than whether it has one under that exact id: a Run written by an older build derived the id differently, and matching on the id would read its prompt as missing. Steering leaves that index — it is typed as a user message but is something said into an already-admitted Turn, so it is never the Turn's own prompt. The same review found the in-process mirror of that crash: `begin()` failing between opening the invocation and recording the prompt runs `failStart` -> `finalize`, whose terminal event seals the run against every later append, recovery's repair included. Before this cutover recovery could still append to `session_messages`, which has no seal; a sealed ledger cannot be repaired, so `finalize` records the prompt itself before sealing, next to the openInvocation call that already keeps the sibling rule "a run cannot end without having begun". The read marker's tail scan now pages past hidden records. It read one bounded page and gave up, so a Turn ending on tool traffic could leave a Session showing unread after it had been read. It never cleared falsely, so this is a badge, not a lost message. Two review points are not taken. The reviewer's fix for the id mismatch was to inline the derived id in `begin()`; that leaves the same string template in two packages, which is still two rules that happen to agree. The reviewer also read the importer's "convert only turns that still have a user row" filter as a silent drop with no producer. It has one: a turn whose only user row was steering belongs to a Turn some durable Root already owns, and converting it stands a second synthetic run beside that one. Ablating the filter fails `does not import Host-handed-off transcript messages as synthetic runs`, so it stays, with its reason written down. storage 1092 pass, runtime-host 1711 pass, runtime 3130 pass, cli 805 pass, 0 failures. Generated-by: Claude Code --- .../__tests__/execution-host-recovery.test.ts | 31 ++++++++++ .../fixtures/execution-host-suite.ts | 18 ++++++ .../session-catalog-coordinator.test.ts | 56 +++++++++++++++++++ .../src/server/hosted-execution-recovery.ts | 36 ++++++------ .../src/server/session-catalog-coordinator.ts | 41 ++++++++++---- .../session-manager-terminal-ledger.test.ts | 35 ++++++++++++ packages/runtime/src/agent-run.ts | 46 +++++++++------ packages/runtime/src/message-authority.ts | 13 +++++ packages/runtime/src/runtime-ledger-repair.ts | 3 + 9 files changed, 230 insertions(+), 49 deletions(-) diff --git a/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts b/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts index 7b6c40030a..911534b6c9 100644 --- a/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts @@ -287,6 +287,37 @@ test('startup recovery closes a legacy non-terminal Run before recording its pro }); }); +test('startup recovery leaves a folded Root prompt the Run already recorded alone', async () => { + await withExecutionRoot(async (fixture) => { + // A folded Root has no single Message identity, so the prompt sits under an + // id recovery cannot rederive. Reading that as "no prompt yet" would record + // the one prompt the model already ran a second time. + const recordedPromptEventId = randomUUID(); + const legacy = await fixture.seedLegacyRootWithoutSourceTranscripts( + 'created', + recordedPromptEventId, + ); + + const host = await fixture.startHost(); + await fixture.stopHost(host); + + assert.deepEqual( + (await fixture.readSessionUserMessages()).map(({ id, turnId, text }) => ({ + id, + turnId, + text, + })), + [ + { + id: recordedPromptEventId, + turnId: legacy.turnId, + text: legacyRootPrompt(legacy).text, + }, + ], + ); + }); +}); + test('startup recovery rejects an unproven legacy Root without creating its missing Run', async () => { await withExecutionRoot(async (fixture) => { const legacy = await fixture.seedLegacyRootWithoutSourceTranscripts('missing'); diff --git a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts index 2c9d4f98ba..0d2c921aea 100644 --- a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts +++ b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts @@ -751,8 +751,15 @@ export class ExecutionFixture { } } + /** + * @param recordedPromptEventId The id the Run already recorded its prompt + * under, for the crash that happened after `begin()` wrote it. An older build + * derived that id differently, so it is a parameter rather than the id + * recovery would derive today. + */ async seedLegacyRootWithoutSourceTranscripts( runState: 'missing' | 'created' | 'terminal' = 'terminal', + recordedPromptEventId?: string, ): Promise<{ turnId: string; runId: string; @@ -837,6 +844,17 @@ export class ExecutionFixture { }, }); } + if (runState !== 'missing' && recordedPromptEventId) { + await stores.runtimeEventStore.appendRuntimeEvent(this.sessionId, runId, { + ...run, + id: recordedPromptEventId, + ts: admittedAt, + partial: false, + role: 'user', + author: 'user', + content: { kind: 'text', ...normalizedInput }, + }); + } if (runState === 'terminal') { const terminalAt = admittedAt + 1; const terminal = buildRecoveredTerminalRuntimeEvent({ diff --git a/packages/runtime-host/src/__tests__/session-catalog-coordinator.test.ts b/packages/runtime-host/src/__tests__/session-catalog-coordinator.test.ts index a4c8983619..dcc21cf9a1 100644 --- a/packages/runtime-host/src/__tests__/session-catalog-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/session-catalog-coordinator.test.ts @@ -195,6 +195,62 @@ test('read marker clears unread only at the ledger transcript tail', async () => assert.equal(caughtUp.lastReadMessageId, 'message-2'); }); +test('read marker pages past a hidden tail to reach the newest visible message', async () => { + // A Turn that ends on tool traffic can put more hidden records at the tail + // than one page holds. Stopping at the page boundary would read the Session + // as never caught up and leave it unread for good. + const hiddenTail = { + throughSequence: 2, + records: [ + { + sequence: 2, + message: { + type: 'turn_state' as const, + id: 'turn-state-1', + turnId: 'turn-1', + ts: 30, + status: 'completed' as const, + partialOutputRetained: false, + }, + }, + ], + nextPosition: 1, + }; + const visiblePage = { + throughSequence: 2, + records: [ + { + sequence: 1, + message: { + type: 'assistant' as const, + id: 'message-2', + turnId: 'turn-1', + ts: 20, + text: 'answer', + modelId: 'fake-model', + }, + }, + ], + nextPosition: null, + }; + const fixture = createFixture({ + header: { hasUnread: true }, + turnIndex: { + readDurableRecords: async (_sessionId, request) => + request.position === undefined ? hiddenTail : visiblePage, + }, + }); + + const outcome = await fixture.coordinator.handlers['session.read_marker.set']( + { sessionId: fixture.sessionId, readThroughMessageId: 'message-2' }, + context, + ); + assert.equal(outcome.ok, true); + if (!outcome.ok || !('hasUnread' in outcome.result)) assert.fail('Read marker failed'); + assert.equal(outcome.result.hasUnread, false); + assert.equal(outcome.result.lastReadMessageId, 'message-2'); +}); + test('metadata replacement preserves execution-semantic labels and ignores injected ones', async () => { const fixture = createFixture({ labels: ['old-user-label', DEEP_RESEARCH_SESSION_LABEL], diff --git a/packages/runtime-host/src/server/hosted-execution-recovery.ts b/packages/runtime-host/src/server/hosted-execution-recovery.ts index 2369be21b8..4fec474291 100644 --- a/packages/runtime-host/src/server/hosted-execution-recovery.ts +++ b/packages/runtime-host/src/server/hosted-execution-recovery.ts @@ -31,7 +31,10 @@ import { import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { StoredMessage } from '@maka/core/session'; import { projectRuntimeEventUserMessage } from '@maka/runtime/runtime-event-read-model'; -import { RuntimeMessageAuthorityInvariantError } from '@maka/runtime/message-authority'; +import { + admittedPromptEventId, + RuntimeMessageAuthorityInvariantError, +} from '@maka/runtime/message-authority'; import { type SessionManager } from '@maka/runtime/session-manager'; import type { ExecutionStoresWriter, RootTurnAdmission } from '@maka/storage/execution-stores'; import type { RootAdmissionOwner } from './root-admission-owner.js'; @@ -82,10 +85,12 @@ export async function prepareHostedExecutionRecovery( const pendingRecoveryClosures: PendingRecoveryClosure[] = []; for (const admission of admissions) { const run = runsById.get(admission.runId); - const admittedMessageId = admittedUserMessageId(admission); - const rootUserMessages = ( - messageIndex.userMessagesByTurnId.get(admission.turnId) ?? [] - ).filter((message) => message.id === admittedMessageId); + const admittedMessageId = admittedPromptEventId(admission.runId, admission.userMessageId); + // Whether the prompt is on the ledger is a question about the Turn, not + // about the id it landed under: a Run written by an older build derived + // that id differently, and matching on the id would read its prompt as + // missing and record a second one. + const rootUserMessages = messageIndex.userMessagesByTurnId.get(admission.turnId) ?? []; const messageIdOwners = messageIndex.messagesById.get(admittedMessageId) ?? []; if (messageIdOwners.length > 1) { throw new Error(`Admitted Turn ${admission.turnId} has a duplicated UserMessage identity`); @@ -248,7 +253,7 @@ async function recordAdmittedUserMessage( const content = requireHostedExecutionMessageContent(admission); const origin = hostedExecutionMessageOrigin(admission.execution); const event: RuntimeEvent = { - id: admittedUserMessageId(admission), + id: admittedPromptEventId(admission.runId, admission.userMessageId), sessionId: admission.sessionId, invocationId: run.invocationId, runId: run.runId, @@ -326,16 +331,6 @@ interface RecoveryExecutionContract { readonly pendingWithoutRun: 'root_replay' | 'domain_replay' | 'host_recovery_closure'; } -/** - * The id the admitted prompt is durable under. A Root folded from several - * queued Messages carries no single admitted Message identity, so recovery - * derives one from the Run — the same crash recovered twice writes the same - * event, and the store dedupes it. - */ -function admittedUserMessageId(admission: RootTurnAdmission): string { - return admission.userMessageId ?? `${admission.runId}-admitted-prompt`; -} - /** * Whether the ledger already carries this admission's message, throwing when * what it carries contradicts the admission. @@ -351,8 +346,7 @@ function verifyUserMessage( const userMessage = rootUserMessages[0]; if (userMessage) { if ( - messageIdOwner !== userMessage || - userMessage.id !== admittedUserMessageId(admission) || + (messageIdOwner !== undefined && messageIdOwner !== userMessage) || !recoveryUserMessageOriginMatches(userMessage, admission.execution) || !messageContentsEqual( normalizeMessageContent(userMessage), @@ -370,10 +364,12 @@ function verifyUserMessage( } /** - * The user messages a Session's ledger holds, as the transcript presents them. + * The prompts a Session's ledger holds, as the transcript presents them. * * Recovery reads the raw events rather than the read model: a Session it is * about to repair may be exactly the one whose projection is still incomplete. + * Steering is excluded — it is typed as a user message but is something said + * into a Turn that was already admitted, so it is never the Turn's own prompt. */ function recoveryUserMessagesFromLedger( events: readonly RuntimeEvent[], @@ -385,7 +381,7 @@ function recoveryUserMessagesFromLedger( event, event.id, ); - if (projected) messages.push(projected); + if (projected && projected.steeringEventId === undefined) messages.push(projected); } return messages; } diff --git a/packages/runtime-host/src/server/session-catalog-coordinator.ts b/packages/runtime-host/src/server/session-catalog-coordinator.ts index 4c3a50135f..138b6e2d4e 100644 --- a/packages/runtime-host/src/server/session-catalog-coordinator.ts +++ b/packages/runtime-host/src/server/session-catalog-coordinator.ts @@ -39,6 +39,7 @@ import { isWorkHubCoordinationSessionTarget, type SessionHeader, type SessionHeaderPatch, + type StoredMessage, } from '@maka/core/session'; import { isSessionNotFoundError, @@ -114,11 +115,7 @@ type SessionTurnIndexReader = Pick< 'readDurableRecords' | 'readDurableTurnContributions' | 'readDurableTurnLandmarks' >; -/** - * How far back a read marker looks for the newest visible message. A Turn ends - * on its assistant text, so the tail of one run is enough; the bound only keeps - * a run of pure tool traffic from walking the whole ledger. - */ +/** One page of the backwards scan a read marker walks to find the newest visible message. */ const SESSION_READ_MARKER_TAIL_MAX_MESSAGES = 64; const SESSION_READ_MARKER_TAIL_MAX_BYTES = 256 * 1024; @@ -868,13 +865,8 @@ export class HostSessionCatalogCoordinator { record: SessionHeaderSnapshot, readThroughMessageId: string, ): Promise { - const tail = await this.#turnIndex.readDurableRecords(record.header.id, { - direction: 'older', - maxMessages: SESSION_READ_MARKER_TAIL_MAX_MESSAGES, - maxStoredBytes: SESSION_READ_MARKER_TAIL_MAX_BYTES, - }); - const latest = tail.records.find(({ message }) => isVisibleSessionMessage(message)); - if (latest?.message.id !== readThroughMessageId) return; + const latest = await this.#newestVisibleMessage(record.header.id); + if (latest?.id !== readThroughMessageId) return; if (record.header.lastReadMessageId === readThroughMessageId && !record.header.hasUnread) { return; } @@ -885,6 +877,31 @@ export class HostSessionCatalogCoordinator { ); } + /** + * The ledger's newest message a client can actually see. A Turn that ends on + * tool traffic can put more hidden records at the tail than one page holds, + * so the scan pages past them instead of reading the Session as never caught + * up and leaving it unread for good. + */ + async #newestVisibleMessage(sessionId: string): Promise { + let throughSequence: number | null | undefined; + let position: number | undefined; + while (true) { + const page = await this.#turnIndex.readDurableRecords(sessionId, { + direction: 'older', + maxMessages: SESSION_READ_MARKER_TAIL_MAX_MESSAGES, + maxStoredBytes: SESSION_READ_MARKER_TAIL_MAX_BYTES, + ...(throughSequence === undefined ? {} : { throughSequence }), + ...(position === undefined ? {} : { position }), + }); + const visible = page.records.find(({ message }) => isVisibleSessionMessage(message)); + if (visible) return visible.message; + if (page.nextPosition === null) return undefined; + throughSequence = page.throughSequence; + position = page.nextPosition; + } + } + async #committedUpdate( sessionId: string, lease: SessionAdmissionLease, diff --git a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts index f2bc41d4fb..205d113bb4 100644 --- a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts +++ b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts @@ -1251,6 +1251,36 @@ describe('SessionManager terminal ledger invariants', () => { await new RuntimeReadModel({ runtimeEventStore: runStore }).getSessionView(session.id); }); + test('a run that failed while recording its prompt records it before sealing', async () => { + const store = new TinySessionStore(); + const runStore = new TinyAgentRunStore({ durability: 'canonical' }); + // The invocation is already open when this append is refused, so the run + // exists with nothing saying what it was asked to do. Its terminal event + // seals it against every later append, crash recovery's included. + runStore.rejectRuntimeEventIdsOnce.add('run-1-admitted-prompt'); + const session = await store.create(makeInput()); + const run = new AgentRun({ + sessionId: session.id, + header: session, + runId: 'run-1', + userInput: { turnId: 'turn-1', text: 'hello' }, + runStore, + runtimeEventStore: runStore, + newId: nextId(), + now: nextNow(41_750), + hooks: inertAgentRunHooks(store), + }); + + await assert.rejects(run.begin()); + await run.finalize(); + + const events = await runStore.readRuntimeEvents(session.id, 'run-1'); + const prompt = events.find((event) => event.role === 'user'); + assert.strictEqual(prompt?.id, 'run-1-admitted-prompt'); + assert.deepEqual(prompt.content, { kind: 'text', text: 'hello' }); + assert.strictEqual(events.filter(isTerminalRuntimeEvent).length, 1); + }); + test('a stop settlement racing finalize commits exactly one terminal run event', async () => { const store = new TinySessionStore(); const settleReachedAppend = deferred(); @@ -2235,6 +2265,8 @@ class TinyAgentRunStore implements AgentRunStore, RuntimeEventStore { private runtimeEventEntries: RuntimeEvent[] = []; /** One-shot append rejections, for latching the store availability. */ failNextRuntimeEventAppends = 0; + /** Event ids the ledger refuses once, the way a transient transition check would. */ + readonly rejectRuntimeEventIdsOnce = new Set(); /** While true every runtime-event read rejects, a store that is down. */ failRuntimeEventReads = false; /** One-shot run-event append rejections, for latching the Run store. */ @@ -2295,6 +2327,9 @@ class TinyAgentRunStore implements AgentRunStore, RuntimeEventStore { if (this.options.rejectRuntimeEventIds?.includes(event.id)) { throw new ToolLedgerRejectionError('orphan_response', event.id); } + if (this.rejectRuntimeEventIdsOnce.delete(event.id)) { + throw new ToolLedgerRejectionError('orphan_response', event.id); + } if (isTerminalRuntimeEvent(event)) await this.options.beforeTerminalRuntimeEventAppend?.(); const eventKey = key(sessionId, runId); this.runtimeEvents.set(eventKey, [...(this.runtimeEvents.get(eventKey) ?? []), clone(event)]); diff --git a/packages/runtime/src/agent-run.ts b/packages/runtime/src/agent-run.ts index 7adc8ea191..df4373f0d0 100644 --- a/packages/runtime/src/agent-run.ts +++ b/packages/runtime/src/agent-run.ts @@ -84,6 +84,7 @@ import { statusFromEvent, turnStatusFromEvent, } from './session-projection-helpers.js'; +import { admittedPromptEventId } from './message-authority.js'; import { commitOrCreateTerminalRunFact } from './terminal-run-commit.js'; import type { RuntimeContinuation } from './runtime-resume.js'; import { @@ -253,6 +254,8 @@ export class AgentRun { private providerStateIdentity: `sha256:${string}` | undefined; private invocationOpening: RuntimeEventInvocationOpenedContent | undefined; private invocationOpeningCommitted = false; + /** Set once `begin()` owes this run's prompt, cleared once the ledger has it. */ + private initialRuntimeEventPending = false; private terminalClaim: | { owner: 'event' | 'stop'; @@ -691,23 +694,9 @@ export class AgentRun { async begin(): Promise { await this.openInvocation(); - let initialRuntimeEventId: string; - - const userMessageTs = this.input.now(); - // The caller's durable message id becomes the initial event's id, so a - // same-id append with different content is refused by the store. - initialRuntimeEventId = this.input.userMessageId ?? this.input.newId(); - this.lastTs = userMessageTs; - - const initialRuntimeEvent = cloneAndFreezeRuntimeSnapshot( - this.buildInitialRuntimeEvent(initialRuntimeEventId, this.lastTs), - ); - await this.recordRuntimeEvents([initialRuntimeEvent], { - requireDurableWrite: this.requiresDurablePersistence(), - }); - await this.commitMessageProjection( - projectRuntimeEventUserMessage(initialRuntimeEvent, initialRuntimeEvent.id), - ); + this.lastTs = this.input.now(); + this.initialRuntimeEventPending = true; + const initialRuntimeEvent = await this.recordInitialRuntimeEvent(this.lastTs); this.active = await this.input.hooks.reserveRun(this.sessionId, this.header, this); @@ -743,6 +732,22 @@ export class AgentRun { }; } + /** Say what this run was asked to do. */ + private async recordInitialRuntimeEvent(ts: number): Promise { + const event = cloneAndFreezeRuntimeSnapshot( + this.buildInitialRuntimeEvent( + admittedPromptEventId(this.runId, this.input.userMessageId), + ts, + ), + ); + await this.recordRuntimeEvents([event], { + requireDurableWrite: this.requiresDurablePersistence(), + }); + this.initialRuntimeEventPending = false; + await this.commitMessageProjection(projectRuntimeEventUserMessage(event, event.id)); + return event; + } + async beginOperation(): Promise { await this.openInvocation(); @@ -1045,6 +1050,13 @@ export class AgentRun { // exception at both ends: its opening rides the continuation-start event, // and a continuation that never committed one has no invocation to end. if (!this.input.commitContinuationStart) await this.openInvocation().catch(() => {}); + // A run also cannot end without saying what it was asked to do. `begin()` + // can fail between opening the invocation and recording its prompt, and + // the terminal event below seals the run against every later append — + // including the one crash recovery would use to repair the same shape. + if (this.initialRuntimeEventPending) { + await this.recordInitialRuntimeEvent(this.lastTs || this.input.now()).catch(() => {}); + } await this.flushRuntimePartialBuffer(true); const lastTs = this.lastTs || this.input.now(); if (this.stopped) this.finalStatus = { status: 'aborted' }; diff --git a/packages/runtime/src/message-authority.ts b/packages/runtime/src/message-authority.ts index ad9df40ad3..52e97b033d 100644 --- a/packages/runtime/src/message-authority.ts +++ b/packages/runtime/src/message-authority.ts @@ -85,6 +85,19 @@ export class RuntimeMessageAuthorityInvariantError extends Error { readonly name = 'RuntimeMessageAuthorityInvariantError'; } +/** + * The id a Root Turn's admitted prompt is durable under. A Root folded from + * several queued Messages carries no single Message identity, so the id comes + * from the Run instead — one rule, so the run that writes the prompt and the + * recovery that rewrites it derive the same id and the store dedupes. + */ +export function admittedPromptEventId( + runId: string, + userMessageId: string | null | undefined, +): string { + return userMessageId ?? `${runId}-admitted-prompt`; +} + export class RuntimeHostedRootConflictError extends Error { readonly name = 'RuntimeHostedRootConflictError'; readonly code = 'session_busy'; diff --git a/packages/runtime/src/runtime-ledger-repair.ts b/packages/runtime/src/runtime-ledger-repair.ts index 061147b5e8..569ccaaf71 100644 --- a/packages/runtime/src/runtime-ledger-repair.ts +++ b/packages/runtime/src/runtime-ledger-repair.ts @@ -79,6 +79,9 @@ export class RuntimeLedgerRepair { .map((invocation) => invocation.turnId), ); const messagesByTurn = groupMessagesByTurn(ledgerMessages); + // A turn whose only user row was steering is not a turn of its own: the + // steering was said into a Turn some durable Root already owns, so + // converting it would stand a second, synthetic run beside that one. const turns = deriveTurnRecords(ledgerMessages).filter((turn) => (messagesByTurn.get(turn.turnId) ?? []).some((message) => message.type === 'user'), ); From 7b0451c67fc14cb6c30de846a32a087d80ed82b6 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 6 Sep 2026 19:03:07 +0800 Subject: [PATCH 08/32] fix(runtime): owe a run's prompt from before its invocation opens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pending flag was set after `openInvocation()`, so the one failure it did not cover was a throw from the opening itself: `finalize` reopens what it can, and a run it manages to open then sealed with a terminal and no prompt — the same hole the previous commit closed, entered from one step earlier. Moving the flag ahead of the opening costs nothing when the invocation never opens: `finalize`'s reopen fails too, and the backfill is a no-op on a run that does not exist. Reported as a residual on #4879 and not asked for; it is one line and it closes the last entrance to a shape that cannot be repaired after the fact. runtime 3131 pass, runtime-host 1723 pass, cli 805 pass, 0 failures. Generated-by: Claude Code --- .../session-manager-terminal-ledger.test.ts | 28 +++++++++++++++++++ packages/runtime/src/agent-run.ts | 4 ++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts index 205d113bb4..0a602b40e8 100644 --- a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts +++ b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts @@ -1281,6 +1281,34 @@ describe('SessionManager terminal ledger invariants', () => { assert.strictEqual(events.filter(isTerminalRuntimeEvent).length, 1); }); + test('a run that failed while opening still records its prompt once finalize reopens it', async () => { + const store = new TinySessionStore(); + const runStore = new TinyAgentRunStore({ durability: 'canonical' }); + // The opening append is what `begin()` fails on here, so the prompt is owed + // from before it — `finalize` reopens the invocation, and a run it can open + // is a run that has to say what it was asked to do. + runStore.rejectRuntimeEventIdsOnce.add('id-1'); + const session = await store.create(makeInput()); + const run = new AgentRun({ + sessionId: session.id, + header: session, + runId: 'run-1', + userInput: { turnId: 'turn-1', text: 'hello' }, + runStore, + runtimeEventStore: runStore, + newId: nextId(), + now: nextNow(41_900), + hooks: inertAgentRunHooks(store), + }); + + await assert.rejects(run.begin()); + await run.finalize(); + + const events = await runStore.readRuntimeEvents(session.id, 'run-1'); + assert.strictEqual(events.find((event) => event.role === 'user')?.id, 'run-1-admitted-prompt'); + assert.strictEqual(events.filter(isTerminalRuntimeEvent).length, 1); + }); + test('a stop settlement racing finalize commits exactly one terminal run event', async () => { const store = new TinySessionStore(); const settleReachedAppend = deferred(); diff --git a/packages/runtime/src/agent-run.ts b/packages/runtime/src/agent-run.ts index df4373f0d0..f9542b2f6a 100644 --- a/packages/runtime/src/agent-run.ts +++ b/packages/runtime/src/agent-run.ts @@ -692,10 +692,12 @@ export class AgentRun { } async begin(): Promise { + // Owed from here, not from after the opening: `openInvocation` can leave the + // invocation open and still throw, and `finalize` reopens what it can. + this.initialRuntimeEventPending = true; await this.openInvocation(); this.lastTs = this.input.now(); - this.initialRuntimeEventPending = true; const initialRuntimeEvent = await this.recordInitialRuntimeEvent(this.lastTs); this.active = await this.input.hooks.reserveRun(this.sessionId, this.header, this); From 354145761e48646a8c6bcb5e1b8add683bd3e220 Mon Sep 17 00:00:00 2001 From: M4n5ter Date: Sun, 6 Sep 2026 20:03:57 +0800 Subject: [PATCH 09/32] fix(runtime): resume transcript migration safely after interruption --- .../__tests__/runtime-ledger-repair.test.ts | 118 ++++++++++++++++-- packages/runtime/src/runtime-ledger-repair.ts | 12 +- packages/runtime/src/session-manager.ts | 9 +- 3 files changed, 124 insertions(+), 15 deletions(-) diff --git a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts index 3a0c41ce3c..6d967b05e9 100644 --- a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts +++ b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts @@ -22,6 +22,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; +import { DatabaseSync } from 'node:sqlite'; import type { StoredMessage } from '@maka/core/session'; import { createSqliteAgentRunStore } from '@maka/storage/agent-run-store'; import { createExternalSessionAdapterRegistry } from '@maka/storage/external-sessions'; @@ -36,6 +37,7 @@ import { import { runtimeInvocationFailureClass } from '../runtime-event-read-model.js'; import { backfillRuntimeEventsFromStoredMessages } from '../runtime-event-backfill.js'; import { RuntimeLedgerRepair } from '../runtime-ledger-repair.js'; +import { BackendRegistry, SessionManager } from '../session-manager.js'; test('repairs imported transcript turns into provider-neutral canonical history', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-transcript-ledger-repair-')); @@ -104,7 +106,6 @@ test('repairs imported transcript turns into provider-neutral canonical history' const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), - now: () => 100, }); await repair.materializeTranscriptLedger(session); @@ -284,7 +285,6 @@ test('an imported snapshot cutoff survives materialization as aborted', async () const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), - now: () => 100, }); await repair.materializeTranscriptLedger(session); @@ -339,7 +339,6 @@ test('does not import Host-handed-off transcript messages as synthetic runs', as const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), - now: () => 100, }); await repair.materializeTranscriptLedger(session); @@ -389,7 +388,6 @@ test('an imported turn with no terminal state is repaired to failed', async () = const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), - now: () => 100, }); await repair.materializeTranscriptLedger(session); @@ -464,18 +462,33 @@ test("converts Maka's own legacy transcript whole, and resumes an interrupted co const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), - now: () => 100, }); - await repair.materializeTranscriptLedger(await sessions.readHeader(session.id)); - // The same transcript converts to the same events, so a second pass — the - // retry after an interrupted one — adds nothing. - await repair.materializeTranscriptLedger(await sessions.readHeader(session.id)); + const append = runtimeEvents.appendRuntimeEvent.bind(runtimeEvents); + runtimeEvents.appendRuntimeEvent = async (sessionId, runId, event) => { + await append(sessionId, runId, event); + if (event.role === 'user') throw new Error('interrupted conversion'); + }; + await assert.rejects( + repair.materializeTranscriptLedger(await sessions.readHeader(session.id)), + /interrupted conversion/, + ); + runtimeEvents.appendRuntimeEvent = append; + const [interrupted] = await runtimeEvents.listSessionInvocations(session.id); + assert.ok(interrupted); + const prefix = await runtimeEvents.readRuntimeEvents(session.id, interrupted.runId); + assert.equal(interrupted.terminalEvent, undefined); + const resumed = new RuntimeLedgerRepair({ + runtimeEventStore: runtimeEvents, + readMessages: (sessionId) => sessions.readMessages(sessionId), + }); + await resumed.materializeTranscriptLedger(await sessions.readHeader(session.id)); const [run] = await runtimeEvents.listSessionInvocations(session.id); assert.ok(run); assert.equal(runtimeInvocationOutcome(run), 'completed'); const events = await runtimeEvents.readRuntimeEvents(session.id, run.runId); + assert.deepEqual(events.slice(0, prefix.length), prefix); assert.deepEqual( events.flatMap((event) => (event.content ? [event.content.kind] : [])), ['invocation_opened', 'text', 'function_call', 'function_response', 'system_note', 'text'], @@ -486,6 +499,92 @@ test("converts Maka's own legacy transcript whole, and resumes an interrupted co } }); +test('startup recovery leaves an interrupted legacy conversion for the importer to finish', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-transcript-restart-')); + const sessions = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + const runtimeEvents = createSqliteRuntimeStore(join(root, 'runtime.sqlite')); + try { + const session = await sessions.create({ + cwd: '/repo', + llmConnectionSlug: 'fake', + model: 'fake-model', + permissionMode: 'ask', + }); + const db = new DatabaseSync(join(root, 'runtime.sqlite')); + try { + db.prepare( + "UPDATE session_metadata SET payload_json = json_remove(payload_json, '$.transcriptLedgerVersion') WHERE session_id = ?", + ).run(session.id); + } finally { + db.close(); + } + await sessions.appendMessages(session.id, [ + { + type: 'user', + id: 'legacy-user', + turnId: 'legacy-turn', + ts: 10, + text: 'Keep this conversation', + }, + { + type: 'assistant', + id: 'legacy-answer', + turnId: 'legacy-turn', + ts: 20, + text: 'The complete original answer', + modelId: 'fake-model', + }, + { + type: 'turn_state', + id: 'legacy-end', + turnId: 'legacy-turn', + ts: 30, + status: 'completed', + partialOutputRetained: true, + }, + ]); + const append = runtimeEvents.appendRuntimeEvent.bind(runtimeEvents); + runtimeEvents.appendRuntimeEvent = async (sessionId, runId, event) => { + await append(sessionId, runId, event); + if (event.role === 'user') throw new Error('interrupted conversion'); + }; + const repair = new RuntimeLedgerRepair({ + runtimeEventStore: runtimeEvents, + readMessages: (id) => sessions.readMessages(id), + }); + await assert.rejects( + repair.materializeTranscriptLedger(await sessions.readHeader(session.id)), + /interrupted conversion/, + ); + runtimeEvents.appendRuntimeEvent = append; + let id = 0; + const manager = new SessionManager({ + store: sessions, + runStore: runs, + runtimeEventStore: runtimeEvents, + backends: new BackendRegistry(), + now: () => 100, + newId: () => `recovery-${++id}`, + }); + await manager.recoverInterruptedSessionsStrict({ sessionStore: sessions, agentRunStore: runs }); + const [pending] = await runtimeEvents.listSessionInvocations(session.id); + assert.ok(pending); + assert.equal(pending.terminalEvent, undefined); + const messages = await manager.getMessages(session.id); + assert.deepEqual( + messages.map((message) => message.id), + ['legacy-user', 'legacy-answer', 'legacy-end'], + ); + assert.equal((await sessions.readHeader(session.id)).transcriptLedgerVersion, 1); + } finally { + runtimeEvents.close(); + await runs.close?.(); + await sessions.close?.(); + await rm(root, { recursive: true, force: true }); + } +}); + test('a resolved Claude transcript replays as the conversation the user kept', async () => { // The whole path, end to end: raw records → lineage resolution → conversion // → Ledger materialization → the replay a continuation would be given. @@ -635,7 +734,6 @@ test('a resolved Claude transcript replays as the conversation the user kept', a const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), - now: () => 100, }); await repair.materializeTranscriptLedger(session); diff --git a/packages/runtime/src/runtime-ledger-repair.ts b/packages/runtime/src/runtime-ledger-repair.ts index 569ccaaf71..f8c7125e98 100644 --- a/packages/runtime/src/runtime-ledger-repair.ts +++ b/packages/runtime/src/runtime-ledger-repair.ts @@ -39,7 +39,6 @@ export interface RuntimeLedgerRepairDeps { runtimeEventStore: RuntimeEventStore; /** The legacy transcript this converter reads; nothing writes back to it. */ readMessages(sessionId: string): Promise; - now: () => number; } export class RuntimeLedgerRepair { @@ -107,7 +106,9 @@ export class RuntimeLedgerRepair { // whole: its tool calls are the ones it would replay. modelHistory: header.externalOrigin ? 'conversation_text' : 'full', newId: transcriptEventIds(runId), - now: this.deps.now, + // The payload must be as repeatable as its id: SQLite dedupes + // complete events, including the backfill provenance timestamps. + now: () => openedAt, }).events, ]; for (const event of events) { @@ -141,6 +142,13 @@ export class RuntimeLedgerRepair { } } +/** Synthetic conversion runs belong to the importer, never execution recovery. */ +export function isTranscriptLedgerInvocation( + invocation: Pick, +): boolean { + return invocation.runId === transcriptRunId(invocation.sessionId, invocation.turnId); +} + function transcriptRunId(sessionId: string, turnId: string): string { const digest = createHash('sha256').update(sessionId).update('\0').update(turnId).digest('hex'); return `transcript-${digest.slice(0, 48)}`; diff --git a/packages/runtime/src/session-manager.ts b/packages/runtime/src/session-manager.ts index 18d05f6cdd..26d98f39f1 100644 --- a/packages/runtime/src/session-manager.ts +++ b/packages/runtime/src/session-manager.ts @@ -153,7 +153,7 @@ import { type RuntimeReadModelSessionView, } from './runtime-read-model.js'; import { inspectAgentRunReadModel, type AgentRunInspectModel } from './agent-run-inspect.js'; -import { RuntimeLedgerRepair } from './runtime-ledger-repair.js'; +import { isTranscriptLedgerInvocation, RuntimeLedgerRepair } from './runtime-ledger-repair.js'; import { buildRecoveredTerminalRuntimeEvent, classifyTerminalRuntimeLedger, @@ -903,7 +903,6 @@ export class SessionManager { this.runtimeLedgerRepair = new RuntimeLedgerRepair({ runtimeEventStore: deps.runtimeEventStore, readMessages: (sessionId) => deps.store.readMessages(sessionId), - now: deps.now, }); } this.runtimeKernel = deps.runtimeKernel ?? new RuntimeKernel({ ...deps }); @@ -4383,7 +4382,11 @@ export class SessionManager { ): Promise<{ hasLedger: boolean; recovered: boolean }> { if (!this.deps.runStore || !this.deps.runtimeEventStore) return { hasLedger: false, recovered: false }; - const runs = await this.listInvocations(sessionId); + // The importer may have committed only a prefix before a restart. Sealing + // that prefix here would make the next read skip the unconverted history. + const runs = (await this.listInvocations(sessionId)).filter( + (run) => !isTranscriptLedgerInvocation(run), + ); if (runs.length === 0) return { hasLedger: false, recovered: false }; const continuationAuthority = runtimeContinuationAuthority(this.deps.runtimeEventStore); const claimOwnedUnsettledRunIds = new Set(); From a5544ad4694b4c27b12cac283bf43362f9fcddde Mon Sep 17 00:00:00 2001 From: M4n5ter Date: Sun, 6 Sep 2026 20:07:46 +0800 Subject: [PATCH 10/32] fix(storage): seek transcript pages before projecting ledger messages --- .../src/__tests__/protocol.test.ts | 4 + .../session-transcript-reader.test.ts | 285 +++++++++++- packages/runtime-host/src/protocol/index.ts | 3 +- .../src/server/session-transcript-reader.ts | 315 ++++++------- .../runtime/src/runtime-event-read-model.ts | 47 +- .../invocation-opening-backfill.test.ts | 5 +- .../__tests__/sqlite-runtime-schema.test.ts | 2 +- packages/storage/src/execution-stores.ts | 13 + .../storage/src/runtime-transcript-query.ts | 436 ++++++++++++++++++ packages/storage/src/sqlite-runtime-schema.ts | 19 +- packages/storage/src/sqlite-runtime-store.ts | 70 ++- 11 files changed, 1006 insertions(+), 193 deletions(-) create mode 100644 packages/storage/src/runtime-transcript-query.ts diff --git a/packages/runtime-host/src/__tests__/protocol.test.ts b/packages/runtime-host/src/__tests__/protocol.test.ts index b025966e9d..a323a02a46 100644 --- a/packages/runtime-host/src/__tests__/protocol.test.ts +++ b/packages/runtime-host/src/__tests__/protocol.test.ts @@ -443,6 +443,10 @@ describe('Runtime Host bootstrap protocol', () => { assert.ok(RUNTIME_HOST_COMPATIBILITY_EPOCH > 117); }); + test('publishes a new compatibility epoch for event-addressed transcript cursors', () => { + assert.ok(RUNTIME_HOST_COMPATIBILITY_EPOCH > 118); + }); + test('selects the highest mutually supported protocol and rejects a gap', () => { assert.equal(negotiateProtocol({ min: 0, max: 0 }, { min: 0, max: 0 }), 0); assert.equal(negotiateProtocol({ min: 1, max: 3 }, { min: 2, max: 4 }), 3); diff --git a/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts b/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts index 69922cc93f..512b04a4d5 100644 --- a/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts +++ b/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts @@ -26,6 +26,9 @@ import { seedInvocation, testInvocationOpening } from '@maka/runtime/test-only/i import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { StoredMessage } from '@maka/core/session'; +import { projectRuntimeEventsToStoredMessages } from '@maka/runtime/runtime-event-read-model'; +import { foldTurnContribution } from '@maka/storage/session-message-projection'; +import type { SessionTurnContribution } from '@maka/storage/execution-stores'; import { type ExecutionStoresWriter, openInteractiveExecutionStoresForWrite, @@ -251,7 +254,8 @@ test('keeps durable history separate from the canonical active overlay', async ( maxBytes: 1024, maxMessages: 10, }); - assert.equal(durable.throughSequence, 1); + assert.equal(durable.throughSequence, await read.readDurableHighWater(session.id)); + assert.ok(durable.throughSequence !== null); assert.deepEqual( durable.fragments.map((fragment) => { const message = JSON.parse(fragment.data.toString('utf8')) as StoredMessage; @@ -268,6 +272,285 @@ test('keeps durable history separate from the canonical active overlay', async ( } }); +test('pages the ledger without materializing off-page Turns or messages', async (t) => { + const base = await mkdtemp(join(tmpdir(), 'maka-transcript-seek-')); + const capability = await resolveStorageRoot({ path: join(base, 'root'), kind: 'interactive' }); + const owner = await tryAcquireInteractiveRootOwner(capability); + assert.ok(owner); + try { + const stores = await openInteractiveExecutionStoresForWrite(owner.lease); + const session = await stores.sessionStore.create({ + cwd: capability.canonicalPath, + llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', + llmConnectionSlug: 'fake', + model: 'fake-model', + permissionMode: 'ask', + }); + const expected: StoredMessage[] = []; + for (let turn = 0; turn < 5; turn++) { + const runId = `run-${turn}`; + const turnId = `turn-${turn}`; + await seedInvocation(stores.runtimeEventStore, { + sessionId: session.id, + runId, + turnId, + openedAt: turn, + }); + let count = 0; + const append = (overrides: Partial) => + stores.runtimeEventStore.appendRuntimeEvent( + session.id, + runId, + runtimeEvent(session.id, { + id: `${runId}-event-${count++}`, + invocationId: runId, + runId, + turnId, + ...overrides, + }), + ); + await append({ + role: 'user', + author: 'user', + content: { kind: 'text', text: `prompt ${turn}` }, + }); + if (turn === 4) { + // More than 5 MiB in a single Turn, outside a tiny head/tail page. + for (let index = 0; index < 180; index++) { + await append({ + role: 'model', + author: 'agent', + content: { kind: 'text', text: 'x'.repeat(32 * 1024) }, + }); + } + await append({ + role: 'model', + author: 'agent', + content: { kind: 'function_call', id: 'tool-1', name: 'Read', args: {} }, + refs: { toolCallId: 'tool-1', stepId: 'assistant-final' }, + }); + await append({ + actions: { + permissionRequest: { + kind: 'tool_permission', + requestId: 'request-1', + toolUseId: 'tool-1', + toolName: 'Read', + category: 'read', + reason: 'custom', + args: {}, + rememberForTurnAllowed: true, + hint: 'original permission hint', + }, + }, + }); + await append({ + actions: { + permissionDecision: { + requestId: 'request-1', + decision: 'allow', + rememberForTurn: true, + }, + }, + refs: { toolCallId: 'tool-1' }, + }); + await append({ + role: 'tool', + author: 'tool', + content: { + kind: 'function_response', + id: 'tool-1', + name: 'Read', + result: { kind: 'text', text: 'result' }, + isError: true, + }, + }); + await append({ + role: 'model', + author: 'agent', + content: { kind: 'thinking', text: 'before text' }, + refs: { providerEventId: 'assistant-final' }, + }); + await append({ + role: 'model', + author: 'agent', + content: { kind: 'text', text: 'final answer 中文' }, + refs: { storedMessageId: 'assistant-final' }, + }); + await append({ + role: 'model', + author: 'agent', + content: { kind: 'thinking', text: ' after text' }, + refs: { providerEventId: 'assistant-final', storedMessageId: 'usage-final' }, + actions: { tokenUsage: { input: 100, output: 25 } }, + }); + await append({ content: { kind: 'system_note', note: 'step_limit' } }); + } else { + await append({ + role: 'model', + author: 'agent', + content: { kind: 'text', text: '\u3000\u00a0' }, + }); + } + await append({ + status: 'failed', + actions: { endInvocation: true, stateDelta: { failureClass: 'tool_step_cap_reached' } }, + }); + const invocation = await stores.runtimeEventStore.readRunInvocation(session.id, runId); + assert.ok(invocation); + const projection = projectRuntimeEventsToStoredMessages( + await stores.runtimeEventStore.readRuntimeEvents(session.id, runId), + { invocations: [invocation] }, + ); + assert.deepEqual(projection.diagnostics, []); + expected.push(...projection.messages); + } + const read = createSessionTranscriptReader({ + stores, + canonicalPermissionOutcomes: { readPermissionOutcome: async () => undefined }, + }); + // Measure actual JSON decoded, not only the eventual response size. Neither + // a small page, the Turn index, nor a lookup miss may decode the 5 MiB Turn. + let decodedBytes = 0; + const parse = JSON.parse; + const measured = t.mock.method(JSON, 'parse', (...args: Parameters) => { + decodedBytes += Buffer.byteLength(args[0]); + return parse(...args); + }); + const through = await read.readDurableHighWater(session.id); + const tail = await read.readDurablePage(session.id, { + direction: 'older', + maxBytes: 1024, + maxMessages: 1, + }); + assert.equal(JSON.parse(tail.fragments[0]!.data.toString()).type, 'system_note'); + const head = await read.readDurablePage(session.id, { + direction: 'newer', + maxBytes: 1024, + maxMessages: 1, + }); + assert.equal(JSON.parse(head.fragments[0]!.data.toString()).text, 'prompt 0'); + assert.deepEqual( + await read.readDurableMessagesById(session.id, { + throughSequence: through, + messageIds: ['missing-stream'], + maxBytes: 1024, + maxMessages: 1, + }), + [], + ); + const landmarks = await read.readDurableTurnLandmarks(session.id, 3); + assert.deepEqual( + landmarks.landmarks.map((item) => item.label), + ['prompt 0', 'prompt 2', 'prompt 4'], + ); + const contributions: SessionTurnContribution[] = []; + let contributionPosition = 0; + for (;;) { + const page = await read.readDurableTurnContributions( + session.id, + through, + contributionPosition, + 2, + ); + contributions.push(...page.contributions); + if (page.nextPosition === null) break; + contributionPosition = page.nextPosition; + } + assert.ok(decodedBytes < 512 * 1024, `decoded ${decodedBytes} bytes for bounded reads`); + measured.mock.restore(); + + const records: Array<{ sequence: number; message: StoredMessage }> = []; + let position = 0; + for (;;) { + const page = await read.readDurableRecords(session.id, { + direction: 'newer', + throughSequence: through, + position, + maxMessages: 2, + maxStoredBytes: 128 * 1024, + }); + records.push(...page.records); + if (page.nextPosition === null) break; + position = page.nextPosition; + } + assert.deepEqual( + records.map((record) => record.message), + expected, + ); + const folded = new Map(); + for (const record of records) { + if (!('turnId' in record.message) || !record.message.turnId) continue; + const turnId = record.message.turnId; + folded.set( + turnId, + foldTurnContribution(folded.get(turnId), turnId, record.sequence, record.message), + ); + } + assert.deepEqual(contributions, [...folded.values()]); + const assistant = records.find((record) => record.message.id === 'assistant-final'); + assert.ok(assistant); + assert.deepEqual( + await read.readDurableMessagesById(session.id, { + throughSequence: through, + messageIds: ['assistant-final'], + maxBytes: 4096, + maxMessages: 1, + }), + [assistant.message], + ); + // Reassemble the same multibyte message in either direction, inside one row. + for (const direction of ['older', 'newer'] as const) { + let byteOffset: number | undefined; + const chunks: Buffer[] = []; + for (;;) { + const page = await read.readDurablePage(session.id, { + direction, + throughSequence: through, + position: assistant.sequence, + ...(byteOffset === undefined ? {} : { byteOffset }), + maxBytes: 37, + maxMessages: 1, + }); + chunks.push(page.fragments[0]!.data); + if (page.next?.position !== assistant.sequence) break; + assert.notEqual(page.next.byteOffset, null); + byteOffset = page.next.byteOffset!; + } + if (direction === 'older') chunks.reverse(); + assert.deepEqual(JSON.parse(Buffer.concat(chunks).toString()), assistant.message); + } + // A later sealed Turn must not alter a previously issued snapshot. + await seedInvocation(stores.runtimeEventStore, { + sessionId: session.id, + runId: 'later', + turnId: 'later', + openedAt: 99, + }); + await stores.runtimeEventStore.appendRuntimeEvent( + session.id, + 'later', + runtimeEvent(session.id, { + id: 'later-terminal', + invocationId: 'later', + runId: 'later', + turnId: 'later', + status: 'completed', + }), + ); + const frozen = await read.readDurablePage(session.id, { + direction: 'older', + throughSequence: through, + maxBytes: 1024, + maxMessages: 1, + }); + assert.deepEqual(frozen.fragments, tail.fragments); + } finally { + await owner.close(); + await rm(base, { recursive: true, force: true }); + } +}); + test('stops scanning a control-only ledger at the cumulative immutable event limit', async () => { const sessionId = 'session-1'; const events = Array.from({ length: 8_193 }, (_, index) => diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts index 4ec1d4da53..a1c40a7709 100644 --- a/packages/runtime-host/src/protocol/index.ts +++ b/packages/runtime-host/src/protocol/index.ts @@ -101,7 +101,8 @@ export const RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION = 1 as const; export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const; // Increment when the same protocol version no longer guarantees safe Client-Host // interoperability. Mismatches are rejected before domain commands are admitted. -export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 121 as const; +export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 122 as const; +// 122: Durable transcript cursors seek Session event ordinals instead of run indexes. // 121: Host diagnostics report `upgradeBlockingActivity`, the Host's // authoritative activity answer for maintenance probes, computed by the same // authority that gates `host.upgrade.prepare`. Older Clients reject the diff --git a/packages/runtime-host/src/server/session-transcript-reader.ts b/packages/runtime-host/src/server/session-transcript-reader.ts index 19c5eec280..87ea53765b 100644 --- a/packages/runtime-host/src/server/session-transcript-reader.ts +++ b/packages/runtime-host/src/server/session-transcript-reader.ts @@ -18,21 +18,19 @@ */ import type { RuntimeEvent } from '@maka/core/runtime-event'; +import { readRunInvocation } from '@maka/core/runtime-event-store'; import type { StoredMessage } from '@maka/core/session'; import { activePresentationRuntimeEvents, affectsRuntimeEventStoredMessageProjection, isHardRuntimeEventReadModelDiagnostic, projectRuntimeEventsToStoredMessages, + projectRuntimeEventUserMessage, } from '@maka/runtime/runtime-event-read-model'; import { type CanonicalPermissionOutcomeReader, type CanonicalPermissionOutcomeRecord, } from '@maka/runtime/interaction-authority'; -import { - isSessionInlineInvocation, - type RuntimeInvocationRecord, -} from '@maka/core/runtime-invocation'; import type { ExecutionStoresWriter, SessionTranscriptMessageLookupRequest, @@ -45,13 +43,14 @@ import type { SessionTurnContributionPage, SessionTurnLandmark, SessionTurnLandmarkSnapshot, + RuntimeTranscriptSource, } from '@maka/storage/execution-stores'; import { foldTurnContribution } from '@maka/storage/session-message-projection'; import { SESSION_TRANSCRIPT_OVERLAY_MAX_MESSAGES, type TurnSnapshot } from '../protocol/index.js'; const PERMISSION_OUTCOME_READ_CONCURRENCY = 8; -/** Sequence room reserved for one invocation's projected transcript rows. */ -const RUN_SEQUENCE_STRIDE = 1 << 20; +/** One event can emit content, a permission, usage, and terminal/notice rows. */ +const EVENT_SEQUENCE_STRIDE = 8; export const ACTIVE_TRANSCRIPT_OVERLAY_MAX_MESSAGES = SESSION_TRANSCRIPT_OVERLAY_MAX_MESSAGES; export const ACTIVE_TRANSCRIPT_OVERLAY_MAX_BYTES = 16 * 1024 * 1024; const ACTIVE_TRANSCRIPT_SOURCE_MAX_EVENTS = ACTIVE_TRANSCRIPT_OVERLAY_MAX_MESSAGES * 2; @@ -104,7 +103,11 @@ export function createSessionTranscriptReader(input: { readActiveOverlay: async (sessionId, rootTurn) => { if (!rootTurn || isTerminalTurn(rootTurn)) return []; - const invocations = await input.stores.runtimeEventStore.listSessionInvocations(sessionId); + const invocation = await readRunInvocation( + input.stores.runtimeEventStore, + sessionId, + rootTurn.runId, + ); const events = await readActiveProjectionEvents(input.stores, sessionId, rootTurn.runId); const canonicalPermissionOutcomes = await readCanonicalPermissionOutcomes( events, @@ -113,7 +116,7 @@ export function createSessionTranscriptReader(input: { const projected = projectRuntimeEventsToStoredMessages( activePresentationRuntimeEvents(events), { - invocations: invocations.filter((invocation) => invocation.runId === rootTurn.runId), + invocations: invocation ? [invocation] : [], canonicalPermissionOutcomes, }, ); @@ -157,89 +160,48 @@ export interface SessionTranscriptReader { } /** - * The settled part of a Session's transcript, read off the RuntimeEvent ledger. - * - * A page is bounded by reading one ended invocation at a time: a Session with a - * thousand Turns costs the same per page as one with three. Sequences are - * `runIndex * RUN_SEQUENCE_STRIDE + indexWithinRun`, which is monotone in the - * order the read model presents runs and derivable from the invocation list - * alone — so locating a page never has to project the Turns before it. They are - * stable for as long as a subscription lives, which is exactly as long as the - * signed cursors that carry them. + * Pages seek immutable Session event ordinals before decoding payloads. Each + * event is projected with just its indexed message context, so neither a long + * Session nor a long Turn has to be loaded to serve a page. The low sequence + * bits distinguish the few rows one event can emit. */ function createDurableLedgerTranscriptReader(input: { stores: ExecutionStoresWriter<'interactive'>; canonicalPermissionOutcomes: CanonicalPermissionOutcomeReader; }) { - const endedInvocations = async (sessionId: string): Promise => - (await input.stores.runtimeEventStore.listSessionInvocations(sessionId)).filter( - (invocation) => isSessionInlineInvocation(invocation.opening) && invocation.terminalEvent, - ); - - const projectRun = async ( - sessionId: string, - invocation: RuntimeInvocationRecord, + const store = input.stores.runtimeEventStore; + const highWater = async (sessionId: string): Promise => { + const ordinal = await store.readTranscriptSourceHighWater(sessionId); + return ordinal === null ? null : ordinal * EVENT_SEQUENCE_STRIDE + EVENT_SEQUENCE_STRIDE - 1; + }; + const projectSource = async ( + source: RuntimeTranscriptSource, ): Promise => { - const events = await input.stores.runtimeEventStore.readRuntimeEvents( - sessionId, - invocation.runId, - ); - const projected = projectRuntimeEventsToStoredMessages(events, { - invocations: [invocation], + const event = source.event; + const projected = projectRuntimeEventsToStoredMessages(source.events, { + invocations: [source.invocation], canonicalPermissionOutcomes: await readCanonicalPermissionOutcomes( - events, + source.events, input.canonicalPermissionOutcomes, ), + context: { + messageId: event.refs?.storedMessageId ?? event.refs?.providerEventId ?? event.id, + ...(source.contentOrder ? { contentOrder: source.contentOrder } : {}), + ...(source.permissionRequest ? { permissionRequest: source.permissionRequest } : {}), + ...(source.toolName ? { toolName: source.toolName } : {}), + ...(event.refs?.toolCallId ? { toolUseId: event.refs.toolCallId } : {}), + hasRetainedOutput: source.hasRetainedOutput, + }, }); if (projected.diagnostics.some(isHardRuntimeEventReadModelDiagnostic)) { throw new Error('Durable RuntimeEvent transcript projection is incomplete'); } - if (projected.messages.length > RUN_SEQUENCE_STRIDE) { - throw new Error('Durable Session Turn exceeds its transcript sequence stride'); + if (projected.messages.length > EVENT_SEQUENCE_STRIDE) { + throw new Error('RuntimeEvent exceeds its transcript sequence stride'); } return projected.messages; }; - /** - * The runs a page must visit, in traversal order, already clipped to the - * watermark and the caller's position. - */ - const traversal = ( - invocations: readonly RuntimeInvocationRecord[], - direction: 'older' | 'newer', - throughSequence: number, - position: number, - ): Array<{ runIndex: number; invocation: RuntimeInvocationRecord }> => { - const highestRunIndex = Math.min(runIndexOf(throughSequence), invocations.length - 1); - const startRunIndex = Math.min(runIndexOf(position), highestRunIndex); - const runs: Array<{ runIndex: number; invocation: RuntimeInvocationRecord }> = []; - if (direction === 'older') { - for (let index = startRunIndex; index >= 0; index -= 1) { - const invocation = invocations[index]; - if (invocation) runs.push({ runIndex: index, invocation }); - } - return runs; - } - for (let index = Math.max(0, startRunIndex); index <= highestRunIndex; index += 1) { - const invocation = invocations[index]; - if (invocation) runs.push({ runIndex: index, invocation }); - } - return runs; - }; - - const highWaterOf = async ( - sessionId: string, - invocations: readonly RuntimeInvocationRecord[], - ): Promise => { - for (let index = invocations.length - 1; index >= 0; index -= 1) { - const invocation = invocations[index]!; - const messages = await projectRun(sessionId, invocation); - if (messages.length > 0) return index * RUN_SEQUENCE_STRIDE + messages.length - 1; - } - return null; - }; - - /** Every projected record of the requested page, ordered for its direction. */ const scan = async function* ( sessionId: string, request: { @@ -248,38 +210,37 @@ function createDurableLedgerTranscriptReader(input: { position?: number; }, ): AsyncGenerator<{ sequence: number; message: StoredMessage }> { - const invocations = await endedInvocations(sessionId); const throughSequence = - request.throughSequence === undefined - ? await highWaterOf(sessionId, invocations) - : request.throughSequence; + request.throughSequence === undefined ? await highWater(sessionId) : request.throughSequence; if (throughSequence === null) return; const position = request.position ?? (request.direction === 'older' ? throughSequence : 0); - for (const { runIndex, invocation } of traversal( - invocations, - request.direction, - throughSequence, - position, - )) { - const messages = await projectRun(sessionId, invocation); - const indexed = messages.map((message, index) => ({ - sequence: runIndex * RUN_SEQUENCE_STRIDE + index, - message, - })); - const selected = indexed.filter( - ({ sequence }) => - sequence <= throughSequence && - (request.direction === 'older' ? sequence <= position : sequence >= position), - ); - if (request.direction === 'older') selected.reverse(); - yield* selected; + let ordinal = ordinalOf(position); + while (ordinal >= 0 && ordinal <= ordinalOf(throughSequence)) { + const source = await store.readTranscriptSource(sessionId, { + direction: request.direction, + throughOrdinal: ordinalOf(throughSequence), + position: ordinal, + }); + if (!source) return; + const messages = await projectSource(source); + const records = messages + .map((message, index) => ({ + sequence: source.ordinal * EVENT_SEQUENCE_STRIDE + index, + message, + })) + .filter( + ({ sequence }) => + sequence <= throughSequence && + (request.direction === 'older' ? sequence <= position : sequence >= position), + ); + if (request.direction === 'older') records.reverse(); + yield* records; + ordinal = source.ordinal + (request.direction === 'older' ? -1 : 1); } }; return { - async readHighWater(sessionId: string): Promise { - return highWaterOf(sessionId, await endedInvocations(sessionId)); - }, + readHighWater: highWater, async readPage( sessionId: string, @@ -363,77 +324,95 @@ function createDurableLedgerTranscriptReader(input: { return { throughSequence, records, nextPosition }; }, - /** - * What each Turn contributed, one ended invocation at a time. - * - * An invocation is a Turn, so the run listing is the index: a page costs the - * runs it actually summarizes, never a scan of the Turns before them. - */ + /** Fold indexed Turn facts, loading only the prompt and terminal payloads. */ async readTurnContributions( sessionId: string, throughSequence: number | null, position: number, maxContributions: number, ): Promise { - const invocations = await endedInvocations(sessionId); - const watermark = throughSequence ?? (await highWaterOf(sessionId, invocations)); + const watermark = throughSequence ?? (await highWater(sessionId)); if (watermark === null) { return { throughSequence: null, contributions: [], nextPosition: null }; } + const turns = await store.readTranscriptTurns( + sessionId, + ordinalOf(watermark), + ordinalOf(position), + maxContributions + 1, + ); const contributions: SessionTurnContribution[] = []; - const lastRunIndex = Math.min(runIndexOf(watermark), invocations.length - 1); - let runIndex = Math.max(0, runIndexOf(position)); - for (; runIndex <= lastRunIndex; runIndex += 1) { - if (contributions.length >= maxContributions) { - return { - throughSequence: watermark, - contributions, - nextPosition: runIndex * RUN_SEQUENCE_STRIDE, - }; + for (const turn of turns.slice(0, maxContributions)) { + let contribution: SessionTurnContribution = { + turnId: turn.invocation.turnId, + firstSequence: Math.max(position, turn.firstOrdinal * EVENT_SEQUENCE_STRIDE), + latestState: null, + userPromptPreview: null, + hasAssistantMessage: turn.hasAssistantMessage, + hasAssistantOutput: turn.hasAssistantOutput, + hasToolResult: turn.hasToolResult, + hasFailedToolResult: turn.hasFailedToolResult, + hasAbortNote: turn.hasAbortNote, + }; + if (turn.user) { + const user = projectRuntimeEventUserMessage(turn.user.event, turn.user.event.id); + if (user) + contribution = foldTurnContribution( + contribution, + turn.invocation.turnId, + turn.user.ordinal * EVENT_SEQUENCE_STRIDE, + user, + ); } - const invocation = invocations[runIndex]; - if (!invocation) continue; - const messages = await projectRun(sessionId, invocation); - let contribution: SessionTurnContribution | undefined; - for (const [index, message] of messages.entries()) { - const sequence = runIndex * RUN_SEQUENCE_STRIDE + index; - if (sequence > watermark || sequence < position) continue; - contribution = foldTurnContribution(contribution, invocation.turnId, sequence, message); + const source = await store.readTranscriptSource(sessionId, { + direction: 'newer', + throughOrdinal: ordinalOf(watermark), + position: turn.terminalOrdinal, + }); + if (source?.ordinal === turn.terminalOrdinal) { + const messages = await projectSource(source); + for (const [index, message] of messages.entries()) { + const sequence = source.ordinal * EVENT_SEQUENCE_STRIDE + index; + if (sequence < position || sequence > watermark) continue; + contribution = foldTurnContribution( + contribution, + turn.invocation.turnId, + sequence, + message, + ); + } } - if (contribution) contributions.push(contribution); + contributions.push(contribution); } - return { throughSequence: watermark, contributions, nextPosition: null }; + const next = turns[maxContributions]; + return { + throughSequence: watermark, + contributions, + nextPosition: next ? next.firstOrdinal * EVENT_SEQUENCE_STRIDE : null, + }; }, - /** Evenly spaced Turn starts, sampled from the run listing itself. */ + /** Evenly spaced Turn starts, selected in SQL before loading their prompts. */ async readTurnLandmarks( sessionId: string, maxLandmarks: number, ): Promise { - const invocations = await endedInvocations(sessionId); - const throughSequence = await highWaterOf(sessionId, invocations); + const throughSequence = await highWater(sessionId); if (throughSequence === null) return { throughSequence: null, landmarks: [] }; - const lastRunIndex = Math.min(runIndexOf(throughSequence), invocations.length - 1); - const count = Math.min(maxLandmarks, lastRunIndex + 1); - const sampled = - count <= 0 - ? [] - : Array.from({ length: count }, (_, index) => - count === 1 ? lastRunIndex : Math.floor((lastRunIndex * index) / (count - 1)), - ); + const turns = await store.readTranscriptLandmarks( + sessionId, + ordinalOf(throughSequence), + maxLandmarks, + ); const landmarks: SessionTurnLandmark[] = []; - for (const runIndex of [...new Set(sampled)]) { - const invocation = invocations[runIndex]; - if (!invocation) continue; - const messages = await projectRun(sessionId, invocation); - const index = messages.findIndex((message) => message.type === 'user'); - const message = index < 0 ? undefined : messages[index]; - if (message?.type !== 'user') continue; - const label = (message.displayText ?? message.text).trim(); + for (const turn of turns) { + if (!turn.user) continue; + const message = projectRuntimeEventUserMessage(turn.user.event, turn.user.event.id); + const label = (message?.displayText ?? message?.text ?? '').trim(); if (!label) continue; landmarks.push({ - turnId: invocation.turnId, - sequence: runIndex * RUN_SEQUENCE_STRIDE + index, + turnId: turn.invocation.turnId, + sequence: turn.user.ordinal * EVENT_SEQUENCE_STRIDE, label, }); } @@ -445,33 +424,33 @@ function createDurableLedgerTranscriptReader(input: { request: SessionTranscriptMessageLookupRequest, ): Promise { if (request.throughSequence === null || request.messageIds.length === 0) return []; - const wanted = new Set(request.messageIds); - const found: StoredMessage[] = []; + const found: Array<{ sequence: number; message: StoredMessage }> = []; let bytes = 0; - // Callers look up streams that were active a moment ago, so a durable copy - // can only be in the run that just sealed. Without this bound the ordinary - // miss — the run is still open — reprojects every Turn in the Session. - let newestRunIndex: number | undefined; - for await (const record of scan(sessionId, { - direction: 'older', - throughSequence: request.throughSequence, - })) { - newestRunIndex ??= runIndexOf(record.sequence); - if (runIndexOf(record.sequence) < newestRunIndex) break; - if (!wanted.delete(record.message.id)) continue; - bytes += Buffer.byteLength(JSON.stringify(record.message), 'utf8'); - if (found.length >= request.maxMessages || bytes > request.maxBytes) break; - found.push(record.message); - if (wanted.size === 0) break; + for (const messageId of new Set(request.messageIds)) { + const source = await store.readTranscriptSource(sessionId, { + direction: 'older', + throughOrdinal: ordinalOf(request.throughSequence), + position: ordinalOf(request.throughSequence), + messageId, + }); + if (!source) continue; + for (const [index, message] of (await projectSource(source)).entries()) { + const sequence = source.ordinal * EVENT_SEQUENCE_STRIDE + index; + if (message.id !== messageId || sequence > request.throughSequence) continue; + bytes += Buffer.byteLength(JSON.stringify(message), 'utf8'); + if (found.length >= request.maxMessages || bytes > request.maxBytes) { + return found.sort((a, b) => a.sequence - b.sequence).map((record) => record.message); + } + found.push({ sequence, message }); + } } - // Restore transcript order: the scan walked backwards to find them. - return found.reverse(); + return found.sort((a, b) => a.sequence - b.sequence).map((record) => record.message); }, }; } -function runIndexOf(sequence: number): number { - return Math.floor(sequence / RUN_SEQUENCE_STRIDE); +function ordinalOf(sequence: number): number { + return Math.floor(sequence / EVENT_SEQUENCE_STRIDE); } function assertActiveOverlayBounded(messages: readonly StoredMessage[]): void { diff --git a/packages/runtime/src/runtime-event-read-model.ts b/packages/runtime/src/runtime-event-read-model.ts index 9aa5f21deb..819d19fceb 100644 --- a/packages/runtime/src/runtime-event-read-model.ts +++ b/packages/runtime/src/runtime-event-read-model.ts @@ -152,6 +152,15 @@ export interface ProjectRuntimeEventsToStoredMessagesOptions { | readonly RuntimeInvocationRecord[] | Readonly>; canonicalPermissionOutcomes?: ReadonlyMap; + /** Facts read by indexed lookup when projecting one durable message. */ + context?: { + messageId: string; + contentOrder?: readonly AssistantStepContentKind[]; + permissionRequest?: RuntimeEvent; + toolName?: string; + toolUseId?: string; + hasRetainedOutput: boolean; + }; } export interface ArchivedToolResultReadModelStatus { @@ -204,6 +213,7 @@ interface ProjectionState { */ thinkingByMessageId: Map; contentOrderByMessageId: Map; + hasRetainedOutput?: boolean; } interface PendingThinking { @@ -228,6 +238,29 @@ export function projectRuntimeEventsToStoredMessages( }; const messages: StoredMessage[] = []; + const context = options.context; + if (context) { + state.hasRetainedOutput = context.hasRetainedOutput; + if (context.contentOrder) + state.contentOrderByMessageId.set(context.messageId, [...context.contentOrder]); + if (context.toolName && context.toolUseId) + state.toolNameByUseId.set(context.toolUseId, context.toolName); + const requestEvent = context.permissionRequest; + const request = requestEvent?.actions?.permissionRequest; + if (request && requestEvent) { + state.permissionRequestById.set(request.requestId, { + requestId: request.requestId, + toolUseId: request.toolUseId, + toolName: request.toolName, + sessionId: requestEvent.sessionId, + runId: requestEvent.runId, + turnId: requestEvent.turnId, + ...(request.hint !== undefined ? { hint: request.hint } : {}), + }); + state.toolNameByUseId.set(request.toolUseId, request.toolName); + } + } + for (const event of events) { recordStepContentOrder(event, state); if (isPartialRuntimeEvent(event)) { @@ -1240,12 +1273,14 @@ function projectTerminalTurnState( } const abortSource = status === 'aborted' ? abortSourceFromRuntime(event) : undefined; const failureClass = status === 'failed' ? failureClassFromRuntimeEvent(event) : undefined; - const partialOutputRetained = messages.some( - (message) => - message.turnId === event.turnId && - ((message.type === 'assistant' && message.text.trim().length > 0) || - message.type === 'tool_result'), - ); + const partialOutputRetained = + state.hasRetainedOutput ?? + messages.some( + (message) => + message.turnId === event.turnId && + ((message.type === 'assistant' && message.text.trim().length > 0) || + message.type === 'tool_result'), + ); messages.push({ type: 'turn_state', id: stableMessageId(event, state, 'turn_state'), diff --git a/packages/storage/src/__tests__/invocation-opening-backfill.test.ts b/packages/storage/src/__tests__/invocation-opening-backfill.test.ts index 3e8a5d1610..9f5e1b3a01 100644 --- a/packages/storage/src/__tests__/invocation-opening-backfill.test.ts +++ b/packages/storage/src/__tests__/invocation-opening-backfill.test.ts @@ -462,7 +462,7 @@ describe('invocation opening fact backfill', () => { 'INSERT INTO core_agent_runs(session_id, run_id, created_at, record_json) VALUES (?, ?, ?, ?)', ).run(corrupt.sessionId, corrupt.runId, corrupt.createdAt, JSON.stringify(corrupt)); assert.throws(() => migrateSqliteRuntimeDatabase(db), /session-1\/run-corrupt-root/); - assert.equal(readUserVersion(db), SQLITE_RUNTIME_SCHEMA_VERSION - 1); + assert.equal(readUserVersion(db), 15); const openings = db .prepare( "SELECT COUNT(*) AS total FROM runtime_events WHERE event_kind = 'invocation_opened'", @@ -509,7 +509,8 @@ function rewindToHeaderEra(db: DatabaseSync): void { 'ALTER TABLE runtime_continuation_claims RENAME COLUMN target_opening_json TO target_run_header_json', ); db.exec('ALTER TABLE core_agent_runs ADD COLUMN record_json TEXT'); - db.exec(`PRAGMA user_version = ${SQLITE_RUNTIME_SCHEMA_VERSION - 1}`); + // Opening facts were introduced in v16, regardless of the current version. + db.exec('PRAGMA user_version = 15'); } function readUserVersion(db: DatabaseSync): number { diff --git a/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts b/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts index 5bdd3984b7..b4e3548893 100644 --- a/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts +++ b/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts @@ -155,7 +155,7 @@ describe('SQLite runtime schema migration', () => { migrateSqliteRuntimeDatabase(db); - assert.equal(SQLITE_RUNTIME_SCHEMA_VERSION, 16); + assert.equal(SQLITE_RUNTIME_SCHEMA_VERSION, 17); assert.equal( ( db diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index 0343df2cc1..1488d2ec72 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -20,6 +20,7 @@ import type { AgentRunEvent, AgentRunEventType, AgentRunProjectionKey } from '@maka/core/agent-run'; import type { RuntimeEvent, ToolBoundaryProtocol } from '@maka/core/runtime-event'; import type { RuntimeContinuationAuthorityStore } from '@maka/core/runtime-event-store'; +import type { RuntimeTranscriptQueries } from './runtime-transcript-query.js'; import type { RuntimeInvocationPageInput, RuntimeInvocationPageResult, @@ -135,8 +136,10 @@ export type { } from './session-store.js'; export type ExecutionSessionWriter = SessionAuthorityStore; +export type { RuntimeTranscriptSource, RuntimeTranscriptTurn } from './runtime-transcript-query.js'; export type ExecutionAgentRunWriter = DurableAgentRunStore; export type ExecutionRuntimeEventWriter = DurableRuntimeEventStore & + RuntimeTranscriptQueries & RuntimeContinuationAuthorityStore & { readonly toolBoundaryProtocol: ToolBoundaryProtocol; commitToolPrepared(input: CommitToolPreparedInput): Promise; @@ -569,6 +572,16 @@ async function createExecutionStoresForWrite runtimeEventStore.readSessionRuntimeEvents(sessionId)), readSessionRuntimeEventEntries: (sessionId) => run(() => runtimeEventStore.readSessionRuntimeEventEntries(sessionId)), + readTranscriptSourceHighWater: (sessionId) => + run(() => runtimeEventStore.readTranscriptSourceHighWater(sessionId)), + readTranscriptSource: (sessionId, request) => + run(() => runtimeEventStore.readTranscriptSource(sessionId, request)), + readTranscriptTurns: (sessionId, throughOrdinal, position, limit) => + run(() => + runtimeEventStore.readTranscriptTurns(sessionId, throughOrdinal, position, limit), + ), + readTranscriptLandmarks: (sessionId, throughOrdinal, limit) => + run(() => runtimeEventStore.readTranscriptLandmarks(sessionId, throughOrdinal, limit)), claimContinuation: (input) => run(() => runtimeEventStore.claimContinuation(input)), readContinuationClaimByBoundary: (boundaryDigest) => run(() => runtimeEventStore.readContinuationClaimByBoundary(boundaryDigest)), diff --git a/packages/storage/src/runtime-transcript-query.ts b/packages/storage/src/runtime-transcript-query.ts new file mode 100644 index 0000000000..ee05fd4809 --- /dev/null +++ b/packages/storage/src/runtime-transcript-query.ts @@ -0,0 +1,436 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { DatabaseSync } from 'node:sqlite'; +import { + decodeRuntimeEvent, + isTerminalRuntimeEvent, + type RuntimeEvent, +} from '@maka/core/runtime-event'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; +import type { AssistantStepContentKind } from '@maka/core/session'; + +/** SQL counterpart of isTerminalRuntimeEvent; shared with the ledger store. */ +export const TERMINAL_RUNTIME_EVENT_SQL = `( + json_extract(payload_json, '$.actions.endInvocation') = 1 + OR json_extract(payload_json, '$.status') IN ('completed', 'failed', 'aborted', 'cancelled') +)`; + +export const TRANSCRIPT_MESSAGE_KEY_SQL = `CASE WHEN event_kind = 'function_call' + THEN json_extract(payload_json, '$.refs.stepId') + ELSE COALESCE(json_extract(payload_json, '$.refs.providerEventId'), json_extract(payload_json, '$.refs.storedMessageId'), event_id) END`; +export const TRANSCRIPT_STORED_ID_SQL = `COALESCE(json_extract(payload_json, '$.refs.storedMessageId'), json_extract(payload_json, '$.refs.providerEventId'), json_extract(payload_json, '$.content.id'), event_id)`; +/** Small indexed facts needed by a terminal row and the Turn index. */ +export const TRANSCRIPT_OUTPUT_SHAPE_SQL = `CASE + WHEN event_kind = 'text' AND json_extract(payload_json, '$.role') = 'model' + THEN CASE WHEN TRIM(json_extract(payload_json, '$.content.text'), char(9,10,11,12,13,32,160,5760,8192,8193,8194,8195,8196,8197,8198,8199,8200,8201,8202,8232,8233,8239,8287,12288,65279)) <> '' THEN 3 ELSE 1 END + WHEN event_kind = 'function_response' THEN CASE WHEN json_extract(payload_json, '$.content.isError') = 1 THEN 12 ELSE 4 END + ELSE 0 END`; + +export interface RuntimeTranscriptSource { + readonly ordinal: number; + readonly event: RuntimeEvent; + /** Only this message's thinking, in ledger order around its text event. */ + readonly events: readonly RuntimeEvent[]; + readonly invocation: RuntimeInvocationRecord; + readonly contentOrder?: readonly AssistantStepContentKind[]; + readonly permissionRequest?: RuntimeEvent; + readonly toolName?: string; + readonly hasRetainedOutput: boolean; +} + +export interface RuntimeTranscriptPosition { + readonly direction: 'older' | 'newer'; + readonly throughOrdinal: number; + readonly position: number; + /** Exact lookup for the bounded live-to-durable handoff. */ + readonly messageId?: string; +} + +export interface RuntimeTranscriptTurn { + readonly firstOrdinal: number; + readonly terminalOrdinal: number; + readonly invocation: RuntimeInvocationRecord; + readonly user?: { ordinal: number; event: RuntimeEvent }; + readonly hasAssistantMessage: boolean; + readonly hasAssistantOutput: boolean; + readonly hasToolResult: boolean; + readonly hasFailedToolResult: boolean; + readonly hasAbortNote: boolean; +} + +export interface RuntimeTranscriptQueries { + readTranscriptSourceHighWater(sessionId: string): Promise; + readTranscriptSource( + sessionId: string, + request: RuntimeTranscriptPosition, + ): Promise; + readTranscriptTurns( + sessionId: string, + throughOrdinal: number, + position: number, + limit: number, + ): Promise; + readTranscriptLandmarks( + sessionId: string, + throughOrdinal: number, + limit: number, + ): Promise; +} + +const messageKey = (alias: string) => + TRANSCRIPT_MESSAGE_KEY_SQL.replaceAll('event_kind', `${alias}.event_kind`) + .replaceAll('payload_json', `${alias}.payload_json`) + .replaceAll('event_id', `${alias}.event_id`); +const terminal = (alias: string) => + TERMINAL_RUNTIME_EVENT_SQL.replaceAll('payload_json', `${alias}.payload_json`); +const opening = `COALESCE(json_extract(opened.payload_json, '$.content'), legacy.opening_json)`; +const joins = ` + FROM runtime_session_event_ordinals o + JOIN runtime_events e ON e.event_id = o.event_id + LEFT JOIN runtime_events opened ON opened.invocation_id = e.invocation_id AND opened.event_kind = 'invocation_opened' + LEFT JOIN runtime_legacy_invocation_openings legacy ON legacy.invocation_id = e.invocation_id`; +const settledInline = ` + ${opening} IS NOT NULL + AND (json_extract(${opening}, '$.lineage.parentRunId') IS NULL + OR (json_extract(${opening}, '$.source.kind') = 'continuation' + AND json_extract(${opening}, '$.lineage.agentId') IS NULL)) + AND EXISTS ( + SELECT 1 FROM runtime_events ended + JOIN runtime_session_event_ordinals ending ON ending.event_id = ended.event_id + WHERE ended.invocation_id = e.invocation_id AND ${terminal('ended')} + AND ending.ordinal <= :throughOrdinal + )`; +// Thinking is context for its text row. An orphan must still reach the +// projector, which reports the missing text instead of silently dropping it. +const transcriptSource = `( + (json_extract(e.payload_json, '$.content') IS NOT NULL + AND e.event_kind <> 'invocation_opened' + AND (e.event_kind <> 'thinking' OR NOT EXISTS ( + SELECT 1 FROM runtime_events text + WHERE text.invocation_id = e.invocation_id AND text.event_kind = 'text' + AND json_extract(text.payload_json, '$.role') = 'model' + AND COALESCE(json_extract(text.payload_json, '$.refs.storedMessageId'), json_extract(text.payload_json, '$.refs.providerEventId'), text.event_id) = ${messageKey('e')} + ))) + OR json_extract(e.payload_json, '$.actions.permissionDecision') IS NOT NULL + OR json_extract(e.payload_json, '$.actions.permissionAnswerAccepted') IS NOT NULL + OR json_extract(e.payload_json, '$.actions.tokenUsage') IS NOT NULL + OR ${terminal('e')} +)`; + +type SourceRow = { + ordinal: number; + event_id: string; + run_id: string; + invocation_id: string; + event_seq: number; +}; + +/** Queries select ledger positions before loading any message payload. No transcript is persisted. */ +export class RuntimeTranscriptQuery { + constructor( + private readonly db: DatabaseSync, + private readonly invocation: (sessionId: string, runId: string) => RuntimeInvocationRecord, + ) {} + + highWater(sessionId: string): number | null { + return ( + this.sourceRow(sessionId, { + direction: 'older', + throughOrdinal: Number.MAX_SAFE_INTEGER, + position: Number.MAX_SAFE_INTEGER, + })?.ordinal ?? null + ); + } + + private sourceRow(sessionId: string, request: RuntimeTranscriptPosition): SourceRow | undefined { + assertOrdinal(request.throughOrdinal); + assertOrdinal(request.position); + if (request.direction !== 'older' && request.direction !== 'newer') + throw new Error('Invalid transcript direction'); + return this.db + .prepare(` + SELECT o.ordinal, e.event_id, e.run_id, e.invocation_id, e.event_seq ${joins} + WHERE o.session_id = :sessionId AND o.ordinal <= :throughOrdinal + ${ + request.messageId === undefined + ? '' + : `AND e.event_id IN ( + SELECT event_id FROM runtime_events WHERE session_id = :sessionId AND (${TRANSCRIPT_STORED_ID_SQL}) = :messageId + UNION SELECT event_id FROM runtime_events WHERE session_id = :sessionId AND event_id = :noticeEventId + )` + } + AND o.ordinal ${request.direction === 'older' ? '<=' : '>='} :position + AND ${settledInline} AND ${transcriptSource} + ORDER BY o.ordinal ${request.direction === 'older' ? 'DESC' : 'ASC'} LIMIT 1 + `) + .get({ + sessionId, + throughOrdinal: request.throughOrdinal, + position: request.position, + ...(request.messageId === undefined + ? {} + : { + messageId: request.messageId, + noticeEventId: request.messageId.endsWith(':step-limit-notice') + ? request.messageId.slice(0, -':step-limit-notice'.length) + : null, + }), + }) as SourceRow | undefined; + } + + source(sessionId: string, request: RuntimeTranscriptPosition): RuntimeTranscriptSource | null { + const row = this.sourceRow(sessionId, request); + if (!row) return null; + const event = this.event(row.event_id); + const invocation = this.invocation(sessionId, row.run_id); + let primary = event; + if (event.content?.kind === 'thinking') { + const text = this.db + .prepare(` + SELECT 1 FROM runtime_events WHERE invocation_id = ? AND event_kind = 'text' + AND json_extract(payload_json, '$.role') = 'model' + AND COALESCE(json_extract(payload_json, '$.refs.storedMessageId'), json_extract(payload_json, '$.refs.providerEventId'), event_id) = ? LIMIT 1 + `) + .get( + row.invocation_id, + event.refs?.providerEventId ?? event.refs?.storedMessageId ?? event.id, + ); + if (text) { + // Its thinking is attached at the text position; any actions still own + // their rows at this event's position, exactly once. + const { content: _content, ...actionsOnly } = event; + primary = actionsOnly; + } + } + const events: Array<{ event: RuntimeEvent; sequence: number }> = [ + { event: primary, sequence: row.event_seq }, + ]; + let contentOrder: AssistantStepContentKind[] | undefined; + if (event.role === 'model' && event.content?.kind === 'text') { + const id = event.refs?.storedMessageId ?? event.refs?.providerEventId ?? event.id; + const thinking = this.db + .prepare(` + SELECT e.event_id, e.event_seq FROM runtime_events e + WHERE e.invocation_id = ? AND e.event_kind = 'thinking' AND ${messageKey('e')} = ? + ORDER BY e.event_seq + `) + .all(row.invocation_id, id) as Array<{ event_id: string; event_seq: number }>; + for (const item of thinking) { + const { actions: _actions, status: _status, ...context } = this.event(item.event_id); + events.push({ event: context, sequence: item.event_seq }); + } + const kinds = this.db + .prepare(` + SELECT CASE e.event_kind WHEN 'function_call' THEN 'tools' ELSE e.event_kind END AS kind, + MIN(e.event_seq) AS first_sequence FROM runtime_events e + WHERE e.invocation_id = :invocationId AND e.event_seq <= :sequence + AND e.event_kind IN ('text', 'thinking', 'function_call') AND json_extract(e.payload_json, '$.role') = 'model' + AND ${messageKey('e')} = :messageId + GROUP BY kind ORDER BY first_sequence + `) + .all({ invocationId: row.invocation_id, sequence: row.event_seq, messageId: id }) as Array<{ + kind: AssistantStepContentKind; + }>; + contentOrder = kinds.map((item) => item.kind); + } + const requestId = + event.actions?.permissionDecision?.requestId ?? + event.actions?.permissionAnswerAccepted?.requestId; + const permissionRow = requestId + ? (this.db + .prepare(` + SELECT event_id FROM runtime_events + WHERE invocation_id = ? AND event_seq <= ? AND json_extract(payload_json, '$.actions.permissionRequest.requestId') = ? + ORDER BY event_seq DESC LIMIT 1 + `) + .get(row.invocation_id, row.event_seq, requestId) as { event_id: string } | undefined) + : undefined; + const permissionRequest = permissionRow ? this.event(permissionRow.event_id) : undefined; + const toolUseId = + event.refs?.toolCallId ?? permissionRequest?.actions?.permissionRequest?.toolUseId; + const toolRow = + requestId && toolUseId + ? (this.db + .prepare(` + SELECT json_extract(payload_json, '$.content.name') AS name FROM runtime_events + WHERE invocation_id = ? AND event_seq <= ? AND json_extract(payload_json, '$.content.id') = ? + AND event_kind IN ('function_call', 'function_response') + ORDER BY event_seq DESC LIMIT 1 + `) + .get(row.invocation_id, row.event_seq, toolUseId) as { name: string } | undefined) + : undefined; + const hasRetainedOutput = + isTerminalRuntimeEvent(event) && this.hasShape(row.invocation_id, row.ordinal, 0, '3,4,12'); + return { + ordinal: row.ordinal, + event, + invocation, + events: events.sort((a, b) => a.sequence - b.sequence).map((item) => item.event), + ...(contentOrder ? { contentOrder } : {}), + ...(permissionRequest ? { permissionRequest } : {}), + ...(toolRow?.name ? { toolName: toolRow.name } : {}), + hasRetainedOutput, + }; + } + + turns( + sessionId: string, + throughOrdinal: number, + position: number, + limit: number, + ): RuntimeTranscriptTurn[] { + assertOrdinal(throughOrdinal); + assertOrdinal(position); + const rows = this.db + .prepare(` + SELECT e.run_id, e.invocation_id, o.ordinal ${joins} + WHERE o.session_id = :sessionId AND o.ordinal <= :throughOrdinal + AND e.event_seq = 1 AND ${settledInline} + AND EXISTS (SELECT 1 FROM runtime_events tail JOIN runtime_session_event_ordinals t ON t.event_id = tail.event_id + WHERE tail.invocation_id = e.invocation_id AND t.ordinal >= :position AND t.ordinal <= :throughOrdinal AND ${transcriptSource.replaceAll('e.', 'tail.')}) + ORDER BY o.ordinal LIMIT :limit + `) + .all({ sessionId, throughOrdinal, position, limit }) as SourceRow[]; + return rows.map((row) => this.turn(sessionId, row, throughOrdinal, position)); + } + + landmarks(sessionId: string, throughOrdinal: number, limit: number): RuntimeTranscriptTurn[] { + assertOrdinal(throughOrdinal); + if (limit < 1) return []; + const rows = this.db + .prepare(` + WITH candidates AS ( + SELECT e.run_id, e.invocation_id, o.ordinal, + ROW_NUMBER() OVER (ORDER BY o.ordinal) - 1 AS rank, COUNT(*) OVER () AS total + ${joins} WHERE o.session_id = :sessionId AND o.ordinal <= :throughOrdinal + AND e.event_seq = 1 AND ${settledInline} + ), samples(n) AS ( + SELECT 0 UNION ALL SELECT n + 1 FROM samples WHERE n + 1 < :limit + ) + SELECT DISTINCT run_id, invocation_id, ordinal FROM candidates + JOIN samples ON rank = CASE WHEN :limit = 1 THEN total - 1 + ELSE CAST(n * (total - 1) / (:limit - 1) AS INTEGER) END + ORDER BY ordinal + `) + .all({ sessionId, throughOrdinal, limit }) as SourceRow[]; + return rows.map((row) => this.turn(sessionId, row, throughOrdinal, 0)); + } + + private turn( + sessionId: string, + row: SourceRow, + throughOrdinal: number, + position: number, + ): RuntimeTranscriptTurn { + const bounds = this.db + .prepare(` + SELECT o.ordinal AS first + FROM runtime_events e JOIN runtime_session_event_ordinals o ON o.event_id = e.event_id + WHERE e.invocation_id = ? AND o.ordinal >= ? AND o.ordinal <= ? AND ${transcriptSource} + ORDER BY e.event_seq LIMIT 1 + `) + .get(row.invocation_id, position, throughOrdinal) as { first: number }; + const user = this.db + .prepare(` + SELECT o.ordinal, e.event_id FROM runtime_events e JOIN runtime_session_event_ordinals o ON o.event_id = e.event_id + WHERE e.invocation_id = ? AND o.ordinal >= ? AND o.ordinal <= ? + AND e.event_kind = 'text' AND json_extract(e.payload_json, '$.role') = 'user' + ORDER BY e.event_seq LIMIT 1 + `) + .get(row.invocation_id, position, throughOrdinal) as + | { ordinal: number; event_id: string } + | undefined; + const ended = this.db + .prepare(` + SELECT o.ordinal FROM runtime_events e JOIN runtime_session_event_ordinals o ON o.event_id = e.event_id + WHERE e.invocation_id = ? AND o.ordinal <= ? AND ${terminal('e')} + ORDER BY o.ordinal DESC LIMIT 1 + `) + .get(row.invocation_id, throughOrdinal) as { ordinal: number }; + return { + firstOrdinal: bounds.first, + terminalOrdinal: ended.ordinal, + invocation: this.invocation(sessionId, row.run_id), + ...(user ? { user: { ordinal: user.ordinal, event: this.event(user.event_id) } } : {}), + ...this.flags(row.invocation_id, throughOrdinal, position), + }; + } + + private flags(invocationId: string, throughOrdinal: number, position: number) { + return { + hasAssistantMessage: this.hasShape(invocationId, throughOrdinal, position, '1,3'), + hasAssistantOutput: this.hasShape(invocationId, throughOrdinal, position, '3'), + hasToolResult: this.hasShape(invocationId, throughOrdinal, position, '4,12'), + hasFailedToolResult: this.hasShape(invocationId, throughOrdinal, position, '12'), + hasAbortNote: false, + }; + } + + private hasShape( + invocationId: string, + throughOrdinal: number, + position: number, + shapes: string, + ): boolean { + return ( + this.db + .prepare(` + SELECT 1 FROM runtime_events JOIN runtime_session_event_ordinals o USING (event_id) + WHERE invocation_id = ? AND (${TRANSCRIPT_OUTPUT_SHAPE_SQL}) IN (${shapes}) + AND o.ordinal >= ? AND o.ordinal <= ? LIMIT 1 + `) + .get(invocationId, position, throughOrdinal) !== undefined + ); + } + + private event(id: string): RuntimeEvent { + const row = this.db + .prepare( + 'SELECT event_id, session_id, invocation_id, run_id, turn_id, payload_json FROM runtime_events WHERE event_id = ?', + ) + .get(id) as + | { + event_id: string; + session_id: string; + invocation_id: string; + run_id: string; + turn_id: string; + payload_json: string; + } + | undefined; + if (!row) throw new Error(`Transcript RuntimeEvent ${id} is missing`); + const event = decodeRuntimeEvent(JSON.parse(row.payload_json)); + if ( + event.id !== row.event_id || + event.sessionId !== row.session_id || + event.invocationId !== row.invocation_id || + event.runId !== row.run_id || + event.turnId !== row.turn_id + ) { + throw new Error(`Transcript RuntimeEvent ${id} has inconsistent storage identity`); + } + return event; + } +} + +function assertOrdinal(value: number): void { + if (!Number.isSafeInteger(value) || value < 0) + throw new Error('Invalid transcript event ordinal'); +} diff --git a/packages/storage/src/sqlite-runtime-schema.ts b/packages/storage/src/sqlite-runtime-schema.ts index a10532f026..3419761fa9 100644 --- a/packages/storage/src/sqlite-runtime-schema.ts +++ b/packages/storage/src/sqlite-runtime-schema.ts @@ -25,12 +25,18 @@ import { } from './legacy-run-header.js'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { encodeCanonicalRuntimeEvent } from '@maka/core/canonical-runtime-event'; +import { + TERMINAL_RUNTIME_EVENT_SQL, + TRANSCRIPT_MESSAGE_KEY_SQL, + TRANSCRIPT_OUTPUT_SHAPE_SQL, + TRANSCRIPT_STORED_ID_SQL, +} from './runtime-transcript-query.js'; import { buildInvocationOpenedEvent, buildSyntheticTerminalRuntimeEvent, } from '@maka/core/runtime-invocation'; -export const SQLITE_RUNTIME_SCHEMA_VERSION = 16; +export const SQLITE_RUNTIME_SCHEMA_VERSION = 17; export const RUNTIME_RECOVERY_AUTHORITY_CAPABILITY = 'runtime_recovery_authority'; export const RUNTIME_RECOVERY_AUTHORITY_CAPABILITY_VERSION = 1; export const RUNTIME_CONTINUATION_AUTHORITY_CAPABILITY = 'runtime_continuation_authority'; @@ -579,6 +585,17 @@ const MIGRATIONS: ReadonlyMap = new Map([ ALTER TABLE runtime_continuation_claims_v16 RENAME TO runtime_continuation_claims; `, ], + [ + 17, + ` + CREATE INDEX IF NOT EXISTS runtime_events_transcript_message ON runtime_events(invocation_id, (${TRANSCRIPT_MESSAGE_KEY_SQL}), event_seq); + CREATE INDEX IF NOT EXISTS runtime_events_transcript_output ON runtime_events(invocation_id, (${TRANSCRIPT_OUTPUT_SHAPE_SQL}), event_seq); + CREATE INDEX IF NOT EXISTS runtime_events_transcript_request ON runtime_events(invocation_id, json_extract(payload_json, '$.actions.permissionRequest.requestId'), event_seq); + CREATE INDEX IF NOT EXISTS runtime_events_transcript_tool ON runtime_events(invocation_id, json_extract(payload_json, '$.content.id'), event_seq); + CREATE INDEX IF NOT EXISTS runtime_events_terminal ON runtime_events(invocation_id, event_seq) WHERE ${TERMINAL_RUNTIME_EVENT_SQL}; + CREATE INDEX IF NOT EXISTS runtime_events_transcript_stored_id ON runtime_events(session_id, (${TRANSCRIPT_STORED_ID_SQL})); + `, + ], ]); /** diff --git a/packages/storage/src/sqlite-runtime-store.ts b/packages/storage/src/sqlite-runtime-store.ts index c71dcbfb01..c54fdcf934 100644 --- a/packages/storage/src/sqlite-runtime-store.ts +++ b/packages/storage/src/sqlite-runtime-store.ts @@ -129,24 +129,18 @@ import { import type { OperationalStateDatabaseLease } from './operational-state-store.js'; import { immutableSteeringMessageId, isRuntimeStorageSafeId } from './runtime-event-invariants.js'; import { assertNoReservedWorkspaceAuthorityAppend } from './runtime-event-authority.js'; +import { + RuntimeTranscriptQuery, + TERMINAL_RUNTIME_EVENT_SQL, + type RuntimeTranscriptPosition, + type RuntimeTranscriptSource, + type RuntimeTranscriptTurn, +} from './runtime-transcript-query.js'; export { SQLITE_RUNTIME_SCHEMA_VERSION } from './sqlite-runtime-schema.js'; export type { ToolRecoveryMode } from '@maka/core/runtime-event'; -/** - * `isTerminalRuntimeEvent` asked in SQL. - * - * The TypeScript predicate stays the authority; this only lets a query find the - * terminal event without decoding every row it passes over. Both have to say the - * same thing, so the SQL half is written once here instead of at each query. - */ -const TERMINAL_RUNTIME_EVENT_SQL = `( - json_extract(payload_json, '$.actions.endInvocation') = 1 - OR json_extract(payload_json, '$.status') - IN ('completed', 'failed', 'aborted', 'cancelled') - )`; - const RUNTIME_EVENT_SCAN_BATCH_SIZE = 128; const RUNTIME_PARTIAL_SEGMENT_TARGET_BYTES = 64 * 1024; @@ -548,6 +542,52 @@ export class SqliteRuntimeStore return this.readRuntimeEventsSync(sessionId, runId); } + private transcriptQuery(): RuntimeTranscriptQuery { + return new RuntimeTranscriptQuery(this.db, (sessionId, runId) => { + const opening = this.readInvocationOpeningsSync(sessionId, { direction: 'asc', runId }).at(0); + if (!opening) throw new Error(`Transcript invocation ${runId} is missing`); + return this.completeInvocationRecordSync(opening); + }); + } + + async readTranscriptSourceHighWater(sessionId: string): Promise { + assertRuntimeStorageSafeId(sessionId, 'Invalid session id'); + return this.readTransaction(() => this.transcriptQuery().highWater(sessionId)); + } + + async readTranscriptSource( + sessionId: string, + request: RuntimeTranscriptPosition, + ): Promise { + assertRuntimeStorageSafeId(sessionId, 'Invalid session id'); + return this.readTransaction(() => this.transcriptQuery().source(sessionId, request)); + } + + async readTranscriptTurns( + sessionId: string, + throughOrdinal: number, + position: number, + limit: number, + ): Promise { + assertRuntimeStorageSafeId(sessionId, 'Invalid session id'); + assertInvocationSearchLimit(limit); + return this.readTransaction(() => + this.transcriptQuery().turns(sessionId, throughOrdinal, position, limit), + ); + } + + async readTranscriptLandmarks( + sessionId: string, + throughOrdinal: number, + limit: number, + ): Promise { + assertRuntimeStorageSafeId(sessionId, 'Invalid session id'); + assertInvocationSearchLimit(limit); + return this.readTransaction(() => + this.transcriptQuery().landmarks(sessionId, throughOrdinal, limit), + ); + } + /** * Enumerate a Session's invocations: the opening fact names each one, and its * highest-sequence event says whether it ended. @@ -688,6 +728,8 @@ export class SqliteRuntimeStore 1 AS from_events FROM runtime_events WHERE session_id = :sessionId AND event_kind = 'invocation_opened' + ${options.runId === undefined ? '' : 'AND run_id = :runId'} + ${options.invocationId === undefined ? '' : 'AND invocation_id = :invocationId'} UNION ALL SELECT NULL, @@ -699,6 +741,8 @@ export class SqliteRuntimeStore 0 FROM runtime_legacy_invocation_openings AS legacy WHERE legacy.session_id = :sessionId + ${options.runId === undefined ? '' : 'AND legacy.run_id = :runId'} + ${options.invocationId === undefined ? '' : 'AND legacy.invocation_id = :invocationId'} AND NOT EXISTS ( SELECT 1 FROM runtime_events WHERE runtime_events.invocation_id = legacy.invocation_id From f92f76a9149f10b2a86cd3fc6c4aec66e62c870c Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 02:07:13 +0800 Subject: [PATCH 11/32] fix(runtime): resume a conversion a released build left part-written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A released build derived the transcript run id exactly as this one does but every event id with `newId()`. An interrupted conversion of its therefore leaves an opening this build cannot name, and appending a second one is refused by `runtime_events_one_opening_per_invocation` — so the Session's durable reads threw `UNIQUE constraint failed` on every later attempt, with its legacy rows intact and unreachable. The opening is adopted rather than rewritten: a run needs exactly one, the index refuses a second, and which id it landed under changes nothing a reader sees. Its converted messages are not adoptable the same way — rederiving them would stand a second, deterministic copy of each beside the one already there, and a Session that disagrees with itself is the failure this ledger exists to remove. Deleting the prefix instead would need a delete path into the transcript authority, which costs more than the shape it repairs. So a run holding such messages is sealed as the unfinished conversion it is: the legacy rows stay, and nothing reads that turn as converted whole. Reported by M4n5ter and hqhq1025, both against real SQLite. Ablation: with the adoption removed, `resumes a conversion a released build opened under a random event id` reproduces the reported UNIQUE constraint failure. Generated-by: Claude Code --- .../__tests__/runtime-ledger-repair.test.ts | 125 ++++++++++++++++++ packages/runtime/src/runtime-ledger-repair.ts | 59 ++++++++- 2 files changed, 182 insertions(+), 2 deletions(-) diff --git a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts index 6d967b05e9..200f8612d8 100644 --- a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts +++ b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts @@ -18,6 +18,7 @@ */ import assert from 'node:assert/strict'; +import { randomUUID } from 'node:crypto'; import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -795,3 +796,127 @@ test('a resolved Claude transcript replays as the conversation the user kept', a await rm(root, { recursive: true, force: true }); } }); + +/** One legacy turn, as a released build would have left it for the converter. */ +async function seedLegacyTurn(sessions: ReturnType) { + const ts = Date.now(); + const session = await sessions.create({ + cwd: '/repo', + llmConnectionSlug: 'anthropic', + model: 'claude-opus-5', + permissionMode: 'ask', + }); + await sessions.appendMessages(session.id, [ + { type: 'user', id: 'r-user', turnId: 'turn-1', ts, text: 'run the tests' }, + { + type: 'assistant', + id: 'r-assistant', + turnId: 'turn-1', + ts: ts + 1, + text: 'All green.', + modelId: 'claude-opus-5', + }, + { + type: 'turn_state', + id: 'r-state', + turnId: 'turn-1', + ts: ts + 2, + status: 'completed', + partialOutputRetained: true, + }, + ]); + return session; +} + +test('resumes a conversion a released build opened under a random event id', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-released-prefix-')); + const sessions = createSessionStore(root); + const runtimeEvents = createSqliteRuntimeStore(join(root, 'runtime.sqlite')); + + try { + const session = await seedLegacyTurn(sessions); + const deps = { + runtimeEventStore: runtimeEvents, + readMessages: (sessionId: string) => sessions.readMessages(sessionId), + }; + + // A released build derived the run id the same way but every event id with + // `newId()`, so its interrupted conversion left an opening this build + // cannot name. `runtime_events_one_opening_per_invocation` refuses a second + // one, so the retry has to read what the run already holds. + const append = runtimeEvents.appendRuntimeEvent.bind(runtimeEvents); + runtimeEvents.appendRuntimeEvent = async (sessionId, runId, event) => { + await append(sessionId, runId, { ...event, id: randomUUID() }); + throw new Error('interrupted conversion'); + }; + await assert.rejects( + new RuntimeLedgerRepair(deps).materializeTranscriptLedger( + await sessions.readHeader(session.id), + ), + /interrupted conversion/, + ); + runtimeEvents.appendRuntimeEvent = append; + + await new RuntimeLedgerRepair(deps).materializeTranscriptLedger( + await sessions.readHeader(session.id), + ); + + const [run] = await runtimeEvents.listSessionInvocations(session.id); + assert.ok(run); + assert.equal(runtimeInvocationOutcome(run), 'completed'); + const events = await runtimeEvents.readRuntimeEvents(session.id, run.runId); + assert.deepEqual( + events.flatMap((event) => (event.content ? [event.content.kind] : [])), + ['invocation_opened', 'text', 'text'], + ); + } finally { + runtimeEvents.close(); + await sessions.close?.(); + await rm(root, { recursive: true, force: true }); + } +}); + +test('seals a released conversion that had already converted messages', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-released-partial-')); + const sessions = createSessionStore(root); + const runtimeEvents = createSqliteRuntimeStore(join(root, 'runtime.sqlite')); + + try { + const session = await seedLegacyTurn(sessions); + const deps = { + runtimeEventStore: runtimeEvents, + readMessages: (sessionId: string) => sessions.readMessages(sessionId), + }; + + const append = runtimeEvents.appendRuntimeEvent.bind(runtimeEvents); + let written = 0; + runtimeEvents.appendRuntimeEvent = async (sessionId, runId, event) => { + await append(sessionId, runId, { ...event, id: randomUUID() }); + written += 1; + if (written === 2) throw new Error('interrupted conversion'); + }; + await assert.rejects( + new RuntimeLedgerRepair(deps).materializeTranscriptLedger( + await sessions.readHeader(session.id), + ), + /interrupted conversion/, + ); + runtimeEvents.appendRuntimeEvent = append; + + await new RuntimeLedgerRepair(deps).materializeTranscriptLedger( + await sessions.readHeader(session.id), + ); + + const [run] = await runtimeEvents.listSessionInvocations(session.id); + assert.ok(run); + // The prefix cannot be finished and must not be doubled: one user text, not two. + assert.equal(runtimeInvocationOutcome(run), 'failed'); + assert.equal(runtimeInvocationFailureClass(run), 'missing_terminal_event'); + const events = await runtimeEvents.readRuntimeEvents(session.id, run.runId); + assert.equal(events.filter((event) => event.content?.kind === 'text').length, 1); + } finally { + runtimeEvents.close(); + await sessions.close?.(); + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/packages/runtime/src/runtime-ledger-repair.ts b/packages/runtime/src/runtime-ledger-repair.ts index f8c7125e98..9e9226802e 100644 --- a/packages/runtime/src/runtime-ledger-repair.ts +++ b/packages/runtime/src/runtime-ledger-repair.ts @@ -68,8 +68,9 @@ export class RuntimeLedgerRepair { // the same turn would make the Session read as two. The one exception is // this converter's own run: an interrupted import re-derives it, and the // deterministic ids let the store dedupe what already landed. + const inlineInvocations = await this.listInlineInvocations(sessionId); const ownedTurnIds = new Set( - (await this.listInlineInvocations(sessionId)) + inlineInvocations .filter( (invocation) => invocation.terminalEvent || @@ -77,6 +78,7 @@ export class RuntimeLedgerRepair { ) .map((invocation) => invocation.turnId), ); + const startedRunIds = new Set(inlineInvocations.map((invocation) => invocation.runId)); const messagesByTurn = groupMessagesByTurn(ledgerMessages); // A turn whose only user row was steering is not a turn of its own: the // steering was said into a Turn some durable Root already owns, so @@ -94,8 +96,33 @@ export class RuntimeLedgerRepair { const runId = transcriptRunId(sessionId, turn.turnId); const openedAt = firstOpenedAt + index; const run = { sessionId, runId, turnId: turn.turnId, invocationId: runId }; + // A build before the ids were derived converted under random ones, so + // an interrupted run of its can hold events this build cannot rederive. + const started = startedRunIds.has(runId) + ? await this.deps.runtimeEventStore.readRuntimeEvents(sessionId, runId) + : []; + const undeducible = started.filter((event) => !isDerivedTranscriptEventId(runId, event.id)); + // Its opening is the one such event that can be adopted: the run needs + // exactly one, `runtime_events_one_opening_per_invocation` refuses a + // second, and which id it landed under changes nothing a reader sees. + const adoptedOpening = + undeducible.length === 1 && undeducible[0]?.content?.kind === 'invocation_opened'; + if (undeducible.length > 0 && !adoptedOpening) { + // Its converted messages cannot be adopted the same way: rederiving + // them would stand a second, deterministic copy of each beside the + // one already there, and a Session that disagrees with itself is the + // failure this ledger exists to remove. The conversion can neither be + // finished nor withdrawn, so it is sealed as the unfinished thing it + // is — the legacy rows stay, and no one reads this turn as converted. + await this.deps.runtimeEventStore.appendRuntimeEvent( + sessionId, + runId, + abandonedTranscriptTerminalEvent({ run, openedAt }), + ); + continue; + } const events = [ - transcriptOpeningEvent({ header, run, openedAt }), + ...(adoptedOpening ? [] : [transcriptOpeningEvent({ header, run, openedAt })]), ...backfillRuntimeEventsFromStoredMessages({ run, outcome: transcriptOutcome(turn, turnMessages, openedAt), @@ -159,6 +186,34 @@ function transcriptRunId(sessionId: string, turnId: string): string { * emits them. The run id is already derived from the Session and turn, so the * same transcript always produces the same ids and a re-run appends nothing. */ +/** Whether this build's converter is the one that could have written that id. */ +function isDerivedTranscriptEventId(runId: string, eventId: string): boolean { + return eventId === `${runId}-opened` || new RegExp(`^${runId}-e\\d+$`).test(eventId); +} + +/** + * The terminal fact of a conversion that a released build left part-written. + * Its id sits outside the derived sequence so it cannot collide with an event + * that prefix already holds. + */ +function abandonedTranscriptTerminalEvent(input: { + run: { sessionId: string; runId: string; turnId: string; invocationId: string }; + openedAt: number; +}): RuntimeEvent { + return backfillRuntimeEventsFromStoredMessages({ + run: input.run, + outcome: { + status: 'failed', + ts: input.openedAt, + failureClass: 'missing_terminal_event', + }, + messages: [], + modelHistory: 'conversation_text', + newId: () => `${input.run.runId}-abandoned`, + now: () => input.openedAt, + }).events[0] as RuntimeEvent; +} + function transcriptEventIds(runId: string): () => string { let seq = 0; return () => { From 8c29d2d0e575de4ff2e012726eee2af32067fea4 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 02:16:08 +0800 Subject: [PATCH 12/32] fix(runtime): convert a legacy transcript a page at a time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every durable page goes through `ensureTranscriptLedgerForRead`, and the converter read the Session's whole `session_messages` array before it wrote anything. So the first page of a legacy Session required loading all of it — the bound #4791 asks for ("a page must not require loading the full Session") and the one the base kept, since its reader paged over the same table. The converter now walks the rows forward a page at a time. A turn is only whole once a row of another turn follows it, so the last turn of a page is carried into the next rather than converted from a prefix of itself: peak memory is one page plus one turn, not one history. Durable progress needs no new record — a turn is skipped once its invocation has a terminal, which is what already made an interrupted import resumable. `openedAt` can no longer come from a count of every turn, because a paged conversion never holds one. It is derived from where the turn starts in the transcript instead, which keeps both properties the count gave it: every imported opening still sorts ahead of the Session's own runs, and turns keep the transcript's order. The high-water sequence rides along with each page so that derivation costs no read of its own. Reported by jackwener. Generated-by: Claude Code --- .../runtime-event-read-model.test.ts | 17 +++ .../runtime-kernel-interaction.test.ts | 10 ++ .../__tests__/runtime-ledger-repair.test.ts | 101 ++++++++++++++-- .../session-manager-terminal-ledger.test.ts | 17 +++ .../src/__tests__/session-manager.test.ts | 17 +++ packages/runtime/src/runtime-ledger-repair.ts | 109 +++++++++++++----- packages/runtime/src/session-manager.ts | 18 ++- packages/storage/src/execution-stores.ts | 2 + packages/storage/src/session-store.ts | 39 +++++++ .../src/sqlite-session-metadata-store.ts | 57 +++++++++ 10 files changed, 347 insertions(+), 40 deletions(-) diff --git a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts index 68044efb3f..af88e86651 100644 --- a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts +++ b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts @@ -2424,6 +2424,23 @@ class ReadOnlyStore implements SessionStore { return [...this.messages]; } + async readMessagesAfter( + _sessionId: string, + request: { afterSequence?: number; maxMessages: number }, + ): Promise<{ + records: readonly { sequence: number; message: StoredMessage }[]; + highWaterSequence: number | null; + }> { + this.readMessagesCalls += 1; + return { + records: this.messages + .map((message, sequence) => ({ sequence, message })) + .filter(({ sequence }) => sequence > (request.afterSequence ?? -1)) + .slice(0, request.maxMessages), + highWaterSequence: this.messages.length > 0 ? this.messages.length - 1 : null, + }; + } + async listTurns(_sessionId: string): Promise { return deriveTurnRecords(this.messages); } diff --git a/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts b/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts index bbe96c8e1d..34faf6750e 100644 --- a/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts +++ b/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts @@ -624,6 +624,16 @@ function memoryStore(): SessionStore { list: async () => [], readHeader: async () => header, readMessages: async () => [...messages], + readMessagesAfter: async ( + _sessionId: string, + request: { afterSequence?: number; maxMessages: number }, + ) => ({ + records: messages + .map((message, sequence) => ({ sequence, message })) + .filter(({ sequence }) => sequence > (request.afterSequence ?? -1)) + .slice(0, request.maxMessages), + highWaterSequence: messages.length > 0 ? messages.length - 1 : null, + }), updateHeader: async (_sessionId, patch) => { header = { ...header, ...patch }; return header; diff --git a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts index 200f8612d8..d16260e85e 100644 --- a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts +++ b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts @@ -106,7 +106,7 @@ test('repairs imported transcript turns into provider-neutral canonical history' assert.equal(session.transcriptLedgerVersion, 0); const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, - readMessages: (sessionId) => sessions.readMessages(sessionId), + readMessagesAfter: (sessionId, request) => sessions.readMessagesAfter(sessionId, request), }); await repair.materializeTranscriptLedger(session); @@ -285,7 +285,7 @@ test('an imported snapshot cutoff survives materialization as aborted', async () ); const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, - readMessages: (sessionId) => sessions.readMessages(sessionId), + readMessagesAfter: (sessionId, request) => sessions.readMessagesAfter(sessionId, request), }); await repair.materializeTranscriptLedger(session); @@ -339,7 +339,7 @@ test('does not import Host-handed-off transcript messages as synthetic runs', as ); const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, - readMessages: (sessionId) => sessions.readMessages(sessionId), + readMessagesAfter: (sessionId, request) => sessions.readMessagesAfter(sessionId, request), }); await repair.materializeTranscriptLedger(session); @@ -388,7 +388,7 @@ test('an imported turn with no terminal state is repaired to failed', async () = ); const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, - readMessages: (sessionId) => sessions.readMessages(sessionId), + readMessagesAfter: (sessionId, request) => sessions.readMessagesAfter(sessionId, request), }); await repair.materializeTranscriptLedger(session); @@ -462,7 +462,7 @@ test("converts Maka's own legacy transcript whole, and resumes an interrupted co const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, - readMessages: (sessionId) => sessions.readMessages(sessionId), + readMessagesAfter: (sessionId, request) => sessions.readMessagesAfter(sessionId, request), }); const append = runtimeEvents.appendRuntimeEvent.bind(runtimeEvents); @@ -481,7 +481,7 @@ test("converts Maka's own legacy transcript whole, and resumes an interrupted co assert.equal(interrupted.terminalEvent, undefined); const resumed = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, - readMessages: (sessionId) => sessions.readMessages(sessionId), + readMessagesAfter: (sessionId, request) => sessions.readMessagesAfter(sessionId, request), }); await resumed.materializeTranscriptLedger(await sessions.readHeader(session.id)); @@ -552,7 +552,7 @@ test('startup recovery leaves an interrupted legacy conversion for the importer }; const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, - readMessages: (id) => sessions.readMessages(id), + readMessagesAfter: (id, request) => sessions.readMessagesAfter(id, request), }); await assert.rejects( repair.materializeTranscriptLedger(await sessions.readHeader(session.id)), @@ -734,7 +734,7 @@ test('a resolved Claude transcript replays as the conversation the user kept', a ); const repair = new RuntimeLedgerRepair({ runtimeEventStore: runtimeEvents, - readMessages: (sessionId) => sessions.readMessages(sessionId), + readMessagesAfter: (sessionId, request) => sessions.readMessagesAfter(sessionId, request), }); await repair.materializeTranscriptLedger(session); @@ -837,7 +837,10 @@ test('resumes a conversion a released build opened under a random event id', asy const session = await seedLegacyTurn(sessions); const deps = { runtimeEventStore: runtimeEvents, - readMessages: (sessionId: string) => sessions.readMessages(sessionId), + readMessagesAfter: ( + sessionId: string, + request: { maxMessages: number; maxStoredBytes: number }, + ) => sessions.readMessagesAfter(sessionId, request), }; // A released build derived the run id the same way but every event id with @@ -885,7 +888,10 @@ test('seals a released conversion that had already converted messages', async () const session = await seedLegacyTurn(sessions); const deps = { runtimeEventStore: runtimeEvents, - readMessages: (sessionId: string) => sessions.readMessages(sessionId), + readMessagesAfter: ( + sessionId: string, + request: { maxMessages: number; maxStoredBytes: number }, + ) => sessions.readMessagesAfter(sessionId, request), }; const append = runtimeEvents.appendRuntimeEvent.bind(runtimeEvents); @@ -920,3 +926,78 @@ test('seals a released conversion that had already converted messages', async () await rm(root, { recursive: true, force: true }); } }); + +test('converts a legacy transcript larger than one page without reading it whole', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-paged-conversion-')); + const sessions = createSessionStore(root); + const runtimeEvents = createSqliteRuntimeStore(join(root, 'runtime.sqlite')); + + try { + const ts = Date.now(); + const session = await sessions.create({ + cwd: '/repo', + llmConnectionSlug: 'anthropic', + model: 'claude-opus-5', + permissionMode: 'ask', + }); + const turnCount = 200; + for (let turn = 0; turn < turnCount; turn += 1) { + await sessions.appendMessages(session.id, [ + { + type: 'user', + id: `p-user-${turn}`, + turnId: `turn-${turn}`, + ts: ts + turn * 3, + text: `ask ${turn}`, + }, + { + type: 'assistant', + id: `p-assistant-${turn}`, + turnId: `turn-${turn}`, + ts: ts + turn * 3 + 1, + text: `answer ${turn}`, + modelId: 'claude-opus-5', + }, + { + type: 'turn_state', + id: `p-state-${turn}`, + turnId: `turn-${turn}`, + ts: ts + turn * 3 + 2, + status: 'completed', + partialOutputRetained: true, + }, + ]); + } + + // The whole transcript is 600 rows. A conversion that still read it whole + // would ask for all of them at once, and the Session cannot serve its first + // transcript page until this finishes. + let largestRead = 0; + await new RuntimeLedgerRepair({ + runtimeEventStore: runtimeEvents, + readMessagesAfter: async (sessionId, request) => { + const page = await sessions.readMessagesAfter(sessionId, request); + largestRead = Math.max(largestRead, page.records.length); + return page; + }, + }).materializeTranscriptLedger(await sessions.readHeader(session.id)); + + assert.ok(largestRead < turnCount * 3, `read ${largestRead} rows in one page`); + const invocations = await runtimeEvents.listSessionInvocations(session.id); + assert.equal(invocations.length, turnCount); + assert.ok(invocations.every((run) => runtimeInvocationOutcome(run) === 'completed')); + // Every imported opening still sorts ahead of anything the Session does + // natively, and turns keep the order the transcript had. + const openedAt = invocations.map((run) => run.openedAt); + assert.ok(openedAt.every((value) => value < session.createdAt)); + assert.deepEqual( + openedAt, + [...openedAt].sort((left, right) => left - right), + ); + assert.equal(new Set(openedAt).size, turnCount); + } finally { + runtimeEvents.close(); + await sessions.close?.(); + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts index 0a602b40e8..64e533a3b7 100644 --- a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts +++ b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts @@ -2246,6 +2246,23 @@ class TinySessionStore implements SessionStore { return clone(this.messages.get(sessionId) ?? []); } + async readMessagesAfter( + sessionId: string, + request: { afterSequence?: number; maxMessages: number }, + ): Promise<{ + records: readonly { sequence: number; message: StoredMessage }[]; + highWaterSequence: number | null; + }> { + const all = clone(this.messages.get(sessionId) ?? []); + return { + records: all + .map((message, sequence) => ({ sequence, message })) + .filter(({ sequence }) => sequence > (request.afterSequence ?? -1)) + .slice(0, request.maxMessages), + highWaterSequence: all.length > 0 ? all.length - 1 : null, + }; + } + async listTurns(sessionId: string): Promise { return deriveTurnRecords(await this.readMessages(sessionId)); } diff --git a/packages/runtime/src/__tests__/session-manager.test.ts b/packages/runtime/src/__tests__/session-manager.test.ts index 9e8776e619..beb9cfa921 100644 --- a/packages/runtime/src/__tests__/session-manager.test.ts +++ b/packages/runtime/src/__tests__/session-manager.test.ts @@ -12933,6 +12933,23 @@ class MemorySessionStore implements SessionStore { return [...(this.messages.get(sessionId) ?? [])]; } + async readMessagesAfter( + sessionId: string, + request: { afterSequence?: number; maxMessages: number }, + ): Promise<{ + records: readonly { sequence: number; message: StoredMessage }[]; + highWaterSequence: number | null; + }> { + const all = await this.readMessages(sessionId); + return { + records: all + .map((message, sequence) => ({ sequence, message })) + .filter(({ sequence }) => sequence > (request.afterSequence ?? -1)) + .slice(0, request.maxMessages), + highWaterSequence: all.length > 0 ? all.length - 1 : null, + }; + } + async listTurns(sessionId: string): Promise { if (this.failListTurnsFor.has(sessionId)) throw new Error(`Cannot list turns for ${sessionId}`); return deriveTurnRecords(await this.readMessages(sessionId)); diff --git a/packages/runtime/src/runtime-ledger-repair.ts b/packages/runtime/src/runtime-ledger-repair.ts index 9e9226802e..fe2ea2eeb6 100644 --- a/packages/runtime/src/runtime-ledger-repair.ts +++ b/packages/runtime/src/runtime-ledger-repair.ts @@ -37,10 +37,25 @@ import type { RuntimeEventBackfillOutcome } from './runtime-event-backfill.js'; export interface RuntimeLedgerRepairDeps { runtimeEventStore: RuntimeEventStore; - /** The legacy transcript this converter reads; nothing writes back to it. */ - readMessages(sessionId: string): Promise; + /** + * One forward page of the legacy transcript this converter reads; nothing + * writes back to it. It is read a page at a time because a Session cannot + * serve its first transcript page until this finishes, and a Session's + * history is not a bound. + */ + readMessagesAfter( + sessionId: string, + request: { afterSequence?: number; maxMessages: number; maxStoredBytes: number }, + ): Promise<{ + records: readonly { sequence: number; message: StoredMessage }[]; + highWaterSequence: number | null; + }>; } +/** How much of a legacy transcript one conversion page holds. */ +const TRANSCRIPT_CONVERSION_PAGE_MAX_MESSAGES = 256; +const TRANSCRIPT_CONVERSION_PAGE_MAX_BYTES = 4 * 1024 * 1024; + export class RuntimeLedgerRepair { private readonly queues = new Map>(); @@ -58,10 +73,6 @@ export class RuntimeLedgerRepair { async materializeTranscriptLedger(header: SessionHeader): Promise { const sessionId = header.id; return this.withRepairQueue(sessionId, async () => { - const messages = await this.deps.readMessages(sessionId); - const ledgerMessages = messages.filter( - (message) => message.type !== 'user' || message.steeringEventId === undefined, - ); // A turn the ledger already owns is not converted again. Its own run is // the authority even when it never ended — a crashed turn is settled by // recovery on that run, and a second, transcript-derived invocation for @@ -79,22 +90,24 @@ export class RuntimeLedgerRepair { .map((invocation) => invocation.turnId), ); const startedRunIds = new Set(inlineInvocations.map((invocation) => invocation.runId)); - const messagesByTurn = groupMessagesByTurn(ledgerMessages); - // A turn whose only user row was steering is not a turn of its own: the - // steering was said into a Turn some durable Root already owns, so - // converting it would stand a second, synthetic run beside that one. - const turns = deriveTurnRecords(ledgerMessages).filter((turn) => - (messagesByTurn.get(turn.turnId) ?? []).some((message) => message.type === 'user'), - ); - if (turns.length === 0) return; - - const firstOpenedAt = Math.max(0, header.createdAt - turns.length); - for (const [index, turn] of turns.entries()) { + for await (const scanned of this.readTurnsInPages(sessionId)) { + const turnMessages = scanned.messages; + // A turn whose only user row was steering is not a turn of its own: the + // steering was said into a Turn some durable Root already owns, so + // converting it would stand a second, synthetic run beside that one. + if (!turnMessages.some((message) => message.type === 'user')) continue; + const [turn] = deriveTurnRecords(turnMessages); + if (!turn) continue; if (ownedTurnIds.has(turn.turnId)) continue; - const turnMessages = messagesByTurn.get(turn.turnId) ?? []; const runId = transcriptRunId(sessionId, turn.turnId); - const openedAt = firstOpenedAt + index; + // Ordered by where the turn starts in the transcript rather than by its + // index among all turns: a paged conversion never holds that count, and + // both keep every imported opening ahead of the Session's own runs. + const openedAt = Math.max( + 0, + header.createdAt - 1 - (scanned.highWater - scanned.firstSequence), + ); const run = { sessionId, runId, turnId: turn.turnId, invocationId: runId }; // A build before the ids were derived converted under random ones, so // an interrupted run of its can hold events this build cannot rederive. @@ -145,6 +158,51 @@ export class RuntimeLedgerRepair { }); } + /** + * The Session's legacy rows, one turn at a time, read a page at a time. + * + * A turn is only complete once a row of another turn follows it, so the rows + * of the page's last turn are carried into the next page rather than + * converted early. Peak memory is therefore one page plus one turn — the same + * bound the transcript reader keeps, and not the Session's whole history. + */ + private async *readTurnsInPages( + sessionId: string, + ): AsyncGenerator<{ messages: StoredMessage[]; firstSequence: number; highWater: number }> { + let carried: { messages: StoredMessage[]; firstSequence: number } | undefined; + let afterSequence: number | undefined; + while (true) { + const page = await this.deps.readMessagesAfter(sessionId, { + ...(afterSequence === undefined ? {} : { afterSequence }), + maxMessages: TRANSCRIPT_CONVERSION_PAGE_MAX_MESSAGES, + maxStoredBytes: TRANSCRIPT_CONVERSION_PAGE_MAX_BYTES, + }); + const highWater = page.highWaterSequence; + if (highWater === null) return; + const scanned = page.records.filter( + ({ message }) => message.type !== 'user' || message.steeringEventId === undefined, + ); + const grouped = new Map(); + if (carried) grouped.set(turnIdOf(carried.messages[0]) ?? '', carried); + for (const { sequence, message } of scanned) { + const turnId = turnIdOf(message); + if (!turnId) continue; + const bucket = grouped.get(turnId); + if (bucket) bucket.messages.push(message); + else grouped.set(turnId, { messages: [message], firstSequence: sequence }); + } + const turns = [...grouped.values()]; + const lastSequence = page.records.at(-1)?.sequence; + // The last turn of a page may continue into the next one, so it is held + // back rather than converted from a prefix of its own rows. A page with + // nothing left to read ends the scan, and what was held back is whole. + carried = lastSequence === undefined ? undefined : turns.pop(); + for (const turn of turns) yield { ...turn, highWater }; + if (lastSequence === undefined) return; + afterSequence = lastSequence; + } + } + private async listInlineInvocations(sessionId: string): Promise { return (await this.deps.runtimeEventStore.listSessionInvocations(sessionId)).filter( (invocation) => isSessionInlineInvocation(invocation.opening), @@ -295,14 +353,7 @@ function transcriptOutcomeStatus(status: TurnRecord['status']): RuntimeInvocatio return 'cancelled'; } -function groupMessagesByTurn(messages: readonly StoredMessage[]): Map { - const grouped = new Map(); - for (const message of messages) { - const turnId = 'turnId' in message ? message.turnId : undefined; - if (!turnId) continue; - const bucket = grouped.get(turnId) ?? []; - bucket.push(message); - grouped.set(turnId, bucket); - } - return grouped; +function turnIdOf(message: StoredMessage | undefined): string | undefined { + if (!message) return undefined; + return 'turnId' in message ? message.turnId : undefined; } diff --git a/packages/runtime/src/session-manager.ts b/packages/runtime/src/session-manager.ts index 26d98f39f1..d0a49d111e 100644 --- a/packages/runtime/src/session-manager.ts +++ b/packages/runtime/src/session-manager.ts @@ -615,6 +615,14 @@ export interface SessionStore { readHeader(sessionId: string): Promise; /** The legacy transcript, read only to convert it onto the ledger. */ readMessages(sessionId: string): Promise; + /** One forward page of the legacy rows the transcript converter lifts. */ + readMessagesAfter( + sessionId: string, + request: { afterSequence?: number; maxMessages: number; maxStoredBytes: number }, + ): Promise<{ + records: readonly { sequence: number; message: StoredMessage }[]; + highWaterSequence: number | null; + }>; /** Commit the Session-list facts a durable message carries. */ commitMessageCatalogProjection?( sessionId: string, @@ -902,7 +910,7 @@ export class SessionManager { if (deps.runStore && deps.runtimeEventStore) { this.runtimeLedgerRepair = new RuntimeLedgerRepair({ runtimeEventStore: deps.runtimeEventStore, - readMessages: (sessionId) => deps.store.readMessages(sessionId), + readMessagesAfter: (sessionId, request) => deps.store.readMessagesAfter(sessionId, request), }); } this.runtimeKernel = deps.runtimeKernel ?? new RuntimeKernel({ ...deps }); @@ -4267,6 +4275,14 @@ export class SessionManager { if (header.transcriptLedgerVersion === 0 && source !== 'import') { throw new Error('Imported Session history is still being prepared'); } + // Version 1 says a conversion ran, not that every legacy fact reached the + // ledger. A released build set it on the first send and went on writing + // context notes to the transcript alone, so those notes stay behind on + // Sessions it touched. Re-running the converter cannot reach them: they + // belong to turns a real run already sealed, and a sealed run refuses the + // append. They are hidden from the model and describe context, so they are + // the accepted cost of the cutover — do not read this marker as proof that + // nothing is left in `session_messages`. if (header.transcriptLedgerVersion !== 1) { await repair.materializeTranscriptLedger(header); await this.updateHeader(sessionId, { transcriptLedgerVersion: 1 }); diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index 1488d2ec72..1559699703 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -443,6 +443,8 @@ async function createExecutionStoresForWrite run(() => sessionStore.listTurnsSnapshot(sessionId)), readHeader: (sessionId) => run(() => sessionStore.readHeader(sessionId)), readMessages: (sessionId) => run(() => sessionStore.readMessages(sessionId)), + readMessagesAfter: (sessionId, request) => + run(() => sessionStore.readMessagesAfter(sessionId, request)), listTurns: (sessionId) => run(() => sessionStore.listTurns(sessionId)), appendMessage: (sessionId, message) => run(() => sessionStore.appendMessage(sessionId, message)), diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index 992cb8322f..d065667d7d 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -225,6 +225,33 @@ export interface SessionTranscriptMessageLookupRequest { readonly maxMessages: number; } +/** + * One forward page of a Session's legacy rows, for the converter that lifts + * them onto the ledger. Nothing else reads `session_messages` any more, so this + * is a migration scan rather than a transcript read. + */ +export interface SessionMessageScanRequest { + /** Exclusive lower bound; omit to start at the first row. */ + readonly afterSequence?: number; + readonly maxStoredBytes: number; + readonly maxMessages: number; +} + +export interface SessionMessageScanRecord { + readonly sequence: number; + readonly message: StoredMessage; +} + +export interface SessionMessageScanPage { + readonly records: readonly SessionMessageScanRecord[]; + /** + * The Session's last legacy sequence. It rides along with every page so the + * converter can place a turn relative to the whole transcript without a read + * that is proportional to it. + */ + readonly highWaterSequence: number | null; +} + export interface SessionTranscriptPageRequest { readonly direction: 'older' | 'newer'; /** Inclusive durable high-water mark. Omit only for the first read. */ @@ -307,6 +334,10 @@ export interface SessionStore { listTurnsSnapshot(sessionId: string): Promise; readHeader(sessionId: string): Promise; readMessages(sessionId: string): Promise; + readMessagesAfter( + sessionId: string, + request: SessionMessageScanRequest, + ): Promise; listTurns(sessionId: string): Promise; appendMessage(sessionId: string, message: StoredMessage): Promise; appendMessages(sessionId: string, messages: StoredMessage[]): Promise; @@ -956,6 +987,14 @@ class SqliteSessionStore implements SessionAuthorityStore { return this.readMessagesSnapshot(sessionId); } + async readMessagesAfter( + sessionId: string, + request: SessionMessageScanRequest, + ): Promise { + await this.ensureReady(); + return this.metadata.readMessagesAfter(sessionId, request); + } + async listTurns(sessionId: string): Promise { return deriveTurnRecords(await this.readMessages(sessionId)); } diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index 159d7547f8..7d0b4dd659 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -142,6 +142,9 @@ import { normalizeSessionHeader, SessionNotFoundError, type ExternalSessionImportLookupResult, + type SessionMessageScanPage, + type SessionMessageScanRecord, + type SessionMessageScanRequest, type SessionTranscriptMessageLookupRequest, } from './session-store.js'; import { @@ -2552,6 +2555,60 @@ export class SqliteSessionMetadataStore { return this.readMessagesWith(sessionId, decodeStoredMessage); } + async readMessagesAfter( + sessionId: string, + request: SessionMessageScanRequest, + ): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + if (!Number.isSafeInteger(request.maxMessages) || request.maxMessages < 1) { + throw new Error('Invalid Session message count limit'); + } + if (!Number.isSafeInteger(request.maxStoredBytes) || request.maxStoredBytes < 1) { + throw new Error('Invalid Session message byte limit'); + } + return this.readTransaction(() => { + if (!this.readRecordSync(sessionId)) throw new SessionNotFoundError(sessionId); + const rows = this.db + .prepare(` + SELECT sequence, record_json + FROM session_messages + WHERE session_id = ? AND sequence > ? + ORDER BY sequence + LIMIT ? + `) + .all(sessionId, request.afterSequence ?? -1, request.maxMessages) as Array<{ + sequence?: unknown; + record_json?: unknown; + }>; + const records: SessionMessageScanRecord[] = []; + let storedBytes = 0; + for (const row of rows) { + const sequence = requireStoredMessageSequence(row.sequence, sessionId); + const recordJson = String(row.record_json); + // The first record of a page is always taken, so a single row larger + // than the budget still makes progress instead of stalling the scan. + if (records.length > 0 && storedBytes + recordJson.length > request.maxStoredBytes) break; + storedBytes += recordJson.length; + try { + records.push({ + sequence, + message: decodeStoredMessage(JSON.parse(recordJson) as unknown), + }); + } catch (error) { + throw new StoredSessionMessageIncompatibleError(sessionId, sequence, { cause: error }); + } + } + const highWater = this.db + .prepare('SELECT MAX(sequence) AS high_water FROM session_messages WHERE session_id = ?') + .get(sessionId) as { high_water?: unknown }; + return { + records, + highWaterSequence: nullableStoredMessageSequence(highWater.high_water, sessionId), + }; + }); + } + async readTranscriptMessages( sessionId: string, request: SessionTranscriptMessageLookupRequest, From 8041671bea08410425df5c1f5d2158f04b1dcc47 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 08:06:19 +0800 Subject: [PATCH 13/32] refactor(runtime): make the read model the only transcript definition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The durable transcript reader asked two things at once: storage decided which RuntimeEvents produce rows and how they group into messages, and the read model decided what those rows contain. The SQL half restated message identity, output shape, terminal status and the thinking-to-text attachment rule, then fed its answers back into the projector through a `context` escape hatch that pre-seeded the projector's own state maps. Nothing checked the two agreed, and four of those expressions were baked into schema v17 indexes, so changing a transcript rule meant editing two languages and writing a migration. Storage now selects Turns by Session ordinal and hands over their events. The read model projects a Turn whole and reports which event each message came from, which is what gives a page its sequence numbers. One definition, in one language, with no index to migrate when it changes. The cost is a weaker bound: a page loads the Turns it takes rows from, where it used to load only the indexed neighbours of one event. That is still not the Session, which is what #4791 asked for; the stronger "not even one long Turn" promise was added by this PR itself, had no demand behind it, and was paid for with the duplicate definition. The reader test now states the per-Turn bound directly and still proves the discriminating case: a page at one end of a Session must not decode a 5 MiB Turn at the other. Sampled real Sessions put the largest Turn at 420 events / 843 KiB, twenty times inside the bound the live overlay already holds. If a Turn ever does outgrow it, the answer is a rebuildable projection index owned by the read model, not a second definition of what a transcript is. Falls out of the same change: - Turn contributions carried five derived booleans. `hasAbortNote` was unreachable — `abort` is a retired note kind no writer emits and the converter drops. The other four only fed a status inference for Turns with no recorded `turn_state`, and every settled Turn has one, so the desktop never saw an inferred status; it trusts `statusSource` and discards them. `projectSessionTurnContribution` now returns nothing rather than guessing a status no reader believes. - `compareRuntimeReadModelMessages` and its semantic-message mirror of the whole StoredMessage schema lost their production caller when this PR retired the dual read; only tests still reached them. - Runtime's `SessionStore.readMessages` became dead when the converter moved to `readMessagesAfter`; it survived only as an obligation on four test fakes. Two defects the reworked reader surfaced and fixes: a Turn's `firstSequence` was taken from its opening fact, which has no row, so every Turn-index cursor sat eight sequences early; and a message-id lookup that missed walked back through the whole Session instead of stopping at the newest Turns the live handoff can name. Generated-by: Claude Code --- apps/desktop/src/main/runtime-host-client.ts | 2 +- .../session-transcript-reader.test.ts | 63 ++- .../src/__tests__/session-turns.test.ts | 47 +- packages/runtime-host/src/protocol/index.ts | 4 +- .../src/protocol/session-turns.ts | 86 +--- .../src/server/session-transcript-reader.ts | 225 +++++---- .../runtime-event-read-model.test.ts | 152 +----- .../runtime-kernel-interaction.test.ts | 1 - .../runtime/src/runtime-event-read-model.ts | 238 +-------- packages/runtime/src/session-manager.ts | 2 - packages/storage/src/execution-stores.ts | 17 +- .../storage/src/runtime-transcript-query.ts | 478 ++++++------------ .../storage/src/session-message-projection.ts | 21 +- packages/storage/src/session-store.ts | 5 - packages/storage/src/sqlite-runtime-schema.ts | 12 +- packages/storage/src/sqlite-runtime-store.ts | 32 +- 16 files changed, 416 insertions(+), 969 deletions(-) diff --git a/apps/desktop/src/main/runtime-host-client.ts b/apps/desktop/src/main/runtime-host-client.ts index 7abc9327e4..9994a82619 100644 --- a/apps/desktop/src/main/runtime-host-client.ts +++ b/apps/desktop/src/main/runtime-host-client.ts @@ -1648,7 +1648,7 @@ export class DesktopRuntimeHostClient { } return [...contributions.values()] .sort((left, right) => left.firstSequence - right.firstSequence) - .map(projectSessionTurnContribution); + .flatMap((contribution) => projectSessionTurnContribution(contribution) ?? []); } async listSessionTurnLandmarks( diff --git a/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts b/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts index 512b04a4d5..7c35f17706 100644 --- a/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts +++ b/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts @@ -272,7 +272,7 @@ test('keeps durable history separate from the canonical active overlay', async ( } }); -test('pages the ledger without materializing off-page Turns or messages', async (t) => { +test('pages the ledger without materializing Turns it takes no rows from', async (t) => { const base = await mkdtemp(join(tmpdir(), 'maka-transcript-seek-')); const capability = await resolveStorageRoot({ path: join(base, 'root'), kind: 'interactive' }); const owner = await tryAcquireInteractiveRootOwner(capability); @@ -409,37 +409,52 @@ test('pages the ledger without materializing off-page Turns or messages', async stores, canonicalPermissionOutcomes: { readPermissionOutcome: async () => undefined }, }); - // Measure actual JSON decoded, not only the eventual response size. Neither - // a small page, the Turn index, nor a lookup miss may decode the 5 MiB Turn. + // Measure actual JSON decoded, not only the eventual response size. + // + // A read decodes the Turns it takes rows from, and no others. That bound is + // per Turn rather than per row: a Turn is projected whole because a row's + // meaning depends on the rest of its Turn. What must still hold is that no + // read walks the Session — so a page at one end must not touch the 5 MiB + // Turn at the other, and a Turn index page must cost only its own Turns. + const SMALL_TURN_BUDGET = 512 * 1024; + const ONE_BIG_TURN_BUDGET = 8 * 1024 * 1024; let decodedBytes = 0; const parse = JSON.parse; const measured = t.mock.method(JSON, 'parse', (...args: Parameters) => { decodedBytes += Buffer.byteLength(args[0]); return parse(...args); }); + const decoding = async (label: string, budget: number, run: () => Promise) => { + decodedBytes = 0; + const result = await run(); + assert.ok(decodedBytes < budget, `${label} decoded ${decodedBytes} bytes`); + return result; + }; const through = await read.readDurableHighWater(session.id); - const tail = await read.readDurablePage(session.id, { - direction: 'older', - maxBytes: 1024, - maxMessages: 1, - }); + const tail = await decoding('tail page', ONE_BIG_TURN_BUDGET, () => + read.readDurablePage(session.id, { direction: 'older', maxBytes: 1024, maxMessages: 1 }), + ); assert.equal(JSON.parse(tail.fragments[0]!.data.toString()).type, 'system_note'); - const head = await read.readDurablePage(session.id, { - direction: 'newer', - maxBytes: 1024, - maxMessages: 1, - }); + // The discriminating read: the first Turn is small and sits at the far end + // of the Session from the 5 MiB one, so serving it may not decode that Turn. + const head = await decoding('head page', SMALL_TURN_BUDGET, () => + read.readDurablePage(session.id, { direction: 'newer', maxBytes: 1024, maxMessages: 1 }), + ); assert.equal(JSON.parse(head.fragments[0]!.data.toString()).text, 'prompt 0'); assert.deepEqual( - await read.readDurableMessagesById(session.id, { - throughSequence: through, - messageIds: ['missing-stream'], - maxBytes: 1024, - maxMessages: 1, - }), + await decoding('lookup miss', ONE_BIG_TURN_BUDGET, () => + read.readDurableMessagesById(session.id, { + throughSequence: through, + messageIds: ['missing-stream'], + maxBytes: 1024, + maxMessages: 1, + }), + ), [], ); - const landmarks = await read.readDurableTurnLandmarks(session.id, 3); + const landmarks = await decoding('landmarks', SMALL_TURN_BUDGET, () => + read.readDurableTurnLandmarks(session.id, 3), + ); assert.deepEqual( landmarks.landmarks.map((item) => item.label), ['prompt 0', 'prompt 2', 'prompt 4'], @@ -447,17 +462,13 @@ test('pages the ledger without materializing off-page Turns or messages', async const contributions: SessionTurnContribution[] = []; let contributionPosition = 0; for (;;) { - const page = await read.readDurableTurnContributions( - session.id, - through, - contributionPosition, - 2, + const page = await decoding('turn index page', ONE_BIG_TURN_BUDGET, () => + read.readDurableTurnContributions(session.id, through, contributionPosition, 2), ); contributions.push(...page.contributions); if (page.nextPosition === null) break; contributionPosition = page.nextPosition; } - assert.ok(decodedBytes < 512 * 1024, `decoded ${decodedBytes} bytes for bounded reads`); measured.mock.restore(); const records: Array<{ sequence: number; message: StoredMessage }> = []; diff --git a/packages/runtime-host/src/__tests__/session-turns.test.ts b/packages/runtime-host/src/__tests__/session-turns.test.ts index 945e36e780..769d4db33b 100644 --- a/packages/runtime-host/src/__tests__/session-turns.test.ts +++ b/packages/runtime-host/src/__tests__/session-turns.test.ts @@ -48,26 +48,43 @@ test('keeps a full sampled landmark index inside its encoded result budget', () assert.doesNotThrow(() => decodeSessionTurnLandmarksQueryResult(result)); }); -test('keeps legacy assistant presence distinct from retained output', () => { - assert.deepEqual( +test('publishes no Turn until its recorded state is on the page', () => { + assert.strictEqual( projectSessionTurnContribution({ turnId: 'turn-1', firstSequence: 0, latestState: null, userPromptPreview: 'hello', - hasAssistantMessage: true, - hasAssistantOutput: false, - hasToolResult: false, - hasFailedToolResult: true, - hasAbortNote: false, + }), + undefined, + ); +}); + +test('takes retained output from the recorded turn state', () => { + assert.deepEqual( + projectSessionTurnContribution({ + turnId: 'turn-1', + firstSequence: 0, + latestState: { + sequence: 4, + message: { + type: 'turn_state', + id: 'state-1', + turnId: 'turn-1', + ts: 1, + status: 'failed', + partialOutputRetained: true, + }, + }, + userPromptPreview: 'hello', }), { turnId: 'turn-1', firstSequence: 0, userPromptPreview: 'hello', - status: 'completed', - statusSource: 'inferred', - partialOutputRetained: false, + status: 'failed', + statusSource: 'recorded', + partialOutputRetained: true, }, ); }); @@ -89,11 +106,6 @@ test('bounds turn diagnostics before publishing a contribution', () => { }, }, userPromptPreview: 'hello', - hasAssistantMessage: false, - hasAssistantOutput: false, - hasToolResult: false, - hasFailedToolResult: false, - hasAbortNote: false, }); assert.ok( @@ -128,11 +140,6 @@ test('rejects invalid turn-state references before publishing a contribution', ( }, }, userPromptPreview: null, - hasAssistantMessage: false, - hasAssistantOutput: false, - hasToolResult: false, - hasFailedToolResult: false, - hasAbortNote: false, }), ); }); diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts index a1c40a7709..10923b13ab 100644 --- a/packages/runtime-host/src/protocol/index.ts +++ b/packages/runtime-host/src/protocol/index.ts @@ -101,7 +101,9 @@ export const RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION = 1 as const; export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const; // Increment when the same protocol version no longer guarantees safe Client-Host // interoperability. Mismatches are rejected before domain commands are admitted. -export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 122 as const; +export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 123 as const; +// 123: Session Turn contributions carry only the Turn's recorded state. Older +// peers require the derived shape booleans this projection no longer sends. // 122: Durable transcript cursors seek Session event ordinals instead of run indexes. // 121: Host diagnostics report `upgradeBlockingActivity`, the Host's // authoritative activity answer for maintenance probes, computed by the same diff --git a/packages/runtime-host/src/protocol/session-turns.ts b/packages/runtime-host/src/protocol/session-turns.ts index c615b7152a..c34b48bc85 100644 --- a/packages/runtime-host/src/protocol/session-turns.ts +++ b/packages/runtime-host/src/protocol/session-turns.ts @@ -72,11 +72,6 @@ export interface SessionTurnContribution { readonly message: TurnStateMessage; } | null; readonly userPromptPreview: string | null; - readonly hasAssistantMessage: boolean; - readonly hasAssistantOutput: boolean; - readonly hasToolResult: boolean; - readonly hasFailedToolResult: boolean; - readonly hasAbortNote: boolean; } export interface SessionTurnsQueryInput { @@ -109,11 +104,6 @@ export function mergeSessionTurnContributions( ? next.latestState : current.latestState, userPromptPreview: current.userPromptPreview ?? next.userPromptPreview, - hasAssistantMessage: current.hasAssistantMessage || next.hasAssistantMessage, - hasAssistantOutput: current.hasAssistantOutput || next.hasAssistantOutput, - hasToolResult: current.hasToolResult || next.hasToolResult, - hasFailedToolResult: current.hasFailedToolResult || next.hasFailedToolResult, - hasAbortNote: current.hasAbortNote || next.hasAbortNote, }; } @@ -135,11 +125,6 @@ export function projectSessionTurnContributionForWire( contribution.userPromptPreview === null ? null : truncateUtf8(contribution.userPromptPreview, SESSION_TURN_PROMPT_PREVIEW_MAX_BYTES), - hasAssistantMessage: contribution.hasAssistantMessage, - hasAssistantOutput: contribution.hasAssistantOutput, - hasToolResult: contribution.hasToolResult, - hasFailedToolResult: contribution.hasFailedToolResult, - hasAbortNote: contribution.hasAbortNote, }; } @@ -183,46 +168,36 @@ function projectTurnStateMessageForWire(message: TurnStateMessage): TurnStateMes }; } -export function projectSessionTurnContribution(contribution: SessionTurnContribution): TurnRecord { +/** + * The Turn a contribution describes, or nothing when its ending is not on the + * page yet. + * + * A Turn's status is read off the `turn_state` its terminal projects; a + * contribution without one has not been folded up to its ending, and a status + * guessed from the rows that did arrive is one no reader trusts anyway. + */ +export function projectSessionTurnContribution( + contribution: SessionTurnContribution, +): TurnRecord | undefined { const state = contribution.latestState?.message; - const partialOutputRetained = contribution.hasAssistantOutput || contribution.hasToolResult; - if (state) { - return { - turnId: contribution.turnId, - firstSequence: contribution.firstSequence, - ...(contribution.userPromptPreview - ? { userPromptPreview: contribution.userPromptPreview } - : {}), - status: state.status, - statusSource: 'recorded', - ...(state.parentTurnId ? { parentTurnId: state.parentTurnId } : {}), - ...(state.retriedFromTurnId ? { retriedFromTurnId: state.retriedFromTurnId } : {}), - ...(state.regeneratedFromTurnId - ? { regeneratedFromTurnId: state.regeneratedFromTurnId } - : {}), - ...(state.branchOfTurnId ? { branchOfTurnId: state.branchOfTurnId } : {}), - ...(state.parentSessionId ? { parentSessionId: state.parentSessionId } : {}), - ...(state.abortedAt !== undefined ? { abortedAt: state.abortedAt } : {}), - ...(state.abortSource ? { abortSource: state.abortSource } : {}), - ...(state.errorClass ? { errorClass: state.errorClass } : {}), - partialOutputRetained: state.partialOutputRetained || partialOutputRetained, - }; - } + if (!state) return undefined; return { turnId: contribution.turnId, firstSequence: contribution.firstSequence, ...(contribution.userPromptPreview ? { userPromptPreview: contribution.userPromptPreview } : {}), - status: contribution.hasAbortNote - ? 'aborted' - : contribution.hasAssistantMessage - ? 'completed' - : contribution.hasFailedToolResult - ? 'failed' - : 'completed', - statusSource: 'inferred', - partialOutputRetained, + status: state.status, + statusSource: 'recorded', + ...(state.parentTurnId ? { parentTurnId: state.parentTurnId } : {}), + ...(state.retriedFromTurnId ? { retriedFromTurnId: state.retriedFromTurnId } : {}), + ...(state.regeneratedFromTurnId ? { regeneratedFromTurnId: state.regeneratedFromTurnId } : {}), + ...(state.branchOfTurnId ? { branchOfTurnId: state.branchOfTurnId } : {}), + ...(state.parentSessionId ? { parentSessionId: state.parentSessionId } : {}), + ...(state.abortedAt !== undefined ? { abortedAt: state.abortedAt } : {}), + ...(state.abortSource ? { abortSource: state.abortSource } : {}), + ...(state.errorClass ? { errorClass: state.errorClass } : {}), + partialOutputRetained: state.partialOutputRetained, }; } @@ -394,11 +369,6 @@ function decodeSessionTurnContribution(value: unknown): SessionTurnContribution 'firstSequence', 'latestState', 'userPromptPreview', - 'hasAssistantMessage', - 'hasAssistantOutput', - 'hasToolResult', - 'hasFailedToolResult', - 'hasAbortNote', ]); let latestState: SessionTurnContribution['latestState'] = null; if (contribution.latestState !== null) { @@ -441,15 +411,5 @@ function decodeSessionTurnContribution(value: unknown): SessionTurnContribution 'Session turn prompt preview', SESSION_TURN_PROMPT_PREVIEW_MAX_BYTES, ), - hasAssistantMessage: requireBoolean(contribution.hasAssistantMessage), - hasAssistantOutput: requireBoolean(contribution.hasAssistantOutput), - hasToolResult: requireBoolean(contribution.hasToolResult), - hasFailedToolResult: requireBoolean(contribution.hasFailedToolResult), - hasAbortNote: requireBoolean(contribution.hasAbortNote), }; } - -function requireBoolean(value: unknown): boolean { - if (typeof value !== 'boolean') throw invalidProtocolFrame('Invalid Session turn contribution'); - return value; -} diff --git a/packages/runtime-host/src/server/session-transcript-reader.ts b/packages/runtime-host/src/server/session-transcript-reader.ts index 87ea53765b..c7eec19801 100644 --- a/packages/runtime-host/src/server/session-transcript-reader.ts +++ b/packages/runtime-host/src/server/session-transcript-reader.ts @@ -43,7 +43,7 @@ import type { SessionTurnContributionPage, SessionTurnLandmark, SessionTurnLandmarkSnapshot, - RuntimeTranscriptSource, + RuntimeTranscriptInvocation, } from '@maka/storage/execution-stores'; import { foldTurnContribution } from '@maka/storage/session-message-projection'; import { SESSION_TRANSCRIPT_OVERLAY_MAX_MESSAGES, type TurnSnapshot } from '../protocol/index.js'; @@ -55,6 +55,22 @@ export const ACTIVE_TRANSCRIPT_OVERLAY_MAX_MESSAGES = SESSION_TRANSCRIPT_OVERLAY export const ACTIVE_TRANSCRIPT_OVERLAY_MAX_BYTES = 16 * 1024 * 1024; const ACTIVE_TRANSCRIPT_SOURCE_MAX_EVENTS = ACTIVE_TRANSCRIPT_OVERLAY_MAX_MESSAGES * 2; const ACTIVE_TRANSCRIPT_SCAN_BATCH_MAX_BYTES = 256 * 1024; +/** + * What one durable page may read of a Turn, matching the bound the live + * overlay already holds for a run. A Turn past it is refused rather than + * half-projected: a prefix of a Turn is not a smaller transcript of it. + */ +const DURABLE_TRANSCRIPT_TURN_MAX_EVENTS = ACTIVE_TRANSCRIPT_SOURCE_MAX_EVENTS; +const DURABLE_TRANSCRIPT_TURN_MAX_BYTES = ACTIVE_TRANSCRIPT_OVERLAY_MAX_BYTES; +/** Turns per storage round trip: one, so a page loads no Turn it cannot use. */ +const TRANSCRIPT_TURN_SCAN_LIMIT = 1; +/** + * How far back the live-to-durable handoff looks for a message id. The ids come + * from assistant streams the subscriber is still watching, so they are in the + * newest Turn or the one it continued from; an id that is in neither is treated + * as absent rather than searched for down the Session. + */ +const TRANSCRIPT_LOOKUP_MAX_TURNS = 2; export function createSessionTranscriptReader(input: { stores: ExecutionStoresWriter<'interactive'>; @@ -160,10 +176,12 @@ export interface SessionTranscriptReader { } /** - * Pages seek immutable Session event ordinals before decoding payloads. Each - * event is projected with just its indexed message context, so neither a long - * Session nor a long Turn has to be loaded to serve a page. The low sequence - * bits distinguish the few rows one event can emit. + * Pages seek immutable Session event ordinals before decoding payloads. One + * Turn is projected at a time, so a page costs one Turn rather than the + * Session. The low sequence bits distinguish the few rows one event emits. + * + * What an event becomes is asked only of the read model. Storage selects Turns + * by ordinal and hands over their events; it never classifies one. */ function createDurableLedgerTranscriptReader(input: { stores: ExecutionStoresWriter<'interactive'>; @@ -171,71 +189,89 @@ function createDurableLedgerTranscriptReader(input: { }) { const store = input.stores.runtimeEventStore; const highWater = async (sessionId: string): Promise => { - const ordinal = await store.readTranscriptSourceHighWater(sessionId); + const ordinal = await store.readTranscriptHighWater(sessionId); return ordinal === null ? null : ordinal * EVENT_SEQUENCE_STRIDE + EVENT_SEQUENCE_STRIDE - 1; }; - const projectSource = async ( - source: RuntimeTranscriptSource, - ): Promise => { - const event = source.event; - const projected = projectRuntimeEventsToStoredMessages(source.events, { - invocations: [source.invocation], + + /** One Turn's rows, each at the sequence its own event sits at. */ + const projectTurn = async ( + turn: RuntimeTranscriptInvocation, + ): Promise<{ sequence: number; message: StoredMessage }[]> => { + const events = turn.events.map((entry) => entry.event); + const projected = projectRuntimeEventsToStoredMessages(events, { + invocations: [turn.invocation], canonicalPermissionOutcomes: await readCanonicalPermissionOutcomes( - source.events, + events, input.canonicalPermissionOutcomes, ), - context: { - messageId: event.refs?.storedMessageId ?? event.refs?.providerEventId ?? event.id, - ...(source.contentOrder ? { contentOrder: source.contentOrder } : {}), - ...(source.permissionRequest ? { permissionRequest: source.permissionRequest } : {}), - ...(source.toolName ? { toolName: source.toolName } : {}), - ...(event.refs?.toolCallId ? { toolUseId: event.refs.toolCallId } : {}), - hasRetainedOutput: source.hasRetainedOutput, - }, }); if (projected.diagnostics.some(isHardRuntimeEventReadModelDiagnostic)) { throw new Error('Durable RuntimeEvent transcript projection is incomplete'); } - if (projected.messages.length > EVENT_SEQUENCE_STRIDE) { - throw new Error('RuntimeEvent exceeds its transcript sequence stride'); - } - return projected.messages; + const ordinals = new Map(turn.events.map((entry) => [entry.event.id, entry.ordinal])); + const emitted = new Map(); + return projected.messages.map((message, index) => { + const ordinal = ordinals.get(projected.sourceEventIds[index]!); + if (ordinal === undefined) { + throw new Error('Durable transcript message has no source RuntimeEvent'); + } + const offset = emitted.get(ordinal) ?? 0; + if (offset >= EVENT_SEQUENCE_STRIDE) { + throw new Error('RuntimeEvent exceeds its transcript sequence stride'); + } + emitted.set(ordinal, offset + 1); + return { sequence: ordinal * EVENT_SEQUENCE_STRIDE + offset, message }; + }); }; + const readTurns = async ( + sessionId: string, + request: { direction: 'older' | 'newer'; throughOrdinal: number; position: number }, + ): Promise => + store.readTranscriptInvocations(sessionId, { + ...request, + limit: TRANSCRIPT_TURN_SCAN_LIMIT, + maxEvents: DURABLE_TRANSCRIPT_TURN_MAX_EVENTS, + maxBytes: DURABLE_TRANSCRIPT_TURN_MAX_BYTES, + }); + const scan = async function* ( sessionId: string, request: { direction: 'older' | 'newer'; throughSequence?: number | null; position?: number; + /** Stops the walk after this many Turns, for a read that may find nothing. */ + maxTurns?: number; }, ): AsyncGenerator<{ sequence: number; message: StoredMessage }> { const throughSequence = request.throughSequence === undefined ? await highWater(sessionId) : request.throughSequence; if (throughSequence === null) return; const position = request.position ?? (request.direction === 'older' ? throughSequence : 0); + const throughOrdinal = ordinalOf(throughSequence); let ordinal = ordinalOf(position); - while (ordinal >= 0 && ordinal <= ordinalOf(throughSequence)) { - const source = await store.readTranscriptSource(sessionId, { + let walked = 0; + while (ordinal >= 0 && ordinal <= throughOrdinal) { + const turns = await readTurns(sessionId, { direction: request.direction, - throughOrdinal: ordinalOf(throughSequence), + throughOrdinal, position: ordinal, }); - if (!source) return; - const messages = await projectSource(source); - const records = messages - .map((message, index) => ({ - sequence: source.ordinal * EVENT_SEQUENCE_STRIDE + index, - message, - })) - .filter( + if (turns.length === 0) return; + for (const turn of turns) { + if (request.maxTurns !== undefined && walked >= request.maxTurns) return; + walked += 1; + const records = (await projectTurn(turn)).filter( ({ sequence }) => sequence <= throughSequence && (request.direction === 'older' ? sequence <= position : sequence >= position), ); - if (request.direction === 'older') records.reverse(); - yield* records; - ordinal = source.ordinal + (request.direction === 'older' ? -1 : 1); + if (request.direction === 'older') records.reverse(); + yield* records; + } + const edge = turns.at(-1)!; + ordinal = request.direction === 'older' ? edge.firstOrdinal - 1 : edge.lastOrdinal + 1; } }; @@ -324,7 +360,7 @@ function createDurableLedgerTranscriptReader(input: { return { throughSequence, records, nextPosition }; }, - /** Fold indexed Turn facts, loading only the prompt and terminal payloads. */ + /** One row per Turn, folded from the Turn's own projected messages. */ async readTurnContributions( sessionId: string, throughSequence: number | null, @@ -335,54 +371,29 @@ function createDurableLedgerTranscriptReader(input: { if (watermark === null) { return { throughSequence: null, contributions: [], nextPosition: null }; } - const turns = await store.readTranscriptTurns( - sessionId, - ordinalOf(watermark), - ordinalOf(position), - maxContributions + 1, - ); + const turns = await store.readTranscriptInvocations(sessionId, { + direction: 'newer', + throughOrdinal: ordinalOf(watermark), + position: ordinalOf(position), + limit: maxContributions + 1, + maxEvents: DURABLE_TRANSCRIPT_TURN_MAX_EVENTS, + maxBytes: DURABLE_TRANSCRIPT_TURN_MAX_BYTES, + }); const contributions: SessionTurnContribution[] = []; for (const turn of turns.slice(0, maxContributions)) { - let contribution: SessionTurnContribution = { - turnId: turn.invocation.turnId, - firstSequence: Math.max(position, turn.firstOrdinal * EVENT_SEQUENCE_STRIDE), - latestState: null, - userPromptPreview: null, - hasAssistantMessage: turn.hasAssistantMessage, - hasAssistantOutput: turn.hasAssistantOutput, - hasToolResult: turn.hasToolResult, - hasFailedToolResult: turn.hasFailedToolResult, - hasAbortNote: turn.hasAbortNote, - }; - if (turn.user) { - const user = projectRuntimeEventUserMessage(turn.user.event, turn.user.event.id); - if (user) - contribution = foldTurnContribution( - contribution, - turn.invocation.turnId, - turn.user.ordinal * EVENT_SEQUENCE_STRIDE, - user, - ); - } - const source = await store.readTranscriptSource(sessionId, { - direction: 'newer', - throughOrdinal: ordinalOf(watermark), - position: turn.terminalOrdinal, - }); - if (source?.ordinal === turn.terminalOrdinal) { - const messages = await projectSource(source); - for (const [index, message] of messages.entries()) { - const sequence = source.ordinal * EVENT_SEQUENCE_STRIDE + index; - if (sequence < position || sequence > watermark) continue; - contribution = foldTurnContribution( - contribution, - turn.invocation.turnId, - sequence, - message, - ); - } + // Folded from the Turn's own rows, so `firstSequence` lands on its first + // row rather than on the opening fact, which has no row at all. + let contribution: SessionTurnContribution | undefined; + for (const { sequence, message } of await projectTurn(turn)) { + if (sequence < position || sequence > watermark) continue; + contribution = foldTurnContribution( + contribution, + turn.invocation.turnId, + sequence, + message, + ); } - contributions.push(contribution); + if (contribution) contributions.push(contribution); } const next = turns[maxContributions]; return { @@ -406,43 +417,47 @@ function createDurableLedgerTranscriptReader(input: { ); const landmarks: SessionTurnLandmark[] = []; for (const turn of turns) { - if (!turn.user) continue; - const message = projectRuntimeEventUserMessage(turn.user.event, turn.user.event.id); + if (!turn.prompt) continue; + const message = projectRuntimeEventUserMessage(turn.prompt.event, turn.prompt.event.id); const label = (message?.displayText ?? message?.text ?? '').trim(); if (!label) continue; landmarks.push({ turnId: turn.invocation.turnId, - sequence: turn.user.ordinal * EVENT_SEQUENCE_STRIDE, + sequence: turn.prompt.ordinal * EVENT_SEQUENCE_STRIDE, label, }); } return { throughSequence, landmarks }; }, + /** + * The durable rows behind a set of message ids. + * + * The ids come from the assistant streams a subscriber is still watching, + * so they belong to the Session's newest Turns. The scan walks back from + * the watermark a Turn at a time and stops as soon as every id is found, + * rather than keeping an index from message id to event. An id that is not + * there stops the walk after the newest Turns instead of reading the + * Session: the handoff shows what the tail holds, not everything it could. + */ async readMessagesById( sessionId: string, request: SessionTranscriptMessageLookupRequest, ): Promise { if (request.throughSequence === null || request.messageIds.length === 0) return []; + const wanted = new Set(request.messageIds); const found: Array<{ sequence: number; message: StoredMessage }> = []; let bytes = 0; - for (const messageId of new Set(request.messageIds)) { - const source = await store.readTranscriptSource(sessionId, { - direction: 'older', - throughOrdinal: ordinalOf(request.throughSequence), - position: ordinalOf(request.throughSequence), - messageId, - }); - if (!source) continue; - for (const [index, message] of (await projectSource(source)).entries()) { - const sequence = source.ordinal * EVENT_SEQUENCE_STRIDE + index; - if (message.id !== messageId || sequence > request.throughSequence) continue; - bytes += Buffer.byteLength(JSON.stringify(message), 'utf8'); - if (found.length >= request.maxMessages || bytes > request.maxBytes) { - return found.sort((a, b) => a.sequence - b.sequence).map((record) => record.message); - } - found.push({ sequence, message }); - } + for await (const record of scan(sessionId, { + direction: 'older', + throughSequence: request.throughSequence, + maxTurns: TRANSCRIPT_LOOKUP_MAX_TURNS, + })) { + if (!wanted.delete(record.message.id)) continue; + bytes += Buffer.byteLength(JSON.stringify(record.message), 'utf8'); + if (found.length >= request.maxMessages || bytes > request.maxBytes) break; + found.push(record); + if (wanted.size === 0) break; } return found.sort((a, b) => a.sequence - b.sequence).map((record) => record.message); }, diff --git a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts index af88e86651..4afde952e7 100644 --- a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts +++ b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts @@ -26,7 +26,6 @@ import { runtimeEventHasModelVisibleContent } from '@maka/core/runtime-event'; import type { SessionHeader, SessionSummary, StoredMessage, TurnRecord } from '@maka/core/session'; import { deriveTurnRecords } from '@maka/core/session'; import { - compareRuntimeReadModelMessages, isHardRuntimeEventReadModelDiagnostic, isUnclaimedRuntimeEventDiagnostic, projectRuntimeEventsToStoredMessages, @@ -349,17 +348,6 @@ describe('projectRuntimeEventsToStoredMessages', () => { displayText: typed, }, ]); - const compare = compareRuntimeReadModelMessages(out.messages, [ - { - type: 'user', - id: 'user-skill', - turnId, - ts: ts + 1, - text: envelope, - displayText: typed, - }, - ]); - assert.deepStrictEqual(compare.diagnostics, []); }); test('full RuntimeEvent turn projects legacy-compatible rows', () => { @@ -1388,7 +1376,6 @@ describe('projectRuntimeEventsToStoredMessages', () => { assert.deepStrictEqual(out.messages, legacy); assert.deepStrictEqual(out.diagnostics, []); - assert.strictEqual(compareRuntimeReadModelMessages(out.messages, legacy).compatible, true); }); test('per-step thinking pairs each step assistant row by its own message id', () => { @@ -2197,84 +2184,8 @@ describe('RuntimeEventActions projection coverage', () => { } }); -describe('compareRuntimeReadModelMessages', () => { - test('treats nested JSON with different property order as compatible', () => { - const projected = projectRuntimeEventsToStoredMessages( - [ - ev({ - id: 'evt-tool-call-json', - role: 'model', - author: 'agent', - content: { - kind: 'function_call', - id: 'tool-json', - name: 'JsonTool', - args: { beta: 2, alpha: { z: 3, a: 1 } }, - }, - }), - ev({ - id: 'evt-tool-result-json', - role: 'tool', - author: 'tool', - content: { - kind: 'function_response', - id: 'tool-json', - name: 'JsonTool', - result: { kind: 'json', value: { outer: { y: 2, x: 1 }, list: [{ b: 2, a: 1 }] } }, - }, - }), - ], - { invocations: [invocation] }, - ); - const legacy: StoredMessage[] = [ - { - type: 'tool_call', - id: 'tool-json', - turnId, - ts, - toolName: 'JsonTool', - args: { alpha: { a: 1, z: 3 }, beta: 2 }, - }, - { - type: 'tool_result', - id: 'different-result-id', - turnId, - ts, - toolUseId: 'tool-json', - isError: false, - content: { kind: 'json', value: { list: [{ a: 1, b: 2 }], outer: { x: 1, y: 2 } } }, - }, - ]; - - const result = compareRuntimeReadModelMessages(projected.messages, legacy); - - assert.strictEqual(result.compatible, true); - assert.deepStrictEqual(result.diagnostics, []); - }); - - test('rejects a mismatched tool activity kind', () => { - const projected: StoredMessage[] = [ - { - type: 'tool_call', - id: 'tool-kind', - turnId, - ts, - toolName: 'CustomTool', - activityKind: 'read', - args: {}, - }, - ]; - const legacy: StoredMessage[] = [ - { - ...(projected[0] as Extract), - activityKind: 'command', - }, - ]; - - assert.strictEqual(compareRuntimeReadModelMessages(projected, legacy).compatible, false); - }); - - test('carries the cross-turn request anchor both ways and compares on it', () => { +describe('token usage projection', () => { + test('carries the cross-turn request anchor both ways', () => { const lastRequestAnchor = { inputTokens: 120, outputTokens: 30 }; const anchored = ev({ id: 'evt-token-anchor', @@ -2307,65 +2218,6 @@ describe('compareRuntimeReadModelMessages', () => { ?.lastRequestAnchor, lastRequestAnchor, ); - - assert.strictEqual( - compareRuntimeReadModelMessages( - [usage as StoredMessage], - [ - { - ...(usage as Extract), - lastRequestAnchor: undefined, - }, - ], - ).compatible, - false, - ); - }); - - test('rejects mismatched replay-critical token usage fields', () => { - const usage: Extract = { - type: 'token_usage', - id: 'usage-1', - turnId, - ts, - input: 100, - output: 25, - runtimeSteps: 3, - contextRemaining: 9000, - providerRequestTraceId: 'provider-trace-1', - }; - - assert.strictEqual( - compareRuntimeReadModelMessages([usage], [{ ...usage, runtimeSteps: 4 }]).compatible, - false, - ); - assert.strictEqual( - compareRuntimeReadModelMessages([usage], [{ ...usage, contextRemaining: 8000 }]).compatible, - false, - ); - assert.strictEqual( - compareRuntimeReadModelMessages( - [usage], - [{ ...usage, providerRequestTraceId: 'provider-trace-2' }], - ).compatible, - false, - ); - }); - - test('rejects missing tool result and assistant text cases', () => { - const projected = projectRuntimeEventsToStoredMessages(baseEvents(), { - invocations: [invocation], - }); - const missing = projected.messages.filter( - (message) => message.type !== 'tool_result' && message.type !== 'assistant', - ); - const result = compareRuntimeReadModelMessages(missing, equivalentLegacyMessages()); - - assert.strictEqual(result.compatible, false); - assert.deepStrictEqual( - result.diagnostics.map((diag) => diag.code), - ['missing_legacy_message', 'missing_legacy_message'], - ); }); }); diff --git a/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts b/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts index 34faf6750e..df2b0bd172 100644 --- a/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts +++ b/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts @@ -623,7 +623,6 @@ function memoryStore(): SessionStore { }, list: async () => [], readHeader: async () => header, - readMessages: async () => [...messages], readMessagesAfter: async ( _sessionId: string, request: { afterSequence?: number; maxMessages: number }, diff --git a/packages/runtime/src/runtime-event-read-model.ts b/packages/runtime/src/runtime-event-read-model.ts index 819d19fceb..578d0d5137 100644 --- a/packages/runtime/src/runtime-event-read-model.ts +++ b/packages/runtime/src/runtime-event-read-model.ts @@ -57,7 +57,6 @@ export type RuntimeEventReadModelDiagnosticCode = | 'archived_tool_result_placeholder' | 'generated_id' | 'tool_use_id_mismatch' - | 'missing_legacy_message' | 'unexpected_projected_message'; /** @@ -82,7 +81,6 @@ const RUNTIME_EVENT_READ_MODEL_DIAGNOSTIC_SEVERITY: Record< archived_tool_result_placeholder: 'soft', generated_id: 'soft', tool_use_id_mismatch: 'hard', - missing_legacy_message: 'soft', unexpected_projected_message: 'soft', }; @@ -145,6 +143,8 @@ export interface RuntimeEventReadModelDiagnostic { export interface RuntimeEventReadModelProjection { messages: StoredMessage[]; diagnostics: RuntimeEventReadModelDiagnostic[]; + /** The id of the event each message was projected from, by position. */ + sourceEventIds: string[]; } export interface ProjectRuntimeEventsToStoredMessagesOptions { @@ -152,15 +152,6 @@ export interface ProjectRuntimeEventsToStoredMessagesOptions { | readonly RuntimeInvocationRecord[] | Readonly>; canonicalPermissionOutcomes?: ReadonlyMap; - /** Facts read by indexed lookup when projecting one durable message. */ - context?: { - messageId: string; - contentOrder?: readonly AssistantStepContentKind[]; - permissionRequest?: RuntimeEvent; - toolName?: string; - toolUseId?: string; - hasRetainedOutput: boolean; - }; } export interface ArchivedToolResultReadModelStatus { @@ -168,11 +159,6 @@ export interface ArchivedToolResultReadModelStatus { status: Extract['status']; } -export interface RuntimeReadModelCompatibilityResult { - compatible: boolean; - diagnostics: RuntimeEventReadModelDiagnostic[]; -} - export interface RuntimeEventTerminalFact { runId: string; turnId: string; @@ -213,7 +199,6 @@ interface ProjectionState { */ thinkingByMessageId: Map; contentOrderByMessageId: Map; - hasRetainedOutput?: boolean; } interface PendingThinking { @@ -237,31 +222,23 @@ export function projectRuntimeEventsToStoredMessages( contentOrderByMessageId: new Map(), }; const messages: StoredMessage[] = []; - - const context = options.context; - if (context) { - state.hasRetainedOutput = context.hasRetainedOutput; - if (context.contentOrder) - state.contentOrderByMessageId.set(context.messageId, [...context.contentOrder]); - if (context.toolName && context.toolUseId) - state.toolNameByUseId.set(context.toolUseId, context.toolName); - const requestEvent = context.permissionRequest; - const request = requestEvent?.actions?.permissionRequest; - if (request && requestEvent) { - state.permissionRequestById.set(request.requestId, { - requestId: request.requestId, - toolUseId: request.toolUseId, - toolName: request.toolName, - sessionId: requestEvent.sessionId, - runId: requestEvent.runId, - turnId: requestEvent.turnId, - ...(request.hint !== undefined ? { hint: request.hint } : {}), - }); - state.toolNameByUseId.set(request.toolUseId, request.toolName); - } - } + /** + * Which event each message came out of, by position. + * + * A message belongs to the event being read when it was appended: nothing + * rewrites an earlier message, so the rows that appear while one event is + * handled are exactly that event's rows. A durable reader numbers its pages + * from this, which is why it is recorded here rather than rediscovered. + */ + const sourceEventIds: string[] = []; + let reading: RuntimeEvent | undefined; + const attributeEmitted = (): void => { + while (sourceEventIds.length < messages.length) sourceEventIds.push(reading!.id); + }; for (const event of events) { + attributeEmitted(); + reading = event; recordStepContentOrder(event, state); if (isPartialRuntimeEvent(event)) { diagnostic(state, event, 'partial_skipped', 'partial RuntimeEvent skipped'); @@ -470,7 +447,8 @@ export function projectRuntimeEventsToStoredMessages( } } - return { messages, diagnostics: state.diagnostics }; + attributeEmitted(); + return { messages, diagnostics: state.diagnostics, sourceEventIds }; } /** @@ -584,39 +562,6 @@ export function applyArchivedToolResultReadModelStatuses( }); } -export function compareRuntimeReadModelMessages( - projected: readonly StoredMessage[], - legacy: readonly StoredMessage[], -): RuntimeReadModelCompatibilityResult { - const diagnostics: RuntimeEventReadModelDiagnostic[] = []; - const projectedCounts = countSemanticMessages(projected); - const legacyCounts = countSemanticMessages(legacy); - - for (const [key, count] of legacyCounts) { - const projectedCount = projectedCounts.get(key) ?? 0; - if (projectedCount < count) { - diagnostics.push({ - code: 'missing_legacy_message', - message: 'projected RuntimeEvent read model is missing a legacy semantic message', - detail: JSON.parse(key) as unknown, - }); - } - } - - for (const [key, count] of projectedCounts) { - const legacyCount = legacyCounts.get(key) ?? 0; - if (legacyCount < count) { - diagnostics.push({ - code: 'unexpected_projected_message', - message: 'projected RuntimeEvent read model has no matching legacy semantic message', - detail: JSON.parse(key) as unknown, - }); - } - } - - return { compatible: diagnostics.length === 0, diagnostics }; -} - export function classifyRuntimeEventTerminalFact( invocation: Pick, events: readonly RuntimeEvent[], @@ -1273,14 +1218,12 @@ function projectTerminalTurnState( } const abortSource = status === 'aborted' ? abortSourceFromRuntime(event) : undefined; const failureClass = status === 'failed' ? failureClassFromRuntimeEvent(event) : undefined; - const partialOutputRetained = - state.hasRetainedOutput ?? - messages.some( - (message) => - message.turnId === event.turnId && - ((message.type === 'assistant' && message.text.trim().length > 0) || - message.type === 'tool_result'), - ); + const partialOutputRetained = messages.some( + (message) => + message.turnId === event.turnId && + ((message.type === 'assistant' && message.text.trim().length > 0) || + message.type === 'tool_result'), + ); messages.push({ type: 'turn_state', id: stableMessageId(event, state, 'turn_state'), @@ -1612,134 +1555,3 @@ function isRuntimeEventDiagnosticDetail( typeof detail.turnId === 'string' ); } - -function countSemanticMessages(messages: readonly StoredMessage[]): Map { - const counts = new Map(); - for (const message of messages) { - const key = stableSemanticKey(semanticMessage(message)); - counts.set(key, (counts.get(key) ?? 0) + 1); - } - return counts; -} - -function stableSemanticKey(value: unknown): string { - return JSON.stringify(sortSemanticValue(value)); -} - -function sortSemanticValue(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map(sortSemanticValue); - } - if (!value || typeof value !== 'object') { - return value; - } - return Object.fromEntries( - Object.keys(value as Record) - .sort() - .map((key) => [key, sortSemanticValue((value as Record)[key])]), - ); -} - -function semanticMessage(message: StoredMessage): unknown { - switch (message.type) { - case 'user': - return { - type: message.type, - turnId: message.turnId, - text: message.text, - displayText: message.displayText, - origin: message.origin, - attachments: message.attachments ?? [], - directoryReferences: message.directoryReferences, - quotes: message.quotes ?? [], - }; - case 'assistant': - return { - type: message.type, - turnId: message.turnId, - text: message.text, - modelId: message.modelId, - thinking: message.thinking, - }; - case 'tool_call': - return { - type: message.type, - turnId: message.turnId, - toolUseId: message.id, - toolName: message.toolName, - activityKind: message.activityKind, - displayName: message.displayName, - intent: message.intent, - args: message.args, - }; - case 'tool_result': - return { - type: message.type, - turnId: message.turnId, - toolUseId: message.toolUseId, - isError: message.isError, - content: message.content, - durationMs: message.durationMs, - }; - case 'permission_decision': - return { - type: message.type, - turnId: message.turnId, - toolUseId: message.toolUseId, - toolName: message.toolName, - decision: message.decision, - rememberForTurn: message.rememberForTurn, - hint: message.hint, - }; - case 'token_usage': - return { - type: message.type, - turnId: message.turnId, - input: message.input, - output: message.output, - cacheHitInput: message.cacheHitInput, - cacheMissInput: message.cacheMissInput, - cacheMissInputSource: message.cacheMissInputSource, - cacheWriteInput: message.cacheWriteInput, - reasoning: message.reasoning, - total: message.total, - rawFinishReason: message.rawFinishReason, - runtimeSteps: message.runtimeSteps, - cacheRead: message.cacheRead, - cacheCreation: message.cacheCreation, - costUsd: message.costUsd, - systemPromptHash: message.systemPromptHash, - contextRemaining: message.contextRemaining, - prefixHash: message.prefixHash, - prefixChangeReason: message.prefixChangeReason, - requestShapeHash: message.requestShapeHash, - requestShapeChangeReason: message.requestShapeChangeReason, - promptSegments: message.promptSegments, - contextBudget: message.contextBudget, - providerRequestTraceId: message.providerRequestTraceId, - lastRequestAnchor: message.lastRequestAnchor, - }; - case 'turn_state': - return { - type: message.type, - turnId: message.turnId, - status: message.status, - parentTurnId: message.parentTurnId, - retriedFromTurnId: message.retriedFromTurnId, - regeneratedFromTurnId: message.regeneratedFromTurnId, - branchOfTurnId: message.branchOfTurnId, - parentSessionId: message.parentSessionId, - abortedAt: message.abortedAt, - abortSource: message.abortSource, - errorClass: message.errorClass, - partialOutputRetained: message.partialOutputRetained, - }; - case 'system_note': - return { - type: message.type, - turnId: message.turnId, - kind: message.kind, - data: message.data, - }; - } -} diff --git a/packages/runtime/src/session-manager.ts b/packages/runtime/src/session-manager.ts index d0a49d111e..1178834dcf 100644 --- a/packages/runtime/src/session-manager.ts +++ b/packages/runtime/src/session-manager.ts @@ -613,8 +613,6 @@ export interface SessionStore { ): Promise; list(filter?: SessionListFilter): Promise; readHeader(sessionId: string): Promise; - /** The legacy transcript, read only to convert it onto the ledger. */ - readMessages(sessionId: string): Promise; /** One forward page of the legacy rows the transcript converter lifts. */ readMessagesAfter( sessionId: string, diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index 1559699703..2e1c06f300 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -136,7 +136,10 @@ export type { } from './session-store.js'; export type ExecutionSessionWriter = SessionAuthorityStore; -export type { RuntimeTranscriptSource, RuntimeTranscriptTurn } from './runtime-transcript-query.js'; +export type { + RuntimeTranscriptInvocation, + RuntimeTranscriptLandmark, +} from './runtime-transcript-query.js'; export type ExecutionAgentRunWriter = DurableAgentRunStore; export type ExecutionRuntimeEventWriter = DurableRuntimeEventStore & RuntimeTranscriptQueries & @@ -574,14 +577,10 @@ async function createExecutionStoresForWrite runtimeEventStore.readSessionRuntimeEvents(sessionId)), readSessionRuntimeEventEntries: (sessionId) => run(() => runtimeEventStore.readSessionRuntimeEventEntries(sessionId)), - readTranscriptSourceHighWater: (sessionId) => - run(() => runtimeEventStore.readTranscriptSourceHighWater(sessionId)), - readTranscriptSource: (sessionId, request) => - run(() => runtimeEventStore.readTranscriptSource(sessionId, request)), - readTranscriptTurns: (sessionId, throughOrdinal, position, limit) => - run(() => - runtimeEventStore.readTranscriptTurns(sessionId, throughOrdinal, position, limit), - ), + readTranscriptHighWater: (sessionId) => + run(() => runtimeEventStore.readTranscriptHighWater(sessionId)), + readTranscriptInvocations: (sessionId, request) => + run(() => runtimeEventStore.readTranscriptInvocations(sessionId, request)), readTranscriptLandmarks: (sessionId, throughOrdinal, limit) => run(() => runtimeEventStore.readTranscriptLandmarks(sessionId, throughOrdinal, limit)), claimContinuation: (input) => run(() => runtimeEventStore.claimContinuation(input)), diff --git a/packages/storage/src/runtime-transcript-query.ts b/packages/storage/src/runtime-transcript-query.ts index ee05fd4809..79fcd5c38b 100644 --- a/packages/storage/src/runtime-transcript-query.ts +++ b/packages/storage/src/runtime-transcript-query.ts @@ -18,13 +18,8 @@ */ import type { DatabaseSync } from 'node:sqlite'; -import { - decodeRuntimeEvent, - isTerminalRuntimeEvent, - type RuntimeEvent, -} from '@maka/core/runtime-event'; +import { decodeRuntimeEvent, type RuntimeEvent } from '@maka/core/runtime-event'; import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; -import type { AssistantStepContentKind } from '@maka/core/session'; /** SQL counterpart of isTerminalRuntimeEvent; shared with the ledger store. */ export const TERMINAL_RUNTIME_EVENT_SQL = `( @@ -32,80 +27,69 @@ export const TERMINAL_RUNTIME_EVENT_SQL = `( OR json_extract(payload_json, '$.status') IN ('completed', 'failed', 'aborted', 'cancelled') )`; -export const TRANSCRIPT_MESSAGE_KEY_SQL = `CASE WHEN event_kind = 'function_call' - THEN json_extract(payload_json, '$.refs.stepId') - ELSE COALESCE(json_extract(payload_json, '$.refs.providerEventId'), json_extract(payload_json, '$.refs.storedMessageId'), event_id) END`; -export const TRANSCRIPT_STORED_ID_SQL = `COALESCE(json_extract(payload_json, '$.refs.storedMessageId'), json_extract(payload_json, '$.refs.providerEventId'), json_extract(payload_json, '$.content.id'), event_id)`; -/** Small indexed facts needed by a terminal row and the Turn index. */ -export const TRANSCRIPT_OUTPUT_SHAPE_SQL = `CASE - WHEN event_kind = 'text' AND json_extract(payload_json, '$.role') = 'model' - THEN CASE WHEN TRIM(json_extract(payload_json, '$.content.text'), char(9,10,11,12,13,32,160,5760,8192,8193,8194,8195,8196,8197,8198,8199,8200,8201,8202,8232,8233,8239,8287,12288,65279)) <> '' THEN 3 ELSE 1 END - WHEN event_kind = 'function_response' THEN CASE WHEN json_extract(payload_json, '$.content.isError') = 1 THEN 12 ELSE 4 END - ELSE 0 END`; +/** + * One invocation's events, in ledger order, carrying the Session ordinal each + * one sits at. + * + * The transcript rows of a Turn come from projecting these together: what a + * RuntimeEvent becomes is decided by the read model alone, so nothing here + * classifies an event or decides whether it produces a row. + */ +export interface RuntimeTranscriptInvocation { + readonly invocation: RuntimeInvocationRecord; + readonly firstOrdinal: number; + readonly lastOrdinal: number; + readonly events: readonly { readonly ordinal: number; readonly event: RuntimeEvent }[]; +} -export interface RuntimeTranscriptSource { - readonly ordinal: number; - readonly event: RuntimeEvent; - /** Only this message's thinking, in ledger order around its text event. */ - readonly events: readonly RuntimeEvent[]; +/** An invocation start, with the prompt event a landmark is labelled by. */ +export interface RuntimeTranscriptLandmark { readonly invocation: RuntimeInvocationRecord; - readonly contentOrder?: readonly AssistantStepContentKind[]; - readonly permissionRequest?: RuntimeEvent; - readonly toolName?: string; - readonly hasRetainedOutput: boolean; + readonly firstOrdinal: number; + readonly prompt?: { readonly ordinal: number; readonly event: RuntimeEvent }; } -export interface RuntimeTranscriptPosition { +export interface RuntimeTranscriptInvocationRequest { readonly direction: 'older' | 'newer'; readonly throughOrdinal: number; + /** Ordinal the walk starts from, inclusive, in `direction`. */ readonly position: number; - /** Exact lookup for the bounded live-to-durable handoff. */ - readonly messageId?: string; -} - -export interface RuntimeTranscriptTurn { - readonly firstOrdinal: number; - readonly terminalOrdinal: number; - readonly invocation: RuntimeInvocationRecord; - readonly user?: { ordinal: number; event: RuntimeEvent }; - readonly hasAssistantMessage: boolean; - readonly hasAssistantOutput: boolean; - readonly hasToolResult: boolean; - readonly hasFailedToolResult: boolean; - readonly hasAbortNote: boolean; + readonly limit: number; + /** Refused rather than truncated: half a Turn projects to a wrong transcript. */ + readonly maxEvents: number; + readonly maxBytes: number; } export interface RuntimeTranscriptQueries { - readTranscriptSourceHighWater(sessionId: string): Promise; - readTranscriptSource( - sessionId: string, - request: RuntimeTranscriptPosition, - ): Promise; - readTranscriptTurns( + readTranscriptHighWater(sessionId: string): Promise; + readTranscriptInvocations( sessionId: string, - throughOrdinal: number, - position: number, - limit: number, - ): Promise; + request: RuntimeTranscriptInvocationRequest, + ): Promise; readTranscriptLandmarks( sessionId: string, throughOrdinal: number, limit: number, - ): Promise; + ): Promise; +} + +export class RuntimeTranscriptOversizedTurnError extends Error { + readonly name = 'RuntimeTranscriptOversizedTurnError'; } -const messageKey = (alias: string) => - TRANSCRIPT_MESSAGE_KEY_SQL.replaceAll('event_kind', `${alias}.event_kind`) - .replaceAll('payload_json', `${alias}.payload_json`) - .replaceAll('event_id', `${alias}.event_id`); -const terminal = (alias: string) => - TERMINAL_RUNTIME_EVENT_SQL.replaceAll('payload_json', `${alias}.payload_json`); -const opening = `COALESCE(json_extract(opened.payload_json, '$.content'), legacy.opening_json)`; const joins = ` FROM runtime_session_event_ordinals o JOIN runtime_events e ON e.event_id = o.event_id LEFT JOIN runtime_events opened ON opened.invocation_id = e.invocation_id AND opened.event_kind = 'invocation_opened' LEFT JOIN runtime_legacy_invocation_openings legacy ON legacy.invocation_id = e.invocation_id`; +const opening = `COALESCE(json_extract(opened.payload_json, '$.content'), legacy.opening_json)`; +/** + * A Turn the Session transcript shows: one this Session ran itself rather than + * on behalf of a subagent, and one that has already ended. + * + * This is a fact about the invocation, not about any row it produces — which + * rows it produces is the read model's question, and is not asked here. + */ const settledInline = ` ${opening} IS NOT NULL AND (json_extract(${opening}, '$.lineage.parentRunId') IS NULL @@ -114,35 +98,14 @@ const settledInline = ` AND EXISTS ( SELECT 1 FROM runtime_events ended JOIN runtime_session_event_ordinals ending ON ending.event_id = ended.event_id - WHERE ended.invocation_id = e.invocation_id AND ${terminal('ended')} + WHERE ended.invocation_id = e.invocation_id + AND ${TERMINAL_RUNTIME_EVENT_SQL.replaceAll('payload_json', 'ended.payload_json')} AND ending.ordinal <= :throughOrdinal )`; -// Thinking is context for its text row. An orphan must still reach the -// projector, which reports the missing text instead of silently dropping it. -const transcriptSource = `( - (json_extract(e.payload_json, '$.content') IS NOT NULL - AND e.event_kind <> 'invocation_opened' - AND (e.event_kind <> 'thinking' OR NOT EXISTS ( - SELECT 1 FROM runtime_events text - WHERE text.invocation_id = e.invocation_id AND text.event_kind = 'text' - AND json_extract(text.payload_json, '$.role') = 'model' - AND COALESCE(json_extract(text.payload_json, '$.refs.storedMessageId'), json_extract(text.payload_json, '$.refs.providerEventId'), text.event_id) = ${messageKey('e')} - ))) - OR json_extract(e.payload_json, '$.actions.permissionDecision') IS NOT NULL - OR json_extract(e.payload_json, '$.actions.permissionAnswerAccepted') IS NOT NULL - OR json_extract(e.payload_json, '$.actions.tokenUsage') IS NOT NULL - OR ${terminal('e')} -)`; -type SourceRow = { - ordinal: number; - event_id: string; - run_id: string; - invocation_id: string; - event_seq: number; -}; +type InvocationRow = { invocation_id: string; run_id: string; first: number; last: number }; -/** Queries select ledger positions before loading any message payload. No transcript is persisted. */ +/** Selects invocations by Session ordinal. Payloads are decoded, never classified. */ export class RuntimeTranscriptQuery { constructor( private readonly db: DatabaseSync, @@ -150,254 +113,125 @@ export class RuntimeTranscriptQuery { ) {} highWater(sessionId: string): number | null { - return ( - this.sourceRow(sessionId, { - direction: 'older', - throughOrdinal: Number.MAX_SAFE_INTEGER, - position: Number.MAX_SAFE_INTEGER, - })?.ordinal ?? null - ); + const row = this.db + .prepare(` + SELECT MAX(o.ordinal) AS ordinal ${joins} + WHERE o.session_id = :sessionId AND o.ordinal <= :throughOrdinal AND ${settledInline} + `) + .get({ sessionId, throughOrdinal: Number.MAX_SAFE_INTEGER }) as { ordinal?: unknown }; + return typeof row.ordinal === 'number' ? row.ordinal : null; } - private sourceRow(sessionId: string, request: RuntimeTranscriptPosition): SourceRow | undefined { + invocations( + sessionId: string, + request: RuntimeTranscriptInvocationRequest, + ): RuntimeTranscriptInvocation[] { assertOrdinal(request.throughOrdinal); assertOrdinal(request.position); - if (request.direction !== 'older' && request.direction !== 'newer') + if (request.direction !== 'older' && request.direction !== 'newer') { throw new Error('Invalid transcript direction'); - return this.db + } + // An invocation is selected by where its own events sit, so a walk that + // starts inside a Turn still finds that Turn and can serve its rows. + const rows = this.db .prepare(` - SELECT o.ordinal, e.event_id, e.run_id, e.invocation_id, e.event_seq ${joins} - WHERE o.session_id = :sessionId AND o.ordinal <= :throughOrdinal - ${ - request.messageId === undefined - ? '' - : `AND e.event_id IN ( - SELECT event_id FROM runtime_events WHERE session_id = :sessionId AND (${TRANSCRIPT_STORED_ID_SQL}) = :messageId - UNION SELECT event_id FROM runtime_events WHERE session_id = :sessionId AND event_id = :noticeEventId - )` - } - AND o.ordinal ${request.direction === 'older' ? '<=' : '>='} :position - AND ${settledInline} AND ${transcriptSource} - ORDER BY o.ordinal ${request.direction === 'older' ? 'DESC' : 'ASC'} LIMIT 1 + SELECT e.invocation_id, e.run_id, MIN(o.ordinal) AS first, MAX(o.ordinal) AS last ${joins} + WHERE o.session_id = :sessionId AND o.ordinal <= :throughOrdinal AND ${settledInline} + GROUP BY e.invocation_id + HAVING ${request.direction === 'older' ? 'first <= :position' : 'last >= :position'} + ORDER BY first ${request.direction === 'older' ? 'DESC' : 'ASC'} + LIMIT :limit `) - .get({ + .all({ sessionId, throughOrdinal: request.throughOrdinal, position: request.position, - ...(request.messageId === undefined - ? {} - : { - messageId: request.messageId, - noticeEventId: request.messageId.endsWith(':step-limit-notice') - ? request.messageId.slice(0, -':step-limit-notice'.length) - : null, - }), - }) as SourceRow | undefined; - } - - source(sessionId: string, request: RuntimeTranscriptPosition): RuntimeTranscriptSource | null { - const row = this.sourceRow(sessionId, request); - if (!row) return null; - const event = this.event(row.event_id); - const invocation = this.invocation(sessionId, row.run_id); - let primary = event; - if (event.content?.kind === 'thinking') { - const text = this.db - .prepare(` - SELECT 1 FROM runtime_events WHERE invocation_id = ? AND event_kind = 'text' - AND json_extract(payload_json, '$.role') = 'model' - AND COALESCE(json_extract(payload_json, '$.refs.storedMessageId'), json_extract(payload_json, '$.refs.providerEventId'), event_id) = ? LIMIT 1 - `) - .get( - row.invocation_id, - event.refs?.providerEventId ?? event.refs?.storedMessageId ?? event.id, - ); - if (text) { - // Its thinking is attached at the text position; any actions still own - // their rows at this event's position, exactly once. - const { content: _content, ...actionsOnly } = event; - primary = actionsOnly; - } - } - const events: Array<{ event: RuntimeEvent; sequence: number }> = [ - { event: primary, sequence: row.event_seq }, - ]; - let contentOrder: AssistantStepContentKind[] | undefined; - if (event.role === 'model' && event.content?.kind === 'text') { - const id = event.refs?.storedMessageId ?? event.refs?.providerEventId ?? event.id; - const thinking = this.db - .prepare(` - SELECT e.event_id, e.event_seq FROM runtime_events e - WHERE e.invocation_id = ? AND e.event_kind = 'thinking' AND ${messageKey('e')} = ? - ORDER BY e.event_seq - `) - .all(row.invocation_id, id) as Array<{ event_id: string; event_seq: number }>; - for (const item of thinking) { - const { actions: _actions, status: _status, ...context } = this.event(item.event_id); - events.push({ event: context, sequence: item.event_seq }); - } - const kinds = this.db - .prepare(` - SELECT CASE e.event_kind WHEN 'function_call' THEN 'tools' ELSE e.event_kind END AS kind, - MIN(e.event_seq) AS first_sequence FROM runtime_events e - WHERE e.invocation_id = :invocationId AND e.event_seq <= :sequence - AND e.event_kind IN ('text', 'thinking', 'function_call') AND json_extract(e.payload_json, '$.role') = 'model' - AND ${messageKey('e')} = :messageId - GROUP BY kind ORDER BY first_sequence - `) - .all({ invocationId: row.invocation_id, sequence: row.event_seq, messageId: id }) as Array<{ - kind: AssistantStepContentKind; - }>; - contentOrder = kinds.map((item) => item.kind); - } - const requestId = - event.actions?.permissionDecision?.requestId ?? - event.actions?.permissionAnswerAccepted?.requestId; - const permissionRow = requestId - ? (this.db - .prepare(` - SELECT event_id FROM runtime_events - WHERE invocation_id = ? AND event_seq <= ? AND json_extract(payload_json, '$.actions.permissionRequest.requestId') = ? - ORDER BY event_seq DESC LIMIT 1 - `) - .get(row.invocation_id, row.event_seq, requestId) as { event_id: string } | undefined) - : undefined; - const permissionRequest = permissionRow ? this.event(permissionRow.event_id) : undefined; - const toolUseId = - event.refs?.toolCallId ?? permissionRequest?.actions?.permissionRequest?.toolUseId; - const toolRow = - requestId && toolUseId - ? (this.db - .prepare(` - SELECT json_extract(payload_json, '$.content.name') AS name FROM runtime_events - WHERE invocation_id = ? AND event_seq <= ? AND json_extract(payload_json, '$.content.id') = ? - AND event_kind IN ('function_call', 'function_response') - ORDER BY event_seq DESC LIMIT 1 - `) - .get(row.invocation_id, row.event_seq, toolUseId) as { name: string } | undefined) - : undefined; - const hasRetainedOutput = - isTerminalRuntimeEvent(event) && this.hasShape(row.invocation_id, row.ordinal, 0, '3,4,12'); - return { - ordinal: row.ordinal, - event, - invocation, - events: events.sort((a, b) => a.sequence - b.sequence).map((item) => item.event), - ...(contentOrder ? { contentOrder } : {}), - ...(permissionRequest ? { permissionRequest } : {}), - ...(toolRow?.name ? { toolName: toolRow.name } : {}), - hasRetainedOutput, - }; - } - - turns( - sessionId: string, - throughOrdinal: number, - position: number, - limit: number, - ): RuntimeTranscriptTurn[] { - assertOrdinal(throughOrdinal); - assertOrdinal(position); - const rows = this.db - .prepare(` - SELECT e.run_id, e.invocation_id, o.ordinal ${joins} - WHERE o.session_id = :sessionId AND o.ordinal <= :throughOrdinal - AND e.event_seq = 1 AND ${settledInline} - AND EXISTS (SELECT 1 FROM runtime_events tail JOIN runtime_session_event_ordinals t ON t.event_id = tail.event_id - WHERE tail.invocation_id = e.invocation_id AND t.ordinal >= :position AND t.ordinal <= :throughOrdinal AND ${transcriptSource.replaceAll('e.', 'tail.')}) - ORDER BY o.ordinal LIMIT :limit - `) - .all({ sessionId, throughOrdinal, position, limit }) as SourceRow[]; - return rows.map((row) => this.turn(sessionId, row, throughOrdinal, position)); + limit: request.limit, + }) as InvocationRow[]; + return rows.map((row) => ({ + invocation: this.invocation(sessionId, row.run_id), + firstOrdinal: row.first, + lastOrdinal: row.last, + events: this.events(row.invocation_id, request), + })); } - landmarks(sessionId: string, throughOrdinal: number, limit: number): RuntimeTranscriptTurn[] { + landmarks(sessionId: string, throughOrdinal: number, limit: number): RuntimeTranscriptLandmark[] { assertOrdinal(throughOrdinal); if (limit < 1) return []; + // Evenly spaced Turn starts, chosen before any payload is read. const rows = this.db .prepare(` WITH candidates AS ( - SELECT e.run_id, e.invocation_id, o.ordinal, + SELECT e.invocation_id, e.run_id, o.ordinal, ROW_NUMBER() OVER (ORDER BY o.ordinal) - 1 AS rank, COUNT(*) OVER () AS total ${joins} WHERE o.session_id = :sessionId AND o.ordinal <= :throughOrdinal AND e.event_seq = 1 AND ${settledInline} ), samples(n) AS ( SELECT 0 UNION ALL SELECT n + 1 FROM samples WHERE n + 1 < :limit ) - SELECT DISTINCT run_id, invocation_id, ordinal FROM candidates + SELECT DISTINCT invocation_id, run_id, ordinal FROM candidates JOIN samples ON rank = CASE WHEN :limit = 1 THEN total - 1 ELSE CAST(n * (total - 1) / (:limit - 1) AS INTEGER) END ORDER BY ordinal `) - .all({ sessionId, throughOrdinal, limit }) as SourceRow[]; - return rows.map((row) => this.turn(sessionId, row, throughOrdinal, 0)); + .all({ sessionId, throughOrdinal, limit }) as Array<{ + invocation_id: string; + run_id: string; + ordinal: number; + }>; + return rows.map((row) => { + // The prompt is the Turn's first user text event, which is what the read + // model projects a user message from. Only that one event is loaded: a + // landmark is a label, and projecting whole Turns to build a scrollbar + // would read most of the Session. + const prompt = this.db + .prepare(` + SELECT o.ordinal, e.event_id FROM runtime_events e + JOIN runtime_session_event_ordinals o ON o.event_id = e.event_id + WHERE e.invocation_id = ? AND o.ordinal <= ? + AND e.event_kind = 'text' AND json_extract(e.payload_json, '$.role') = 'user' + ORDER BY e.event_seq LIMIT 1 + `) + .get(row.invocation_id, throughOrdinal) as + | { ordinal: number; event_id: string } + | undefined; + return { + invocation: this.invocation(sessionId, row.run_id), + firstOrdinal: row.ordinal, + ...(prompt + ? { prompt: { ordinal: prompt.ordinal, event: this.event(prompt.event_id) } } + : {}), + }; + }); } - private turn( - sessionId: string, - row: SourceRow, - throughOrdinal: number, - position: number, - ): RuntimeTranscriptTurn { - const bounds = this.db + private events( + invocationId: string, + limits: { maxEvents: number; maxBytes: number }, + ): RuntimeTranscriptInvocation['events'] { + const rows = this.db .prepare(` - SELECT o.ordinal AS first + SELECT o.ordinal, e.event_id, e.session_id, e.invocation_id, e.run_id, e.turn_id, e.payload_json FROM runtime_events e JOIN runtime_session_event_ordinals o ON o.event_id = e.event_id - WHERE e.invocation_id = ? AND o.ordinal >= ? AND o.ordinal <= ? AND ${transcriptSource} - ORDER BY e.event_seq LIMIT 1 - `) - .get(row.invocation_id, position, throughOrdinal) as { first: number }; - const user = this.db - .prepare(` - SELECT o.ordinal, e.event_id FROM runtime_events e JOIN runtime_session_event_ordinals o ON o.event_id = e.event_id - WHERE e.invocation_id = ? AND o.ordinal >= ? AND o.ordinal <= ? - AND e.event_kind = 'text' AND json_extract(e.payload_json, '$.role') = 'user' - ORDER BY e.event_seq LIMIT 1 - `) - .get(row.invocation_id, position, throughOrdinal) as - | { ordinal: number; event_id: string } - | undefined; - const ended = this.db - .prepare(` - SELECT o.ordinal FROM runtime_events e JOIN runtime_session_event_ordinals o ON o.event_id = e.event_id - WHERE e.invocation_id = ? AND o.ordinal <= ? AND ${terminal('e')} - ORDER BY o.ordinal DESC LIMIT 1 + WHERE e.invocation_id = ? ORDER BY e.event_seq `) - .get(row.invocation_id, throughOrdinal) as { ordinal: number }; - return { - firstOrdinal: bounds.first, - terminalOrdinal: ended.ordinal, - invocation: this.invocation(sessionId, row.run_id), - ...(user ? { user: { ordinal: user.ordinal, event: this.event(user.event_id) } } : {}), - ...this.flags(row.invocation_id, throughOrdinal, position), - }; - } - - private flags(invocationId: string, throughOrdinal: number, position: number) { - return { - hasAssistantMessage: this.hasShape(invocationId, throughOrdinal, position, '1,3'), - hasAssistantOutput: this.hasShape(invocationId, throughOrdinal, position, '3'), - hasToolResult: this.hasShape(invocationId, throughOrdinal, position, '4,12'), - hasFailedToolResult: this.hasShape(invocationId, throughOrdinal, position, '12'), - hasAbortNote: false, - }; - } - - private hasShape( - invocationId: string, - throughOrdinal: number, - position: number, - shapes: string, - ): boolean { - return ( - this.db - .prepare(` - SELECT 1 FROM runtime_events JOIN runtime_session_event_ordinals o USING (event_id) - WHERE invocation_id = ? AND (${TRANSCRIPT_OUTPUT_SHAPE_SQL}) IN (${shapes}) - AND o.ordinal >= ? AND o.ordinal <= ? LIMIT 1 - `) - .get(invocationId, position, throughOrdinal) !== undefined - ); + .all(invocationId) as Array; + if (rows.length > limits.maxEvents) { + throw new RuntimeTranscriptOversizedTurnError( + `Turn ${invocationId} holds more RuntimeEvents than a transcript page may read`, + ); + } + let bytes = 0; + return rows.map((row) => { + bytes += row.payload_json.length; + if (bytes > limits.maxBytes) { + throw new RuntimeTranscriptOversizedTurnError( + `Turn ${invocationId} holds more RuntimeEvent bytes than a transcript page may read`, + ); + } + return { ordinal: row.ordinal, event: decodeStoredEvent(row) }; + }); } private event(id: string): RuntimeEvent { @@ -405,29 +239,33 @@ export class RuntimeTranscriptQuery { .prepare( 'SELECT event_id, session_id, invocation_id, run_id, turn_id, payload_json FROM runtime_events WHERE event_id = ?', ) - .get(id) as - | { - event_id: string; - session_id: string; - invocation_id: string; - run_id: string; - turn_id: string; - payload_json: string; - } - | undefined; + .get(id) as StoredEventRow | undefined; if (!row) throw new Error(`Transcript RuntimeEvent ${id} is missing`); - const event = decodeRuntimeEvent(JSON.parse(row.payload_json)); - if ( - event.id !== row.event_id || - event.sessionId !== row.session_id || - event.invocationId !== row.invocation_id || - event.runId !== row.run_id || - event.turnId !== row.turn_id - ) { - throw new Error(`Transcript RuntimeEvent ${id} has inconsistent storage identity`); - } - return event; + return decodeStoredEvent(row); + } +} + +type StoredEventRow = { + event_id: string; + session_id: string; + invocation_id: string; + run_id: string; + turn_id: string; + payload_json: string; +}; + +function decodeStoredEvent(row: StoredEventRow): RuntimeEvent { + const event = decodeRuntimeEvent(JSON.parse(row.payload_json)); + if ( + event.id !== row.event_id || + event.sessionId !== row.session_id || + event.invocationId !== row.invocation_id || + event.runId !== row.run_id || + event.turnId !== row.turn_id + ) { + throw new Error(`Transcript RuntimeEvent ${row.event_id} has inconsistent storage identity`); } + return event; } function assertOrdinal(value: number): void { diff --git a/packages/storage/src/session-message-projection.ts b/packages/storage/src/session-message-projection.ts index 94fc0fb3a2..8188f50b8b 100644 --- a/packages/storage/src/session-message-projection.ts +++ b/packages/storage/src/session-message-projection.ts @@ -89,12 +89,7 @@ function truncatePreview(text: string, maxLength = 96): string { return `${chars.slice(0, maxLength - 1).join('')}…`; } -/** - * One Turn's summary, folded message by message in transcript order. - * - * Both transcript authorities fold the same way: the sqlite catalog over its - * own rows, and the ledger reader over the messages a run projects. - */ +/** One Turn's summary, folded message by message in transcript order. */ export function foldTurnContribution( current: SessionTurnContribution | undefined, turnId: string, @@ -106,25 +101,11 @@ export function foldTurnContribution( firstSequence: sequence, latestState: null, userPromptPreview: null, - hasAssistantMessage: false, - hasAssistantOutput: false, - hasToolResult: false, - hasFailedToolResult: false, - hasAbortNote: false, }; const userPrompt = message.type === 'user' ? (message.displayText ?? message.text).trim() : ''; return { ...contribution, latestState: message.type === 'turn_state' ? { sequence, message } : contribution.latestState, userPromptPreview: contribution.userPromptPreview ?? (userPrompt || null), - hasAssistantMessage: contribution.hasAssistantMessage || message.type === 'assistant', - hasAssistantOutput: - contribution.hasAssistantOutput || - (message.type === 'assistant' && message.text.trim().length > 0), - hasToolResult: contribution.hasToolResult || message.type === 'tool_result', - hasFailedToolResult: - contribution.hasFailedToolResult || (message.type === 'tool_result' && message.isError), - hasAbortNote: - contribution.hasAbortNote || (message.type === 'system_note' && message.kind === 'abort'), }; } diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index d065667d7d..648dc6e81c 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -297,11 +297,6 @@ export interface SessionTurnContribution { readonly message: TurnStateMessage; } | null; readonly userPromptPreview: string | null; - readonly hasAssistantMessage: boolean; - readonly hasAssistantOutput: boolean; - readonly hasToolResult: boolean; - readonly hasFailedToolResult: boolean; - readonly hasAbortNote: boolean; } export interface SessionTurnContributionPage { diff --git a/packages/storage/src/sqlite-runtime-schema.ts b/packages/storage/src/sqlite-runtime-schema.ts index 3419761fa9..d8399f17ab 100644 --- a/packages/storage/src/sqlite-runtime-schema.ts +++ b/packages/storage/src/sqlite-runtime-schema.ts @@ -25,12 +25,7 @@ import { } from './legacy-run-header.js'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { encodeCanonicalRuntimeEvent } from '@maka/core/canonical-runtime-event'; -import { - TERMINAL_RUNTIME_EVENT_SQL, - TRANSCRIPT_MESSAGE_KEY_SQL, - TRANSCRIPT_OUTPUT_SHAPE_SQL, - TRANSCRIPT_STORED_ID_SQL, -} from './runtime-transcript-query.js'; +import { TERMINAL_RUNTIME_EVENT_SQL } from './runtime-transcript-query.js'; import { buildInvocationOpenedEvent, buildSyntheticTerminalRuntimeEvent, @@ -588,12 +583,7 @@ const MIGRATIONS: ReadonlyMap = new Map([ [ 17, ` - CREATE INDEX IF NOT EXISTS runtime_events_transcript_message ON runtime_events(invocation_id, (${TRANSCRIPT_MESSAGE_KEY_SQL}), event_seq); - CREATE INDEX IF NOT EXISTS runtime_events_transcript_output ON runtime_events(invocation_id, (${TRANSCRIPT_OUTPUT_SHAPE_SQL}), event_seq); - CREATE INDEX IF NOT EXISTS runtime_events_transcript_request ON runtime_events(invocation_id, json_extract(payload_json, '$.actions.permissionRequest.requestId'), event_seq); - CREATE INDEX IF NOT EXISTS runtime_events_transcript_tool ON runtime_events(invocation_id, json_extract(payload_json, '$.content.id'), event_seq); CREATE INDEX IF NOT EXISTS runtime_events_terminal ON runtime_events(invocation_id, event_seq) WHERE ${TERMINAL_RUNTIME_EVENT_SQL}; - CREATE INDEX IF NOT EXISTS runtime_events_transcript_stored_id ON runtime_events(session_id, (${TRANSCRIPT_STORED_ID_SQL})); `, ], ]); diff --git a/packages/storage/src/sqlite-runtime-store.ts b/packages/storage/src/sqlite-runtime-store.ts index c54fdcf934..563daececc 100644 --- a/packages/storage/src/sqlite-runtime-store.ts +++ b/packages/storage/src/sqlite-runtime-store.ts @@ -132,9 +132,9 @@ import { assertNoReservedWorkspaceAuthorityAppend } from './runtime-event-author import { RuntimeTranscriptQuery, TERMINAL_RUNTIME_EVENT_SQL, - type RuntimeTranscriptPosition, - type RuntimeTranscriptSource, - type RuntimeTranscriptTurn, + type RuntimeTranscriptInvocation, + type RuntimeTranscriptInvocationRequest, + type RuntimeTranscriptLandmark, } from './runtime-transcript-query.js'; export { SQLITE_RUNTIME_SCHEMA_VERSION } from './sqlite-runtime-schema.js'; @@ -550,37 +550,25 @@ export class SqliteRuntimeStore }); } - async readTranscriptSourceHighWater(sessionId: string): Promise { + async readTranscriptHighWater(sessionId: string): Promise { assertRuntimeStorageSafeId(sessionId, 'Invalid session id'); return this.readTransaction(() => this.transcriptQuery().highWater(sessionId)); } - async readTranscriptSource( + async readTranscriptInvocations( sessionId: string, - request: RuntimeTranscriptPosition, - ): Promise { + request: RuntimeTranscriptInvocationRequest, + ): Promise { assertRuntimeStorageSafeId(sessionId, 'Invalid session id'); - return this.readTransaction(() => this.transcriptQuery().source(sessionId, request)); - } - - async readTranscriptTurns( - sessionId: string, - throughOrdinal: number, - position: number, - limit: number, - ): Promise { - assertRuntimeStorageSafeId(sessionId, 'Invalid session id'); - assertInvocationSearchLimit(limit); - return this.readTransaction(() => - this.transcriptQuery().turns(sessionId, throughOrdinal, position, limit), - ); + assertInvocationSearchLimit(request.limit); + return this.readTransaction(() => this.transcriptQuery().invocations(sessionId, request)); } async readTranscriptLandmarks( sessionId: string, throughOrdinal: number, limit: number, - ): Promise { + ): Promise { assertRuntimeStorageSafeId(sessionId, 'Invalid session id'); assertInvocationSearchLimit(limit); return this.readTransaction(() => From a32e71a2c31dd5d6b139d836708465c96e78e344 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 08:22:55 +0800 Subject: [PATCH 14/32] fix(runtime): keep the compaction notes on the ledger after the rebase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit main's #4850 fix writes both compaction decision notes the moment a stage reports the fold, so a stopped turn still records why. This branch had moved turn-scoped notes off `appendMessage` onto the invocation's own ledger, and the two changes met in the same block: the note helper wrote StoredMessages directly, and the settlement fallback still used the pre-#4850 shape. Route the decision-time notes through `recordSystemNote` and let it report whether the append landed, so the per-send flags keep #4850's contract — a failed early write leaves the settlement fallback armed. Generated-by: Claude Code --- packages/runtime/src/__tests__/ai-sdk-backend.test.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts index 6baf558e54..c98dbfed28 100644 --- a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts +++ b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts @@ -5507,6 +5507,9 @@ describe('AiSdkBackend model history', () => { appendMessage: async (message: StoredMessage) => { appended.push(message as unknown as { type: string; kind?: string; data?: unknown }); }, + recordSystemNote: async (kind, _turnId, data) => { + appended.push({ type: 'system_note', kind, data }); + }, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -5584,6 +5587,9 @@ describe('AiSdkBackend model history', () => { appendMessage: async (message: StoredMessage) => { appended.push(message as unknown as { type: string; kind?: string }); }, + recordSystemNote: async (kind) => { + appended.push({ type: 'system_note', kind }); + }, connection: connection(), apiKey: 'sk-test', modelId: 'mock-model-id', @@ -5646,7 +5652,10 @@ describe('AiSdkBackend model history', () => { sessionId: 'session-1', header: header(), appendMessage: async (message: StoredMessage) => { - const candidate = message as unknown as { type: string; kind?: string }; + persisted.push(message as unknown as { type: string; kind?: string }); + }, + recordSystemNote: async (kind) => { + const candidate = { type: 'system_note', kind }; if (isFailOpenNote(candidate)) { noteWriteAttempts += 1; if (failNextNoteWrite) { From 69e110eb0f6655b79c5c18c40df67cfeff015b11 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 08:26:00 +0800 Subject: [PATCH 15/32] fix(runtime): resume a released conversion by the rows it already converted A conversion a released build left part-written could hold events under ids this build cannot rederive. The converter answered that by sealing the run with a synthetic `missing_terminal_event` terminal: the turn was never finished, its legacy rows stayed unread, and the seal was permanent. That was wrong on its own premise. Every backfilled event names the legacy row it came from in `refs.storedMessageId`, so the prefix is identifiable without derived ids, and the run holds no terminal, so it still accepts appends. Derive the turn whole as a fresh conversion would, then skip only the events whose row already has that many on the run. A row half-converted by a crash between two of its events keeps the rest, a re-run appends nothing, and the ids stay the ones a fresh conversion would mint. `isDerivedTranscriptEventId` and `abandonedTranscriptTerminalEvent` go with the branch they existed for. Also stop retaining the invocation inventory across the scan: the ownership answer is two sets of ids, and the paged scan outlives the records. Generated-by: Claude Code --- .../__tests__/runtime-ledger-repair.test.ts | 13 ++- packages/runtime/src/runtime-ledger-repair.ts | 108 +++++++----------- 2 files changed, 51 insertions(+), 70 deletions(-) diff --git a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts index d16260e85e..6e62459e0d 100644 --- a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts +++ b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts @@ -879,7 +879,7 @@ test('resumes a conversion a released build opened under a random event id', asy } }); -test('seals a released conversion that had already converted messages', async () => { +test('finishes a released conversion that had already converted messages', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-released-partial-')); const sessions = createSessionStore(root); const runtimeEvents = createSqliteRuntimeStore(join(root, 'runtime.sqlite')); @@ -915,11 +915,14 @@ test('seals a released conversion that had already converted messages', async () const [run] = await runtimeEvents.listSessionInvocations(session.id); assert.ok(run); - // The prefix cannot be finished and must not be doubled: one user text, not two. - assert.equal(runtimeInvocationOutcome(run), 'failed'); - assert.equal(runtimeInvocationFailureClass(run), 'missing_terminal_event'); + // The prefix is resumed by the legacy row each event names, so the turn + // converts whole and no row it already carried is converted twice. + assert.equal(runtimeInvocationOutcome(run), 'completed'); const events = await runtimeEvents.readRuntimeEvents(session.id, run.runId); - assert.equal(events.filter((event) => event.content?.kind === 'text').length, 1); + assert.deepEqual( + events.flatMap((event) => (event.content ? [event.content.kind] : [])), + ['invocation_opened', 'text', 'text'], + ); } finally { runtimeEvents.close(); await sessions.close?.(); diff --git a/packages/runtime/src/runtime-ledger-repair.ts b/packages/runtime/src/runtime-ledger-repair.ts index fe2ea2eeb6..7f382d9131 100644 --- a/packages/runtime/src/runtime-ledger-repair.ts +++ b/packages/runtime/src/runtime-ledger-repair.ts @@ -79,17 +79,7 @@ export class RuntimeLedgerRepair { // the same turn would make the Session read as two. The one exception is // this converter's own run: an interrupted import re-derives it, and the // deterministic ids let the store dedupe what already landed. - const inlineInvocations = await this.listInlineInvocations(sessionId); - const ownedTurnIds = new Set( - inlineInvocations - .filter( - (invocation) => - invocation.terminalEvent || - invocation.runId !== transcriptRunId(sessionId, invocation.turnId), - ) - .map((invocation) => invocation.turnId), - ); - const startedRunIds = new Set(inlineInvocations.map((invocation) => invocation.runId)); + const { ownedTurnIds, startedRunIds } = await this.readLedgerOwnership(sessionId); for await (const scanned of this.readTurnsInPages(sessionId)) { const turnMessages = scanned.messages; @@ -111,31 +101,19 @@ export class RuntimeLedgerRepair { const run = { sessionId, runId, turnId: turn.turnId, invocationId: runId }; // A build before the ids were derived converted under random ones, so // an interrupted run of its can hold events this build cannot rederive. + // What it can read is which legacy row each of them came from, and that + // is the identity the conversion resumes on. const started = startedRunIds.has(runId) ? await this.deps.runtimeEventStore.readRuntimeEvents(sessionId, runId) : []; - const undeducible = started.filter((event) => !isDerivedTranscriptEventId(runId, event.id)); - // Its opening is the one such event that can be adopted: the run needs - // exactly one, `runtime_events_one_opening_per_invocation` refuses a - // second, and which id it landed under changes nothing a reader sees. - const adoptedOpening = - undeducible.length === 1 && undeducible[0]?.content?.kind === 'invocation_opened'; - if (undeducible.length > 0 && !adoptedOpening) { - // Its converted messages cannot be adopted the same way: rederiving - // them would stand a second, deterministic copy of each beside the - // one already there, and a Session that disagrees with itself is the - // failure this ledger exists to remove. The conversion can neither be - // finished nor withdrawn, so it is sealed as the unfinished thing it - // is — the legacy rows stay, and no one reads this turn as converted. - await this.deps.runtimeEventStore.appendRuntimeEvent( - sessionId, - runId, - abandonedTranscriptTerminalEvent({ run, openedAt }), - ); - continue; + const converted = new Map(); + for (const event of started) { + const rowId = event.refs?.storedMessageId; + if (rowId) converted.set(rowId, (converted.get(rowId) ?? 0) + 1); } - const events = [ - ...(adoptedOpening ? [] : [transcriptOpeningEvent({ header, run, openedAt })]), + const hasOpening = started.some((event) => event.content?.kind === 'invocation_opened'); + const derived = [ + ...(hasOpening ? [] : [transcriptOpeningEvent({ header, run, openedAt })]), ...backfillRuntimeEventsFromStoredMessages({ run, outcome: transcriptOutcome(turn, turnMessages, openedAt), @@ -151,7 +129,18 @@ export class RuntimeLedgerRepair { now: () => openedAt, }).events, ]; - for (const event of events) { + // The whole turn is derived either way, so the ids stay the ones a + // fresh conversion would mint; only the events whose row already has + // that many on the run are dropped. A row half-converted by a crash + // between two of its events keeps the rest. + const seen = new Map(); + for (const event of derived) { + const rowId = event.refs?.storedMessageId; + if (rowId !== undefined) { + const index = seen.get(rowId) ?? 0; + seen.set(rowId, index + 1); + if (index < (converted.get(rowId) ?? 0)) continue; + } await this.deps.runtimeEventStore.appendRuntimeEvent(sessionId, runId, event); } } @@ -203,10 +192,27 @@ export class RuntimeLedgerRepair { } } - private async listInlineInvocations(sessionId: string): Promise { - return (await this.deps.runtimeEventStore.listSessionInvocations(sessionId)).filter( - (invocation) => isSessionInlineInvocation(invocation.opening), - ); + /** + * Which turns the ledger already owns and which runs it has started, as ids + * rather than records: the inventory is one row per invocation and the scan + * that follows outlives it, so nothing keeps the records themselves. + */ + private async readLedgerOwnership( + sessionId: string, + ): Promise<{ ownedTurnIds: Set; startedRunIds: Set }> { + const ownedTurnIds = new Set(); + const startedRunIds = new Set(); + for (const invocation of await this.deps.runtimeEventStore.listSessionInvocations(sessionId)) { + if (!isSessionInlineInvocation(invocation.opening)) continue; + startedRunIds.add(invocation.runId); + if ( + invocation.terminalEvent || + invocation.runId !== transcriptRunId(sessionId, invocation.turnId) + ) { + ownedTurnIds.add(invocation.turnId); + } + } + return { ownedTurnIds, startedRunIds }; } private async withRepairQueue(key: string, operation: () => Promise): Promise { @@ -244,34 +250,6 @@ function transcriptRunId(sessionId: string, turnId: string): string { * emits them. The run id is already derived from the Session and turn, so the * same transcript always produces the same ids and a re-run appends nothing. */ -/** Whether this build's converter is the one that could have written that id. */ -function isDerivedTranscriptEventId(runId: string, eventId: string): boolean { - return eventId === `${runId}-opened` || new RegExp(`^${runId}-e\\d+$`).test(eventId); -} - -/** - * The terminal fact of a conversion that a released build left part-written. - * Its id sits outside the derived sequence so it cannot collide with an event - * that prefix already holds. - */ -function abandonedTranscriptTerminalEvent(input: { - run: { sessionId: string; runId: string; turnId: string; invocationId: string }; - openedAt: number; -}): RuntimeEvent { - return backfillRuntimeEventsFromStoredMessages({ - run: input.run, - outcome: { - status: 'failed', - ts: input.openedAt, - failureClass: 'missing_terminal_event', - }, - messages: [], - modelHistory: 'conversation_text', - newId: () => `${input.run.runId}-abandoned`, - now: () => input.openedAt, - }).events[0] as RuntimeEvent; -} - function transcriptEventIds(runId: string): () => string { let seq = 0; return () => { From 09f0bf2780aa1c0335658f76bdf5669bf5293d47 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 08:41:48 +0800 Subject: [PATCH 16/32] test(runtime): guard the absence this cutover is named for Nothing named the property directly: restoring the second write broke other tests incidentally, which is a symptom, not a guard. Drive an ordinary send through SessionManager and assert `session_messages` is empty while the prompt is still readable from the ledger. Verified it discriminates: re-adding a `store.appendMessage` on the send path fails this test. Generated-by: Claude Code --- .../session-manager-terminal-ledger.test.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts index 64e533a3b7..fab95a5e12 100644 --- a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts +++ b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts @@ -246,6 +246,29 @@ describe('SessionManager terminal ledger invariants', () => { assert.strictEqual(turnState.errorClass, 'tool_failed'); }); + test('an ordinary send leaves session_messages empty', async () => { + const store = new TinySessionStore(); + const { manager, session } = await makeHarness( + [ + { type: 'text_delta', messageId: 'message-1', text: 'hi' }, + { type: 'complete', stopReason: 'end_turn' }, + ], + { store }, + ); + + await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' })); + + // The absence this cutover is named for: restore the second write anywhere + // on the send path and this row count stops being zero. + assert.deepStrictEqual(await store.readMessages(session.id), []); + // ...and the prompt is still readable, from the ledger alone. + const messages = await manager.getMessages(session.id); + assert.strictEqual( + messages.some((message) => message.type === 'user' && message.text === 'hello'), + true, + ); + }); + test('stopSession keeps renderer abortSource on terminal facts and run headers', async () => { const store = new TinySessionStore(); const runStore = new TinyAgentRunStore(); From 1491feef8f6537ad75e810964414f24a60763717 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 08:52:24 +0800 Subject: [PATCH 17/32] test(desktop): read a Turn's retained-output flag from its recorded state The Host no longer sends the five derived shape booleans, so a Turn's `partialOutputRetained` comes from the `turn_state` row it recorded. This fixture kept sending the booleans and expected the derived answer, which disagreed with the state it supplied in the same page. Generated-by: Claude Code --- .../src/main/__tests__/runtime-host-client.test.ts | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/apps/desktop/src/main/__tests__/runtime-host-client.test.ts b/apps/desktop/src/main/__tests__/runtime-host-client.test.ts index 0b82aef122..931598cb24 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-client.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-client.test.ts @@ -78,11 +78,6 @@ test('derives turn records from bounded contribution pages', async () => { firstSequence: 0, latestState: null, userPromptPreview: 'hello', - hasAssistantMessage: true, - hasAssistantOutput: true, - hasToolResult: false, - hasFailedToolResult: false, - hasAbortNote: false, }], nextPosition: 2, }; @@ -105,11 +100,6 @@ test('derives turn records from bounded contribution pages', async () => { }, }, userPromptPreview: null, - hasAssistantMessage: false, - hasAssistantOutput: false, - hasToolResult: true, - hasFailedToolResult: false, - hasAbortNote: false, }], nextPosition: null, }; @@ -124,7 +114,7 @@ test('derives turn records from bounded contribution pages', async () => { userPromptPreview: 'hello', status: 'completed', statusSource: 'recorded', - partialOutputRetained: true, + partialOutputRetained: false, }]); assert.deepEqual(positions, [0, 2]); await client.close(); From fe4e36600a7f74c91a74967589ca606ac4abcfdf Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 09:55:51 +0800 Subject: [PATCH 18/32] fix(storage): reassemble chunked records in the paged transcript scan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `readMessagesAfter` read `record_json` straight out of `session_messages`. A record over the chunk threshold stores only a marker there, so the scan handed the transcript conversion a marker instead of a message and the conversion refused the Session — every Desktop window seeded with a large assistant message lost its whole transcript. It now joins `session_message_payloads` and decodes through the same reassembly the by-id reads use, and sizes a record by its chunk total rather than by the marker's length so the byte budget still means bytes. Generated-by: Claude Code --- .../src/__tests__/session-store.test.ts | 11 +++++ .../src/sqlite-session-metadata-store.ts | 40 ++++++++++--------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/packages/storage/src/__tests__/session-store.test.ts b/packages/storage/src/__tests__/session-store.test.ts index ba594bcd34..dd1a519ba0 100644 --- a/packages/storage/src/__tests__/session-store.test.ts +++ b/packages/storage/src/__tests__/session-store.test.ts @@ -731,6 +731,17 @@ describe('SQLite SessionStore', () => { sessionId = session.id; await store.appendMessages(session.id, [message, smallMessage]); assert.deepEqual(await store.readMessages(session.id), [message, smallMessage]); + // The paged scan the transcript conversion reads through must reassemble + // a chunked record too: inline it is only a marker, which decodes as + // nothing a transcript can carry. + const page = await store.readMessagesAfter(session.id, { + maxMessages: 8, + maxStoredBytes: 4 * 1024 * 1024, + }); + assert.deepEqual( + page.records.map((record) => record.message), + [message, smallMessage], + ); } finally { await store.close?.(); } diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index 7d0b4dd659..af482d5e25 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -2571,33 +2571,35 @@ export class SqliteSessionMetadataStore { if (!this.readRecordSync(sessionId)) throw new SessionNotFoundError(sessionId); const rows = this.db .prepare(` - SELECT sequence, record_json - FROM session_messages - WHERE session_id = ? AND sequence > ? - ORDER BY sequence + SELECT message.sequence, message.record_json, payload.record_bytes, payload.sha256 + FROM session_messages AS message + LEFT JOIN session_message_payloads AS payload + ON payload.session_id = message.session_id AND payload.sequence = message.sequence + WHERE message.session_id = ? AND message.sequence > ? + ORDER BY message.sequence LIMIT ? `) - .all(sessionId, request.afterSequence ?? -1, request.maxMessages) as Array<{ - sequence?: unknown; - record_json?: unknown; - }>; + .all( + sessionId, + request.afterSequence ?? -1, + request.maxMessages, + ) as StoredSessionMessagePayloadRow[]; const records: SessionMessageScanRecord[] = []; let storedBytes = 0; for (const row of rows) { const sequence = requireStoredMessageSequence(row.sequence, sessionId); - const recordJson = String(row.record_json); + // A record too large for one row is stored in chunks, with only a + // marker inline; its size is the chunk total, not the marker's. + const recordBytes = + typeof row.record_bytes === 'number' ? row.record_bytes : String(row.record_json).length; // The first record of a page is always taken, so a single row larger // than the budget still makes progress instead of stalling the scan. - if (records.length > 0 && storedBytes + recordJson.length > request.maxStoredBytes) break; - storedBytes += recordJson.length; - try { - records.push({ - sequence, - message: decodeStoredMessage(JSON.parse(recordJson) as unknown), - }); - } catch (error) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence, { cause: error }); - } + if (records.length > 0 && storedBytes + recordBytes > request.maxStoredBytes) break; + storedBytes += recordBytes; + records.push({ + sequence, + message: decodeStoredMessageRecordRow(this.db, sessionId, row), + }); } const highWater = this.db .prepare('SELECT MAX(sequence) AS high_water FROM session_messages WHERE session_id = ?') From d45be867e6f7993e76dfe0595a24c8de524d8576 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 09:55:58 +0800 Subject: [PATCH 19/32] refactor(runtime-host): drop the durable-coverage claim from transcript bootstraps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `durableCoverage: 'complete'` promised that consecutive durable sequences differ by one, and the client enforced it. A durable sequence is now an event ordinal times its stride, so the gaps are the representation and no projection can make that promise — every Desktop range load of a real Session failed as a sequence gap. The field, its validation and the client-side identity check are gone. Overlay identities are still dense and still checked; the durable side has nothing left to claim. Runtime Host compatibility epoch 123 -> 124. Generated-by: Claude Code --- .../src/__tests__/connection-session.test.ts | 1 - .../session-subscription-client.test.ts | 57 +------------------ .../session-transcript-protocol.test.ts | 2 - .../src/client/session-subscription.ts | 3 - packages/runtime-host/src/protocol/index.ts | 5 +- .../src/protocol/session-transcript.ts | 7 --- .../src/server/session-transcript-pager.ts | 1 - 7 files changed, 5 insertions(+), 71 deletions(-) diff --git a/packages/runtime-host/src/__tests__/connection-session.test.ts b/packages/runtime-host/src/__tests__/connection-session.test.ts index 67a35a2b28..6460f8dde7 100644 --- a/packages/runtime-host/src/__tests__/connection-session.test.ts +++ b/packages/runtime-host/src/__tests__/connection-session.test.ts @@ -1639,7 +1639,6 @@ function transcriptBootstrapFor(sessionId: string) { const contents = Buffer.from('t'.repeat(16 * 1024)); return { throughSequence: 0, - durableCoverage: 'complete' as const, overlayMessageCount: 0, durable: { kind: 'page' as const, diff --git a/packages/runtime-host/src/__tests__/session-subscription-client.test.ts b/packages/runtime-host/src/__tests__/session-subscription-client.test.ts index f6a7e5b551..4fc7bed7e9 100644 --- a/packages/runtime-host/src/__tests__/session-subscription-client.test.ts +++ b/packages/runtime-host/src/__tests__/session-subscription-client.test.ts @@ -394,7 +394,6 @@ test('reassembles a large message from bounded backward pages', async () => { const openRequest = await acceptConnectionAndReadOpen(transport, hostEpoch, rootId); const opened = openResult(hostEpoch, 'subscription-fragmented', { throughSequence: 0, - durableCoverage: 'complete', overlayMessageCount: 0, durable: transcriptPage({ rawBytes: encoded.byteLength - splitAt, @@ -475,7 +474,6 @@ test('decodes one bounded page without walking the remaining transcript', async const subscription = new ClientSessionSubscription( openResult('host-1', 'subscription-bounded-page', { throughSequence: 4, - durableCoverage: 'complete', overlayMessageCount: 0, durable: { ...transcriptPage({ @@ -588,7 +586,6 @@ test('assembles the complete edge Turn while paging newer transcript', async () const subscription = new ClientSessionSubscription( openResult('host-1', 'subscription-newer-turn', { throughSequence: 1, - durableCoverage: 'complete', overlayMessageCount: 0, durable: initial, overlay: { ...transcriptPage({ source: 'overlay' }), throughSequence: 1 }, @@ -877,51 +874,7 @@ test('keeps the connection usable when close wins the overlay release race', asy ); }); -test('rejects a durable sequence gap', async () => { - const message = Buffer.from( - JSON.stringify({ - type: 'user', - id: 'user-1', - turnId: 'turn-1', - ts: 1, - text: 'hello', - }), - 'utf8', - ); - const fragment = { - kind: 'durable' as const, - sequence: 0, - byteOffset: 0, - totalBytes: message.byteLength, - payloadDigest: null, - data: message.toString('base64'), - }; - const gap = new ClientSessionSubscription( - openResult('host-1', 'subscription-gap', { - throughSequence: 1, - durableCoverage: 'complete', - overlayMessageCount: 0, - durable: { - ...transcriptPage({ - rawBytes: message.byteLength, - fragments: [{ ...fragment, sequence: 1 }], - }), - throughSequence: 1, - }, - overlay: { ...transcriptPage({ source: 'overlay' }), throughSequence: 1 }, - }), - async () => undefined, - async () => { - throw new Error('unexpected page request'); - }, - ); - await assert.rejects( - () => gap.loadTranscript(decodeStoredMessage), - hasSubscriptionReason('correlation_changed'), - ); -}); - -test('loads a projected durable transcript with intentionally sparse sequences', async () => { +test('loads a durable transcript whose sequences are sparse', async () => { const messages = [0, 2].map((sequence) => Buffer.from( JSON.stringify({ @@ -937,7 +890,6 @@ test('loads a projected durable transcript with intentionally sparse sequences', const subscription = new ClientSessionSubscription( openResult('host-1', 'subscription-projected', { throughSequence: 2, - durableCoverage: 'projected', overlayMessageCount: 0, durable: { ...transcriptPage({ @@ -983,7 +935,6 @@ test('rejects a durable message that does not match its payload digest', async ( const subscription = new ClientSessionSubscription( openResult('host-1', 'subscription-digest-mismatch', { throughSequence: 0, - durableCoverage: 'complete', overlayMessageCount: 0, durable: transcriptPage({ rawBytes: message.byteLength, @@ -1048,7 +999,6 @@ test('rejects a transcript cursor that does not advance', async () => { const subscription = new ClientSessionSubscription( openResult('host-1', 'subscription-stuck-cursor', { throughSequence: 0, - durableCoverage: 'complete', overlayMessageCount: 0, durable: repeated, overlay: transcriptPage({ source: 'overlay' }), @@ -1078,7 +1028,6 @@ test('rejects an overlay that terminates before its declared high-water', async const subscription = new ClientSessionSubscription( openResult('host-1', 'subscription-truncated-overlay', { throughSequence: null, - durableCoverage: 'complete', overlayMessageCount: 2, durable: { ...transcriptPage(), throughSequence: null }, overlay: { @@ -1170,7 +1119,6 @@ test('acknowledges a complete overlay before waiting for durable continuation pa const subscription = new ClientSessionSubscription( openResult('host-1', 'subscription-overlay-release-before-durable', { throughSequence: 0, - durableCoverage: 'complete', overlayMessageCount: 1, durable: transcriptPage({ rawBytes: durableMessage.byteLength - split, @@ -1244,7 +1192,6 @@ test('close stops transcript pagination after the in-flight page', async () => { const subscription = new ClientSessionSubscription( openResult('host-1', 'subscription-closing', { throughSequence: 0, - durableCoverage: 'complete', overlayMessageCount: 0, durable: transcriptPage({ rawBytes: Math.floor(message.byteLength / 2), @@ -1583,7 +1530,6 @@ function openResult( function transcriptBootstrap(message: Buffer): SessionTranscriptBootstrap { return { throughSequence: 0, - durableCoverage: 'complete', overlayMessageCount: 0, durable: transcriptPage({ rawBytes: message.byteLength, @@ -1605,7 +1551,6 @@ function transcriptBootstrap(message: Buffer): SessionTranscriptBootstrap { function overlayBootstrap(message: Buffer): SessionTranscriptBootstrap { return { throughSequence: null, - durableCoverage: 'complete', overlayMessageCount: 1, durable: { ...transcriptPage(), throughSequence: null }, overlay: { diff --git a/packages/runtime-host/src/__tests__/session-transcript-protocol.test.ts b/packages/runtime-host/src/__tests__/session-transcript-protocol.test.ts index da33e1ce1f..a2d25e713c 100644 --- a/packages/runtime-host/src/__tests__/session-transcript-protocol.test.ts +++ b/packages/runtime-host/src/__tests__/session-transcript-protocol.test.ts @@ -72,7 +72,6 @@ test('Session transcript protocol accepts bounded correlated pages and bootstrap const bootstrap = { throughSequence: 3, - durableCoverage: 'complete' as const, overlayMessageCount: 0, durable: { ...page, direction: 'older' as const }, overlay: { @@ -207,7 +206,6 @@ test('Session transcript protocol rejects malformed and uncorrelated values', () () => decodeSessionTranscriptBootstrap({ throughSequence: 3, - durableCoverage: 'complete', overlayMessageCount: 0, durable: page, overlay: { ...page, source: 'overlay', throughSequence: 2 }, diff --git a/packages/runtime-host/src/client/session-subscription.ts b/packages/runtime-host/src/client/session-subscription.ts index a458dc9743..7e47cd8f08 100644 --- a/packages/runtime-host/src/client/session-subscription.ts +++ b/packages/runtime-host/src/client/session-subscription.ts @@ -345,9 +345,6 @@ export class ClientSessionSubscription } const overlay = await this.#consumeTranscriptOverlay(bootstrap); const durable = await this.#loadTranscriptSource(bootstrap.durable); - if (bootstrap.durableCoverage === 'complete') { - assertCompleteIdentities(durable, bootstrap.throughSequence); - } const messages = durable.map((entry) => entry.value); const indexById = new Map(); for (const [index, message] of messages.entries()) { diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts index 10923b13ab..89f54e208a 100644 --- a/packages/runtime-host/src/protocol/index.ts +++ b/packages/runtime-host/src/protocol/index.ts @@ -101,7 +101,10 @@ export const RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION = 1 as const; export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const; // Increment when the same protocol version no longer guarantees safe Client-Host // interoperability. Mismatches are rejected before domain commands are admitted. -export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 123 as const; +export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 124 as const; +// 124: Session transcript bootstraps drop `durableCoverage`. A durable sequence +// is an event ordinal times its stride, so no projection has contiguous +// sequences any more and the claim the field made is unavailable to make. // 123: Session Turn contributions carry only the Turn's recorded state. Older // peers require the derived shape booleans this projection no longer sends. // 122: Durable transcript cursors seek Session event ordinals instead of run indexes. diff --git a/packages/runtime-host/src/protocol/session-transcript.ts b/packages/runtime-host/src/protocol/session-transcript.ts index e50b0d03e9..a1e6285c43 100644 --- a/packages/runtime-host/src/protocol/session-transcript.ts +++ b/packages/runtime-host/src/protocol/session-transcript.ts @@ -75,8 +75,6 @@ export interface SessionTranscriptPage { export interface SessionTranscriptBootstrap { readonly throughSequence: number | null; - /** Whether every durable sequence is present or policy projection may leave gaps. */ - readonly durableCoverage: 'complete' | 'projected'; readonly overlayMessageCount: number; readonly durable: SessionTranscriptPage; readonly overlay: SessionTranscriptPage; @@ -194,7 +192,6 @@ export function decodeSessionTranscriptPageInput(value: unknown): SessionTranscr export function decodeSessionTranscriptBootstrap(value: unknown): SessionTranscriptBootstrap { const bootstrap = requireExactRecord(value, 'Session transcript bootstrap', [ 'throughSequence', - 'durableCoverage', 'overlayMessageCount', 'durable', 'overlay', @@ -203,9 +200,6 @@ export function decodeSessionTranscriptBootstrap(value: unknown): SessionTranscr bootstrap.throughSequence === null ? null : requireCount(bootstrap.throughSequence, 'Session transcript watermark'); - if (bootstrap.durableCoverage !== 'complete' && bootstrap.durableCoverage !== 'projected') { - throw invalidProtocolFrame('Invalid Session transcript durable coverage'); - } const overlayMessageCount = requireCount( bootstrap.overlayMessageCount, 'Session transcript overlay message count', @@ -230,7 +224,6 @@ export function decodeSessionTranscriptBootstrap(value: unknown): SessionTranscr } return { throughSequence, - durableCoverage: bootstrap.durableCoverage, overlayMessageCount, durable, overlay, diff --git a/packages/runtime-host/src/server/session-transcript-pager.ts b/packages/runtime-host/src/server/session-transcript-pager.ts index b6cd126825..0d29e12b88 100644 --- a/packages/runtime-host/src/server/session-transcript-pager.ts +++ b/packages/runtime-host/src/server/session-transcript-pager.ts @@ -140,7 +140,6 @@ export async function createSessionTranscriptBootstrap(input: { }); const bootstrap: SessionTranscriptBootstrap = { throughSequence: input.throughSequence, - durableCoverage: projection === 'shared' ? 'projected' : 'complete', overlayMessageCount: overlayMessages.length, durable: pageFromSelection( state, From ad351d2c551f9abf8012e20e39cc36d77a18c64b Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 09:56:08 +0800 Subject: [PATCH 20/32] fix(desktop): ask for one older row instead of assuming a transcript starts at zero `#hasOlder` after a jump was `sequence > 0`, and the catch-up refused a sequence that was not exactly the previous one plus one. Both read a durable sequence as a dense counter. It is an event ordinal times its stride, so the oldest row of a Session sits at no fixed number and two adjacent rows are never adjacent integers: a reader who jumped into history was told there was nothing above them. `loadAround` now asks for one bounded row older than its anchor and lets the answer decide. A jump is user-initiated, so the extra read is paid once per jump, and only that read can answer the question at all. Generated-by: Claude Code --- .../desktop-transcript-range-store.test.ts | 70 +++++++++++-------- .../__tests__/runtime-host-client.test.ts | 1 - .../runtime-host-session-observer.test.ts | 2 - .../runtime-host-session-test-fixture.ts | 1 - .../src/main/desktop-transcript-replica.ts | 29 +++++--- 5 files changed, 62 insertions(+), 41 deletions(-) diff --git a/apps/desktop/src/main/__tests__/desktop-transcript-range-store.test.ts b/apps/desktop/src/main/__tests__/desktop-transcript-range-store.test.ts index 3b66889a7f..6d8b1cb4ea 100644 --- a/apps/desktop/src/main/__tests__/desktop-transcript-range-store.test.ts +++ b/apps/desktop/src/main/__tests__/desktop-transcript-range-store.test.ts @@ -262,7 +262,6 @@ test('bounds the default active transcript range by Turn identities', async () = events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: messages.length - 1, - durableCoverage: 'complete', overlayMessageCount: 0, durable: bootstrapPage, overlay: { ...transcriptPage('older', null, messages.length - 1), source: 'overlay' }, @@ -297,7 +296,6 @@ test('bounds the default active transcript range by presentation bytes', async ( events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: messages.length - 1, - durableCoverage: 'complete', overlayMessageCount: 0, durable: bootstrapPage, overlay: { ...transcriptPage('older', null, messages.length - 1), source: 'overlay' }, @@ -336,7 +334,6 @@ test('keeps an oversized latest Turn visible after bootstrap eviction', async () events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: latest.identity, - durableCoverage: 'complete', overlayMessageCount: 0, durable: bootstrapPage, overlay: { ...transcriptPage('older', null, latest.identity), source: 'overlay' }, @@ -377,7 +374,6 @@ test('keeps an oversized latest Turn visible before a trailing session note', as events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: trailingNote.identity, - durableCoverage: 'complete', overlayMessageCount: 0, durable: bootstrapPage, overlay: { @@ -439,7 +435,6 @@ test('keeps an oversized latest Turn when returning from history to a trailing s events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: trailingNote.identity, - durableCoverage: 'complete', overlayMessageCount: 0, durable: bootstrapPage, overlay: { @@ -499,7 +494,6 @@ test('keeps a bounded contiguous window while moving between history and the tai events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: 4, - durableCoverage: 'complete', overlayMessageCount: 0, durable: bootstrapPage, overlay: { ...page(null), source: 'overlay' }, @@ -549,7 +543,6 @@ test('retains the reading anchor while an older page replaces the far edges', as events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: 7, - durableCoverage: 'complete', overlayMessageCount: 0, durable: bootstrapPage, overlay: { ...transcriptPage('older', null, 7), source: 'overlay' }, @@ -575,11 +568,10 @@ test('retains the reading anchor while an older page replaces the far edges', as assert.equal(snapshot.hasNewer, true); }); -for (const { coverage, textBytes } of (['complete', 'projected'] as const).flatMap((coverage) => - [0, 300 * 1024, 600 * 1024].map((textBytes) => ({ coverage, textBytes })), +for (const { stride, textBytes } of [1, 3].flatMap((stride) => + [0, 300 * 1024, 600 * 1024].map((textBytes) => ({ stride, textBytes })), )) { - test(`scrolls both ways through bounded ${coverage} history with ${textBytes}-byte Turns`, async () => { - const stride = coverage === 'projected' ? 3 : 1; + test(`scrolls both ways through bounded stride-${stride} history with ${textBytes}-byte Turns`, async () => { const messages = Array.from({ length: 40 }, (_, index) => ({ identity: index * stride, message: { ...assistantMessage('x'.repeat(textBytes), `assistant-${index}`), turnId: `turn-${index}` }, @@ -609,7 +601,6 @@ for (const { coverage, textBytes } of (['complete', 'projected'] as const).flatM async close() {}, transcriptBootstrap: { throughSequence: through, - durableCoverage: coverage, overlayMessageCount: 0, durable: makePage('older', null), overlay: { ...transcriptPage('older', null, through), source: 'overlay' }, @@ -707,7 +698,6 @@ test('delivers a mid-session tail append even while a history window is resident events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: 4, - durableCoverage: 'complete', overlayMessageCount: 0, durable: bootstrapPage, overlay: { ...page(null), source: 'overlay' }, @@ -758,7 +748,6 @@ test('advances a projected transcript across hidden durable records', async () = events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: 1, - durableCoverage: 'projected', overlayMessageCount: 0, durable: bootstrapPage, overlay: { ...transcriptPage('older', null, 1), source: 'overlay' }, @@ -815,7 +804,6 @@ test('keeps an oversized streaming Turn visible when its overlay settles', async events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: older.identity, - durableCoverage: 'complete', overlayMessageCount: 1, durable: bootstrapPage, overlay: { ...transcriptPage('older', null, older.identity), source: 'overlay' }, @@ -867,7 +855,6 @@ test('keeps an oversized settled Turn visible before a trailing session note', a events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: older.identity, - durableCoverage: 'complete', overlayMessageCount: 1, durable: bootstrapPage, overlay: { ...bootstrapPage, source: 'overlay' }, @@ -932,7 +919,6 @@ test('does not resurrect a discarded replica when a tail re-anchor is in flight' events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: 4, - durableCoverage: 'complete', overlayMessageCount: 0, durable: bootstrapPage, overlay: { ...page(null), source: 'overlay' }, @@ -1020,7 +1006,6 @@ for (const direction of ['older', 'newer'] as const) { events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: 4, - durableCoverage: 'complete', overlayMessageCount: 0, durable: bootstrapPage, overlay: { ...page(null), source: 'overlay' }, @@ -1098,7 +1083,6 @@ test('does not drive a discarded replica terminal when a contiguous catch-up is events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: 4, - durableCoverage: 'complete', overlayMessageCount: 0, durable: bootstrapPage, overlay: { ...page(null, 4), source: 'overlay' }, @@ -1146,14 +1130,13 @@ test('loads a history target with newer messages available below it', async () = })); const bootstrapPage = transcriptPage('older', null, 4); const aroundPage = transcriptPage('newer', 'newer', 4); - let aroundInput: { direction: string; anchorSequence: number | null } | undefined; + const inputs: Array<{ direction: string; anchorSequence: number | null }> = []; const handle = runtimeHostSessionFixture({ snapshot: continuitySnapshot(), transcript: Promise.resolve([]), events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: 4, - durableCoverage: 'complete', overlayMessageCount: 0, durable: bootstrapPage, overlay: { ...transcriptPage('older', null, 4), source: 'overlay' }, @@ -1163,8 +1146,8 @@ test('loads a history target with newer messages available below it', async () = ? { messages: messages.slice(4), nextCursor: null } : { messages: messages.slice(0, 3), nextCursor: 'newer' }, loadTranscriptPage: async (input) => { - aroundInput = input; - return aroundPage; + inputs.push(input); + return input.direction === 'older' ? olderProbePage(0, false) : aroundPage; }, async close() {}, }); @@ -1174,8 +1157,14 @@ test('loads a history target with newer messages available below it', async () = await replica.loadAround(0, 128 * 1024); - assert.equal(aroundInput?.direction, 'newer'); - assert.equal(aroundInput?.anchorSequence, null); + assert.deepEqual( + inputs.map(({ direction, anchorSequence }) => ({ direction, anchorSequence })), + [ + { direction: 'newer', anchorSequence: null }, + // Nothing older than the anchor exists, and only this read can say so. + { direction: 'older', anchorSequence: 0 }, + ], + ); assert.deepEqual(replica.snapshot().durable.map(({ sequence }) => sequence), [0, 1, 2]); assert.equal(replica.snapshot().hasOlder, false); assert.equal(replica.snapshot().hasNewer, true); @@ -1206,7 +1195,6 @@ test('keeps an oversized transcript sparse while moving between indexed prompts' events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: 15, - durableCoverage: 'complete', overlayMessageCount: 0, durable: bootstrapPage, overlay: { ...transcriptPage('older', null, 15), source: 'overlay' }, @@ -1223,7 +1211,10 @@ test('keeps an oversized transcript sparse while moving between indexed prompts' anchorSequence: input.anchorSequence, maxBytes: input.maxBytes, }); - if (input.direction === 'older') return latestPage; + if (input.direction === 'older') { + if (input.maxBytes > 1) return latestPage; + return olderProbePage(input.anchorSequence!, input.anchorSequence !== 0); + } return input.anchorSequence === null ? historicalPage : intermediatePage; }, async close() {}, @@ -1268,9 +1259,13 @@ test('keeps an oversized transcript sparse while moving between indexed prompts' assert.equal(rendererStore.range().hasNewer, false); assertRangeFitsBudget(rendererStore); + // Every jump that is not to the tail pays one extra single-byte read, the + // only thing that can say whether the anchor has anything older than it. assert.deepEqual(requests, [ { direction: 'newer', anchorSequence: null, maxBytes: DESKTOP_TRANSCRIPT_RANGE_MAX_BYTES }, + { direction: 'older', anchorSequence: 0, maxBytes: 1 }, { direction: 'newer', anchorSequence: 5, maxBytes: DESKTOP_TRANSCRIPT_RANGE_MAX_BYTES }, + { direction: 'older', anchorSequence: 6, maxBytes: 1 }, { direction: 'older', anchorSequence: 16, maxBytes: DESKTOP_TRANSCRIPT_RANGE_MAX_BYTES }, ]); replica.close(); @@ -1310,7 +1305,6 @@ test('keeps history resident when an active overlay uses its own cache budget', events: { async *[Symbol.asyncIterator]() {} }, transcriptBootstrap: { throughSequence: 1, - durableCoverage: 'complete', overlayMessageCount: 1, durable: bootstrapPage, overlay: { ...transcriptPage('older', null, 1), source: 'overlay' }, @@ -1601,6 +1595,26 @@ function transcriptPage( }; } +/** The one-byte read `loadAround` uses to ask whether `sequence` has anything + * older than it: only the presence of a fragment answers, not its content. */ +function olderProbePage(sequence: number, exists: boolean) { + return { + ...transcriptPage('older', null, sequence), + fragments: exists + ? [ + { + kind: 'durable' as const, + sequence, + byteOffset: 0, + totalBytes: 1, + payloadDigest: null, + data: '', + }, + ] + : [], + }; +} + function syntheticLargeTranscript(): Array<{ identity: number; message: StoredMessage }> { return Array.from({ length: 8 }, (_, index) => { const number = index + 1; diff --git a/apps/desktop/src/main/__tests__/runtime-host-client.test.ts b/apps/desktop/src/main/__tests__/runtime-host-client.test.ts index 931598cb24..cc73b7b4db 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-client.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-client.test.ts @@ -153,7 +153,6 @@ function subscription( activeAssistantStreams: [], transcriptBootstrap: { throughSequence: null, - durableCoverage: 'complete', overlayMessageCount: 0, durable: emptyTranscriptPage(sessionId, 'durable'), overlay: emptyTranscriptPage(sessionId, 'overlay'), diff --git a/apps/desktop/src/main/__tests__/runtime-host-session-observer.test.ts b/apps/desktop/src/main/__tests__/runtime-host-session-observer.test.ts index f32b62a2c9..c018004f62 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-session-observer.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-session-observer.test.ts @@ -694,7 +694,6 @@ test('fences transcript range failures across same-source replica recovery', asy events, transcriptBootstrap: { throughSequence: 1, - durableCoverage: 'complete', overlayMessageCount: 0, durable: bootstrap, overlay: { ...bootstrap, source: 'overlay', nextCursor: null }, @@ -990,7 +989,6 @@ test('finishes transcript open and replays a stale range request after replaceme events, transcriptBootstrap: { throughSequence: 0, - durableCoverage: 'complete', overlayMessageCount: 0, durable: { kind: 'page', diff --git a/apps/desktop/src/main/__tests__/runtime-host-session-test-fixture.ts b/apps/desktop/src/main/__tests__/runtime-host-session-test-fixture.ts index 8a22cd60a3..108d47bfb9 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-session-test-fixture.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-session-test-fixture.ts @@ -46,7 +46,6 @@ export function runtimeHostSessionFixture(input: { activeAssistantStreams: input.activeAssistantStreams ?? [], transcriptBootstrap: input.transcriptBootstrap ?? { throughSequence: null, - durableCoverage: 'complete', overlayMessageCount: 0, durable: emptyPage(sessionId, 'durable'), overlay: emptyPage(sessionId, 'overlay'), diff --git a/apps/desktop/src/main/desktop-transcript-replica.ts b/apps/desktop/src/main/desktop-transcript-replica.ts index c327c3838f..0d6ace1efe 100644 --- a/apps/desktop/src/main/desktop-transcript-replica.ts +++ b/apps/desktop/src/main/desktop-transcript-replica.ts @@ -82,7 +82,6 @@ export class DesktopTranscriptReplica { readonly generation: string; readonly hostEpoch: string; readonly #handle: DesktopRuntimeHostSession; - readonly #durableCoverage: DesktopRuntimeHostSession['transcriptBootstrap']['durableCoverage']; readonly #maxResidentBytes: number; readonly #maxResidentTurns: number; readonly #maxOverlayBytes: number; @@ -111,7 +110,6 @@ export class DesktopTranscriptReplica { options: DesktopTranscriptReplicaOptions, ) { this.#handle = handle; - this.#durableCoverage = handle.transcriptBootstrap.durableCoverage; this.sessionId = handle.snapshot.session.sessionId; this.generation = options.generation ?? randomUUID(); this.hostEpoch = handle.hostEpoch; @@ -316,6 +314,20 @@ export class DesktopTranscriptReplica { anchorSequence: loadTail ? sequence + 1 : sequence === 0 ? null : sequence - 1, maxBytes, }); + // A durable sequence is an event ordinal times its stride, so the oldest row + // of a Session is at no fixed number and `sequence > 0` cannot answer this. + // Ask for one row older than the anchor instead; a jump is user-initiated, + // so the extra bounded read is paid once per jump. + const older = loadTail + ? null + : await this.#handle.loadTranscriptPage({ + source: 'durable', + direction: 'older', + throughSequence, + cursor: null, + anchorSequence: sequence, + maxBytes: 1, + }); await this.#withDecodedPage(page, (decoded) => { this.#assertOpen(); // `#resident` can flip to false across the `await` above (a concurrent @@ -338,7 +350,7 @@ export class DesktopTranscriptReplica { this.#clearDurable(); const completedOverlayMessageIds = this.#installDurable(decoded.messages); this.#durableThrough = throughSequence; - this.#hasOlder = loadTail ? decoded.nextCursor !== null : sequence > 0; + this.#hasOlder = loadTail ? decoded.nextCursor !== null : older!.fragments.length > 0; this.#hasNewer = loadTail ? false : decoded.nextCursor !== null; evictedDurableSequences.push( ...this.#evictToBudget( @@ -472,9 +484,6 @@ export class DesktopTranscriptReplica { // drives the session terminal. A discarded replica has no watermark to // meet, so return cleanly and let a later resume re-catch-up. if (!this.#resident) return; - if (this.#durableCoverage === 'complete' && nextSequence !== target + 1) { - throw correlationError('Desktop transcript catch-up ended before its watermark'); - } this.#durableThrough = target; this.#publish([], [], []); } @@ -532,10 +541,12 @@ export class DesktopTranscriptReplica { } } + /** + * A durable sequence is an event ordinal times its stride, so the next row is + * only ever at or after the previous one plus one — never exactly there. + */ #matchesCoverageStep(sequence: number, firstPossibleSequence: number): boolean { - return this.#durableCoverage === 'complete' - ? sequence === firstPossibleSequence - : sequence >= firstPossibleSequence; + return sequence >= firstPossibleSequence; } #publish( From 0d886e9f3a8ac721ffd29859e357d49aa955a610 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 09:56:08 +0800 Subject: [PATCH 21/32] test(desktop): scroll until the transcript range moves The paging test sent one wheel gesture and required a page to land. How many gestures that takes is how tall the resident range happens to be, which is not what the test is about; the bootstrap now lands a slightly taller range and the single gesture stopped 240px short of the trigger. Generated-by: Claude Code --- apps/desktop/e2e/transcript-scroll-cost.spec.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/desktop/e2e/transcript-scroll-cost.spec.ts b/apps/desktop/e2e/transcript-scroll-cost.spec.ts index 01899a42a5..4a9dcb1530 100644 --- a/apps/desktop/e2e/transcript-scroll-cost.spec.ts +++ b/apps/desktop/e2e/transcript-scroll-cost.spec.ts @@ -252,10 +252,14 @@ test('paging back through the whole history keeps the mounted range bounded', as const firstBefore = await turns.first().getAttribute('data-turn-id'); if (firstBefore === 'turn-prompt-rail-1') break; // The product asks for history on an upward wheel near the start, so the - // gesture that pages is the gesture a reader makes. - await wheel(page, cdp, { ticks: 12, deltaY: -120 }); + // gesture that pages is the gesture a reader makes. How many gestures it + // takes is how tall the resident range happens to be, which is not what + // this test is about — keep scrolling until the range moves. await expect - .poll(async () => turns.first().getAttribute('data-turn-id')) + .poll(async () => { + await wheel(page, cdp, { ticks: 12, deltaY: -120 }); + return turns.first().getAttribute('data-turn-id'); + }) .not.toBe(firstBefore); pages += 1; mountedMax = Math.max(mountedMax, await turns.count()); From 4cddf6f15ec15e066dab0149475458ca19600e76 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 11:08:28 +0800 Subject: [PATCH 22/32] fix(runtime-host): read the WorkHub Coordination transcript from its own rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every other Session's transcript is what its runs did, so the ledger holds all of it. The Coordination Session's is not. Only `workhub.coordination.answer` admits a root Turn, and nothing in the renderer calls it; `act`, `prepareStop`, `prepareReplacement` and `record` all append rows under a Turn id no admission ever minted. A ledger row must hang on an invocation, so those rows have none to reach and no conversion can lift them — this Session's whole timeline disappeared. The page, record-scan and by-id reads become generic over an ordered record source, and the Coordination Session gets one backed by its own rows. It is written to be deleted: the WorkHub's own ADR already requires a Coordination Turn per action, and once that holds the Session reads like any other. `readMessagesAfter` grows a backward bound so the new source can page towards older rows. Without it a backward page scans from sequence zero, which is the full replay #4647 removed. Refs #3492 Generated-by: Claude Code --- .../session-transcript-reader.test.ts | 89 ++++- .../src/server/session-transcript-reader.ts | 345 ++++++++++++------ packages/storage/src/session-store.ts | 12 +- .../src/sqlite-session-metadata-store.ts | 10 +- 4 files changed, 342 insertions(+), 114 deletions(-) diff --git a/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts b/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts index 7c35f17706..8bbf475115 100644 --- a/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts +++ b/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts @@ -25,7 +25,7 @@ import test from 'node:test'; import { seedInvocation, testInvocationOpening } from '@maka/runtime/test-only/invocation-fixture'; import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { RuntimeEvent } from '@maka/core/runtime-event'; -import type { StoredMessage } from '@maka/core/session'; +import { WORKHUB_COORDINATION_SESSION_ID, type StoredMessage } from '@maka/core/session'; import { projectRuntimeEventsToStoredMessages } from '@maka/runtime/runtime-event-read-model'; import { foldTurnContribution } from '@maka/storage/session-message-projection'; import type { SessionTurnContribution } from '@maka/storage/execution-stores'; @@ -662,6 +662,93 @@ test('stops an oversized active projection before retaining the full RuntimeEven assert.equal(visited, 8_193); }); +test('reads the WorkHub Coordination transcript from its own rows', async () => { + const rows: Array<{ sequence: number; message: StoredMessage }> = [ + { + sequence: 0, + message: { + type: 'user', + id: 'wha_1-user', + turnId: 'wha_1', + ts: 1, + text: 'continue this work', + }, + }, + { + sequence: 1, + message: { + type: 'workhub_coordination', + id: 'wha_1', + turnId: 'wha_1', + ts: 2, + schemaVersion: 1, + kind: 'delegation_assigned', + actionId: 'wha_1', + actionFingerprint: `sha256:${'0'.repeat(64)}`, + coordinationTurnId: 'wha_1', + targetSessionId: 'session-target', + targetTurnId: 'turn-target', + targetMessageId: 'whm_1', + targetSessionName: 'Target', + delegationId: 'whd_1', + disposition: 'delegate_existing', + userText: 'continue this work', + }, + }, + ]; + let ledgerReads = 0; + const stores = { + agentRunStore: {}, + // Any ledger read is the defect: this Session's Turns are never admitted, + // so nothing ever converts these rows and a ledger read returns nothing. + runtimeEventStore: new Proxy( + {}, + { + get: () => () => { + ledgerReads += 1; + return Promise.resolve([]); + }, + }, + ), + sessionStore: { + readTranscriptHighWaterSnapshot: async () => rows.at(-1)!.sequence, + readMessagesAfter: async ( + _sessionId: string, + request: { afterSequence?: number; beforeSequence?: number; maxMessages: number }, + ) => ({ + records: + request.beforeSequence === undefined + ? rows.filter(({ sequence }) => sequence > (request.afterSequence ?? -1)) + : rows.filter(({ sequence }) => sequence < request.beforeSequence!).reverse(), + highWaterSequence: rows.at(-1)!.sequence, + }), + }, + } as unknown as ExecutionStoresWriter<'interactive'>; + const read = createSessionTranscriptReader({ + stores, + canonicalPermissionOutcomes: { readPermissionOutcome: async () => undefined }, + ensureTranscriptLedger: async () => assert.fail('the Coordination Session has no conversion'), + }); + + const page = await read.readDurableRecords(WORKHUB_COORDINATION_SESSION_ID, { + direction: 'newer', + maxMessages: 8, + maxStoredBytes: 64 * 1024, + }); + + assert.deepEqual( + page.records.map(({ message }) => message.id), + ['wha_1-user', 'wha_1'], + ); + assert.equal(ledgerReads, 0); + assert.deepEqual( + (await read.readDurableTurnLandmarks(WORKHUB_COORDINATION_SESSION_ID, 4)).landmarks.map( + ({ turnId }) => turnId, + ), + ['wha_1'], + ); +}); + function runtimeEvent(sessionId: string, overrides: Partial): RuntimeEvent { return { id: 'event-1', diff --git a/packages/runtime-host/src/server/session-transcript-reader.ts b/packages/runtime-host/src/server/session-transcript-reader.ts index c7eec19801..1bb6675238 100644 --- a/packages/runtime-host/src/server/session-transcript-reader.ts +++ b/packages/runtime-host/src/server/session-transcript-reader.ts @@ -19,7 +19,7 @@ import type { RuntimeEvent } from '@maka/core/runtime-event'; import { readRunInvocation } from '@maka/core/runtime-event-store'; -import type { StoredMessage } from '@maka/core/session'; +import { WORKHUB_COORDINATION_SESSION_ID, type StoredMessage } from '@maka/core/session'; import { activePresentationRuntimeEvents, affectsRuntimeEventStoredMessageProjection, @@ -71,6 +71,9 @@ const TRANSCRIPT_TURN_SCAN_LIMIT = 1; * as absent rather than searched for down the Session. */ const TRANSCRIPT_LOOKUP_MAX_TURNS = 2; +/** One storage round trip of Coordination rows, sized like one ledger Turn. */ +const COORDINATION_TRANSCRIPT_SCAN_LIMIT = 64; +const COORDINATION_TRANSCRIPT_SCAN_MAX_BYTES = DURABLE_TRANSCRIPT_TURN_MAX_BYTES; export function createSessionTranscriptReader(input: { stores: ExecutionStoresWriter<'interactive'>; @@ -82,40 +85,34 @@ export function createSessionTranscriptReader(input: { */ ensureTranscriptLedger?: (sessionId: string) => Promise; }): SessionTranscriptReader { - const durable = createDurableLedgerTranscriptReader(input); - const prepared = async (sessionId: string): Promise => { + const ledger = createDurableLedgerTranscriptReader(input); + const coordination = createCoordinationTranscriptReader(input.stores); + const isCoordination = (sessionId: string): boolean => + sessionId === WORKHUB_COORDINATION_SESSION_ID; + // Only a ledger-backed Session has a conversion; the Coordination Session's + // rows are the transcript, not something a run left behind. + const prepared = async (sessionId: string): Promise => { + if (isCoordination(sessionId)) return coordination; await input.ensureTranscriptLedger?.(sessionId); + return ledger; }; return { - readDurableHighWater: async (sessionId) => { - await prepared(sessionId); - return durable.readHighWater(sessionId); - }, - readDurablePage: async (sessionId, request) => { - await prepared(sessionId); - return durable.readPage(sessionId, request); - }, - readDurableRecords: async (sessionId, request) => { - await prepared(sessionId); - return durable.readRecords(sessionId, request); - }, - readDurableMessagesById: async (sessionId, request) => { - await prepared(sessionId); - return durable.readMessagesById(sessionId, request); - }, - readDurableTurnContributions: async ( - sessionId, - throughSequence, - position, - maxContributions, - ) => { - await prepared(sessionId); - return durable.readTurnContributions(sessionId, throughSequence, position, maxContributions); - }, - readDurableTurnLandmarks: async (sessionId, maxLandmarks) => { - await prepared(sessionId); - return durable.readTurnLandmarks(sessionId, maxLandmarks); - }, + readDurableHighWater: async (sessionId) => (await prepared(sessionId)).readHighWater(sessionId), + readDurablePage: async (sessionId, request) => + (await prepared(sessionId)).readPage(sessionId, request), + readDurableRecords: async (sessionId, request) => + (await prepared(sessionId)).readRecords(sessionId, request), + readDurableMessagesById: async (sessionId, request) => + (await prepared(sessionId)).readMessagesById(sessionId, request), + readDurableTurnContributions: async (sessionId, throughSequence, position, maxContributions) => + (await prepared(sessionId)).readTurnContributions( + sessionId, + throughSequence, + position, + maxContributions, + ), + readDurableTurnLandmarks: async (sessionId, maxLandmarks) => + (await prepared(sessionId)).readTurnLandmarks(sessionId, maxLandmarks), readActiveOverlay: async (sessionId, rootTurn) => { if (!rootTurn || isTerminalTurn(rootTurn)) return []; @@ -278,13 +275,109 @@ function createDurableLedgerTranscriptReader(input: { return { readHighWater: highWater, + ...pagedTranscriptReads({ readHighWater: highWater, scan }), + + /** One row per Turn, folded from the Turn's own projected messages. */ + async readTurnContributions( + sessionId: string, + throughSequence: number | null, + position: number, + maxContributions: number, + ): Promise { + const watermark = throughSequence ?? (await highWater(sessionId)); + if (watermark === null) { + return { throughSequence: null, contributions: [], nextPosition: null }; + } + const turns = await store.readTranscriptInvocations(sessionId, { + direction: 'newer', + throughOrdinal: ordinalOf(watermark), + position: ordinalOf(position), + limit: maxContributions + 1, + maxEvents: DURABLE_TRANSCRIPT_TURN_MAX_EVENTS, + maxBytes: DURABLE_TRANSCRIPT_TURN_MAX_BYTES, + }); + const contributions: SessionTurnContribution[] = []; + for (const turn of turns.slice(0, maxContributions)) { + // Folded from the Turn's own rows, so `firstSequence` lands on its first + // row rather than on the opening fact, which has no row at all. + let contribution: SessionTurnContribution | undefined; + for (const { sequence, message } of await projectTurn(turn)) { + if (sequence < position || sequence > watermark) continue; + contribution = foldTurnContribution( + contribution, + turn.invocation.turnId, + sequence, + message, + ); + } + if (contribution) contributions.push(contribution); + } + const next = turns[maxContributions]; + return { + throughSequence: watermark, + contributions, + nextPosition: next ? next.firstOrdinal * EVENT_SEQUENCE_STRIDE : null, + }; + }, + + /** Evenly spaced Turn starts, selected in SQL before loading their prompts. */ + async readTurnLandmarks( + sessionId: string, + maxLandmarks: number, + ): Promise { + const throughSequence = await highWater(sessionId); + if (throughSequence === null) return { throughSequence: null, landmarks: [] }; + const turns = await store.readTranscriptLandmarks( + sessionId, + ordinalOf(throughSequence), + maxLandmarks, + ); + const landmarks: SessionTurnLandmark[] = []; + for (const turn of turns) { + if (!turn.prompt) continue; + const message = projectRuntimeEventUserMessage(turn.prompt.event, turn.prompt.event.id); + const label = (message?.displayText ?? message?.text ?? '').trim(); + if (!label) continue; + landmarks.push({ + turnId: turn.invocation.turnId, + sequence: turn.prompt.ordinal * EVENT_SEQUENCE_STRIDE, + label, + }); + } + return { throughSequence, landmarks }; + }, + }; +} + +/** An ordered, bounded walk over one Session's transcript records. */ +interface TranscriptRecordSource { + readHighWater(sessionId: string): Promise; + scan( + sessionId: string, + request: { + direction: 'older' | 'newer'; + throughSequence?: number | null; + position?: number; + /** Stops the walk after this many Turns, for a read that may find nothing. */ + maxTurns?: number; + }, + ): AsyncGenerator<{ sequence: number; message: StoredMessage }>; +} + +/** + * The reads that are the same whatever produces the records: a byte-bounded + * page, a record scan, and a lookup by message id. Each walks one source's + * ordered records and never asks where they came from. + */ +function pagedTranscriptReads(source: TranscriptRecordSource) { + return { async readPage( sessionId: string, request: SessionTranscriptPageRequest, ): Promise { const throughSequence = request.throughSequence === undefined - ? await this.readHighWater(sessionId) + ? await source.readHighWater(sessionId) : request.throughSequence; if (throughSequence === null) { return { throughSequence: null, fragments: [], rawBytes: 0, next: null }; @@ -293,7 +386,7 @@ function createDurableLedgerTranscriptReader(input: { let rawBytes = 0; let next: SessionTranscriptStoragePage['next'] = null; let truncated = false; - for await (const record of scan(sessionId, { ...request, throughSequence })) { + for await (const record of source.scan(sessionId, { ...request, throughSequence })) { if (fragments.length >= request.maxMessages || rawBytes >= request.maxBytes) { truncated = true; next = { position: record.sequence, byteOffset: null }; @@ -341,7 +434,7 @@ function createDurableLedgerTranscriptReader(input: { ): Promise { const throughSequence = request.throughSequence === undefined - ? await this.readHighWater(sessionId) + ? await source.readHighWater(sessionId) : request.throughSequence; if (throughSequence === null) { return { throughSequence: null, records: [], nextPosition: null }; @@ -349,7 +442,7 @@ function createDurableLedgerTranscriptReader(input: { const records: Array<{ sequence: number; message: StoredMessage }> = []; let storedBytes = 0; let nextPosition: number | null = null; - for await (const record of scan(sessionId, { ...request, throughSequence })) { + for await (const record of source.scan(sessionId, { ...request, throughSequence })) { if (records.length >= request.maxMessages || storedBytes >= request.maxStoredBytes) { nextPosition = record.sequence; break; @@ -360,76 +453,6 @@ function createDurableLedgerTranscriptReader(input: { return { throughSequence, records, nextPosition }; }, - /** One row per Turn, folded from the Turn's own projected messages. */ - async readTurnContributions( - sessionId: string, - throughSequence: number | null, - position: number, - maxContributions: number, - ): Promise { - const watermark = throughSequence ?? (await highWater(sessionId)); - if (watermark === null) { - return { throughSequence: null, contributions: [], nextPosition: null }; - } - const turns = await store.readTranscriptInvocations(sessionId, { - direction: 'newer', - throughOrdinal: ordinalOf(watermark), - position: ordinalOf(position), - limit: maxContributions + 1, - maxEvents: DURABLE_TRANSCRIPT_TURN_MAX_EVENTS, - maxBytes: DURABLE_TRANSCRIPT_TURN_MAX_BYTES, - }); - const contributions: SessionTurnContribution[] = []; - for (const turn of turns.slice(0, maxContributions)) { - // Folded from the Turn's own rows, so `firstSequence` lands on its first - // row rather than on the opening fact, which has no row at all. - let contribution: SessionTurnContribution | undefined; - for (const { sequence, message } of await projectTurn(turn)) { - if (sequence < position || sequence > watermark) continue; - contribution = foldTurnContribution( - contribution, - turn.invocation.turnId, - sequence, - message, - ); - } - if (contribution) contributions.push(contribution); - } - const next = turns[maxContributions]; - return { - throughSequence: watermark, - contributions, - nextPosition: next ? next.firstOrdinal * EVENT_SEQUENCE_STRIDE : null, - }; - }, - - /** Evenly spaced Turn starts, selected in SQL before loading their prompts. */ - async readTurnLandmarks( - sessionId: string, - maxLandmarks: number, - ): Promise { - const throughSequence = await highWater(sessionId); - if (throughSequence === null) return { throughSequence: null, landmarks: [] }; - const turns = await store.readTranscriptLandmarks( - sessionId, - ordinalOf(throughSequence), - maxLandmarks, - ); - const landmarks: SessionTurnLandmark[] = []; - for (const turn of turns) { - if (!turn.prompt) continue; - const message = projectRuntimeEventUserMessage(turn.prompt.event, turn.prompt.event.id); - const label = (message?.displayText ?? message?.text ?? '').trim(); - if (!label) continue; - landmarks.push({ - turnId: turn.invocation.turnId, - sequence: turn.prompt.ordinal * EVENT_SEQUENCE_STRIDE, - label, - }); - } - return { throughSequence, landmarks }; - }, - /** * The durable rows behind a set of message ids. * @@ -448,7 +471,7 @@ function createDurableLedgerTranscriptReader(input: { const wanted = new Set(request.messageIds); const found: Array<{ sequence: number; message: StoredMessage }> = []; let bytes = 0; - for await (const record of scan(sessionId, { + for await (const record of source.scan(sessionId, { direction: 'older', throughSequence: request.throughSequence, maxTurns: TRANSCRIPT_LOOKUP_MAX_TURNS, @@ -464,6 +487,114 @@ function createDurableLedgerTranscriptReader(input: { }; } +/** + * The WorkHub Coordination Session's transcript, read from the rows the WorkHub + * writes. + * + * Every other Session's transcript is what its runs did, so the ledger holds + * all of it. The Coordination Session's is not: a delegation, a stop and a + * routing summary are appended under a Turn id that no root Turn admission ever + * minted, so there is no invocation for the ledger to hang them on and no + * conversion that could lift them. `workhub.coordination.answer` is the one + * path that would admit a real Turn and nothing in the renderer calls it. + * + * Delete this source once the WorkHub admits a Coordination Turn for every + * action, which its own ADR already requires (#3492, + * `docs/architecture/workhub-coordination-session-adr.md`): the Session then + * reads like any other and this reader has nothing left to do. + */ +function createCoordinationTranscriptReader(stores: ExecutionStoresWriter<'interactive'>) { + const store = stores.sessionStore; + const highWater = (sessionId: string): Promise => + store.readTranscriptHighWaterSnapshot(sessionId); + + const scan = async function* ( + sessionId: string, + request: { + direction: 'older' | 'newer'; + throughSequence?: number | null; + position?: number; + }, + ): AsyncGenerator<{ sequence: number; message: StoredMessage }> { + const throughSequence = + request.throughSequence === undefined ? await highWater(sessionId) : request.throughSequence; + if (throughSequence === null) return; + const older = request.direction === 'older'; + const position = request.position ?? (older ? throughSequence : 0); + let cursor = older ? Math.min(position, throughSequence) + 1 : position - 1; + for (;;) { + const page = await store.readMessagesAfter(sessionId, { + ...(older ? { beforeSequence: cursor } : { afterSequence: cursor }), + maxMessages: COORDINATION_TRANSCRIPT_SCAN_LIMIT, + maxStoredBytes: COORDINATION_TRANSCRIPT_SCAN_MAX_BYTES, + }); + if (page.records.length === 0) return; + for (const record of page.records) { + if (!older && record.sequence > throughSequence) return; + yield record; + } + cursor = page.records.at(-1)!.sequence; + } + }; + + const source: TranscriptRecordSource = { readHighWater: highWater, scan }; + return { + readHighWater: highWater, + + ...pagedTranscriptReads(source), + + /** Folded from the rows themselves; these Turns have nothing else. */ + async readTurnContributions( + sessionId: string, + throughSequence: number | null, + position: number, + maxContributions: number, + ): Promise { + const watermark = throughSequence ?? (await highWater(sessionId)); + if (watermark === null) { + return { throughSequence: null, contributions: [], nextPosition: null }; + } + const byTurn = new Map(); + let nextPosition: number | null = null; + for await (const { sequence, message } of scan(sessionId, { + direction: 'newer', + throughSequence: watermark, + position, + })) { + const turnId = message.turnId; + if (turnId === undefined) continue; + if (!byTurn.has(turnId) && byTurn.size === maxContributions) { + nextPosition = sequence; + break; + } + byTurn.set(turnId, foldTurnContribution(byTurn.get(turnId), turnId, sequence, message)); + } + return { throughSequence: watermark, contributions: [...byTurn.values()], nextPosition }; + }, + + /** Every prompt, in order: this transcript has no index to sample from. */ + async readTurnLandmarks( + sessionId: string, + maxLandmarks: number, + ): Promise { + const throughSequence = await highWater(sessionId); + if (throughSequence === null) return { throughSequence: null, landmarks: [] }; + const landmarks: SessionTurnLandmark[] = []; + for await (const { sequence, message } of scan(sessionId, { + direction: 'newer', + throughSequence, + })) { + if (message.type !== 'user' || message.turnId === undefined) continue; + const label = (message.displayText ?? message.text ?? '').trim(); + if (!label) continue; + landmarks.push({ turnId: message.turnId, sequence, label }); + if (landmarks.length === maxLandmarks) break; + } + return { throughSequence, landmarks }; + }, + }; +} + function ordinalOf(sequence: number): number { return Math.floor(sequence / EVENT_SEQUENCE_STRIDE); } diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index 648dc6e81c..48273d33a4 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -226,13 +226,19 @@ export interface SessionTranscriptMessageLookupRequest { } /** - * One forward page of a Session's legacy rows, for the converter that lifts - * them onto the ledger. Nothing else reads `session_messages` any more, so this - * is a migration scan rather than a transcript read. + * One page of a Session's legacy rows, for the converter that lifts them onto + * the ledger and for the WorkHub Coordination Session, whose transcript no run + * produces and so has no ledger to read. */ export interface SessionMessageScanRequest { /** Exclusive lower bound; omit to start at the first row. */ readonly afterSequence?: number; + /** + * Walk towards older rows instead, from this exclusive upper bound. Records + * then come back newest first, so the byte budget truncates at the older end, + * which is the end the walk is heading for. Pass at most one bound. + */ + readonly beforeSequence?: number; readonly maxStoredBytes: number; readonly maxMessages: number; } diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index af482d5e25..94b297a13a 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -2567,6 +2567,10 @@ export class SqliteSessionMetadataStore { if (!Number.isSafeInteger(request.maxStoredBytes) || request.maxStoredBytes < 1) { throw new Error('Invalid Session message byte limit'); } + if (request.afterSequence !== undefined && request.beforeSequence !== undefined) { + throw new Error('Invalid Session message scan bounds'); + } + const backward = request.beforeSequence !== undefined; return this.readTransaction(() => { if (!this.readRecordSync(sessionId)) throw new SessionNotFoundError(sessionId); const rows = this.db @@ -2575,13 +2579,13 @@ export class SqliteSessionMetadataStore { FROM session_messages AS message LEFT JOIN session_message_payloads AS payload ON payload.session_id = message.session_id AND payload.sequence = message.sequence - WHERE message.session_id = ? AND message.sequence > ? - ORDER BY message.sequence + WHERE message.session_id = ? AND message.sequence ${backward ? '<' : '>'} ? + ORDER BY message.sequence ${backward ? 'DESC' : 'ASC'} LIMIT ? `) .all( sessionId, - request.afterSequence ?? -1, + backward ? request.beforeSequence : (request.afterSequence ?? -1), request.maxMessages, ) as StoredSessionMessagePayloadRow[]; const records: SessionMessageScanRecord[] = []; From 86f1d76087a50b289bfd68d154fbf34c72a03d09 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 11:52:37 +0800 Subject: [PATCH 23/32] fix(runtime): bound a transcript Turn before reading it and keep the prompt's catalog facts owed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings from review, both introduced by this PR's own new code. `RuntimeTranscriptQuery#events` read a whole invocation with `.all()` and only then refused an oversized Turn, so the limits that exist to cap what one Turn may pull into memory were checked after that memory was already paid: a probe materialized 20,973,944 bytes against a 16,777,216-byte budget. It now walks the rows and refuses on the row that crosses either limit. The byte budget also counted `payload_json.length` — UTF-16 code units — which admitted a CJK Turn three times the size it was asked to bound; it counts stored bytes. `recordInitialRuntimeEvent` cleared `initialRuntimeEventPending` between the ledger append and the catalog commit, and recovery's `recordAdmittedUserMessage` runs only when the prompt event is missing. A crash or a throw between those two writes therefore left the prompt on the ledger with no catalog projection, and nothing recomputed it: the connection lock is one-way, so the Session could still rebind its LLM connection after having run a Turn. On main the transcript row and the lock were one `appendMessage` transaction, so this window is new here. The flag now clears only after the projection lands, and recovery commits the projection whether or not the ledger already holds the message — it is idempotent, the lock is one-way, and the early return on a sealed run keeps it away from Turns that have their own assistant preview. Two other findings from the same review are not changed here. The mixed-history ordering claim (imported turns sorting after native ones because Session ordinals are `MAX(ordinal)+1` regardless of `openedAt`) describes a state no supported path produces: `sendMessage` converts inside `admitTurn`, the two `agentId` senders target sessions created at `transcriptLedgerVersion: 1`, and staging sessions cannot start a Turn at all. `openedAt` orders the invocation inventory, which is what it is for. The transcript pager's `GROUP BY invocation_id` over every ordinal within the watermark is real — 2.6/25/77 ms at 100/1,000/3,000 rounds — but removing it needs a persisted per-invocation ordinal range, which is new state and a migration, so it is deferred rather than folded in here. Generated-by: Claude Code --- .../__tests__/root-turn-coordinator.test.ts | 86 +++++++++++++++++++ .../src/server/hosted-execution-recovery.ts | 24 ++++-- packages/runtime/src/agent-run.ts | 5 +- .../__tests__/sqlite-runtime-store.test.ts | 82 ++++++++++++++++++ .../storage/src/runtime-transcript-query.ts | 26 +++--- 5 files changed, 202 insertions(+), 21 deletions(-) diff --git a/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts b/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts index 65cb7330cd..d5eaeaae54 100644 --- a/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts @@ -575,6 +575,92 @@ test('startup recovery closes a ScheduledTask Run after its pending fire was set } }); +test('startup recovery commits the catalog facts a crashed Turn wrote no projection for', async () => { + const fixture = await createFailureFixture({ + registerBackend: (backends) => + backends.register('ai-sdk', (context) => new FakeBackend(context)), + }); + const turnId = 'turn-catalog-projection-lost'; + const runId = 'run-catalog-projection-lost'; + const userMessageId = 'message-catalog-projection-lost'; + let recovery: RootTurnCoordinator | undefined; + try { + await fixture.coordinator.close(); + const session = await fixture.stores.sessionStore.readHeaderSnapshot(fixture.sessionId); + assert.equal(session.connectionLocked, false); + const admittedAt = Date.now(); + const admission = await fixture.stores.agentRunStore.admitRootTurn({ + sessionId: fixture.sessionId, + turnId, + proposedRunId: runId, + proposedUserMessageId: userMessageId, + execution: { kind: 'external_message' }, + previousRootTurnId: null, + normalizedInput: { text: 'Say what the catalog never heard.' }, + sourceMessages: [], + admittedAt, + }); + assert.equal(admission.kind, 'admitted'); + await seedInvocation(fixture.stores.runtimeEventStore, { + sessionId: fixture.sessionId, + invocationId: runId, + runId, + turnId, + openedAt: admittedAt, + opening: { + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: session.llmConnectionId!, + llmConnectionSlug: session.llmConnectionSlug, + modelId: session.model, + }, + configuration: { + cwd: session.cwd, + permissionMode: session.permissionMode, + collaborationMode: session.collaborationMode ?? 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + root: { kind: 'user' }, + }, + }); + // The ledger append and the catalog commit are two writes; this is the state + // a crash between them leaves, and the one the message-missing check misses. + await fixture.stores.runtimeEventStore.appendRuntimeEvent(fixture.sessionId, runId, { + id: userMessageId, + sessionId: fixture.sessionId, + invocationId: runId, + runId, + turnId, + ts: admittedAt, + partial: false, + role: 'user', + author: 'user', + content: { kind: 'text', text: 'Say what the catalog never heard.' }, + }); + + recovery = fixture.createRecoveryCoordinator(); + await recovery.prepareRecovery(); + await fixture.manager.recoverInterruptedSessionsStrict(fixture.stores); + await recovery.recover(); + + const recovered = await fixture.stores.sessionStore.readHeaderSnapshot(fixture.sessionId); + assert.equal(recovered.connectionLocked, true); + assert.equal( + (await fixture.stores.runtimeEventStore.readRuntimeEvents(fixture.sessionId, runId)).filter( + (event) => event.id === userMessageId, + ).length, + 1, + ); + } finally { + await recovery?.close(); + await fixture.messages.close(); + await fixture.dispose(); + } +}); + test('a failed exact Capability retry does not poison the parked continuation binding', async () => { const capabilities = new HostClientCapabilityCoordinator({ ...clientCapabilityCoordinatorTestAdmission(), diff --git a/packages/runtime-host/src/server/hosted-execution-recovery.ts b/packages/runtime-host/src/server/hosted-execution-recovery.ts index 4fec474291..6a8cb4beb7 100644 --- a/packages/runtime-host/src/server/hosted-execution-recovery.ts +++ b/packages/runtime-host/src/server/hosted-execution-recovery.ts @@ -148,11 +148,9 @@ export async function prepareHostedExecutionRecovery( admission.execution, ); } - if ( - executionContract.requiresUserMessage && - !verifyUserMessage(admission, rootUserMessages, messageIdOwner) - ) { - await recordAdmittedUserMessage(input.stores, admission, run); + if (executionContract.requiresUserMessage) { + const recorded = verifyUserMessage(admission, rootUserMessages, messageIdOwner); + await recordAdmittedUserMessage(input.stores, admission, run, recorded); } continue; } @@ -182,9 +180,12 @@ export async function prepareHostedExecutionRecovery( admission.turnId, admission.execution, ); - if (!verifyUserMessage(admission, rootUserMessages, messageIdOwner)) { - await recordAdmittedUserMessage(input.stores, admission, run); - } + await recordAdmittedUserMessage( + input.stores, + admission, + run, + verifyUserMessage(admission, rootUserMessages, messageIdOwner), + ); } if (replayAdmissions.length > 1) { throw new Error(`Session ${session.id} has multiple admitted Turns without Runs`); @@ -248,6 +249,7 @@ async function recordAdmittedUserMessage( stores: ExecutionStoresWriter<'interactive'>, admission: RootTurnAdmission, run: RuntimeInvocationRecord, + ledgerHasMessage: boolean, ): Promise { if (run.terminalEvent) return; const content = requireHostedExecutionMessageContent(admission); @@ -264,9 +266,13 @@ async function recordAdmittedUserMessage( author: origin ? 'host' : 'user', content: { kind: 'text', ...content, ...(origin ? { origin } : {}) }, }; - await stores.runtimeEventStore.appendRuntimeEvent(admission.sessionId, run.runId, event); + if (!ledgerHasMessage) { + await stores.runtimeEventStore.appendRuntimeEvent(admission.sessionId, run.runId, event); + } // The Turn never reached the commit that carries these, and no later path // recomputes them: the connection lock is one-way and the preview is a write. + // Committed even when the ledger already holds the message, because the two + // are separate writes and a crash between them leaves exactly that state. const message = projectRuntimeEventUserMessage(event, event.id); if (message) { await stores.sessionStore.commitMessageCatalogProjection(admission.sessionId, message); diff --git a/packages/runtime/src/agent-run.ts b/packages/runtime/src/agent-run.ts index f9542b2f6a..1c6e4c17d2 100644 --- a/packages/runtime/src/agent-run.ts +++ b/packages/runtime/src/agent-run.ts @@ -745,8 +745,11 @@ export class AgentRun { await this.recordRuntimeEvents([event], { requireDurableWrite: this.requiresDurablePersistence(), }); - this.initialRuntimeEventPending = false; + // Owed until the catalog carries it too, not just until the ledger does: + // the projection is where the connection lock latches, and re-recording the + // event is free because its id is derived and the store dedupes it. await this.commitMessageProjection(projectRuntimeEventUserMessage(event, event.id)); + this.initialRuntimeEventPending = false; return event; } diff --git a/packages/storage/src/__tests__/sqlite-runtime-store.test.ts b/packages/storage/src/__tests__/sqlite-runtime-store.test.ts index 6b99977949..59b9cfbc73 100644 --- a/packages/storage/src/__tests__/sqlite-runtime-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-runtime-store.test.ts @@ -26,6 +26,8 @@ import { describe, it } from 'node:test'; import { DEFAULT_TOOL_MODE } from '@maka/core/tool-mode'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { RunSealedError } from '@maka/core/runtime-event-store'; +import { buildInvocationOpenedEvent } from '@maka/core/runtime-invocation'; +import { RuntimeTranscriptOversizedTurnError } from '../runtime-transcript-query.js'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; import { buildImmutableRuntimePrefix, @@ -111,6 +113,86 @@ describe('SqliteRuntimeStore', () => { }); }); + it('bounds a transcript Turn by the bytes it stores, not by its JSON string length', async () => { + await withStore(async (store) => { + const run = { + sessionId: 'session-1', + invocationId: 'invocation-1', + runId: 'run-1', + turnId: 'turn-1', + }; + await store.appendRuntimeEvent( + run.sessionId, + run.runId, + buildInvocationOpenedEvent({ + id: 'oversized-opening', + run, + openedAt: 1, + opening: { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'fake-connection', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd: '/tmp', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: DEFAULT_TOOL_MODE, + }, + root: { kind: 'user' }, + source: { kind: 'fresh' }, + }, + }), + ); + // Every character here is three stored bytes, so a budget read as UTF-16 + // code units admits a Turn three times the size it was asked to bound. + const text = '本'.repeat(4_000); + await store.appendRuntimeEvent(run.sessionId, run.runId, { + id: 'oversized-prompt', + ...run, + ts: 2, + partial: false, + role: 'user', + author: 'user', + content: { kind: 'text', text }, + }); + await store.appendRuntimeEvent(run.sessionId, run.runId, { + id: 'oversized-terminal', + ...run, + ts: 3, + partial: false, + role: 'system', + author: 'system', + status: 'completed', + actions: { endInvocation: true }, + }); + + const request = { + direction: 'newer' as const, + throughOrdinal: Number.MAX_SAFE_INTEGER, + position: 1, + limit: 8, + maxEvents: 64, + }; + await assert.rejects( + store.readTranscriptInvocations(run.sessionId, { ...request, maxBytes: 6_000 }), + (error: unknown) => error instanceof RuntimeTranscriptOversizedTurnError, + ); + const served = await store.readTranscriptInvocations(run.sessionId, { + ...request, + maxBytes: 64_000, + }); + assert.equal(served.length, 1); + }); + }); + it('assigns stable Session ordinals in commit order across Runs', async () => { await withStore(async (store, dbPath) => { const first = functionCallEvent({ id: 'ordinal-1', ts: 20 }); diff --git a/packages/storage/src/runtime-transcript-query.ts b/packages/storage/src/runtime-transcript-query.ts index 79fcd5c38b..7df157fa36 100644 --- a/packages/storage/src/runtime-transcript-query.ts +++ b/packages/storage/src/runtime-transcript-query.ts @@ -210,28 +210,32 @@ export class RuntimeTranscriptQuery { invocationId: string, limits: { maxEvents: number; maxBytes: number }, ): RuntimeTranscriptInvocation['events'] { - const rows = this.db + // Walked row by row: the limits cap what one Turn may pull into memory, so + // a check after `.all()` has already paid the cost it was meant to refuse. + const cursor = this.db .prepare(` SELECT o.ordinal, e.event_id, e.session_id, e.invocation_id, e.run_id, e.turn_id, e.payload_json FROM runtime_events e JOIN runtime_session_event_ordinals o ON o.event_id = e.event_id WHERE e.invocation_id = ? ORDER BY e.event_seq `) - .all(invocationId) as Array; - if (rows.length > limits.maxEvents) { - throw new RuntimeTranscriptOversizedTurnError( - `Turn ${invocationId} holds more RuntimeEvents than a transcript page may read`, - ); - } + .iterate(invocationId) as Iterable; + const events: Array = []; let bytes = 0; - return rows.map((row) => { - bytes += row.payload_json.length; + for (const row of cursor) { + if (events.length === limits.maxEvents) { + throw new RuntimeTranscriptOversizedTurnError( + `Turn ${invocationId} holds more RuntimeEvents than a transcript page may read`, + ); + } + bytes += Buffer.byteLength(row.payload_json); if (bytes > limits.maxBytes) { throw new RuntimeTranscriptOversizedTurnError( `Turn ${invocationId} holds more RuntimeEvent bytes than a transcript page may read`, ); } - return { ordinal: row.ordinal, event: decodeStoredEvent(row) }; - }); + events.push({ ordinal: row.ordinal, event: decodeStoredEvent(row) }); + } + return events; } private event(id: string): RuntimeEvent { From 41ae92cee8ff284e1c460f6938d05e87285cbcc6 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 15:14:50 +0800 Subject: [PATCH 24/32] fix(storage): keep a replayed projection from moving the catalog preview back 86f1d7608 made startup recovery commit a crashed Turn's catalog projection unconditionally, on the argument that the early return on `run.terminalEvent` kept it away from Turns that already carry a newer preview. That holds only for sealed Turns. A steering message commits its own projection on a live run (agent-run.ts:651), so prompt -> steering -> crash -> recovery replays the older prompt projection over the newer steering preview. `updateCatalogProjectionSync` already refuses to move `lastMessageAt` backwards via `maxTimestamp`; the preview was left an unconditional replace. Fixed at that same line rather than at the recovery call site: recovery is not the only replay path, and the ordering rule belongs to the writer that already owns the timestamp half of it. The connection lock still latches unconditionally, which is the part recovery actually owes -- it is a one-way latch, so an older message can only ever set it. Ablation: reverting the guard fails the new test with actual 'the original prompt' / expected 'the steering said later'. Generated-by: Claude Code --- .../src/__tests__/session-store.test.ts | 36 +++++++++++++++++++ .../src/sqlite-session-metadata-store.ts | 10 +++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/packages/storage/src/__tests__/session-store.test.ts b/packages/storage/src/__tests__/session-store.test.ts index dd1a519ba0..a712b3b01e 100644 --- a/packages/storage/src/__tests__/session-store.test.ts +++ b/packages/storage/src/__tests__/session-store.test.ts @@ -437,6 +437,42 @@ describe('SQLite SessionStore', () => { } }); + test('a replayed older message latches the connection without moving the preview back', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-session-preview-replay-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + const prompt = { + type: 'user' as const, + id: 'message-prompt', + turnId: 'turn-1', + ts: 10, + text: 'the original prompt', + }; + await store.commitMessageCatalogProjection(session.id, prompt); + await store.commitMessageCatalogProjection(session.id, { + ...prompt, + id: 'message-steering', + ts: 20, + text: 'the steering said later', + }); + + // Recovery replays the prompt when the ledger holds it but the catalog + // does not; on a Turn still running, a steering line is already on show. + await store.updateHeader(session.id, { connectionLocked: false }); + await store.commitMessageCatalogProjection(session.id, prompt); + + const page = await store.listCatalogPage(undefined, undefined, 10); + if (page.kind !== 'page') assert.fail('expected a catalog page'); + assert.equal(page.records[0]?.summary.lastMessagePreview, 'the steering said later'); + assert.equal(page.records[0]?.activityAt, 20); + assert.equal((await store.readHeader(session.id)).connectionLocked, true); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + test('keeps staging imports outside the catalog pagination domain', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-session-staging-catalog-')); const store = createSessionStore(root); diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index 94b297a13a..9e24d422a3 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -4994,6 +4994,14 @@ export class SqliteSessionMetadataStore { const current = this.readRecordSync(sessionId); if (!current) throw new SessionNotFoundError(sessionId); const lastMessageAt = maxTimestamp(current.header.lastMessageAt, projection.lastMessageAt); + // The preview refuses to move backwards for the same reason the timestamp + // does: a message older than the one on show is a repair of something the + // catalog already passed, and recovery replays exactly those. + const stale = + !replacePreview && + projection.lastMessageAt !== undefined && + current.header.lastMessageAt !== undefined && + projection.lastMessageAt < current.header.lastMessageAt; this.updateHeaderSync( sessionId, { @@ -5002,7 +5010,7 @@ export class SqliteSessionMetadataStore { }, { skipNoop: true, - ...(replacePreview || projection.lastMessagePreview !== undefined + ...(!stale && (replacePreview || projection.lastMessagePreview !== undefined) ? { catalogPreview: { kind: 'replace', From 957c269235de9ba4caad1c0653dca3899e4c9cd2 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 16:05:13 +0800 Subject: [PATCH 25/32] fix(runtime): mint a converted Session's ordinals in the order its turns were said A Session's event ordinals are minted at append time, which is the conversation's order for every run this build starts. It is not the order of a turn converted from the legacy transcript: that turn was said before runs already on the ledger, and it is appended after them. The durable reader orders by ordinal, so a mixed Session read its older question after its newer one. The state is reachable on an ordinary upgrade, and reviewers reproduced it end to end: a released build imported a transcript and then sent on that Session without converting first -- its `sendMessage` had no conversion gate -- so the native run took the Session's ordinals while the imported turns held none. The guards on this build's send path prevent a mixed Session from being created from here on; they cannot retire a database an earlier release already wrote. Fixed where the order is decided rather than at the converter: conversion ends by renumbering the Session's ordinals in the order its invocations opened. `openedAt` was already computed to put imported openings ahead of the Session's own runs and no reader consulted it -- this is what makes the reader agree, so the value stops being a second, dead expression of the same intent. Renumbering is safe here because it is a recomputation, not a move: ordinals are only read on this build and only through `ensureTranscriptLedgerForRead`, which converts first, and a Session at transcript ledger version 0 cannot start a Turn while the conversion holds its queue. Nothing has read these numbers yet. Rejected: enforcing the rule inside `appendRuntimeEvent` so no caller has to remember it. It would have to compare openings by wall clock, and a backwards clock step would then reorder a live Session under readers holding its cursors. Rejected: refusing a mixed history at the compatibility boundary, which trades a silent misordering for a Session nobody can open. The [P3] left open alongside this one -- a single row is decoded before `Buffer.byteLength` measures it -- stays open. Refusing before the string crosses into JS needs a second scan per Turn to read `octet_length`, which buys back at most one row of the 50 KB that `tool-output.ts` already caps. Ablation: without the renumber the new test fails with 'the newer question' ordered ahead of 'the older question'. Generated-by: Claude Code --- packages/core/src/runtime-event-store.ts | 12 ++ .../src/__tests__/agent-run-inspect.test.ts | 4 + .../__tests__/runtime-ledger-repair.test.ts | 114 ++++++++++++++++++ .../session-manager-terminal-ledger.test.ts | 8 ++ .../src/__tests__/session-manager.test.ts | 10 ++ packages/runtime/src/runtime-ledger-repair.ts | 7 ++ packages/storage/src/execution-stores.ts | 2 + packages/storage/src/sqlite-runtime-store.ts | 52 ++++++++ 8 files changed, 209 insertions(+) diff --git a/packages/core/src/runtime-event-store.ts b/packages/core/src/runtime-event-store.ts index 9fa17f304e..fb1c594568 100644 --- a/packages/core/src/runtime-event-store.ts +++ b/packages/core/src/runtime-event-store.ts @@ -157,6 +157,18 @@ export interface RuntimeEventStore { upToEventSeq?: number; }): Promise; readSessionRuntimeEvents(sessionId: string): Promise; + /** + * Renumber a Session's event ordinals in the order its invocations opened. + * + * Ordinals are minted at append time, which is the conversation's order for + * every run this build starts. It is not the order of a run converted from + * the legacy transcript: that turn was said before runs already on the + * ledger, and it is appended after them. The transcript conversion is the + * only caller and the only writer that can know this, and it runs while the + * Session still has no ordinal reader, so these numbers are recomputed + * rather than moved out from under anyone. + */ + resequenceSessionEventOrdinals(sessionId: string): Promise; } /** One invocation by run id, through the store's fast path when it has one. */ diff --git a/packages/runtime/src/__tests__/agent-run-inspect.test.ts b/packages/runtime/src/__tests__/agent-run-inspect.test.ts index 64a6bf19fc..245d409752 100644 --- a/packages/runtime/src/__tests__/agent-run-inspect.test.ts +++ b/packages/runtime/src/__tests__/agent-run-inspect.test.ts @@ -225,6 +225,10 @@ class MemoryAgentRunStore implements AgentRunStore, RuntimeEventStore { .map((event, index) => ({ ordinal: index + 1, event: copyRuntimeEvent(event) })); } + // Ordinals here are positions in the append log, read off it every time, so + // there is nothing stored for a resequence to move. + async resequenceSessionEventOrdinals(_sessionId: string): Promise {} + async readSessionRuntimeEvents(sessionId: string): Promise { const ordered: Array<{ event: RuntimeEvent; runId: string; eventIndex: number }> = []; for (const [eventKey, events] of this.runtimeEvents.entries()) { diff --git a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts index 6e62459e0d..8c8702e5c4 100644 --- a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts +++ b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts @@ -33,6 +33,7 @@ import { buildRuntimeEventModelReplayPlan } from '../model-history.js'; import { buildPriorRuntimeContext } from '../prior-run-context.js'; import { buildInvocationOpenedEvent, + buildSyntheticTerminalRuntimeEvent, runtimeInvocationOutcome, } from '@maka/core/runtime-invocation'; import { runtimeInvocationFailureClass } from '../runtime-event-read-model.js'; @@ -500,6 +501,119 @@ test("converts Maka's own legacy transcript whole, and resumes an interrupted co } }); +test('converts an imported turn ahead of a run the Session already sent', async () => { + // The state a released build leaves behind: it imported a transcript and then + // sent on that Session without converting first, so the native run took the + // Session's ordinals before the older imported turn was ever on the ledger. + const root = await mkdtemp(join(tmpdir(), 'maka-transcript-mixed-')); + const sessions = createSessionStore(root); + const runtimeEvents = createSqliteRuntimeStore(join(root, 'runtime.sqlite')); + try { + const ts = Date.now(); + const session = await sessions.createImportedSession( + { + cwd: '/repo', + llmConnectionSlug: 'anthropic', + model: 'claude-opus-5', + permissionMode: 'ask', + }, + [ + { type: 'user', id: 'i-user', turnId: 'turn-old', ts, text: 'the older question' }, + { + type: 'assistant', + id: 'i-assistant', + turnId: 'turn-old', + ts: ts + 1, + text: 'the older answer', + modelId: 'claude-opus-5', + }, + { + type: 'turn_state', + id: 'i-state', + turnId: 'turn-old', + ts: ts + 2, + status: 'completed', + partialOutputRetained: true, + }, + ], + { adapterId: 'claude-code', sourceSessionId: 'imported-source' }, + ); + const run = { sessionId: session.id, runId: 'native-run', turnId: 'turn-new' }; + const sentAt = ts + 1_000; + await sessions.appendMessages(session.id, [ + { type: 'user', id: 'n-user', turnId: 'turn-new', ts: sentAt, text: 'the newer question' }, + ]); + for (const event of [ + buildInvocationOpenedEvent({ + id: 'native-opening', + run: { ...run, invocationId: run.runId }, + openedAt: sentAt, + opening: { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: { + provenance: 'unknown', + backendKind: 'ai-sdk', + llmConnectionSlug: 'anthropic', + modelId: 'claude-opus-5', + }, + configuration: { + cwd: '/repo', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + root: { kind: 'user' }, + source: { kind: 'fresh' }, + }, + }), + { + id: 'native-user', + sessionId: session.id, + invocationId: run.runId, + runId: run.runId, + turnId: run.turnId, + ts: sentAt, + partial: false, + role: 'user', + author: 'user', + modelVisibility: 'visible', + content: { kind: 'text', text: 'the newer question' }, + } as const, + buildSyntheticTerminalRuntimeEvent({ + id: 'native-terminal', + invocationId: run.runId, + run, + status: 'completed', + ts: sentAt + 1, + }), + ]) { + await runtimeEvents.appendRuntimeEvent(session.id, run.runId, event); + } + + const repair = new RuntimeLedgerRepair({ + runtimeEventStore: runtimeEvents, + readMessagesAfter: (sessionId, request) => sessions.readMessagesAfter(sessionId, request), + }); + await repair.materializeTranscriptLedger(await sessions.readHeader(session.id)); + + const entries = await runtimeEvents.readSessionRuntimeEventEntries(session.id); + assert.deepEqual( + entries.flatMap(({ event }) => (event.content?.kind === 'text' ? [event.content.text] : [])), + ['the older question', 'the older answer', 'the newer question'], + ); + assert.deepEqual( + entries.map(({ ordinal }) => ordinal), + entries.map((_, index) => index + 1), + ); + } finally { + await runtimeEvents.close?.(); + await rm(root, { recursive: true, force: true }); + } +}); + test('startup recovery leaves an interrupted legacy conversion for the importer to finish', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-transcript-restart-')); const sessions = createSessionStore(root); diff --git a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts index fab95a5e12..522383ca65 100644 --- a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts +++ b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts @@ -2436,6 +2436,10 @@ class TinyAgentRunStore implements AgentRunStore, RuntimeEventStore { .map((event, index) => ({ ordinal: index + 1, event: clone(event) })); } + // Ordinals here are positions in the append log, read off it every time, so + // there is nothing stored for a resequence to move. + async resequenceSessionEventOrdinals(_sessionId: string): Promise {} + async readSessionRuntimeEvents(sessionId: string): Promise { const ordered: Array<{ event: RuntimeEvent; runId: string; eventIndex: number }> = []; for (const [eventKey, events] of this.runtimeEvents.entries()) { @@ -2510,6 +2514,10 @@ class BatchingRuntimeEventStore implements RuntimeEventStore { .map((event, index) => ({ ordinal: index + 1, event: clone(event) })); } + // Ordinals here are positions in the append log, read off it every time, so + // there is nothing stored for a resequence to move. + async resequenceSessionEventOrdinals(_sessionId: string): Promise {} + async listSessionInvocations(sessionId: string): Promise { return runtimeInvocationsFromSessionEvents(sessionId, clone(this.events)); } diff --git a/packages/runtime/src/__tests__/session-manager.test.ts b/packages/runtime/src/__tests__/session-manager.test.ts index 9b0f8d94b9..f898f2ece5 100644 --- a/packages/runtime/src/__tests__/session-manager.test.ts +++ b/packages/runtime/src/__tests__/session-manager.test.ts @@ -5086,6 +5086,8 @@ describe('SessionManager permission mode updates', () => { readRuntimeEvents: (sessionId, runId) => durableEvents.readRuntimeEvents(sessionId, runId), readSessionRuntimeEventEntries: (sessionId) => durableEvents.readSessionRuntimeEventEntries(sessionId), + resequenceSessionEventOrdinals: (sessionId) => + durableEvents.resequenceSessionEventOrdinals(sessionId), readSessionRuntimeEvents: (sessionId) => durableEvents.readSessionRuntimeEvents(sessionId), listSessionInvocations: (sessionId) => durableEvents.listSessionInvocations(sessionId), }; @@ -13420,6 +13422,10 @@ class MemoryAgentRunStore .map((event, index) => ({ ordinal: index + 1, event: copyRuntimeEvent(event) })); } + // Ordinals here are positions in the append log, read off it every time, so + // there is nothing stored for a resequence to move. + async resequenceSessionEventOrdinals(_sessionId: string): Promise {} + replaceRuntimeEvent( sessionId: string, runId: string, @@ -13803,6 +13809,10 @@ class MemoryRuntimeEventStore implements RuntimeEventStore { .map((event, index) => ({ ordinal: index + 1, event: copyRuntimeEvent(event) })); } + // Ordinals here are positions in the append log, read off it every time, so + // there is nothing stored for a resequence to move. + async resequenceSessionEventOrdinals(_sessionId: string): Promise {} + async readSessionRuntimeEvents(sessionId: string): Promise { const ordered: Array<{ event: RuntimeEvent; runId: string; eventIndex: number }> = []; for (const [eventKey, events] of this.runtimeEvents.entries()) { diff --git a/packages/runtime/src/runtime-ledger-repair.ts b/packages/runtime/src/runtime-ledger-repair.ts index 7f382d9131..78e9f787f6 100644 --- a/packages/runtime/src/runtime-ledger-repair.ts +++ b/packages/runtime/src/runtime-ledger-repair.ts @@ -144,6 +144,13 @@ export class RuntimeLedgerRepair { await this.deps.runtimeEventStore.appendRuntimeEvent(sessionId, runId, event); } } + // Appending gave every converted event an ordinal above the Session's + // existing runs, which is the wrong order whenever the transcript holds a + // turn older than a run already on the ledger. A released build could + // leave exactly that: it sent on an imported Session without converting + // first. `openedAt` above already says where each imported turn belongs; + // this is what makes the reader agree. + await this.deps.runtimeEventStore.resequenceSessionEventOrdinals(sessionId); }); } diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index 8819a7cd53..9d5bc3542b 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -579,6 +579,8 @@ async function createExecutionStoresForWrite runtimeEventStore.readSessionRuntimeEvents(sessionId)), readSessionRuntimeEventEntries: (sessionId) => run(() => runtimeEventStore.readSessionRuntimeEventEntries(sessionId)), + resequenceSessionEventOrdinals: (sessionId) => + run(() => runtimeEventStore.resequenceSessionEventOrdinals(sessionId)), readTranscriptHighWater: (sessionId) => run(() => runtimeEventStore.readTranscriptHighWater(sessionId)), readTranscriptInvocations: (sessionId, request) => diff --git a/packages/storage/src/sqlite-runtime-store.ts b/packages/storage/src/sqlite-runtime-store.ts index 7ff54569f7..e08a9736fd 100644 --- a/packages/storage/src/sqlite-runtime-store.ts +++ b/packages/storage/src/sqlite-runtime-store.ts @@ -1487,6 +1487,58 @@ export class SqliteRuntimeStore }); } + async resequenceSessionEventOrdinals(sessionId: string): Promise { + assertRuntimeStorageSafeId(sessionId, 'Invalid session id'); + this.transaction(() => { + // Lifted above the range first: the second statement renumbers into the + // space these rows occupy, and (session_id, ordinal) is a primary key. + // Shifting up rather than below zero keeps every intermediate value + // inside the table's own `ordinal > 0`, and lands them past the 1..N the + // renumber assigns, since the count cannot exceed the maximum. + const { shift } = this.db + .prepare(` + SELECT COALESCE(MAX(ordinal), 0) AS shift + FROM runtime_session_event_ordinals + WHERE session_id = ? + `) + .get(sessionId) as { shift: number }; + this.db + .prepare(` + UPDATE runtime_session_event_ordinals + SET ordinal = ordinal + :shift + WHERE session_id = :sessionId + `) + .run({ sessionId, shift }); + this.db + .prepare(` + WITH opening AS ( + SELECT invocation_id, CAST(json_extract(payload_json, '$.ts') AS INTEGER) AS opened_at + FROM runtime_events + WHERE session_id = :sessionId AND event_kind = 'invocation_opened' + ), + ordered AS ( + SELECT + o.event_id AS event_id, + ROW_NUMBER() OVER ( + ORDER BY COALESCE(opening.opened_at, e.committed_at), e.invocation_id, o.ordinal + ) AS ordinal + FROM runtime_session_event_ordinals o + JOIN runtime_events e ON e.event_id = o.event_id + LEFT JOIN opening ON opening.invocation_id = e.invocation_id + WHERE o.session_id = :sessionId + ) + UPDATE runtime_session_event_ordinals + SET ordinal = ( + SELECT ordered.ordinal + FROM ordered + WHERE ordered.event_id = runtime_session_event_ordinals.event_id + ) + WHERE session_id = :sessionId + `) + .run({ sessionId }); + }); + } + async #commitWorkspaceBaseline( input: WorkspaceBaselineAuthorityInput, rootId: string, From 16f70e8725b6adaf2ab61d999928f3631e607c03 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 17:28:50 +0800 Subject: [PATCH 26/32] perf(storage): page a transcript off the ordinal index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Selecting a transcript page grouped every ordinal row inside the watermark by invocation and sorted the groups, so each page cost the whole Session and reading one end to end was quadratic in its length. The grouping was never needed: an invocation's first event is its opening and its last is its terminal, and both already carry an ordinal. Walking back — the direction history is read in — now takes the ordinal index directly, descending from the caller's position and stopping at the page. Walking forward cannot: "has an event at or after this ordinal" is a claim about an invocation's ending, and no ordinal index answers it without assuming Turns never interleave, which the runtime does not promise. That direction reads a spine of one row per invocation instead, which costs the Session in Turns rather than in events. The record each page row carries is now looked up by invocation rather than by run. Both opening shelves key on invocation_id — the ledger through its unique partial index, the migrated shelf through its primary key — while a run lookup had to scan the Session's openings, which left the page cost growing with the Session even after the selection stopped. Measured through readTranscriptInvocations on synthetic Sessions of 200/500/1000/2000 rounds, fetching the last invocation alone: 1.8 / 4.4 / 9.2 / 19.8 ms before, 0.2 ms flat after. Paging a whole Session backwards 100 Turns per page is 8 ms per page at every length. Generated-by: Claude Code --- .../storage/src/runtime-transcript-query.ts | 193 ++++++++++++++---- packages/storage/src/sqlite-runtime-store.ts | 11 +- 2 files changed, 157 insertions(+), 47 deletions(-) diff --git a/packages/storage/src/runtime-transcript-query.ts b/packages/storage/src/runtime-transcript-query.ts index 7df157fa36..b20546b5d6 100644 --- a/packages/storage/src/runtime-transcript-query.ts +++ b/packages/storage/src/runtime-transcript-query.ts @@ -77,12 +77,41 @@ export class RuntimeTranscriptOversizedTurnError extends Error { readonly name = 'RuntimeTranscriptOversizedTurnError'; } -const joins = ` - FROM runtime_session_event_ordinals o - JOIN runtime_events e ON e.event_id = o.event_id - LEFT JOIN runtime_events opened ON opened.invocation_id = e.invocation_id AND opened.event_kind = 'invocation_opened' - LEFT JOIN runtime_legacy_invocation_openings legacy ON legacy.invocation_id = e.invocation_id`; -const opening = `COALESCE(json_extract(opened.payload_json, '$.content'), legacy.opening_json)`; +/** + * Every invocation of a Session, one row each, with the ordinal it starts at. + * + * An invocation's first event is its opening, so the opening's ordinal is the + * invocation's — no scan of the events between is needed to learn where a Turn + * begins. A database migrated from run headers keeps its opening beside the + * ledger instead of in it, and records which of its own events came first; + * that anchor is the same fact, read from where that Session put it. + */ +const spine = ` + spine AS ( + SELECT e.invocation_id AS invocation_id, o.ordinal AS first, + json_extract(e.payload_json, '$.content') AS opening + FROM runtime_events e + JOIN runtime_session_event_ordinals o ON o.event_id = e.event_id + WHERE e.session_id = :sessionId AND e.event_kind = 'invocation_opened' + UNION ALL + SELECT legacy.invocation_id, o.ordinal, legacy.opening_json + FROM runtime_legacy_invocation_openings legacy + JOIN runtime_session_event_ordinals o ON o.event_id = legacy.anchor_event_id + WHERE legacy.session_id = :sessionId + AND NOT EXISTS ( + SELECT 1 FROM runtime_events opened + WHERE opened.invocation_id = legacy.invocation_id + AND opened.event_kind = 'invocation_opened' + ) + ), + ended AS ( + SELECT t.invocation_id AS invocation_id, MIN(o.ordinal) AS last + FROM runtime_events t + JOIN runtime_session_event_ordinals o ON o.event_id = t.event_id + WHERE t.session_id = :sessionId + AND ${TERMINAL_RUNTIME_EVENT_SQL.replaceAll('payload_json', 't.payload_json')} + GROUP BY t.invocation_id + )`; /** * A Turn the Session transcript shows: one this Session ran itself rather than * on behalf of a subagent, and one that has already ended. @@ -91,32 +120,42 @@ const opening = `COALESCE(json_extract(opened.payload_json, '$.content'), legacy * rows it produces is the read model's question, and is not asked here. */ const settledInline = ` - ${opening} IS NOT NULL - AND (json_extract(${opening}, '$.lineage.parentRunId') IS NULL - OR (json_extract(${opening}, '$.source.kind') = 'continuation' - AND json_extract(${opening}, '$.lineage.agentId') IS NULL)) - AND EXISTS ( - SELECT 1 FROM runtime_events ended - JOIN runtime_session_event_ordinals ending ON ending.event_id = ended.event_id - WHERE ended.invocation_id = e.invocation_id - AND ${TERMINAL_RUNTIME_EVENT_SQL.replaceAll('payload_json', 'ended.payload_json')} - AND ending.ordinal <= :throughOrdinal - )`; + spine.opening IS NOT NULL + AND (json_extract(spine.opening, '$.lineage.parentRunId') IS NULL + OR (json_extract(spine.opening, '$.source.kind') = 'continuation' + AND json_extract(spine.opening, '$.lineage.agentId') IS NULL)) + AND ended.last <= :throughOrdinal`; +const settledSpine = ` + FROM spine JOIN ended ON ended.invocation_id = spine.invocation_id + WHERE ${settledInline}`; +/** The same two facts, read off the ordinal index instead of a materialized spine. */ +const inlineOpening = (payload: string) => ` + (json_extract(${payload}, '$.lineage.parentRunId') IS NULL + OR (json_extract(${payload}, '$.source.kind') = 'continuation' + AND json_extract(${payload}, '$.lineage.agentId') IS NULL))`; +const endingOrdinal = (invocation: string) => ` + (SELECT MIN(o2.ordinal) FROM runtime_events t + JOIN runtime_session_event_ordinals o2 ON o2.event_id = t.event_id + WHERE t.invocation_id = ${invocation} + AND ${TERMINAL_RUNTIME_EVENT_SQL.replaceAll('payload_json', 't.payload_json')})`; -type InvocationRow = { invocation_id: string; run_id: string; first: number; last: number }; +type InvocationRow = { invocation_id: string; first: number; last: number }; /** Selects invocations by Session ordinal. Payloads are decoded, never classified. */ export class RuntimeTranscriptQuery { constructor( private readonly db: DatabaseSync, - private readonly invocation: (sessionId: string, runId: string) => RuntimeInvocationRecord, + private readonly invocation: ( + sessionId: string, + invocationId: string, + ) => RuntimeInvocationRecord, ) {} highWater(sessionId: string): number | null { const row = this.db .prepare(` - SELECT MAX(o.ordinal) AS ordinal ${joins} - WHERE o.session_id = :sessionId AND o.ordinal <= :throughOrdinal AND ${settledInline} + WITH ${spine} + SELECT MAX(ended.last) AS ordinal ${settledSpine} `) .get({ sessionId, throughOrdinal: Number.MAX_SAFE_INTEGER }) as { ordinal?: unknown }; return typeof row.ordinal === 'number' ? row.ordinal : null; @@ -132,24 +171,37 @@ export class RuntimeTranscriptQuery { throw new Error('Invalid transcript direction'); } // An invocation is selected by where its own events sit, so a walk that - // starts inside a Turn still finds that Turn and can serve its rows. - const rows = this.db - .prepare(` - SELECT e.invocation_id, e.run_id, MIN(o.ordinal) AS first, MAX(o.ordinal) AS last ${joins} - WHERE o.session_id = :sessionId AND o.ordinal <= :throughOrdinal AND ${settledInline} - GROUP BY e.invocation_id - HAVING ${request.direction === 'older' ? 'first <= :position' : 'last >= :position'} - ORDER BY first ${request.direction === 'older' ? 'DESC' : 'ASC'} + // starts inside a Turn still finds that Turn and can serve its rows. Both + // ends are the invocation's own two events — its opening and its ending — + // rather than the extremes of everything between them. + // + // Walking back is the direction a Session's history is read in, and it + // takes the ordinal index directly: openings descending from `position`, + // stopping at the page. What it costs is the page, not the Session. + // + // Walking forward cannot: "has an event at or after `position`" is a claim + // about an invocation's ending, and no ordinal index answers it without + // assuming Turns never interleave. It reads the spine instead, which costs + // the Session in Turns rather than in events. + const rows = + request.direction === 'older' + ? this.olderInvocations(sessionId, request) + : (this.db + .prepare(` + WITH ${spine} + SELECT spine.invocation_id, spine.first AS first, ended.last AS last + ${settledSpine} AND ended.last >= :position + ORDER BY spine.first ASC LIMIT :limit `) - .all({ - sessionId, - throughOrdinal: request.throughOrdinal, - position: request.position, - limit: request.limit, - }) as InvocationRow[]; + .all({ + sessionId, + throughOrdinal: request.throughOrdinal, + position: request.position, + limit: request.limit, + }) as InvocationRow[]); return rows.map((row) => ({ - invocation: this.invocation(sessionId, row.run_id), + invocation: this.invocation(sessionId, row.invocation_id), firstOrdinal: row.first, lastOrdinal: row.last, events: this.events(row.invocation_id, request), @@ -162,22 +214,20 @@ export class RuntimeTranscriptQuery { // Evenly spaced Turn starts, chosen before any payload is read. const rows = this.db .prepare(` - WITH candidates AS ( - SELECT e.invocation_id, e.run_id, o.ordinal, - ROW_NUMBER() OVER (ORDER BY o.ordinal) - 1 AS rank, COUNT(*) OVER () AS total - ${joins} WHERE o.session_id = :sessionId AND o.ordinal <= :throughOrdinal - AND e.event_seq = 1 AND ${settledInline} + WITH ${spine}, candidates AS ( + SELECT spine.invocation_id AS invocation_id, spine.first AS ordinal, + ROW_NUMBER() OVER (ORDER BY spine.first) - 1 AS rank, COUNT(*) OVER () AS total + ${settledSpine} ), samples(n) AS ( SELECT 0 UNION ALL SELECT n + 1 FROM samples WHERE n + 1 < :limit ) - SELECT DISTINCT invocation_id, run_id, ordinal FROM candidates + SELECT DISTINCT invocation_id, ordinal FROM candidates JOIN samples ON rank = CASE WHEN :limit = 1 THEN total - 1 ELSE CAST(n * (total - 1) / (:limit - 1) AS INTEGER) END ORDER BY ordinal `) .all({ sessionId, throughOrdinal, limit }) as Array<{ invocation_id: string; - run_id: string; ordinal: number; }>; return rows.map((row) => { @@ -197,7 +247,7 @@ export class RuntimeTranscriptQuery { | { ordinal: number; event_id: string } | undefined; return { - invocation: this.invocation(sessionId, row.run_id), + invocation: this.invocation(sessionId, row.invocation_id), firstOrdinal: row.ordinal, ...(prompt ? { prompt: { ordinal: prompt.ordinal, event: this.event(prompt.event_id) } } @@ -206,6 +256,61 @@ export class RuntimeTranscriptQuery { }); } + /** + * The page walking back from `position`, taken off the ordinal index. + * + * Openings that live in the ledger are read in ordinal order and the walk + * stops at the page. A Session migrated from run headers keeps some openings + * beside the ledger, ordered by the anchor event each one names rather than + * by an ordinal of its own; that side is read separately and merged, so the + * common Session pays nothing for a table its history never wrote to. + */ + private olderInvocations( + sessionId: string, + request: RuntimeTranscriptInvocationRequest, + ): InvocationRow[] { + const bind = { + sessionId, + throughOrdinal: request.throughOrdinal, + position: request.position, + limit: request.limit, + }; + const ledger = this.db + .prepare(` + SELECT e.invocation_id AS invocation_id, o.ordinal AS first, + ${endingOrdinal('e.invocation_id')} AS last + FROM runtime_session_event_ordinals o + JOIN runtime_events e ON e.event_id = o.event_id + WHERE o.session_id = :sessionId AND o.ordinal <= :position + AND e.event_kind = 'invocation_opened' + AND ${inlineOpening("json_extract(e.payload_json, '$.content')")} + AND ${endingOrdinal('e.invocation_id')} <= :throughOrdinal + ORDER BY o.ordinal DESC + LIMIT :limit + `) + .all(bind) as InvocationRow[]; + const migrated = this.db + .prepare(` + SELECT legacy.invocation_id AS invocation_id, o.ordinal AS first, + ${endingOrdinal('legacy.invocation_id')} AS last + FROM runtime_legacy_invocation_openings legacy + JOIN runtime_session_event_ordinals o ON o.event_id = legacy.anchor_event_id + WHERE legacy.session_id = :sessionId AND o.ordinal <= :position + AND ${inlineOpening('legacy.opening_json')} + AND ${endingOrdinal('legacy.invocation_id')} <= :throughOrdinal + AND NOT EXISTS ( + SELECT 1 FROM runtime_events opened + WHERE opened.invocation_id = legacy.invocation_id + AND opened.event_kind = 'invocation_opened' + ) + ORDER BY o.ordinal DESC + LIMIT :limit + `) + .all(bind) as InvocationRow[]; + if (migrated.length === 0) return ledger; + return [...ledger, ...migrated].sort((a, b) => b.first - a.first).slice(0, request.limit); + } + private events( invocationId: string, limits: { maxEvents: number; maxBytes: number }, diff --git a/packages/storage/src/sqlite-runtime-store.ts b/packages/storage/src/sqlite-runtime-store.ts index e08a9736fd..3a7add048f 100644 --- a/packages/storage/src/sqlite-runtime-store.ts +++ b/packages/storage/src/sqlite-runtime-store.ts @@ -544,9 +544,14 @@ export class SqliteRuntimeStore } private transcriptQuery(): RuntimeTranscriptQuery { - return new RuntimeTranscriptQuery(this.db, (sessionId, runId) => { - const opening = this.readInvocationOpeningsSync(sessionId, { direction: 'asc', runId }).at(0); - if (!opening) throw new Error(`Transcript invocation ${runId} is missing`); + return new RuntimeTranscriptQuery(this.db, (sessionId, invocationId) => { + // By invocation rather than by run: both shelves key their opening on it, + // so a page's records cost the page instead of the Session's Turns. + const opening = this.readInvocationOpeningsSync(sessionId, { + direction: 'asc', + invocationId, + }).at(0); + if (!opening) throw new Error(`Transcript invocation ${invocationId} is missing`); return this.completeInvocationRecordSync(opening); }); } From 94c24b08206526b4a060a3ad5e0c8d0499e4b84f Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 21:22:01 +0800 Subject: [PATCH 27/32] perf(storage): materialize the ordinal renumber instead of recomputing it `resequenceSessionEventOrdinals` reads its new ordinals from a windowed `ordered` CTE inside a correlated subquery, one lookup per row. SQLite treats such a CTE as a view by default and re-evaluates the whole `ROW_NUMBER() OVER (...)` for every row it is asked about, so a repair costs the Session squared: 5.2 s at 500 rounds and 22.9 s at 1000. `AS MATERIALIZED` computes it once into a transient table. Same rows, same order, same output: 84 ms and 357 ms for the same two Sessions. Generated-by: Claude Code --- packages/storage/src/sqlite-runtime-store.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/storage/src/sqlite-runtime-store.ts b/packages/storage/src/sqlite-runtime-store.ts index 3a7add048f..5fc9b60881 100644 --- a/packages/storage/src/sqlite-runtime-store.ts +++ b/packages/storage/src/sqlite-runtime-store.ts @@ -1521,7 +1521,7 @@ export class SqliteRuntimeStore FROM runtime_events WHERE session_id = :sessionId AND event_kind = 'invocation_opened' ), - ordered AS ( + ordered AS MATERIALIZED ( SELECT o.event_id AS event_id, ROW_NUMBER() OVER ( From 3e8aad4b0e7a759f863c20110d5412ceca6c297a Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 21:22:27 +0800 Subject: [PATCH 28/32] perf(storage): page the transcript forward off the ordinal index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backward paging already walked openings on `runtime_session_event_ordinals` and stopped at the page. Forward paging could not join it: "has an event at or after `position`" is a claim about where an invocation *ends*, and the query answered it by materializing a spine of every opening and every ending in the Session. A page then cost the Session in Turns — 156, 235 and 552 ms per page at 200, 500 and 2000 rounds. An ending is an event of its invocation like any other, so it has an ordinal too. Forward paging now walks endings ascending from `position` and stops at the page, the mirror of what backward paging does with openings: 9-11 ms per page, flat across the same Session lengths. `highWater` is the first row of that same walk taken descending, and `landmarks` — which samples the whole Session by definition — keeps its full pass without a spine to build it on. Neither direction assumes Turns do not overlap. That mattered: the runtime permits concurrent visible invocations (RuntimeKernel holds a Set of execution claims per Session and only the Host coordinator admits one root Turn at a time), and `conversation-copy` has fixtures that interleave two of them deliberately. A reader that assumed otherwise would silently drop a Turn from a forward page. Two things fall out of the rewrite: - The SQL lineage predicate now matches `isSessionInlineInvocation`, its TS twin: `source.kind <> 'fresh'` rather than `= 'continuation'`. The old form excluded handoff-sourced continuations the TS side includes. - `readInvocationOpeningsSync`'s outer `invocation_id`/`run_id` filter was dead — both UNION branches already filter on those columns inline. The new test asserts the property rather than a duration: it runs EXPLAIN QUERY PLAN on every statement the three bounded reads actually execute and refuses any full scan. It fails on the spine with `SCAN spine`. Generated-by: Claude Code --- .../__tests__/sqlite-runtime-store.test.ts | 126 +++++++++- .../storage/src/runtime-transcript-query.ts | 221 +++++++++--------- packages/storage/src/sqlite-runtime-store.ts | 8 +- 3 files changed, 244 insertions(+), 111 deletions(-) diff --git a/packages/storage/src/__tests__/sqlite-runtime-store.test.ts b/packages/storage/src/__tests__/sqlite-runtime-store.test.ts index 86b49e9cfe..55d0655d21 100644 --- a/packages/storage/src/__tests__/sqlite-runtime-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-runtime-store.test.ts @@ -29,7 +29,11 @@ import { encodeCanonicalRuntimeEvent } from '@maka/core/canonical-runtime-event' import { RunSealedError } from '@maka/core/runtime-event-store'; import { buildInvocationOpenedEvent } from '@maka/core/runtime-invocation'; import { readLogicalRuntimeExecution } from '@maka/core/runtime-logical-execution'; -import { RuntimeTranscriptOversizedTurnError } from '../runtime-transcript-query.js'; +import { + RuntimeTranscriptOversizedTurnError, + RuntimeTranscriptQuery, +} from '../runtime-transcript-query.js'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; import { buildImmutableRuntimePrefix, @@ -195,6 +199,45 @@ describe('SqliteRuntimeStore', () => { }); }); + it('pages the transcript without reading rows the page does not contain', async () => { + await withStore(async (store, dbPath) => { + for (let turn = 0; turn < 4; turn += 1) await appendSettledTurn(store, turn); + store.close(); + const db = new DatabaseSync(dbPath); + try { + const executed: { sql: string; bind: unknown[] }[] = []; + const query = new RuntimeTranscriptQuery( + watchStatements(db, executed), + () => + ({ + sessionId: 'session-1', + }) as unknown as RuntimeInvocationRecord, + ); + const request = { + throughOrdinal: Number.MAX_SAFE_INTEGER, + position: 6, + limit: 1, + maxEvents: 64, + maxBytes: 64_000, + }; + query.highWater('session-1'); + query.invocations('session-1', { ...request, direction: 'older' }); + query.invocations('session-1', { ...request, direction: 'newer' }); + // A full scan is how a page starts costing the Session it sits in: the + // rows it walks are every Turn's, not the page's. + for (const { sql, bind } of executed) { + const plan = db.prepare(`EXPLAIN QUERY PLAN ${sql}`).all(...(bind as [])) as unknown as { + detail: string; + }[]; + const scans = plan.filter((step) => step.detail.startsWith('SCAN')); + assert.deepEqual(scans, [], `${scans[0]?.detail} in ${sql}`); + } + } finally { + db.close(); + } + }); + }); + it('assigns stable Session ordinals in commit order across Runs', async () => { await withStore(async (store, dbPath) => { const first = functionCallEvent({ id: 'ordinal-1', ts: 20 }); @@ -2360,6 +2403,87 @@ function continuationStartEvent( }; } +/** A DatabaseSync that records what each statement was actually run with. */ +function watchStatements( + db: DatabaseSync, + executed: { sql: string; bind: unknown[] }[], +): DatabaseSync { + return { + prepare(sql: string) { + const statement = db.prepare(sql); + const record = + (call: (...bind: unknown[]) => T) => + (...bind: unknown[]) => { + executed.push({ sql, bind }); + return call(...bind); + }; + return { + all: record((...bind) => statement.all(...(bind as []))), + get: record((...bind) => statement.get(...(bind as []))), + iterate: record((...bind) => statement.iterate(...(bind as []))), + }; + }, + } as unknown as DatabaseSync; +} + +async function appendSettledTurn(store: Store, index: number): Promise { + const run = { + sessionId: 'session-1', + invocationId: `invocation-${index}`, + runId: `run-${index}`, + turnId: `turn-${index}`, + }; + await store.appendRuntimeEvent( + run.sessionId, + run.runId, + buildInvocationOpenedEvent({ + id: `opened-${index}`, + run, + openedAt: index * 10, + opening: { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'fake-connection', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd: '/tmp', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: DEFAULT_TOOL_MODE, + }, + root: { kind: 'user' }, + source: { kind: 'fresh' }, + }, + }), + ); + await store.appendRuntimeEvent(run.sessionId, run.runId, { + id: `prompt-${index}`, + ...run, + ts: index * 10 + 1, + partial: false, + role: 'user', + author: 'user', + content: { kind: 'text', text: `turn ${index}` }, + }); + await store.appendRuntimeEvent(run.sessionId, run.runId, { + id: `terminal-${index}`, + ...run, + ts: index * 10 + 2, + partial: false, + role: 'system', + author: 'system', + status: 'completed', + actions: { endInvocation: true }, + }); +} + function functionCallEvent(overrides: Partial = {}): RuntimeEvent { return { id: 'call-event-1', diff --git a/packages/storage/src/runtime-transcript-query.ts b/packages/storage/src/runtime-transcript-query.ts index b20546b5d6..1e453d4234 100644 --- a/packages/storage/src/runtime-transcript-query.ts +++ b/packages/storage/src/runtime-transcript-query.ts @@ -77,67 +77,59 @@ export class RuntimeTranscriptOversizedTurnError extends Error { readonly name = 'RuntimeTranscriptOversizedTurnError'; } -/** - * Every invocation of a Session, one row each, with the ordinal it starts at. - * - * An invocation's first event is its opening, so the opening's ordinal is the - * invocation's — no scan of the events between is needed to learn where a Turn - * begins. A database migrated from run headers keeps its opening beside the - * ledger instead of in it, and records which of its own events came first; - * that anchor is the same fact, read from where that Session put it. - */ -const spine = ` - spine AS ( - SELECT e.invocation_id AS invocation_id, o.ordinal AS first, - json_extract(e.payload_json, '$.content') AS opening - FROM runtime_events e - JOIN runtime_session_event_ordinals o ON o.event_id = e.event_id - WHERE e.session_id = :sessionId AND e.event_kind = 'invocation_opened' - UNION ALL - SELECT legacy.invocation_id, o.ordinal, legacy.opening_json - FROM runtime_legacy_invocation_openings legacy - JOIN runtime_session_event_ordinals o ON o.event_id = legacy.anchor_event_id - WHERE legacy.session_id = :sessionId - AND NOT EXISTS ( - SELECT 1 FROM runtime_events opened - WHERE opened.invocation_id = legacy.invocation_id - AND opened.event_kind = 'invocation_opened' - ) - ), - ended AS ( - SELECT t.invocation_id AS invocation_id, MIN(o.ordinal) AS last - FROM runtime_events t - JOIN runtime_session_event_ordinals o ON o.event_id = t.event_id - WHERE t.session_id = :sessionId - AND ${TERMINAL_RUNTIME_EVENT_SQL.replaceAll('payload_json', 't.payload_json')} - GROUP BY t.invocation_id - )`; /** * A Turn the Session transcript shows: one this Session ran itself rather than - * on behalf of a subagent, and one that has already ended. + * on behalf of a subagent. * * This is a fact about the invocation, not about any row it produces — which * rows it produces is the read model's question, and is not asked here. */ -const settledInline = ` - spine.opening IS NOT NULL - AND (json_extract(spine.opening, '$.lineage.parentRunId') IS NULL - OR (json_extract(spine.opening, '$.source.kind') = 'continuation' - AND json_extract(spine.opening, '$.lineage.agentId') IS NULL)) - AND ended.last <= :throughOrdinal`; -const settledSpine = ` - FROM spine JOIN ended ON ended.invocation_id = spine.invocation_id - WHERE ${settledInline}`; -/** The same two facts, read off the ordinal index instead of a materialized spine. */ -const inlineOpening = (payload: string) => ` - (json_extract(${payload}, '$.lineage.parentRunId') IS NULL - OR (json_extract(${payload}, '$.source.kind') = 'continuation' - AND json_extract(${payload}, '$.lineage.agentId') IS NULL))`; +const visibleOpening = (payload: string) => ` + (${payload} IS NOT NULL + AND (json_extract(${payload}, '$.lineage.parentRunId') IS NULL + OR (json_extract(${payload}, '$.source.kind') <> 'fresh' + AND json_extract(${payload}, '$.lineage.agentId') IS NULL)))`; +/** Where an invocation ends; NULL while it is still running. */ const endingOrdinal = (invocation: string) => ` (SELECT MIN(o2.ordinal) FROM runtime_events t JOIN runtime_session_event_ordinals o2 ON o2.event_id = t.event_id WHERE t.invocation_id = ${invocation} AND ${TERMINAL_RUNTIME_EVENT_SQL.replaceAll('payload_json', 't.payload_json')})`; +/** + * A Session migrated from run headers keeps some openings beside the ledger + * rather than in it, ordered by the anchor event each one names. + */ +const migratedOpening = ` + FROM runtime_legacy_invocation_openings legacy + JOIN runtime_session_event_ordinals o ON o.event_id = legacy.anchor_event_id + WHERE legacy.session_id = :sessionId + AND ${visibleOpening('legacy.opening_json')} + AND NOT EXISTS ( + SELECT 1 FROM runtime_events opened + WHERE opened.invocation_id = legacy.invocation_id + AND opened.event_kind = 'invocation_opened' + )`; +const ledgerOpening = ` + FROM runtime_session_event_ordinals o + JOIN runtime_events e ON e.event_id = o.event_id + WHERE o.session_id = :sessionId + AND e.event_kind = 'invocation_opened' + AND ${visibleOpening("json_extract(e.payload_json, '$.content')")}`; +/** Either shelf's opening for one invocation, reached from an event it owns. */ +const openingOrdinal = (invocation: string) => ` + COALESCE( + (SELECT o3.ordinal FROM runtime_events op + JOIN runtime_session_event_ordinals o3 ON o3.event_id = op.event_id + WHERE op.invocation_id = ${invocation} AND op.event_kind = 'invocation_opened'), + (SELECT o3.ordinal FROM runtime_legacy_invocation_openings lg + JOIN runtime_session_event_ordinals o3 ON o3.event_id = lg.anchor_event_id + WHERE lg.invocation_id = ${invocation}))`; +const openingContent = (invocation: string) => ` + COALESCE( + (SELECT json_extract(op.payload_json, '$.content') FROM runtime_events op + WHERE op.invocation_id = ${invocation} AND op.event_kind = 'invocation_opened'), + (SELECT lg.opening_json FROM runtime_legacy_invocation_openings lg + WHERE lg.invocation_id = ${invocation}))`; type InvocationRow = { invocation_id: string; first: number; last: number }; @@ -152,13 +144,14 @@ export class RuntimeTranscriptQuery { ) {} highWater(sessionId: string): number | null { - const row = this.db - .prepare(` - WITH ${spine} - SELECT MAX(ended.last) AS ordinal ${settledSpine} - `) - .get({ sessionId, throughOrdinal: Number.MAX_SAFE_INTEGER }) as { ordinal?: unknown }; - return typeof row.ordinal === 'number' ? row.ordinal : null; + // The furthest a transcript reaches is the last ending on it. + const [row] = this.byEnding(sessionId, { + order: 'DESC', + from: 0, + throughOrdinal: Number.MAX_SAFE_INTEGER, + limit: 1, + }); + return row?.last ?? null; } invocations( @@ -175,31 +168,19 @@ export class RuntimeTranscriptQuery { // ends are the invocation's own two events — its opening and its ending — // rather than the extremes of everything between them. // - // Walking back is the direction a Session's history is read in, and it - // takes the ordinal index directly: openings descending from `position`, - // stopping at the page. What it costs is the page, not the Session. - // - // Walking forward cannot: "has an event at or after `position`" is a claim - // about an invocation's ending, and no ordinal index answers it without - // assuming Turns never interleave. It reads the spine instead, which costs - // the Session in Turns rather than in events. + // Each direction walks the end of the Turn that `position` bounds, which + // is the one the ordinal index can seek to: backward that is the opening, + // forward the ending. Neither assumes Turns do not overlap, and each stops + // at the page, so a page costs the page rather than the Session. const rows = request.direction === 'older' - ? this.olderInvocations(sessionId, request) - : (this.db - .prepare(` - WITH ${spine} - SELECT spine.invocation_id, spine.first AS first, ended.last AS last - ${settledSpine} AND ended.last >= :position - ORDER BY spine.first ASC - LIMIT :limit - `) - .all({ - sessionId, - throughOrdinal: request.throughOrdinal, - position: request.position, - limit: request.limit, - }) as InvocationRow[]); + ? this.byOpening(sessionId, request) + : this.byEnding(sessionId, { + order: 'ASC', + from: request.position, + throughOrdinal: request.throughOrdinal, + limit: request.limit, + }).sort((a, b) => a.first - b.first); return rows.map((row) => ({ invocation: this.invocation(sessionId, row.invocation_id), firstOrdinal: row.first, @@ -214,10 +195,16 @@ export class RuntimeTranscriptQuery { // Evenly spaced Turn starts, chosen before any payload is read. const rows = this.db .prepare(` - WITH ${spine}, candidates AS ( - SELECT spine.invocation_id AS invocation_id, spine.first AS ordinal, - ROW_NUMBER() OVER (ORDER BY spine.first) - 1 AS rank, COUNT(*) OVER () AS total - ${settledSpine} + WITH settled AS ( + SELECT e.invocation_id AS invocation_id, o.ordinal AS ordinal ${ledgerOpening} + AND ${endingOrdinal('e.invocation_id')} <= :throughOrdinal + UNION ALL + SELECT legacy.invocation_id, o.ordinal ${migratedOpening} + AND ${endingOrdinal('legacy.invocation_id')} <= :throughOrdinal + ), candidates AS ( + SELECT invocation_id, ordinal, + ROW_NUMBER() OVER (ORDER BY ordinal) - 1 AS rank, COUNT(*) OVER () AS total + FROM settled ), samples(n) AS ( SELECT 0 UNION ALL SELECT n + 1 FROM samples WHERE n + 1 < :limit ) @@ -257,33 +244,29 @@ export class RuntimeTranscriptQuery { } /** - * The page walking back from `position`, taken off the ordinal index. + * The page of settled visible invocations that opened at or before + * `position`, newest first. * - * Openings that live in the ledger are read in ordinal order and the walk - * stops at the page. A Session migrated from run headers keeps some openings - * beside the ledger, ordered by the anchor event each one names rather than - * by an ordinal of its own; that side is read separately and merged, so the + * The two shelves are read as separate statements and merged rather than + * unioned, so each keeps its own index walk and stops at the page — and the * common Session pays nothing for a table its history never wrote to. */ - private olderInvocations( + private byOpening( sessionId: string, request: RuntimeTranscriptInvocationRequest, ): InvocationRow[] { const bind = { sessionId, - throughOrdinal: request.throughOrdinal, position: request.position, + throughOrdinal: request.throughOrdinal, limit: request.limit, }; const ledger = this.db .prepare(` SELECT e.invocation_id AS invocation_id, o.ordinal AS first, ${endingOrdinal('e.invocation_id')} AS last - FROM runtime_session_event_ordinals o - JOIN runtime_events e ON e.event_id = o.event_id - WHERE o.session_id = :sessionId AND o.ordinal <= :position - AND e.event_kind = 'invocation_opened' - AND ${inlineOpening("json_extract(e.payload_json, '$.content')")} + ${ledgerOpening} + AND o.ordinal <= :position AND ${endingOrdinal('e.invocation_id')} <= :throughOrdinal ORDER BY o.ordinal DESC LIMIT :limit @@ -293,16 +276,9 @@ export class RuntimeTranscriptQuery { .prepare(` SELECT legacy.invocation_id AS invocation_id, o.ordinal AS first, ${endingOrdinal('legacy.invocation_id')} AS last - FROM runtime_legacy_invocation_openings legacy - JOIN runtime_session_event_ordinals o ON o.event_id = legacy.anchor_event_id - WHERE legacy.session_id = :sessionId AND o.ordinal <= :position - AND ${inlineOpening('legacy.opening_json')} + ${migratedOpening} + AND o.ordinal <= :position AND ${endingOrdinal('legacy.invocation_id')} <= :throughOrdinal - AND NOT EXISTS ( - SELECT 1 FROM runtime_events opened - WHERE opened.invocation_id = legacy.invocation_id - AND opened.event_kind = 'invocation_opened' - ) ORDER BY o.ordinal DESC LIMIT :limit `) @@ -311,6 +287,41 @@ export class RuntimeTranscriptQuery { return [...ledger, ...migrated].sort((a, b) => b.first - a.first).slice(0, request.limit); } + /** + * The page of settled visible invocations whose ending sits between `from` + * and `throughOrdinal`, in `order` of that ending. + * + * An ending is an event of the invocation like any other, so this walks the + * same ordinal index — one statement, because the ending is on the ledger + * whichever shelf the opening came from. + */ + private byEnding( + sessionId: string, + bounds: { order: 'ASC' | 'DESC'; from: number; throughOrdinal: number; limit: number }, + ): InvocationRow[] { + return this.db + .prepare(` + SELECT ending.invocation_id AS invocation_id, + ${openingOrdinal('ending.invocation_id')} AS first, + o.ordinal AS last + FROM runtime_session_event_ordinals o + JOIN runtime_events ending ON ending.event_id = o.event_id + WHERE o.session_id = :sessionId + AND o.ordinal BETWEEN :from AND :throughOrdinal + AND ${TERMINAL_RUNTIME_EVENT_SQL.replaceAll('payload_json', 'ending.payload_json')} + AND o.ordinal = ${endingOrdinal('ending.invocation_id')} + AND ${visibleOpening(openingContent('ending.invocation_id'))} + ORDER BY o.ordinal ${bounds.order} + LIMIT :limit + `) + .all({ + sessionId, + from: bounds.from, + throughOrdinal: bounds.throughOrdinal, + limit: bounds.limit, + }) as InvocationRow[]; + } + private events( invocationId: string, limits: { maxEvents: number; maxBytes: number }, diff --git a/packages/storage/src/sqlite-runtime-store.ts b/packages/storage/src/sqlite-runtime-store.ts index 5fc9b60881..8e731946c0 100644 --- a/packages/storage/src/sqlite-runtime-store.ts +++ b/packages/storage/src/sqlite-runtime-store.ts @@ -743,9 +743,7 @@ export class SqliteRuntimeStore AND runtime_events.event_kind = 'invocation_opened' ) ) - WHERE (:invocationId IS NULL OR invocation_id = :invocationId) - AND (:runId IS NULL OR run_id = :runId) - AND ( + WHERE ( :beforeOpenedAt IS NULL OR opened_at < :beforeOpenedAt OR (opened_at = :beforeOpenedAt AND invocation_id < :beforeInvocationId) @@ -755,8 +753,8 @@ export class SqliteRuntimeStore `) .all({ sessionId, - invocationId: options.invocationId ?? null, - runId: options.runId ?? null, + ...(options.invocationId === undefined ? {} : { invocationId: options.invocationId }), + ...(options.runId === undefined ? {} : { runId: options.runId }), beforeOpenedAt: options.before?.openedAt ?? null, beforeInvocationId: options.before?.invocationId ?? null, limit: options.limit ?? -1, From 8e20e6d5a11f0c1606c86f8b03d648a283c71804 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 21:35:00 +0800 Subject: [PATCH 29/32] refactor(runtime): drop the retained-output fact nothing reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `partialOutputRetained` claimed a Turn kept some of its output. It was derived twice and consumed nowhere. Twice: `deriveTurnRecords` recomputed it from the Turn's own assistant and tool_result rows and OR'd that with whatever the `turn_state` message carried, while the ledger projector computed the same predicate over the same rows a second time to fill that message in. Two authorities for one fact, reconciled by an OR — the shape that survives only because nobody checks whether they agree. Nowhere: no renderer, presenter, or decision path reads `TurnRecord`'s or `TurnStateMessage`'s copy. The one place that looked like a consumer — `describeFailedTurnExecutionState` — already ignored the hint and derives its guidance from tool activity counts; its test said so. Removing it takes out the projection in `runtime-event-read-model`, the default in `runtime-read-model`, the wire projection in `session-turns` and `shared-session-transcript`, the field on both core types, its shape and decoder entries, and the constant `true` the three external session adapters wrote. Nothing replaces it: a Turn's retained output is its rows, which the transcript already carries. Epoch 127 — older peers require the field on `turn_state` and on the Turn contribution. Generated-by: Claude Code --- .../desktop-session-projection.test.ts | 1 - .../main/__tests__/interrupted-resume.test.ts | 1 - .../quote-companion-disposal.test.ts | 2 +- .../__tests__/quote-companion-retry.test.ts | 4 +- .../__tests__/runtime-host-client.test.ts | 2 - .../runtime-host-session-observer.test.ts | 2 - .../session-status-presentation.test.ts | 4 +- .../src/main/__tests__/thread-search.test.ts | 1 - .../__tests__/workhub-session-port.test.ts | 3 -- apps/desktop/stories/app-shell.stories.tsx | 27 ++++++-------- .../stories/session-workbar.stories.tsx | 1 - .../cli/src/__tests__/pi-transcript.test.ts | 4 -- .../runtime-host-run-command.test.ts | 7 ---- .../runtime-host-session-driver.test.ts | 2 - packages/core/src/session.ts | 13 ------- .../session-catalog-coordinator.test.ts | 1 - .../session-transcript-pager.test.ts | 1 - .../src/__tests__/session-turns.test.ts | 37 +------------------ packages/runtime-host/src/protocol/index.ts | 5 ++- .../src/protocol/session-turns.ts | 6 --- .../src/server/shared-session-transcript.ts | 3 -- .../workhub-coordination-coordinator.ts | 1 - .../runtime-event-read-model.test.ts | 6 --- .../__tests__/runtime-ledger-repair.test.ts | 9 ----- .../session-manager-terminal-ledger.test.ts | 1 - .../src/__tests__/session-manager.test.ts | 22 ----------- packages/runtime/src/ai-sdk-turn.ts | 4 +- .../runtime/src/runtime-event-read-model.ts | 7 ---- packages/runtime/src/runtime-read-model.ts | 1 - .../src/claude-code-session-adapter.ts | 4 -- packages/storage/src/codex-session-adapter.ts | 2 - .../storage/src/opencode-session-adapter.ts | 4 -- .../__tests__/live-turn-projection.test.ts | 5 +-- packages/ui/src/__tests__/materialize.test.ts | 7 ---- .../__tests__/transcript-projection.test.ts | 4 +- 35 files changed, 26 insertions(+), 178 deletions(-) diff --git a/apps/desktop/src/main/__tests__/desktop-session-projection.test.ts b/apps/desktop/src/main/__tests__/desktop-session-projection.test.ts index 37be37826e..bf762c4b3b 100644 --- a/apps/desktop/src/main/__tests__/desktop-session-projection.test.ts +++ b/apps/desktop/src/main/__tests__/desktop-session-projection.test.ts @@ -142,7 +142,6 @@ test('projects typed linked Session ids without rewriting opaque tool data', () turnId: 'turn-1', status: 'completed', parentSessionId: 'child-session', - partialOutputRetained: false, }).parentSessionId, linkedSessionId, ); diff --git a/apps/desktop/src/main/__tests__/interrupted-resume.test.ts b/apps/desktop/src/main/__tests__/interrupted-resume.test.ts index dd04ce8b1c..d4f90d1ff1 100644 --- a/apps/desktop/src/main/__tests__/interrupted-resume.test.ts +++ b/apps/desktop/src/main/__tests__/interrupted-resume.test.ts @@ -83,7 +83,6 @@ describe('latest interrupted resume candidate', () => { ts: 2, status: 'failed', errorClass: 'timeout', - partialOutputRetained: false, }, { type: 'tool_call', id: 'call-1', turnId: 'turn-1', ts: 3, toolName: 'Read', args: {} }, { diff --git a/apps/desktop/src/main/__tests__/quote-companion-disposal.test.ts b/apps/desktop/src/main/__tests__/quote-companion-disposal.test.ts index 625380d509..37f7759fd8 100644 --- a/apps/desktop/src/main/__tests__/quote-companion-disposal.test.ts +++ b/apps/desktop/src/main/__tests__/quote-companion-disposal.test.ts @@ -50,7 +50,7 @@ function session(id: string): SessionSummary { } function settledTurn(turnId: string): TurnRecord { - return { turnId, status: 'completed', partialOutputRetained: false }; + return { turnId, status: 'completed' }; } const sourceSession = session('side-chat-disposal-source'); diff --git a/apps/desktop/src/main/__tests__/quote-companion-retry.test.ts b/apps/desktop/src/main/__tests__/quote-companion-retry.test.ts index 00e7a3214a..159cbb3311 100644 --- a/apps/desktop/src/main/__tests__/quote-companion-retry.test.ts +++ b/apps/desktop/src/main/__tests__/quote-companion-retry.test.ts @@ -2088,11 +2088,11 @@ function choiceFor( } function settledTurn(turnId: string): TurnRecord { - return { turnId, status: 'completed', partialOutputRetained: false }; + return { turnId, status: 'completed' }; } function runningTurn(turnId: string): TurnRecord { - return { turnId, status: 'running', partialOutputRetained: false }; + return { turnId, status: 'running' }; } async function waitUntil(predicate: () => boolean, diagnostics?: () => string): Promise { diff --git a/apps/desktop/src/main/__tests__/runtime-host-client.test.ts b/apps/desktop/src/main/__tests__/runtime-host-client.test.ts index cc73b7b4db..a5d56ee0c2 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-client.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-client.test.ts @@ -96,7 +96,6 @@ test('derives turn records from bounded contribution pages', async () => { turnId: 'turn-1', ts: 3, status: 'completed', - partialOutputRetained: false, }, }, userPromptPreview: null, @@ -114,7 +113,6 @@ test('derives turn records from bounded contribution pages', async () => { userPromptPreview: 'hello', status: 'completed', statusSource: 'recorded', - partialOutputRetained: false, }]); assert.deepEqual(positions, [0, 2]); await client.close(); diff --git a/apps/desktop/src/main/__tests__/runtime-host-session-observer.test.ts b/apps/desktop/src/main/__tests__/runtime-host-session-observer.test.ts index c018004f62..58ea887b18 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-session-observer.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-session-observer.test.ts @@ -2164,7 +2164,6 @@ test("finishes a watched predecessor after initial catch-up recovery", async () turnId: "turn-1", ts: 20, status: "completed" as const, - partialOutputRetained: true, }, ]), events: secondEvents, @@ -2320,7 +2319,6 @@ test("reconciles terminal, Goal, interaction, and sidecar state after subscripti turnId: 'turn-1', status: 'completed' as const, statusSource: 'recorded' as const, - partialOutputRetained: true, }], openSession: async () => { openCount += 1; diff --git a/apps/desktop/src/main/__tests__/session-status-presentation.test.ts b/apps/desktop/src/main/__tests__/session-status-presentation.test.ts index fcb8517fec..111c5e707f 100644 --- a/apps/desktop/src/main/__tests__/session-status-presentation.test.ts +++ b/apps/desktop/src/main/__tests__/session-status-presentation.test.ts @@ -79,10 +79,8 @@ describe('failed turn execution state', () => { assert.match(describeFailedTurnExecutionState(state, 'zh-TW') ?? '', /工具執行出錯/); }); - it('does not infer execution guidance from a legacy output hint', () => { + it('offers no execution guidance for a Turn that ran nothing', () => { assert.equal(describeFailedTurnExecutionState(NOTHING_RAN, 'zh-CN'), undefined); - const legacyState = { ...NOTHING_RAN, partialOutputRetained: true }; - assert.equal(describeFailedTurnExecutionState(legacyState, 'zh-CN'), undefined); }); it('prefers the most specific state the turn reached', () => { diff --git a/apps/desktop/src/main/__tests__/thread-search.test.ts b/apps/desktop/src/main/__tests__/thread-search.test.ts index cac84c1e77..dc36bbcb28 100644 --- a/apps/desktop/src/main/__tests__/thread-search.test.ts +++ b/apps/desktop/src/main/__tests__/thread-search.test.ts @@ -609,7 +609,6 @@ describe('thread search text projection', () => { turnId: 't1', ts: 1, status: 'completed', - partialOutputRetained: false, }, { type: 'permission_decision', diff --git a/apps/desktop/src/main/__tests__/workhub-session-port.test.ts b/apps/desktop/src/main/__tests__/workhub-session-port.test.ts index c6fbca465b..91022b9777 100644 --- a/apps/desktop/src/main/__tests__/workhub-session-port.test.ts +++ b/apps/desktop/src/main/__tests__/workhub-session-port.test.ts @@ -123,7 +123,6 @@ test('projects the durable Coordination transcript into the WorkHub conversation turnId: 'turn-1', ts: 12, status: 'completed', - partialOutputRetained: true, }, { type: 'workhub_coordination', @@ -693,7 +692,6 @@ test('projects durable Session messages into an ordered WorkHub conversation', ( turnId: 'turn-1', ts: 14, status: 'completed', - partialOutputRetained: true, }, ], }); @@ -738,7 +736,6 @@ test('desktop adapter rebuilds recent turns from the Session transcript and clos turnId: 'turn-1', ts: 12, status: 'completed', - partialOutputRetained: true, }, ]; let closes = 0; diff --git a/apps/desktop/stories/app-shell.stories.tsx b/apps/desktop/stories/app-shell.stories.tsx index 8c12078423..05850f4c2a 100644 --- a/apps/desktop/stories/app-shell.stories.tsx +++ b/apps/desktop/stories/app-shell.stories.tsx @@ -578,7 +578,7 @@ export const StreamingTurn: Story = { runningStatus: true, messages: [ user('msg-s-1', 'turn-s', 3, '顶层布局的 story 怎么做最稳？'), - { type: 'turn_state', id: 'state-s', turnId: 'turn-s', ts: NOW - 30_000, status: 'running', partialOutputRetained: false }, + { type: 'turn_state', id: 'state-s', turnId: 'turn-s', ts: NOW - 30_000, status: 'running' }, ], liveTurn: { turnId: 'turn-s', phase: 'streamed', steps: [{ @@ -611,7 +611,7 @@ export const RunningStatusDuringToolRun: Story = { runningStatus: true, messages: [ user('msg-t-1', 'turn-t', 2, '把整个测试套件跑一遍，看看那三个失败用例是不是同一个原因。'), - { type: 'turn_state', id: 'state-t', turnId: 'turn-t', ts: NOW - 120_000, status: 'running', partialOutputRetained: false }, + { type: 'turn_state', id: 'state-t', turnId: 'turn-t', ts: NOW - 120_000, status: 'running' }, ], liveTurn: { turnId: 'turn-t', phase: 'streamed', steps: [{ @@ -641,8 +641,7 @@ const NPM_TEST_STDOUT_AT_CANCEL = "\n> maka@0.2.0 test\n> npm run build:test && // Aborting settles the call as a cancelled `terminal` result (isError), and // `toolResultActivityStatus` maps a cancelled terminal to `interrupted`. There is // no `interrupted` turn status (only running/completed/aborted/failed) — the -// tool-level state is derived from the settled result, not asserted. Because the -// turn kept that partial result, `partialOutputRetained` is true. +// tool-level state is derived from the settled result, not asserted. // // `npm test` runs for minutes (build:test then the runner), so a cancel at ~16s is // still inside a running process — it settles `cancelled`/130, not `timed_out`/124 @@ -668,7 +667,6 @@ export const InterruptedToolAfterTurnAbort: Story = { turnId: 'turn-i', ts: NOW - 118_000, status: 'running', - partialOutputRetained: false, }, { type: 'assistant', @@ -725,7 +723,6 @@ export const InterruptedToolAfterTurnAbort: Story = { status: 'aborted', abortedAt: NOW - 98_000, abortSource: 'renderer.stop_button', - partialOutputRetained: true, }, ], }} @@ -744,7 +741,7 @@ export const FailedTurnWithToolError: Story = { chat={{ messages: [ user('msg-f-1', 'turn-f', 5, '把 core 里的类型错误修掉，然后跑一遍类型检查确认。'), - { type: 'turn_state', id: 'state-f-running', turnId: 'turn-f', ts: NOW - 290_000, status: 'running', partialOutputRetained: false }, + { type: 'turn_state', id: 'state-f-running', turnId: 'turn-f', ts: NOW - 290_000, status: 'running' }, { type: 'assistant', id: 'msg-assistant-f', turnId: 'turn-f', ts: NOW - 285_000, text: '先运行类型检查定位问题。', modelId: 'claude-sonnet-4-5' }, { type: 'tool_call', @@ -773,7 +770,7 @@ export const FailedTurnWithToolError: Story = { text: "src/session.ts(88,7): error TS2322: Type 'string' is not assignable to type 'number'.\nnpm run typecheck exited with code 2.", }, }, - { type: 'turn_state', id: 'state-f-failed', turnId: 'turn-f', ts: NOW - 281_000, status: 'failed', errorClass: 'tool_failed', partialOutputRetained: false }, + { type: 'turn_state', id: 'state-f-failed', turnId: 'turn-f', ts: NOW - 281_000, status: 'failed', errorClass: 'tool_failed' }, ], }} /> @@ -794,7 +791,7 @@ export const ProviderStreamTruncated: Story = { chat={{ messages: [ user('msg-st-1', 'turn-st', 4, '检查项目的构建结果。'), { type: 'assistant', id: 'msg-st-answer', turnId: 'turn-st', ts: NOW - 199_000, text: '构建已完成，我继续检查输出。', modelId: 'claude-sonnet-4-5' }, - { type: 'turn_state', id: 'state-st-failed', turnId: 'turn-st', ts: NOW - 198_000, status: 'failed', errorClass: 'stream_truncated', retry: { decision: 'declined', because: 'side_effects' }, partialOutputRetained: true }, + { type: 'turn_state', id: 'state-st-failed', turnId: 'turn-st', ts: NOW - 198_000, status: 'failed', errorClass: 'stream_truncated', retry: { decision: 'declined', because: 'side_effects' } }, ] }} /> ), @@ -814,8 +811,8 @@ export const ProviderRateLimited: Story = { chat={{ messages: [ user('msg-r-1', 'turn-r', 4, '再生成三个对照方案，越详细越好。'), - { type: 'turn_state', id: 'state-r-running', turnId: 'turn-r', ts: NOW - 200_000, status: 'running', partialOutputRetained: false }, - { type: 'turn_state', id: 'state-r-failed', turnId: 'turn-r', ts: NOW - 198_000, status: 'failed', errorClass: 'rate_limit', partialOutputRetained: false }, + { type: 'turn_state', id: 'state-r-running', turnId: 'turn-r', ts: NOW - 200_000, status: 'running' }, + { type: 'turn_state', id: 'state-r-failed', turnId: 'turn-r', ts: NOW - 198_000, status: 'failed', errorClass: 'rate_limit' }, ], }} /> @@ -840,7 +837,7 @@ export const ProviderRetrying: Story = { runningStatus: true, messages: [ user('msg-rr-1', 'turn-rr', 1, '把这份长文档翻译成英文。'), - { type: 'turn_state', id: 'state-rr', turnId: 'turn-rr', ts: NOW - 20_000, status: 'running', partialOutputRetained: false }, + { type: 'turn_state', id: 'state-rr', turnId: 'turn-rr', ts: NOW - 20_000, status: 'running' }, ], liveTurn: { turnId: 'turn-rr', @@ -884,9 +881,9 @@ export const SafeResumeAfterRestart: Story = { safeResumeAction: { pending: false, onResume: noop }, messages: [ user('msg-sr-1', 'turn-sr', 3, '把这份报告整理成要点清单。'), - { type: 'turn_state', id: 'state-sr-running', turnId: 'turn-sr', ts: NOW - 150_000, status: 'running', partialOutputRetained: false }, + { type: 'turn_state', id: 'state-sr-running', turnId: 'turn-sr', ts: NOW - 150_000, status: 'running' }, { type: 'assistant', id: 'msg-assistant-sr', turnId: 'turn-sr', ts: NOW - 148_000, text: '好的，我先通读一遍，抓住主要结论——', modelId: 'claude-sonnet-4-5' }, - { type: 'turn_state', id: 'state-sr-failed', turnId: 'turn-sr', ts: NOW - 146_000, status: 'failed', errorClass: 'app_restarted', partialOutputRetained: true }, + { type: 'turn_state', id: 'state-sr-failed', turnId: 'turn-sr', ts: NOW - 146_000, status: 'failed', errorClass: 'app_restarted' }, ], }} /> @@ -1040,7 +1037,6 @@ export const ComputerUseObservability: Story = { turnId: 'turn-cu', ts: NOW - 40_000, status: 'running', - partialOutputRetained: false, }, ], liveTurn: { @@ -2006,7 +2002,6 @@ function StreamingTailHarness() { turnId: 'turn-tail', ts: NOW - 30_000, status: 'running', - partialOutputRetained: false, }, ], liveTurn: { diff --git a/apps/desktop/stories/session-workbar.stories.tsx b/apps/desktop/stories/session-workbar.stories.tsx index 6c0889d8db..b228c31896 100644 --- a/apps/desktop/stories/session-workbar.stories.tsx +++ b/apps/desktop/stories/session-workbar.stories.tsx @@ -868,7 +868,6 @@ function bridge(options: { { turnId: 'source-turn', status: 'completed', - partialOutputRetained: false, }, ], readSettledMessages: async () => ({ messages: [], settled: true }), diff --git a/packages/cli/src/__tests__/pi-transcript.test.ts b/packages/cli/src/__tests__/pi-transcript.test.ts index 8cd0e2a40e..06ea4215e8 100644 --- a/packages/cli/src/__tests__/pi-transcript.test.ts +++ b/packages/cli/src/__tests__/pi-transcript.test.ts @@ -969,7 +969,6 @@ describe('Maka Pi TUI transcript', () => { turnId: 'turn-1', ts: 2, status: 'running', - partialOutputRetained: true, }, ]); @@ -1055,7 +1054,6 @@ describe('Maka Pi TUI transcript', () => { turnId: 'turn-1', ts: 5, status: 'completed', - partialOutputRetained: true, }, ]); @@ -1419,7 +1417,6 @@ describe('Maka Pi TUI transcript', () => { turnId: 'turn-1', ts: 2, status: 'completed', - partialOutputRetained: false, }, ] satisfies StoredMessage[]); @@ -5129,7 +5126,6 @@ function inFlightBackgroundPollFixture(): { turnId: 'turn-1', ts: 1, status: 'running', - partialOutputRetained: true, }, { type: 'tool_call', id: 'bash-bg', turnId: 'turn-1', ts: 2, toolName: 'Bash', args: {} }, { diff --git a/packages/cli/src/__tests__/runtime-host-run-command.test.ts b/packages/cli/src/__tests__/runtime-host-run-command.test.ts index fc13e5b2eb..26b45e7a3c 100644 --- a/packages/cli/src/__tests__/runtime-host-run-command.test.ts +++ b/packages/cli/src/__tests__/runtime-host-run-command.test.ts @@ -1455,7 +1455,6 @@ function graphMessages(includeTerminal = true): StoredMessage[] { turnId: 'turn-2', ts: 5, status: 'completed', - partialOutputRetained: false, }); } return messages; @@ -1482,7 +1481,6 @@ function sandboxBoundaryMessages( turnId: 'turn-2', ts: 10, status: 'completed', - partialOutputRetained: true, }, ]; } @@ -1502,7 +1500,6 @@ function multipleSandboxFailureMessages(): StoredMessage[] { turnId: 'turn-2', ts: 11, status: 'completed', - partialOutputRetained: true, }, ]; } @@ -1517,7 +1514,6 @@ function abortedGraphMessages(): StoredMessage[] { ts: 5, status: 'aborted', abortSource: 'user_interrupt', - partialOutputRetained: true, }, ]; } @@ -1532,7 +1528,6 @@ function failedGraphMessages(errorClass: string): StoredMessage[] { ts: 5, status: 'failed', errorClass, - partialOutputRetained: true, }, ]; } @@ -1546,7 +1541,6 @@ function failedThenCompletedGraphMessages(): StoredMessage[] { turnId: 'turn-2', ts: 6, status: 'completed', - partialOutputRetained: true, }, ]; } @@ -1583,7 +1577,6 @@ function multiWakeGraphMessages(includeFinalTerminal: boolean): StoredMessage[] turnId: 'turn-3', ts: 8, status: 'completed', - partialOutputRetained: false, }); } return messages; diff --git a/packages/cli/src/__tests__/runtime-host-session-driver.test.ts b/packages/cli/src/__tests__/runtime-host-session-driver.test.ts index 230b1ec72e..3e39593131 100644 --- a/packages/cli/src/__tests__/runtime-host-session-driver.test.ts +++ b/packages/cli/src/__tests__/runtime-host-session-driver.test.ts @@ -2136,7 +2136,6 @@ describe('Runtime Host Maka Session driver', () => { turnId: 'turn-running', ts: 80, status: 'running', - partialOutputRetained: true, }, ]; const subscriptions = [ @@ -3084,7 +3083,6 @@ function turnStateMessage( turnId, ts: 80, status, - partialOutputRetained: true, }; } diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts index e043237838..4796afb437 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session.ts @@ -930,8 +930,6 @@ export interface TurnStateMessage { abortSource?: string; errorClass?: string; retry?: ModelRetryDecision; - /** Legacy retained-output hint; current projections derive this from output contributions. */ - partialOutputRetained?: boolean; } export const WORKHUB_COORDINATION_RECORD_SCHEMA_VERSION = 1 as const; @@ -1141,7 +1139,6 @@ export interface TurnRecord { abortSource?: string; errorClass?: string; retry?: ModelRetryDecision; - partialOutputRetained: boolean; } /** @@ -1273,7 +1270,6 @@ const TOKEN_USAGE_MESSAGE_SHAPE = defineObjectShape()( const TURN_STATE_MESSAGE_SHAPE = defineObjectShape()( ['type', 'id', 'turnId', 'ts', 'status'], [ - 'partialOutputRetained', 'parentTurnId', 'retriedFromTurnId', 'regeneratedFromTurnId', @@ -1554,8 +1550,6 @@ function decodeMessage( hasExactShape(message, TURN_STATE_MESSAGE_SHAPE) && hasMessageEnvelope(message, true) && isTurnStatus(message.status) && - (message.partialOutputRetained === undefined || - typeof message.partialOutputRetained === 'boolean') && isOptionalString(message.parentTurnId) && isOptionalString(message.retriedFromTurnId) && isOptionalString(message.regeneratedFromTurnId) && @@ -1836,11 +1830,6 @@ export function deriveTurnRecords(messages: readonly StoredMessage[]): TurnRecor const latestState = bucket .filter((message): message is TurnStateMessage => message.type === 'turn_state') .at(-1); - const partialOutputRetained = bucket.some( - (message) => - (message.type === 'assistant' && message.text.trim().length > 0) || - message.type === 'tool_result', - ); if (latestState) { return { turnId, @@ -1859,14 +1848,12 @@ export function deriveTurnRecords(messages: readonly StoredMessage[]): TurnRecor ...(latestState.abortSource ? { abortSource: latestState.abortSource } : {}), ...(latestState.errorClass ? { errorClass: latestState.errorClass } : {}), ...(latestState.retry ? { retry: latestState.retry } : {}), - partialOutputRetained: latestState.partialOutputRetained || partialOutputRetained, }; } return { turnId, status: inferLegacyTurnStatus(bucket), statusSource: 'inferred', - partialOutputRetained, }; }); } diff --git a/packages/runtime-host/src/__tests__/session-catalog-coordinator.test.ts b/packages/runtime-host/src/__tests__/session-catalog-coordinator.test.ts index dcc21cf9a1..9b191e3280 100644 --- a/packages/runtime-host/src/__tests__/session-catalog-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/session-catalog-coordinator.test.ts @@ -210,7 +210,6 @@ test('read marker pages past a hidden tail to reach the newest visible message', turnId: 'turn-1', ts: 30, status: 'completed' as const, - partialOutputRetained: false, }, }, ], diff --git a/packages/runtime-host/src/__tests__/session-transcript-pager.test.ts b/packages/runtime-host/src/__tests__/session-transcript-pager.test.ts index 0c4a63a7d6..23410e23f6 100644 --- a/packages/runtime-host/src/__tests__/session-transcript-pager.test.ts +++ b/packages/runtime-host/src/__tests__/session-transcript-pager.test.ts @@ -251,7 +251,6 @@ test('projects durable and active transcript records before sharing them', async status: 'aborted', abortedAt: 5, abortSource: 'stop_button', - partialOutputRetained: true, }, 'session-1', ); diff --git a/packages/runtime-host/src/__tests__/session-turns.test.ts b/packages/runtime-host/src/__tests__/session-turns.test.ts index 89989d0ca2..5e89f7a914 100644 --- a/packages/runtime-host/src/__tests__/session-turns.test.ts +++ b/packages/runtime-host/src/__tests__/session-turns.test.ts @@ -60,7 +60,7 @@ test('publishes no Turn until its recorded state is on the page', () => { ); }); -test('takes retained output from the recorded turn state', () => { +test('takes the published Turn from the recorded turn state', () => { assert.deepEqual( projectSessionTurnContribution({ turnId: 'turn-1', @@ -73,7 +73,6 @@ test('takes retained output from the recorded turn state', () => { turnId: 'turn-1', ts: 1, status: 'failed', - partialOutputRetained: true, }, }, userPromptPreview: 'hello', @@ -84,7 +83,6 @@ test('takes retained output from the recorded turn state', () => { userPromptPreview: 'hello', status: 'failed', statusSource: 'recorded', - partialOutputRetained: true, }, ); }); @@ -101,7 +99,6 @@ test('bounds turn diagnostics before publishing a contribution', () => { turnId: 'turn-1', ts: 1, status: 'failed', - partialOutputRetained: false, errorClass: '失败'.repeat(100_000), retry: { decision: 'declined', because: 'side_effects' }, }, @@ -139,41 +136,9 @@ test('rejects invalid turn-state references before publishing a contribution', ( ts: 1, status: 'completed', parentTurnId: 'x'.repeat(129), - partialOutputRetained: false, }, }, userPromptPreview: null, }), ); }); - -for (const retained of [true, false]) { - test(`carries the recorded retained-output fact: ${retained}`, () => { - const contribution = projectSessionTurnContributionForWire({ - turnId: 'turn-1', - firstSequence: 0, - latestState: { - sequence: 100, - message: { - type: 'turn_state', - id: 'state', - turnId: 'turn-1', - ts: 100, - status: 'failed', - partialOutputRetained: retained, - }, - }, - userPromptPreview: null, - }); - const decoded = decodeSessionTurnsQueryResult({ - sessionId: 'session-1', - throughSequence: 100, - contributions: [contribution], - nextPosition: null, - }); - assert.equal( - projectSessionTurnContribution(decoded.contributions[0]!)?.partialOutputRetained, - retained, - ); - }); -} diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts index 239f069904..9871929a93 100644 --- a/packages/runtime-host/src/protocol/index.ts +++ b/packages/runtime-host/src/protocol/index.ts @@ -101,7 +101,10 @@ export const RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION = 1 as const; export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const; // Increment when the same protocol version no longer guarantees safe Client-Host // interoperability. Mismatches are rejected before domain commands are admitted. -export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 126 as const; +export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 127 as const; +// 127: Turn states and Turn records drop `partialOutputRetained`. The fact was +// derived twice — once from the Turn's output rows, once off the state message +// — and read by nothing; older peers require the field on both. // 126: Session transcript bootstraps drop `durableCoverage`. A durable sequence // is an event ordinal times its stride, so no projection has contiguous // sequences any more and the claim the field made is unavailable to make. diff --git a/packages/runtime-host/src/protocol/session-turns.ts b/packages/runtime-host/src/protocol/session-turns.ts index 7328d6914b..0a3f682381 100644 --- a/packages/runtime-host/src/protocol/session-turns.ts +++ b/packages/runtime-host/src/protocol/session-turns.ts @@ -165,9 +165,6 @@ function projectTurnStateMessageForWire(message: TurnStateMessage): TurnStateMes ? { errorClass: truncateUtf8(message.errorClass, SESSION_TURN_DIAGNOSTIC_MAX_BYTES) } : {}), ...(message.retry ? { retry: message.retry } : {}), - ...(message.partialOutputRetained !== undefined - ? { partialOutputRetained: message.partialOutputRetained } - : {}), }; } @@ -201,9 +198,6 @@ export function projectSessionTurnContribution( ...(state.abortSource ? { abortSource: state.abortSource } : {}), ...(state.errorClass ? { errorClass: state.errorClass } : {}), ...(state.retry ? { retry: state.retry } : {}), - // Absent only on a `turn_state` written before the field existed, where - // nothing else on the contribution can say whether output survived. - partialOutputRetained: state.partialOutputRetained ?? false, }; } diff --git a/packages/runtime-host/src/server/shared-session-transcript.ts b/packages/runtime-host/src/server/shared-session-transcript.ts index e18dee61d7..0995206c7e 100644 --- a/packages/runtime-host/src/server/shared-session-transcript.ts +++ b/packages/runtime-host/src/server/shared-session-transcript.ts @@ -137,9 +137,6 @@ export function projectSharedSessionTranscriptMessage( ...(message.abortSource === undefined ? {} : { abortSource: message.abortSource }), ...(message.errorClass === undefined ? {} : { errorClass: message.errorClass }), ...(message.retry === undefined ? {} : { retry: message.retry }), - ...(message.partialOutputRetained === undefined - ? {} - : { partialOutputRetained: message.partialOutputRetained }), }; case 'token_usage': return { diff --git a/packages/runtime-host/src/server/workhub-coordination-coordinator.ts b/packages/runtime-host/src/server/workhub-coordination-coordinator.ts index 1a33b3095c..6c953ed7e3 100644 --- a/packages/runtime-host/src/server/workhub-coordination-coordinator.ts +++ b/packages/runtime-host/src/server/workhub-coordination-coordinator.ts @@ -884,7 +884,6 @@ function coordinationSummaryMessages(input: WorkHubCoordinationRecordInput): Sto turnId: input.turnId, ts: ts + 2, status: 'completed', - partialOutputRetained: false, }, ]; } diff --git a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts index 3fb0cf2906..880ffdefd3 100644 --- a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts +++ b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts @@ -283,7 +283,6 @@ function equivalentLegacyMessages(): StoredMessage[] { ts: ts + 8, status: 'completed', parentTurnId: 'parent-turn', - partialOutputRetained: true, }, ]; } @@ -395,9 +394,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { type: 'turn_state', status: 'completed', parentTurnId: 'parent-turn', - partialOutputRetained: true, }); - assert.equal(deriveTurnRecords(out.messages)[0]?.partialOutputRetained, true); assert.deepStrictEqual(out.diagnostics, []); }); @@ -1577,7 +1574,6 @@ describe('projectRuntimeEventsToStoredMessages', () => { status: 'failed', parentTurnId: 'parent-turn', errorClass: 'tool_failed', - partialOutputRetained: false, }, ]); assert.deepStrictEqual(out.diagnostics, []); @@ -1619,7 +1615,6 @@ describe('projectRuntimeEventsToStoredMessages', () => { status: 'failed', parentTurnId: 'parent-turn', errorClass: 'context_overflow', - partialOutputRetained: false, }, ); assert.deepStrictEqual(out.diagnostics, []); @@ -1680,7 +1675,6 @@ describe('projectRuntimeEventsToStoredMessages', () => { parentTurnId: 'parent-turn', abortedAt: ts + 9, abortSource: 'renderer.stop_button', - partialOutputRetained: false, }, ]); assert.deepStrictEqual(out.diagnostics, []); diff --git a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts index 8c8702e5c4..33f9d16349 100644 --- a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts +++ b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts @@ -91,7 +91,6 @@ test('repairs imported transcript turns into provider-neutral canonical history' turnId: 'turn-1', ts: externalTs, status: 'completed', - partialOutputRetained: true, }, ]; const session = await sessions.createImportedSession( @@ -152,7 +151,6 @@ test('repairs imported transcript turns into provider-neutral canonical history' turnId: 'turn-2', ts: session.createdAt + 3, status: 'completed', - partialOutputRetained: true, }, ]; const continuedRun = { @@ -271,7 +269,6 @@ test('an imported snapshot cutoff survives materialization as aborted', async () status: 'aborted', abortedAt: ts, abortSource: 'external_session_snapshot', - partialOutputRetained: true, }, ]; const session = await sessions.createImportedSession( @@ -333,7 +330,6 @@ test('does not import Host-handed-off transcript messages as synthetic runs', as turnId: 'host-turn', ts: 11, status: 'completed', - partialOutputRetained: false, }, ], { adapterId: 'test', sourceSessionId: 'host-session' }, @@ -457,7 +453,6 @@ test("converts Maka's own legacy transcript whole, and resumes an interrupted co turnId: 'turn-1', ts: ts + 5, status: 'completed', - partialOutputRetained: true, }, ]); @@ -533,7 +528,6 @@ test('converts an imported turn ahead of a run the Session already sent', async turnId: 'turn-old', ts: ts + 2, status: 'completed', - partialOutputRetained: true, }, ], { adapterId: 'claude-code', sourceSessionId: 'imported-source' }, @@ -656,7 +650,6 @@ test('startup recovery leaves an interrupted legacy conversion for the importer turnId: 'legacy-turn', ts: 30, status: 'completed', - partialOutputRetained: true, }, ]); const append = runtimeEvents.appendRuntimeEvent.bind(runtimeEvents); @@ -936,7 +929,6 @@ async function seedLegacyTurn(sessions: ReturnType) { turnId: 'turn-1', ts: ts + 2, status: 'completed', - partialOutputRetained: true, }, ]); return session; @@ -1081,7 +1073,6 @@ test('converts a legacy transcript larger than one page without reading it whole turnId: `turn-${turn}`, ts: ts + turn * 3 + 2, status: 'completed', - partialOutputRetained: true, }, ]); } diff --git a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts index 1ab07e2b3a..9fc4afd975 100644 --- a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts +++ b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts @@ -272,7 +272,6 @@ describe('SessionManager terminal ledger invariants', () => { (message) => message.type === 'turn_state' && message.turnId === 'turn-1', ); if (turnState?.type !== 'turn_state') throw new Error('failed turn_state was not projected'); - assert.equal(turnState.partialOutputRetained, false); assert.strictEqual(turnState.status, 'failed'); assert.strictEqual(turnState.errorClass, 'stream_truncated'); assert.deepEqual(turnState.retry, { decision: 'declined', because: 'side_effects' }); diff --git a/packages/runtime/src/__tests__/session-manager.test.ts b/packages/runtime/src/__tests__/session-manager.test.ts index f898f2ece5..5dafa68077 100644 --- a/packages/runtime/src/__tests__/session-manager.test.ts +++ b/packages/runtime/src/__tests__/session-manager.test.ts @@ -7414,7 +7414,6 @@ describe('SessionManager permission mode updates', () => { turnId: 'turn-1', ts: 103, status: 'completed', - partialOutputRetained: true, }, { type: 'user', id: 'imported-user-2', turnId: 'turn-2', ts: 104, text: 'Second question' }, { @@ -7431,7 +7430,6 @@ describe('SessionManager permission mode updates', () => { turnId: 'turn-2', ts: 106, status: 'completed', - partialOutputRetained: true, }, ]); @@ -7630,7 +7628,6 @@ describe('SessionManager permission mode updates', () => { turnId: 'turn-1', status: 'completed', statusSource: 'recorded', - partialOutputRetained: true, }, ]); assert.deepStrictEqual( @@ -7945,7 +7942,6 @@ describe('SessionManager permission mode updates', () => { turnId: 'turn-1', ts: 103, status: 'completed', - partialOutputRetained: true, }, ]; await store.appendMessages(session.id, legacyMessages); @@ -7993,7 +7989,6 @@ describe('SessionManager permission mode updates', () => { turnId: 'turn-1', ts: 103, status: 'completed', - partialOutputRetained: false, }, ]); }); @@ -8080,7 +8075,6 @@ describe('SessionManager permission mode updates', () => { ts: 103, status: 'failed', errorClass: 'tool_failed', - partialOutputRetained: true, }); assert.strictEqual(runtimeEvents.filter((event) => event.status === 'failed').length, 1); }); @@ -8396,13 +8390,11 @@ describe('SessionManager permission mode updates', () => { turnId: 'turn-1', status: 'completed', statusSource: 'recorded', - partialOutputRetained: true, }, { turnId: 'turn-2', status: 'running', statusSource: 'recorded', - partialOutputRetained: true, }, ]); }); @@ -8434,7 +8426,6 @@ describe('SessionManager permission mode updates', () => { turnId: header.turnId, ts: 101, status: 'running', - partialOutputRetained: false, }, ]); await runStore.appendRuntimeEvent( @@ -8537,7 +8528,6 @@ describe('SessionManager permission mode updates', () => { turnId: header.turnId, ts: 101, status: 'running', - partialOutputRetained: false, }, ]); await runStore.appendRuntimeEvent( @@ -10790,7 +10780,6 @@ describe('SessionManager permission mode updates', () => { status: 'aborted', abortedAt: 2, abortSource: 'renderer.stop_button', - partialOutputRetained: false, }, ); }); @@ -11295,7 +11284,6 @@ describe('SessionManager permission mode updates', () => { turnId: 'running-turn', ts: 11, status: 'running', - partialOutputRetained: false, }, ]); await store.appendMessages(waiting.id, [ @@ -11306,7 +11294,6 @@ describe('SessionManager permission mode updates', () => { turnId: 'waiting-turn', ts: 21, status: 'running', - partialOutputRetained: false, }, ]); await store.appendMessages(activeStuck.id, [ @@ -11323,7 +11310,6 @@ describe('SessionManager permission mode updates', () => { turnId: 'active-stuck-turn', ts: 31, status: 'running', - partialOutputRetained: false, }, ]); await store.appendMessages(failedThenCompleted.id, [ @@ -11340,7 +11326,6 @@ describe('SessionManager permission mode updates', () => { turnId: 'failed-completed-turn', ts: 33, status: 'running', - partialOutputRetained: false, }, { type: 'turn_state', @@ -11349,7 +11334,6 @@ describe('SessionManager permission mode updates', () => { ts: 34, status: 'failed', errorClass: 'tool_failed', - partialOutputRetained: false, }, { type: 'turn_state', @@ -11357,7 +11341,6 @@ describe('SessionManager permission mode updates', () => { turnId: 'failed-completed-turn', ts: 35, status: 'completed', - partialOutputRetained: false, }, ]); await store.appendMessages(activeDone.id, [ @@ -11368,7 +11351,6 @@ describe('SessionManager permission mode updates', () => { turnId: 'active-turn', ts: 31, status: 'completed', - partialOutputRetained: false, }, ]); @@ -14467,7 +14449,6 @@ async function seedRuntimeReadTurn(input: { turnId: input.turnId, ts: 103, status: 'completed', - partialOutputRetained: true, }, ]; const projectedMessages: StoredMessage[] = [ @@ -14492,7 +14473,6 @@ async function seedRuntimeReadTurn(input: { turnId: input.turnId, ts: 103, status: 'completed', - partialOutputRetained: true, }, ]; await input.store.appendMessages(input.sessionId, legacyMessages); @@ -14588,7 +14568,6 @@ async function seedRuntimeReadTurnWithHeader(input: { : {}), ...(input.header.branchOfTurnId ? { branchOfTurnId: input.header.branchOfTurnId } : {}), ...(input.header.parentSessionId ? { parentSessionId: input.header.parentSessionId } : {}), - partialOutputRetained: true, }, ]); await seedRuntimeRun(input.runStore, header, events); @@ -14849,7 +14828,6 @@ async function seedRunningTurn( turnId, ts: 10, status: 'running', - partialOutputRetained: false, }, ]); } diff --git a/packages/runtime/src/ai-sdk-turn.ts b/packages/runtime/src/ai-sdk-turn.ts index 83a2952199..5f49726698 100644 --- a/packages/runtime/src/ai-sdk-turn.ts +++ b/packages/runtime/src/ai-sdk-turn.ts @@ -2592,8 +2592,8 @@ export class AiSdkTurn { // Flush the in-flight step's partial text/thinking before the terminal // abort/error events. Earlier steps already flushed at their // `finish-step`; this keeps their and this step's streamed-out output on - // BOTH exits — user stop and provider error / watchdog timeout — so - // partialOutputRetained reflects what the user actually saw. + // BOTH exits — user stop and provider error / watchdog timeout — so the + // transcript keeps what the user actually saw. await flushStep().catch(() => {}); if (this.aborted) { queue.push({ diff --git a/packages/runtime/src/runtime-event-read-model.ts b/packages/runtime/src/runtime-event-read-model.ts index 113823f12b..fd01aecfec 100644 --- a/packages/runtime/src/runtime-event-read-model.ts +++ b/packages/runtime/src/runtime-event-read-model.ts @@ -1223,12 +1223,6 @@ function projectTerminalTurnState( } const abortSource = status === 'aborted' ? abortSourceFromRuntime(event) : undefined; const failureClass = status === 'failed' ? failureClassFromRuntimeEvent(event) : undefined; - const partialOutputRetained = messages.some( - (message) => - message.turnId === event.turnId && - ((message.type === 'assistant' && message.text.trim().length > 0) || - message.type === 'tool_result'), - ); messages.push({ type: 'turn_state', id: stableMessageId(event, state, 'turn_state'), @@ -1248,7 +1242,6 @@ function projectTerminalTurnState( ...(status === 'failed' && event.content?.kind === 'error' && event.content.retry ? { retry: event.content.retry } : {}), - partialOutputRetained, }); if (failureClass === 'tool_step_cap_reached') { messages.push({ diff --git a/packages/runtime/src/runtime-read-model.ts b/packages/runtime/src/runtime-read-model.ts index b402c9e2f6..177922d05f 100644 --- a/packages/runtime/src/runtime-read-model.ts +++ b/packages/runtime/src/runtime-read-model.ts @@ -285,7 +285,6 @@ function runningTurnRecords( turnId, status: 'running', statusSource: 'recorded', - partialOutputRetained: false, }); } return marked; diff --git a/packages/storage/src/claude-code-session-adapter.ts b/packages/storage/src/claude-code-session-adapter.ts index 5a491c35e3..0d5cb83b28 100644 --- a/packages/storage/src/claude-code-session-adapter.ts +++ b/packages/storage/src/claude-code-session-adapter.ts @@ -455,7 +455,6 @@ export function convertTranscript( status: 'aborted', abortedAt: turn.lastTs, abortSource: 'claude-code.interrupt', - partialOutputRetained: true, }); } else if (turn.failed) { messages.push({ @@ -465,7 +464,6 @@ export function convertTranscript( ts: turn.lastTs, status: 'failed', errorClass: 'claude_code_api_error', - partialOutputRetained: true, }); } else if (turn.terminalStop) { messages.push({ @@ -474,7 +472,6 @@ export function convertTranscript( turnId: turn.turnId, ts: turn.lastTs, status: 'completed', - partialOutputRetained: true, }); } else { messages.push({ @@ -485,7 +482,6 @@ export function convertTranscript( status: 'aborted', abortedAt: turn.lastTs, abortSource: EXTERNAL_SNAPSHOT_ABORT_SOURCE, - partialOutputRetained: true, }); } turn = undefined; diff --git a/packages/storage/src/codex-session-adapter.ts b/packages/storage/src/codex-session-adapter.ts index e9aa2e982c..16c621c4e7 100644 --- a/packages/storage/src/codex-session-adapter.ts +++ b/packages/storage/src/codex-session-adapter.ts @@ -451,7 +451,6 @@ function convertCodexRollout( ts: timestampFor(record), status: failed ? 'failed' : 'completed', ...(failed ? { errorClass: 'codex_error' } : {}), - partialOutputRetained: true, }); failedTurnIds.delete(turnId); if (activeTurnId === turnId) { @@ -472,7 +471,6 @@ function convertCodexRollout( status: 'aborted', abortedAt: normalizeEpochMs(payload.completed_at) ?? ts, abortSource: stringField(payload, 'reason') ?? 'codex', - partialOutputRetained: true, }); if (activeTurnId === turnId) { activeTurnId = undefined; diff --git a/packages/storage/src/opencode-session-adapter.ts b/packages/storage/src/opencode-session-adapter.ts index 4df11ac326..9fb2fc4e24 100644 --- a/packages/storage/src/opencode-session-adapter.ts +++ b/packages/storage/src/opencode-session-adapter.ts @@ -285,7 +285,6 @@ export function convertTranscript( ts: turn.lastTs, status: 'failed', errorClass: 'opencode_error', - partialOutputRetained: true, }); } else if (turn.aborted) { out.push({ @@ -296,7 +295,6 @@ export function convertTranscript( status: 'aborted', abortedAt: turn.lastTs, abortSource: EXTERNAL_SNAPSHOT_ABORT_SOURCE, - partialOutputRetained: true, }); } else if (turn.closed) { out.push({ @@ -305,7 +303,6 @@ export function convertTranscript( turnId: turn.turnId, ts: turn.lastTs, status: 'completed', - partialOutputRetained: true, }); } else { // A turn whose last assistant step asked for tools and never came back: @@ -319,7 +316,6 @@ export function convertTranscript( status: 'aborted', abortedAt: turn.lastTs, abortSource: EXTERNAL_SNAPSHOT_ABORT_SOURCE, - partialOutputRetained: true, }); } turn = undefined; diff --git a/packages/ui/src/__tests__/live-turn-projection.test.ts b/packages/ui/src/__tests__/live-turn-projection.test.ts index a78c95c49b..037160c972 100644 --- a/packages/ui/src/__tests__/live-turn-projection.test.ts +++ b/packages/ui/src/__tests__/live-turn-projection.test.ts @@ -716,7 +716,7 @@ describe('reconcileTerminalLiveTurn', () => { assert.equal(reconcileTerminalLiveTurn(steeringOnly, []), steeringOnly); assert.deepEqual(reconcileTerminalLiveTurn(withSteering, [{ type: 'turn_state', id: 'state-1', turnId: 'turn-1', ts: 3, - status: 'completed', partialOutputRetained: false, + status: 'completed', }]), toolOnly); }); @@ -874,7 +874,6 @@ describe('reconcileTerminalLiveTurn', () => { turnId: 'turn-1', ts: 4, status: 'completed', - partialOutputRetained: false, }, ]), { turnId: 'turn-1', @@ -1024,7 +1023,7 @@ describe('tool_result_preview live projection', () => { }, { type: 'turn_state', id: 'state-1', turnId: 'turn-1', ts: 3, - status: 'running', partialOutputRetained: true, + status: 'running', }, ], 'en'); const started = applyLiveTurnEvent(undefined, { diff --git a/packages/ui/src/__tests__/materialize.test.ts b/packages/ui/src/__tests__/materialize.test.ts index 4755a516d5..dbcfa56caa 100644 --- a/packages/ui/src/__tests__/materialize.test.ts +++ b/packages/ui/src/__tests__/materialize.test.ts @@ -390,7 +390,6 @@ describe("live content over persisted partial rows", () => { turnId: "t1", ts: 2, status: "running", - partialOutputRetained: false, }, { type: "assistant", @@ -446,7 +445,6 @@ describe("unfinished tools take their status from the turn", () => { turnId: "t1", ts: 2, status: "running", - partialOutputRetained: false, }, { type: "tool_call", @@ -470,7 +468,6 @@ describe("unfinished tools take their status from the turn", () => { turnId: "t1", ts: 2, status: "failed", - partialOutputRetained: false, }, { type: "tool_call", @@ -497,7 +494,6 @@ describe("live tool status over persisted", () => { turnId: "t1", ts: 2, status: "running", - partialOutputRetained: false, }, { type: "tool_call", @@ -544,7 +540,6 @@ describe("live tool status over persisted", () => { turnId: "t1", ts: 2, status: "failed", - partialOutputRetained: false, }, { type: "tool_call", @@ -599,7 +594,6 @@ describe("live tool status over persisted", () => { turnId: "t1", ts: 2, status: "running", - partialOutputRetained: false, }, { type: "tool_call", @@ -659,7 +653,6 @@ describe("live tool status over persisted", () => { turnId: "t1", ts: 2, status: "running", - partialOutputRetained: false, }, { type: "tool_call", diff --git a/packages/ui/src/__tests__/transcript-projection.test.ts b/packages/ui/src/__tests__/transcript-projection.test.ts index c0436e5162..fe39d9aeb4 100644 --- a/packages/ui/src/__tests__/transcript-projection.test.ts +++ b/packages/ui/src/__tests__/transcript-projection.test.ts @@ -418,7 +418,7 @@ describe('turn identity moves across structural change classes', () => { const base: StoredMessage[] = [ { type: 'user', id: 'u1', turnId: 'turn-1', ts: 1, text: 'ask' }, { type: 'assistant', id: 'a1', turnId: 'turn-1', ts: 4, text: 'answer', modelId: 'model-1' }, - { type: 'turn_state', id: 's1', turnId: 'turn-1', ts: 5, status: 'completed', partialOutputRetained: false }, + { type: 'turn_state', id: 's1', turnId: 'turn-1', ts: 5, status: 'completed' }, ]; const cases: Array<{ @@ -435,7 +435,7 @@ describe('turn identity moves across structural change classes', () => { field: 'status', refresh: [ ...base.slice(0, 2), - { type: 'turn_state', id: 's1', turnId: 'turn-1', ts: 5, status: 'failed', partialOutputRetained: false }, + { type: 'turn_state', id: 's1', turnId: 'turn-1', ts: 5, status: 'failed' }, ], }, { From 3a4371a7450b97114ee05cd690b5b4ccadf91779 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 22:31:41 +0800 Subject: [PATCH 30/32] fix(core): keep retired keys readable where the shape is exact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `hasExactShape` rejects any key outside `allowed`, so removing a field from a shape stops every stored record that carries it from decoding at all. Dropping `partialOutputRetained` did exactly that: rows written by every released build failed `decodeMessage`, and a Session made of them could no longer be converted or displayed. Nothing was lost on disk, but the upgrade path was. What a shape emits may shrink freely; what it accepts may only grow. Those are two contracts, and `defineObjectShape()` derives them both from one type — so a type change and a persisted-format change share one syntax, and `Covers` makes the compatible option a type error inside the helper. Name the second contract instead: a third `retired` argument, accepted on read and dropped by `pickShape`. The obligation already had three hand-built copies outside the helper — `withoutRetiredSubagentRuntimeKeys`, and the derived `LAST_REQUEST_ANCHOR_DECODE_SHAPE` and `CONTEXT_BUDGET_SHAPE` — all of which this removes. The field returns to no consumer and to no wire. Generated-by: Claude Code --- .../__tests__/session-retired-fields.test.ts | 38 +++++++++++ packages/core/src/record-schema.ts | 11 ++- packages/core/src/session.ts | 26 ++----- packages/core/src/usage-record-schema.ts | 68 ++++++++----------- 4 files changed, 83 insertions(+), 60 deletions(-) create mode 100644 packages/core/src/__tests__/session-retired-fields.test.ts diff --git a/packages/core/src/__tests__/session-retired-fields.test.ts b/packages/core/src/__tests__/session-retired-fields.test.ts new file mode 100644 index 0000000000..ad03037dbb --- /dev/null +++ b/packages/core/src/__tests__/session-retired-fields.test.ts @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { decodeStoredMessage, type StoredMessage } from '../session.js'; + +const decode = (row: Record): StoredMessage => + decodeStoredMessage(row as unknown as Parameters[0]); + +const turnState = { type: 'turn_state', id: 's1', turnId: 't1', ts: 1, status: 'failed' } as const; + +test('decodes turn states written before partialOutputRetained was retired', () => { + for (const retained of [true, false]) { + assert.deepEqual(decode({ ...turnState, partialOutputRetained: retained }), turnState); + } + assert.deepEqual(decode({ ...turnState }), turnState); +}); + +test('still rejects turn-state keys no released writer produced', () => { + assert.throws(() => decode({ ...turnState, unknownFutureKey: true })); +}); diff --git a/packages/core/src/record-schema.ts b/packages/core/src/record-schema.ts index c451ab52b3..f1564791a5 100644 --- a/packages/core/src/record-schema.ts +++ b/packages/core/src/record-schema.ts @@ -38,11 +38,18 @@ type Covers = export interface ExactObjectShape { readonly required: readonly string[]; readonly allowed: ReadonlySet; + readonly retired?: ReadonlySet; } /** * Defines a JSON object shape while making schema additions a type error until * both the required and optional key lists are updated. + * + * `retired` names keys older writers persisted that this type no longer has. + * They are accepted on read and dropped by {@link pickShape}, so what the shape + * emits may shrink freely while what it accepts only grows. Removing a key from + * `optional` without listing it here makes every stored record carrying it fail + * validation outright. */ export function defineObjectShape() { return < @@ -51,9 +58,11 @@ export function defineObjectShape() { >( required: Required & Covers, Required[number]>, optional: Optional & Covers, Optional[number]>, + retired: readonly string[] = [], ): ExactObjectShape => ({ required, allowed: new Set([...required, ...optional]), + retired: new Set(retired), }); } @@ -64,7 +73,7 @@ export function isRecord(value: unknown): value is Record { export function hasExactShape(value: Record, shape: ExactObjectShape): boolean { return ( shape.required.every((key) => Object.hasOwn(value, key)) && - Object.keys(value).every((key) => shape.allowed.has(key)) + Object.keys(value).every((key) => shape.allowed.has(key) || shape.retired?.has(key) === true) ); } diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts index 4796afb437..c4a5285eac 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session.ts @@ -41,6 +41,7 @@ import { isFiniteNumber, isOptionalString, isRecord, + pickShape, } from './record-schema.js'; import { isPermissionDecisionFields } from './interaction-record-schema.js'; import { isTokenUsageFields, type TokenUsageFields } from './usage-record-schema.js'; @@ -473,17 +474,8 @@ const SUBAGENT_SESSION_RUNTIME_SHAPE = defineObjectShape 'categoryPolicy', ], ['presetId'], + ['permissionCeiling'], ); - -/** - * Keys older child sessions wrote that this type no longer has. - * - * `hasExactShape` rejects unknown keys, so without this a record written before - * the key was dropped would fail validation and make the whole child Session - * unreadable. Nothing reads the values, and they stay in the stored JSON as - * written — this only stops their presence from being treated as corruption. - */ -const RETIRED_SUBAGENT_RUNTIME_KEYS: readonly string[] = ['permissionCeiling']; const SUBAGENT_SESSION_SPAWN_IDENTITY_SHAPE = defineObjectShape()( ['schemaVersion', 'requestFingerprint', 'initialTurnId', 'initialRunId'], [], @@ -531,20 +523,11 @@ export function isSubagentSessionParent(value: unknown): value is SubagentSessio return swarmValid && graphValid && !(value.swarm && value.graph); } -function withoutRetiredSubagentRuntimeKeys( - value: Record, -): Record { - if (!RETIRED_SUBAGENT_RUNTIME_KEYS.some((key) => Object.hasOwn(value, key))) return value; - return Object.fromEntries( - Object.entries(value).filter(([key]) => !RETIRED_SUBAGENT_RUNTIME_KEYS.includes(key)), - ); -} - /** Strict decoder guard for the persisted child execution snapshot. */ export function isSubagentSessionRuntime(value: unknown): value is SubagentSessionRuntime { if ( !isRecord(value) || - !hasExactShape(withoutRetiredSubagentRuntimeKeys(value), SUBAGENT_SESSION_RUNTIME_SHAPE) || + !hasExactShape(value, SUBAGENT_SESSION_RUNTIME_SHAPE) || value.schemaVersion !== SUBAGENT_SESSION_RUNTIME_SCHEMA_VERSION || !Number.isSafeInteger(value.definitionVersion) || (value.definitionVersion as number) < 1 || @@ -1280,6 +1263,7 @@ const TURN_STATE_MESSAGE_SHAPE = defineObjectShape()( 'errorClass', 'retry', ], + ['partialOutputRetained'], ); const WORKHUB_DELEGATION_ASSIGNED_MESSAGE_SHAPE = defineObjectShape()( @@ -1560,7 +1544,7 @@ function decodeMessage( isOptionalString(message.errorClass) && (message.retry === undefined || isModelRetryDecision(message.retry)) ) - return message as unknown as TurnStateMessage; + return pickShape(message as unknown as TurnStateMessage, TURN_STATE_MESSAGE_SHAPE); break; case 'workhub_coordination': if (isWorkHubCoordinationMessage(message)) { diff --git a/packages/core/src/usage-record-schema.ts b/packages/core/src/usage-record-schema.ts index f3c2d0e5cc..64acce4967 100644 --- a/packages/core/src/usage-record-schema.ts +++ b/packages/core/src/usage-record-schema.ts @@ -70,34 +70,6 @@ const COMPACTION_DECISION_SHAPE = defineObjectShape()( - [ - 'enabled', - 'estimatedTokensBefore', - 'estimatedTokensAfter', - 'keptTurns', - 'droppedTurns', - 'keptEvents', - 'droppedEvents', - ], - [ - 'policyName', - 'prunedToolResults', - 'prunedToolResultEstimatedTokensBefore', - 'prunedToolResultEstimatedTokensAfter', - 'archivePlaceholders', - 'archiveWriteFailures', - 'unarchivedToolResults', - 'archivePlaceholderReasonCounts', - 'activePrunedToolResults', - 'activeSupersededToolResults', - 'activeDuplicateToolResults', - 'activeArchiveFailures', - 'activeEstimatedTokensSaved', - 'compactionDecisions', - ], -); - /** * Keys written by retired context-budget implementations. They remain * accepted only so persisted usage records stay readable; current code cannot @@ -176,10 +148,34 @@ const RETIRED_CONTEXT_BUDGET_KEYS = [ 'historyRewriteGate', ] as const; -const CONTEXT_BUDGET_SHAPE = { - required: CURRENT_CONTEXT_BUDGET_SHAPE.required, - allowed: new Set([...CURRENT_CONTEXT_BUDGET_SHAPE.allowed, ...RETIRED_CONTEXT_BUDGET_KEYS]), -}; +const CONTEXT_BUDGET_SHAPE = defineObjectShape()( + [ + 'enabled', + 'estimatedTokensBefore', + 'estimatedTokensAfter', + 'keptTurns', + 'droppedTurns', + 'keptEvents', + 'droppedEvents', + ], + [ + 'policyName', + 'prunedToolResults', + 'prunedToolResultEstimatedTokensBefore', + 'prunedToolResultEstimatedTokensAfter', + 'archivePlaceholders', + 'archiveWriteFailures', + 'unarchivedToolResults', + 'archivePlaceholderReasonCounts', + 'activePrunedToolResults', + 'activeSupersededToolResults', + 'activeDuplicateToolResults', + 'activeArchiveFailures', + 'activeEstimatedTokensSaved', + 'compactionDecisions', + ], + RETIRED_CONTEXT_BUDGET_KEYS, +); const PROMPT_SEGMENT_KINDS = new Set([ 'system_prompt', @@ -323,17 +319,13 @@ export interface LastRequestAnchor { const LAST_REQUEST_ANCHOR_SHAPE = defineObjectShape()( ['inputTokens'], ['outputTokens', 'modelId', 'connectionId'], + ['payloadChars'], ); -const RETIRED_LAST_REQUEST_ANCHOR_KEYS = ['payloadChars'] as const; -const LAST_REQUEST_ANCHOR_DECODE_SHAPE = { - required: LAST_REQUEST_ANCHOR_SHAPE.required, - allowed: new Set([...LAST_REQUEST_ANCHOR_SHAPE.allowed, ...RETIRED_LAST_REQUEST_ANCHOR_KEYS]), -}; export function isLastRequestAnchor(value: unknown): value is LastRequestAnchor { return ( isRecord(value) && - hasExactShape(value, LAST_REQUEST_ANCHOR_DECODE_SHAPE) && + hasExactShape(value, LAST_REQUEST_ANCHOR_SHAPE) && isFiniteNumber(value.inputTokens) && value.inputTokens > 0 && (value.outputTokens === undefined || From e2f304fcef1a0daefb88cca1b3cf844b56b721bb Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 22:31:50 +0800 Subject: [PATCH 31/32] fix(runtime-host): page overlapping Turns in sequence order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A page resumes from one record's sequence and drops everything on the other side of it, so a scalar cursor is only sound over a stream totally ordered by that sequence. The walk emitted Turn by Turn in opening order, which coincides with sequence order only while Turns occupy disjoint ordinal ranges. A nested run breaks that: the inner Turn is taken first, and the outer Turn's later rows are then excluded by the next page's filter and never revisited. One sweep returned them, page-size-1 did not — which is what located the defect in resumption rather than in selection. The invariant this relied on is the one this PR withdrew from the store, because a fixture interleaves two visible root runs on purpose. So make emission monotone instead of assuming disjointness: drain Turns whose ordinal ranges overlap as one cluster, sorted. A Session without overlap yields a cluster of exactly one Turn, and the lookahead read that ends a cluster is the next cluster's first Turn. Generated-by: Claude Code --- .../session-transcript-reader.test.ts | 88 +++++++++++++++++++ .../src/server/session-transcript-reader.ts | 61 +++++++++---- 2 files changed, 131 insertions(+), 18 deletions(-) diff --git a/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts b/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts index 22406b77fc..b288e834ec 100644 --- a/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts +++ b/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts @@ -749,6 +749,94 @@ test('reads the WorkHub Coordination transcript from its own rows', async () => ); }); +test('pages a nested Turn the same way a single sweep reads it', async () => { + const base = await mkdtemp(join(tmpdir(), 'maka-nested-paging-')); + const capability = await resolveStorageRoot({ path: join(base, 'root'), kind: 'interactive' }); + const owner = await tryAcquireInteractiveRootOwner(capability); + assert.ok(owner); + try { + const stores = await openInteractiveExecutionStoresForWrite(owner.lease); + const session = await stores.sessionStore.create({ + cwd: capability.canonicalPath, + llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', + llmConnectionSlug: 'fake', + model: 'fake-model', + permissionMode: 'ask', + }); + let counter = 0; + const append = (runId: string, overrides: Partial) => + stores.runtimeEventStore.appendRuntimeEvent( + session.id, + runId, + runtimeEvent(session.id, { + id: `${runId}-event-${counter++}`, + invocationId: runId, + runId, + turnId: `turn-${runId}`, + ts: counter, + ...overrides, + }), + ); + const text = (runId: string, body: string) => + append(runId, { role: 'model', author: 'agent', content: { kind: 'text', text: body } }); + + // `outer` opens first and ends last; `inner` opens and ends inside it, so + // the two Turns share a stretch of the Session's ordinals. + await seedInvocation(stores.runtimeEventStore, { + sessionId: session.id, + runId: 'outer', + turnId: 'turn-outer', + openedAt: 0, + }); + await text('outer', 'outer before'); + await seedInvocation(stores.runtimeEventStore, { + sessionId: session.id, + runId: 'inner', + turnId: 'turn-inner', + openedAt: 1, + }); + for (let index = 0; index < 4; index++) await text('inner', `inner ${index}`); + await append('inner', { status: 'completed', actions: { endInvocation: true } }); + await text('outer', 'outer after'); + await append('outer', { status: 'completed', actions: { endInvocation: true } }); + + const read = createSessionTranscriptReader({ + stores, + canonicalPermissionOutcomes: { readPermissionOutcome: async () => undefined }, + }); + const throughSequence = await read.readDurableHighWater(session.id); + + for (const direction of ['older', 'newer'] as const) { + const sweep = await read.readDurablePage(session.id, { + direction, + throughSequence, + maxBytes: 1 << 20, + maxMessages: 64, + }); + const swept = sweep.fragments.map((fragment) => fragment.sequence); + + const paged: number[] = []; + let position: number | undefined; + for (let page = 0; page < 32; page++) { + const result = await read.readDurablePage(session.id, { + direction, + throughSequence, + ...(position === undefined ? {} : { position }), + maxBytes: 1 << 20, + maxMessages: 1, + }); + if (result.fragments.length === 0) break; + paged.push(...result.fragments.map((fragment) => fragment.sequence)); + if (result.next?.position === undefined || result.next.position === null) break; + position = result.next.position; + } + assert.deepEqual(paged, swept, direction); + } + } finally { + await rm(base, { recursive: true, force: true }); + } +}); + function runtimeEvent(sessionId: string, overrides: Partial): RuntimeEvent { return { id: 'event-1', diff --git a/packages/runtime-host/src/server/session-transcript-reader.ts b/packages/runtime-host/src/server/session-transcript-reader.ts index fe318b36f0..5977acf25b 100644 --- a/packages/runtime-host/src/server/session-transcript-reader.ts +++ b/packages/runtime-host/src/server/session-transcript-reader.ts @@ -287,28 +287,53 @@ function createDurableLedgerTranscriptReader(input: { if (throughSequence === null) return; const position = request.position ?? (request.direction === 'older' ? throughSequence : 0); const throughOrdinal = ordinalOf(throughSequence); + const older = request.direction === 'older'; + const readTurnAt = async (at: number): Promise => + at < 0 || at > throughOrdinal + ? undefined + : ( + await readTurns(sessionId, { + direction: request.direction, + throughOrdinal, + position: at, + }) + )[0]; let ordinal = ordinalOf(position); let walked = 0; + let carried: RuntimeTranscriptInvocation | undefined; while (ordinal >= 0 && ordinal <= throughOrdinal) { - const turns = await readTurns(sessionId, { - direction: request.direction, - throughOrdinal, - position: ordinal, - }); - if (turns.length === 0) return; - for (const turn of turns) { - if (request.maxTurns !== undefined && walked >= request.maxTurns) return; - walked += 1; - const records = (await projectTurn(turn)).filter( - ({ sequence }) => - sequence <= throughSequence && - (request.direction === 'older' ? sequence <= position : sequence >= position), - ); - if (request.direction === 'older') records.reverse(); - yield* records; + const first = carried ?? (await readTurnAt(ordinal)); + carried = undefined; + if (first === undefined) return; + if (request.maxTurns !== undefined && walked >= request.maxTurns) return; + // A page resumes from one record's sequence and drops everything the other + // side of it, so what this yields has to be monotone in sequence. Turns + // whose ordinal ranges overlap — a nested run inside its parent — are + // therefore drained together instead of one after the other. + const cluster = [first]; + let low = first.firstOrdinal; + let high = first.lastOrdinal; + for (;;) { + const next = await readTurnAt(older ? low - 1 : high + 1); + if (next === undefined) break; + if (older ? next.lastOrdinal < low : next.firstOrdinal > high) { + carried = next; + break; + } + cluster.push(next); + low = Math.min(low, next.firstOrdinal); + high = Math.max(high, next.lastOrdinal); } - const edge = turns.at(-1)!; - ordinal = request.direction === 'older' ? edge.firstOrdinal - 1 : edge.lastOrdinal + 1; + walked += cluster.length; + const records = (await Promise.all(cluster.map(projectTurn))) + .flat() + .filter( + ({ sequence }) => + sequence <= throughSequence && (older ? sequence <= position : sequence >= position), + ) + .sort((a, b) => (older ? b.sequence - a.sequence : a.sequence - b.sequence)); + yield* records; + ordinal = older ? low - 1 : high + 1; } }; From 4e670b7abf3d3aecae71a9c643830a1549055a02 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 7 Sep 2026 23:33:42 +0800 Subject: [PATCH 32/32] fix(storage): keep an undecodable payload from wedging the ledger schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `runtime_events_terminal` is a partial index, and SQLite evaluates a partial index's predicate over every row while building it. `json_extract` fails the whole statement on a malformed payload, so one such row is enough to abort migration 18 — which runs inside the upgrade's single write transaction, so the rollback also undoes the version bump and the next open tries, and fails, the same way. The store never reopens, and retrying cannot repair it. No path was found by which a released build writes a malformed payload: every writer encodes through JSON.stringify. But the costs are not symmetric — being wrong about that costs a permanently unopenable store, and being wrong the other way costs one clause. The guard belongs in the shared predicate rather than in the index alone, so the index and the four queries that read it keep identical text and the index stays usable; the semantics agree, since a row that cannot be decoded is not a terminal fact. Reported by review, verified here: one malformed row fails `CREATE INDEX` with `malformed JSON` and leaves `user_version` at 17, while the guarded predicate builds and selects the same rows over `{"status":"completed"}`, `{"status":null}` and a payload without the key. This also restores main's `tool-result-archive-evidence` test to its own form: the malformed write it makes to reach the reader's corrupt branch is accepted again. Generated-by: Claude Code --- .../__tests__/sqlite-runtime-schema.test.ts | 27 +++++++++++++++++++ .../tool-result-archive-evidence.test.ts | 8 +----- .../storage/src/runtime-transcript-query.ts | 17 +++++++++--- 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts b/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts index 700f38b512..cebfb41c80 100644 --- a/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts +++ b/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts @@ -201,4 +201,31 @@ describe('SQLite runtime schema migration', () => { db.close(); } }); + + it('builds the terminal index over a ledger holding an undecodable payload', () => { + const db = new DatabaseSync(':memory:'); + try { + migrateSqliteRuntimeDatabase(db); + db.prepare( + 'INSERT INTO runtime_events(event_id, session_id, invocation_id, run_id, turn_id, event_seq, event_kind, payload_json, committed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', + ).run('event', 'session', 'invocation', 'run', 'turn', 1, 'text', '{', 1); + // A partial index is rebuilt by evaluating its predicate over every row, + // so one such row would otherwise fail this migration — and the failure + // rolls the version back, leaving the next open to fail the same way. + db.exec( + `DROP INDEX runtime_events_terminal; PRAGMA user_version = ${SQLITE_RUNTIME_SCHEMA_VERSION - 1}`, + ); + migrateSqliteRuntimeDatabase(db); + + assert.equal( + (db.prepare('PRAGMA user_version').get() as { user_version: number }).user_version, + SQLITE_RUNTIME_SCHEMA_VERSION, + ); + assert.ok( + db.prepare("SELECT 1 FROM sqlite_master WHERE name = 'runtime_events_terminal'").get(), + ); + } finally { + db.close(); + } + }); }); diff --git a/packages/storage/src/__tests__/tool-result-archive-evidence.test.ts b/packages/storage/src/__tests__/tool-result-archive-evidence.test.ts index db12e458c0..7d83b7850c 100644 --- a/packages/storage/src/__tests__/tool-result-archive-evidence.test.ts +++ b/packages/storage/src/__tests__/tool-result-archive-evidence.test.ts @@ -260,13 +260,7 @@ test('invalid event JSON and transition envelopes remain corrupt', async (t) => const saved = f.db .prepare("SELECT payload_json FROM runtime_events WHERE event_id = 'response'") .get()!; - // Syntactically malformed JSON cannot be stored at all: the terminal-event - // index reads the payload, so SQLite refuses the write. An undecodable - // payload reaches the same reader branch. - assert.throws(() => - f.db.exec("UPDATE runtime_events SET payload_json = '{' WHERE event_id = 'response'"), - ); - f.db.exec(`UPDATE runtime_events SET payload_json = '{"nope":1}' WHERE event_id = 'response'`); + f.db.exec("UPDATE runtime_events SET payload_json = '{' WHERE event_id = 'response'"); assert.deepEqual(await f.reader.read({ sessionId: 'session', runtimeEventId: 'response' }), { ok: false, reason: 'corrupt', diff --git a/packages/storage/src/runtime-transcript-query.ts b/packages/storage/src/runtime-transcript-query.ts index 1e453d4234..7620d2ff6a 100644 --- a/packages/storage/src/runtime-transcript-query.ts +++ b/packages/storage/src/runtime-transcript-query.ts @@ -21,10 +21,21 @@ import type { DatabaseSync } from 'node:sqlite'; import { decodeRuntimeEvent, type RuntimeEvent } from '@maka/core/runtime-event'; import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; -/** SQL counterpart of isTerminalRuntimeEvent; shared with the ledger store. */ +/** + * SQL counterpart of isTerminalRuntimeEvent; shared with the ledger store. + * + * The `json_valid` guard is what keeps this usable as a partial index: SQLite + * evaluates the index predicate over every row while building it, and + * `json_extract` on a malformed payload fails the whole statement. That would + * abort the migration that creates the index, roll back its version bump, and + * leave the next open to try — and fail — again. + */ export const TERMINAL_RUNTIME_EVENT_SQL = `( - json_extract(payload_json, '$.actions.endInvocation') = 1 - OR json_extract(payload_json, '$.status') IN ('completed', 'failed', 'aborted', 'cancelled') + json_valid(payload_json) + AND ( + json_extract(payload_json, '$.actions.endInvocation') = 1 + OR json_extract(payload_json, '$.status') IN ('completed', 'failed', 'aborted', 'cancelled') + ) )`; /**