diff --git a/hypaware-core/plugins-workspace/openclaw/src/backfill.js b/hypaware-core/plugins-workspace/openclaw/src/backfill.js index 10b1b728..1c13ef24 100644 --- a/hypaware-core/plugins-workspace/openclaw/src/backfill.js +++ b/hypaware-core/plugins-workspace/openclaw/src/backfill.js @@ -34,12 +34,13 @@ import { isPlainObject, sha256Hex, stringValue } from 'hypaware/core/util' * neither consumer may hold its own parse). * * The session file is the authoritative record of a turn: it already carries - * one flattened per-message record with the message's own native id, its - * timestamp, and for an assistant message the model, provider, api, stop - * reason, and usage. So this module builds an `AiGatewayProjectedExchange` - * straight off those fields. It deliberately does NOT route through - * `anthropicMessages()` (projector.js): those parse a wire request/response - * pair, and there is no wire pair here to reconstruct. + * one per-message record with the message's own native id and timestamp on + * the record line, and its role, content and (for an assistant message) the + * model, provider, api, stop reason and usage in the nested `message` + * envelope the LLP 0158 reader normalizes. So this module builds an + * `AiGatewayProjectedExchange` straight off those fields. It deliberately + * does NOT route through `anthropicMessages()` (projector.js): those parse a + * wire request/response pair, and there is no wire pair here to reconstruct. * * That directness is what makes R11 true by construction rather than by * coincidence. A backfilled row carries the record's own `message_id`, so it @@ -492,10 +493,13 @@ function projectedExchangeFromSession(args) { * @returns {AiGatewayProjectedMessage | undefined} */ function projectedMessageFromRecord(message) { - const record = message.record - const rawRole = stringValue(record.role) + // Through the LLP 0158 reader's normalized fields, never `message.record`: + // OpenClaw nests `role`/`content` under the record line's `message` + // envelope, so the top-level read this used to do found neither and dropped + // every record of every real session (#543). + const rawRole = stringValue(message.role) if (!rawRole) return undefined - const content = record.content + const content = message.content // The gateway drops a message whose content normalizes to no blocks; doing it // here keeps `messages_projected` honest about what was actually emitted. if (typeof content === 'string' ? content.length === 0 : !Array.isArray(content) || content.length === 0) { @@ -563,11 +567,12 @@ function messageAttributes(message) { * * Each name is read through a small alias list rather than one spelling: the * session file's `usage` block is OpenClaw's own normalization of whatever its - * provider returned, and which spelling it settles on could not be verified at - * implementation time (no live OpenClaw install was reachable). Accepting the - * Anthropic wire spelling, the OpenAI wire spelling, and the camelCase - * normalized spelling costs a lookup and means an unverified guess cannot - * silently zero out a session's tokens. + * provider returned. A live install has since confirmed the camelCase + * spelling (`input`/`output`/`cacheRead`/`cacheWrite`, LLP 0158 Context), so + * the alias list is no longer a hedge against an unverified guess; it is kept + * because the block is still the client's normalization of a provider payload + * and a version that passed a wire spelling through would otherwise silently + * zero out a session's tokens. * * @ref LLP 0035#one-carrier [implements]: usage rides the assistant record * only, so it lands once per response diff --git a/hypaware-core/plugins-workspace/openclaw/src/session_file.js b/hypaware-core/plugins-workspace/openclaw/src/session_file.js index 6064c59c..11dac6cd 100644 --- a/hypaware-core/plugins-workspace/openclaw/src/session_file.js +++ b/hypaware-core/plugins-workspace/openclaw/src/session_file.js @@ -247,12 +247,23 @@ export async function readOpenclawSessionMessages(filePath) { * Parse one JSONL line into an {@link OpenclawSessionMessage}, or * `undefined` when the line is not a `type: "message"` record. Guards the * same way the header parse does (rules 1-3 above), applied to the fields - * LLP 0158's Context names as present on a message envelope: `id` and a - * timestamp on every message, `model`/`provider`/`api`/`stopReason`/`usage` - * on an assistant one. Every field this function does not normalize (role, - * content blocks, and anything else OpenClaw writes) is still reachable - * through `record`, the untouched envelope, so a caller that needs one is - * not blocked on this reader growing a field for it. + * LLP 0158's Context names, at the level it names them at: `id` on the + * record line, a timestamp, `role` and `content` on every message, and + * `model`/`provider`/`api`/`stopReason`/`usage` on an assistant one. Every + * field this function does not normalize is still reachable through + * `record`, the untouched record line, so a caller that needs one is not + * blocked on this reader growing a field for it. `record` is the LINE, + * though, not the message: `parentId` is on it, but a message-level field + * this reader does not normalize (`idempotencyKey`, `toolCallId`) is at + * `record.message`. Naming the level is the whole point here, since reading + * a message field off the line is exactly #543. + * + * `role` and `content` are normalized here rather than left to `record` + * precisely because their location is the non-obvious part + * ({@link openclawMessageEnvelope}): both consumers used to reach into + * `record.role`/`record.content` themselves, so both read one level too high + * and both dropped every real session (#543). A field whose *address* is the + * thing that is easy to get wrong belongs to the one reader. * * @param {string} line * @returns {OpenclawSessionMessage | undefined} @@ -261,24 +272,124 @@ function parseOpenclawSessionMessage(line) { const row = parseMaybeJson(line) if (!isPlainObject(row)) return undefined if (row.type !== 'message') return undefined + const envelope = openclawMessageEnvelope(row) /** @type {OpenclawSessionMessage} */ const message = { record: row } - const id = nonBlankString(row.id) + // `id` is the one field read the other way round, and deliberately so. + // LLP 0158's verified shape puts message IDENTITY on the record line; the + // nested envelope is OpenClaw's normalization of a provider response, and + // the day a version starts copying the provider's own `msg_...` id into it, + // envelope-first would silently repoint every `message_id` (and so every + // `part_id`) that backfill and settlement agree on. Already-committed rows + // would stop deduping against the new ones and the history would double, + // with nothing raised anywhere. The envelope stays the fallback, so a record + // that states identity only there is still read. + const id = nonBlankString(row.id) ?? nonBlankString(envelope.id) if (id !== undefined) message.id = id - const timestampMs = parseTimestampMs(row.timestamp) + const timestampMs = messageField(envelope, row, 'timestamp', parseTimestampMs) if (timestampMs !== undefined) message.timestampMs = timestampMs - const model = nonBlankString(row.model) + const role = messageField(envelope, row, 'role', nonBlankString) + if (role !== undefined) message.role = role + const content = messageField(envelope, row, 'content', statedValue) + if (content !== undefined) message.content = content + const model = messageField(envelope, row, 'model', nonBlankString) if (model !== undefined) message.model = model - const provider = nonBlankString(row.provider) + const provider = messageField(envelope, row, 'provider', nonBlankString) if (provider !== undefined) message.provider = provider - const api = nonBlankString(row.api) + const api = messageField(envelope, row, 'api', nonBlankString) if (api !== undefined) message.api = api - const stopReason = nonBlankString(row.stopReason) + const stopReason = messageField(envelope, row, 'stopReason', nonBlankString) if (stopReason !== undefined) message.stopReason = stopReason - if (isPlainObject(row.usage)) message.usage = row.usage + const usage = messageField(envelope, row, 'usage', plainObject) + if (usage !== undefined) message.usage = usage return message } +/** + * The message envelope of a `type: "message"` record: the nested `message` + * object, not the record line. + * + * A record line states only what identifies and positions the message + * (`id`, `parentId`, `timestamp`, `type`); the message itself - `role`, + * `content`, and for an assistant turn `model`, `provider`, `api`, + * `stopReason`, `usage` - is one level down under `message`. Read at the top + * level every one of those fields is absent, which is not a loud failure: + * `provider` reads `undefined`, the backfill allowlist resolves the record to + * `unknown`, and the run reports a clean "0 rows" for a session it simply + * failed to read (#543). Fail-closed exclusion and a parse miss are + * indistinguishable at that seam, so the address has to be right here. + * + * The record line is the fallback, not a second address to prefer: a record + * that nests no `message` object states its fields on the line itself, and + * reading them there is better than reading a message with no role and no + * content. A field the envelope does state is never overridden by a + * same-named field on the line. The fallback is per FIELD, not per record + * ({@link messageField}), so a record that nests a partial envelope still + * recovers the rest of its fields from the line rather than reading as a + * message that is missing them. `id` is the single documented exception, + * read line-first because it is identity rather than content + * ({@link parseOpenclawSessionMessage}). + * + * @ref LLP 0158#decision [implements]: where a message's fields live is part + * of the one reader's knowledge, not something each consumer re-derives + * @param {Record} row + * @returns {Record} + */ +function openclawMessageEnvelope(row) { + return isPlainObject(row.message) ? row.message : row +} + +/** + * One message field, read from the envelope first and the record line + * second. Not a substitution across fields (rule 1): it is the same field + * name, looked for at the two levels one record can state it at. + * + * `stated` is the field's own present-value test, and it runs at BOTH + * levels before the fallback decides. Running it only on the result would + * let a blank or wrong-typed envelope value shadow a usable one on the + * line, which would make that value absent (rule 3) and load-bearing at the + * same time: a nested `provider: " "` beside a line-level + * `provider: "anthropic"` would resolve the record to `unknown` and the + * backfill allowlist would exclude it fail-closed, the same silent drop + * #543 was. If a level does not state the field, it does not get a vote. + * + * @template T + * @param {Record} envelope + * @param {Record} row + * @param {string} key + * @param {(value: unknown) => T | undefined} stated + * @returns {T | undefined} + */ +function messageField(envelope, row, key, stated) { + const fromEnvelope = stated(envelope[key]) + return fromEnvelope !== undefined ? fromEnvelope : stated(row[key]) +} + +/** + * `content`'s present-value test. Unlike every other normalized field it has + * no single shape to check: OpenClaw writes a string on some turns and a + * block array on others, and both consumers already accept either, so the + * value passes through as written. Only `null` is refused, so a nulled-out + * envelope field counts as unstated and the record line still gets its turn. + * + * @param {unknown} value + * @returns {unknown} + */ +function statedValue(value) { + return value === null ? undefined : value +} + +/** + * A plain object, else `undefined`: `usage`'s present-value test, so a + * non-object `usage` reads as absent at whichever level wrote it. + * + * @param {unknown} value + * @returns {Record | undefined} + */ +function plainObject(value) { + return isPlainObject(value) ? value : undefined +} + /** * A non-blank string, byte-identical, else `undefined`. Stricter than * `stringValue` (core util) on purpose: a whitespace-only value matches no diff --git a/hypaware-core/plugins-workspace/openclaw/src/settle.js b/hypaware-core/plugins-workspace/openclaw/src/settle.js index 7c1c1de4..dbfb7808 100644 --- a/hypaware-core/plugins-workspace/openclaw/src/settle.js +++ b/hypaware-core/plugins-workspace/openclaw/src/settle.js @@ -385,7 +385,7 @@ function buildOpenclawSessionIndex(candidate, header, messages) { // there is no identity to upgrade a row to, so it never enters the // content index. if (!message.id) continue - const key = sessionMatchKey(roles[i], message.record.content) + const key = sessionMatchKey(roles[i], message.content) if (ambiguous.has(key)) continue if (byContentKey.has(key)) { byContentKey.delete(key) @@ -540,13 +540,16 @@ function readMatchKey(attributes) { /** * A session message record's own role, as written in the file (the raw * value `sessionMatchKey` needs, since it owns the `toolResult` - * reconciliation itself). + * reconciliation itself). Read through the LLP 0158 reader's normalized + * field, never `message.record`: OpenClaw nests `role` under the record + * line's `message` envelope, and reading it a level too high made every + * record of every real session settle as `unknown` (#543). * * @param {OpenclawSessionMessage} message * @returns {string} */ function rawRole(message) { - return stringValue(message.record.role) ?? 'unknown' + return stringValue(message.role) ?? 'unknown' } /** diff --git a/hypaware-core/plugins-workspace/openclaw/src/types.d.ts b/hypaware-core/plugins-workspace/openclaw/src/types.d.ts index 6a076d37..c8fee1ec 100644 --- a/hypaware-core/plugins-workspace/openclaw/src/types.d.ts +++ b/hypaware-core/plugins-workspace/openclaw/src/types.d.ts @@ -15,20 +15,33 @@ export interface OpenclawSessionHeader { /** * One `type: "message"` record recovered from an OpenClaw session * transcript. The normalized fields are the ones LLP 0158's Context names as - * present on every message envelope (`id`, a timestamp) or on an assistant - * envelope specifically (`model`, `provider`, `api`, `stopReason`, `usage`); - * each is a present-value read off the envelope, absent when the field is - * missing, non-string (for the string fields), or blank, following the same - * "unconfirmable is unresolvable" rule the header applies. `record` is the - * full raw envelope, for a caller that needs a field this reader does not - * normalize (e.g. message content/blocks); it is an untyped bag rather than - * `JsonObject` on purpose, the same choice `CodexRolloutItem.payload` makes - * for the same reason (an arbitrary parsed line, not a value this reader - * constructs and can vouch for the shape of). + * present on every message envelope (`id`, a timestamp, `role`, `content`) + * or on an assistant envelope specifically (`model`, `provider`, `api`, + * `stopReason`, `usage`); each is read off the nested `message` envelope, + * falling back to the record line, and is absent when the field is missing, + * non-string (for the string fields), or blank, following the same + * "unconfirmable is unresolvable" rule the header applies. `id` is the one + * field read the other way round, record line first and envelope second, + * because the line is where LLP 0158 rule 7 verified message identity + * lives. `content` is whatever the envelope wrote (a string or a block + * array), passed through unnormalized, so unlike the string fields it + * refuses only an explicit `null` rather than a blank or wrong-typed value. + * `record` is the full raw record LINE, for a caller that + * needs a field this reader does not normalize: `parentId` is on the line + * itself, while a message-level field (`idempotencyKey`, `toolCallId`) is + * at `record.message`, one level down, and reading it off the line is the + * same mistake as #543. It is an untyped bag rather than `JsonObject` on + * purpose, the same choice + * `CodexRolloutItem.payload` makes for the same reason (an arbitrary parsed + * line, not a value this reader constructs and can vouch for the shape of). + * Reaching into `record` for `role`/`content` is the #543 defect: those live + * under `message`, and the reader is the one place that knows it. */ export interface OpenclawSessionMessage { id?: string timestampMs?: number + role?: string + content?: unknown model?: string provider?: string api?: string diff --git a/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md b/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md index 1f790747..25649a14 100644 --- a/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md +++ b/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md @@ -22,9 +22,17 @@ The OpenClaw session file's first line is a header record: `{"type":"session","version":3,"id":"","timestamp":"...","cwd":"..."}` (verified against live files on this machine, 2026-07-30). Subsequent lines carry `model_change`, `thinking_level_change`, `custom`, and `message` -records; each `message` envelope has its own short id, a timestamp, and for -assistant messages the `model`, `provider`, `api`, `stopReason`, and full -`usage` (tokens and cost). +records. + +A `message` record is **two levels deep** (verified against a live +`~/.openclaw/agents/main/sessions/.jsonl`, 2026-07-31, issue #543). The +record line states only what identifies and positions the message, +`['id', 'message', 'parentId', 'timestamp', 'type']`; the message itself is +the nested `message` object, whose assistant-turn keys are +`['api', 'content', 'idempotencyKey', 'model', 'provider', 'role', +'stopReason', 'timestamp', 'usage']`. `usage` is OpenClaw's own +normalization: `input`, `output`, `cacheRead`, `cacheWrite`, `totalTokens`, +`cost`. Two HypAware consumers need this file: @@ -51,6 +59,42 @@ answers wrong: 4. The header read is a bounded prefix read of line 1, so the hot settle path never pays for a large transcript (LLP 0049 R6's affordability argument, applied as LLP 0150#bounded applies it). +5. A message's fields are read from the nested `message` envelope, with the + record line as the fallback, never the other way round. This is the rule + that reads wrong most quietly: read one level too high, every field is + simply absent, and absence is a legal answer everywhere downstream. + `provider: undefined` resolves to `unknown`, the backfill allowlist + excludes it fail-closed, and the run reports a clean `0 rows` for a + session it never managed to read (#543). A parse miss and an intended + exclusion are indistinguishable at that seam, so the address has to be + right in the reader. +6. The fallback is per FIELD, not per record, and rule 3's present-value + test runs at BOTH levels before it decides. A record does not have to + nest nothing to read something off the line; it only has to state that + one field nowhere else. And a nested value that reads as absent (blank, + wrong-typed, `null`) cannot also be the value that suppresses the line: + that would make one value absent and load-bearing at once. A nested + `provider: " "` beside a line-level `provider: "anthropic"` would + otherwise resolve the record to `unknown` and lose it fail-closed, and a + nested `timestamp` that does not parse would drop `message_created_at`, + which re-dates the row to session start, defeats the `--since` window + (a timestamp-less item is kept unconditionally), and puts the settlement + ordinal match outside every window so the turn never dedupes. Same + silent-drop family as #543, one level down. "Reads as absent" is each + field's own answer, not one predicate for all of them: the string fields + refuse blank and non-string alike, while `content` has no single shape to + check (a string on some turns, a block array on others) and so refuses + only an explicit `null`. +7. `id` is the one field read line-first, envelope-fallback, because it is + identity rather than content: the record line is where this document + verified message identity lives, and the nested envelope is OpenClaw's + normalization of a provider response. A version that started copying the + provider's own `msg_...` id into the envelope would, under rule 5, + silently repoint every `message_id` and therefore every `part_id` that + backfill and settlement agree on (LLP 0157 R11); already-committed rows + would stop deduping against new ones and the history would double with + nothing raised. The envelope stays the fallback, so a record that states + identity only there still resolves one. LLP 0150 documented two shipped bugs (#453, #459) caused by exactly this shape: two modules holding copies of the same session-header rules for the @@ -95,12 +139,25 @@ slice two plugins needed was promoted to `src/core/codex/`. - The `cwd` predicate is shared with (or identical in behavior to) the Codex one, so "no usable cwd" means the same thing at every site that feeds the `.hypignore` gate. +- The reader normalizes every field whose *address* is non-obvious, `role` + and `content` included, rather than leaving them to callers to pick out of + the raw record. The raw record line stays exposed for genuinely + caller-specific fields, but "reach into `record` for a message field" is + not a supported read: both callers did exactly that for `role`/`content`, + both reached one level too high, and both dropped every real session + (#543). A field two callers must locate identically is the reader's. - The settlement enricher and the backfill provider both consume this reader. A future consumer that re-derived the fields for itself would be the same defect LLP 0150 removed, in a new place. - The invariant is tested once in core, with each caller keeping a short test pinning the rules at its own seam, following the LLP 0150 Consequences test shape. +- **Fixtures are path-faithful or they prove nothing.** Every test that + writes a session file writes the real two-level record shape, through one + helper per suite, so no fixture can quietly re-invent a flat envelope. + The suite that shipped #543 was green throughout: it asserted a reader + that read flat against a fixture that wrote flat, and neither was a + session file OpenClaw ever produced. ## Consequences diff --git a/test/plugins/openclaw-backfill.test.js b/test/plugins/openclaw-backfill.test.js index 43809465..5320eda8 100644 --- a/test/plugins/openclaw-backfill.test.js +++ b/test/plugins/openclaw-backfill.test.js @@ -35,6 +35,35 @@ async function stageEnv() { } } +/** + * One `type: "message"` line in the shape OpenClaw actually appends: `id`, + * `parentId`, and `timestamp` on the record line, and every message field + * (`role`, `content`, `model`, `provider`, `api`, `stopReason`, `usage`) + * nested under `message`. Tests author records flat and this is the one + * place that puts them where OpenClaw puts them, so no fixture can drift + * back to the invented flat envelope the suite used to assert against + * (#543: a flat fixture made a reader that reads flat look correct while + * every real session projected zero rows). + * + * Verified against a live `~/.openclaw/agents/main/sessions/.jsonl`: + * record keys `['id', 'message', 'parentId', 'timestamp', 'type']`, assistant + * message keys `['api', 'content', 'idempotencyKey', 'model', 'provider', + * 'role', 'stopReason', 'timestamp', 'usage']`. + * + * @param {Record} fields + * @returns {Record} + */ +function messageLine(fields) { + const { id, timestamp, parentId, ...message } = fields + return { + type: 'message', + ...(id !== undefined ? { id } : {}), + ...(timestamp !== undefined ? { timestamp } : {}), + parentId: parentId ?? null, + message: { ...message, ...(timestamp !== undefined ? { timestamp } : {}) }, + } +} + /** * Write one `~/.openclaw/agents//sessions/.jsonl`. * @@ -64,7 +93,7 @@ async function writeSession(env, doc) { ...doc.header, })) } - for (const record of doc.records ?? []) lines.push(JSON.stringify({ type: 'message', ...record })) + for (const record of doc.records ?? []) lines.push(JSON.stringify(messageLine(record))) await fs.writeFile(filePath, lines.join('\n') + '\n', 'utf8') return filePath } @@ -177,6 +206,7 @@ const ASSISTANT_RECORD = { provider: 'anthropic', api: 'anthropic-messages', stopReason: 'end_turn', + idempotencyKey: 'idem-asst-1', usage: { input: 11, output: 7, cacheRead: 3, cacheWrite: 2 }, } @@ -192,6 +222,27 @@ function provider(env, opts = {}) { // Shape and native identity // --------------------------------------------------------------------------- +// @ref LLP 0158#decision [tests]: the message envelope is nested under +// `message`, so a fixture that states those fields flat is not a session file +// OpenClaw ever wrote and cannot prove the reader reads one (#543). +test('the fixture writes the record shape OpenClaw actually appends', async () => { + const env = await stageEnv() + try { + const filePath = await writeSession(env, { header: { cwd: '/work/repo' }, records: [ASSISTANT_RECORD] }) + const lines = (await fs.readFile(filePath, 'utf8')).trim().split('\n') + const record = JSON.parse(lines[1]) + assert.deepEqual(Object.keys(record).sort(), ['id', 'message', 'parentId', 'timestamp', 'type']) + assert.deepEqual( + Object.keys(record.message).sort(), + ['api', 'content', 'idempotencyKey', 'model', 'provider', 'role', 'stopReason', 'timestamp', 'usage'] + ) + assert.equal(record.provider, undefined, 'a real record states no provider at the top level') + assert.equal(record.message.provider, 'anthropic') + } finally { + await env.cleanup() + } +}) + test('projects one item per session file, with the header cwd and native session id', async () => { const env = await stageEnv() try { @@ -485,6 +536,70 @@ test('a claude-cli turn is excluded whole, prompt included, and reported as cove } }) +// The #543 regression case, both sides of the allowlist in one real-shape +// file: a session that mixes an `anthropic` turn with a `claude-cli` turn must +// PARTIALLY project. Before the envelope fix every record of every real +// session read `provider: undefined`, so the whole file resolved to `unknown` +// and the run reported `sessions_projected: 0` with an `excluded_backend` +// event naming a provider no session file ever stated. +test('a mixed real-shape session partially projects: anthropic turns land, claude-cli turns stay excluded', async () => { + const env = await stageEnv() + try { + await writeSession(env, { + header: { cwd: '/work/repo' }, + records: [ + USER_RECORD, + ASSISTANT_RECORD, + { + id: 'msg-user-2', + timestamp: '2026-07-30T10:01:00.000Z', + role: 'user', + content: [{ type: 'text', text: 'delegate this' }], + }, + { + id: 'msg-asst-2', + timestamp: '2026-07-30T10:01:05.000Z', + role: 'assistant', + content: [{ type: 'text', text: 'done' }], + model: 'claude-sonnet-4-6', + provider: 'claude-cli', + api: 'anthropic-messages', + stopReason: 'end_turn', + usage: { input: 4, output: 2, cacheRead: 0, cacheWrite: 0, totalTokens: 6, cost: 0 }, + }, + ], + }) + const { ctx, entries } = runContext() + const { items, events } = await collect(provider(env).run(ctx)) + + assert.equal(items.length, 1, 'the anthropic half of the session must project') + assert.deepEqual( + value(items[0]).messages.map((/** @type {any} */ m) => m.message_id), + ['msg-user-1', 'msg-asst-1'] + ) + const excluded = events.filter((e) => e.event === 'excluded_backend') + assert.deepEqual(excluded.map((e) => e.attributes?.provider), ['claude-cli']) + assert.equal(excluded[0].attributes?.record_count, 2) + assert.equal(excluded[0].attributes?.covered_by, 'claude_transcript') + + const complete = entries.find((e) => e.message === 'openclaw.backfill.scan_complete') + assert.equal(complete.fields.sessions_projected, 1) + assert.equal(complete.fields.messages_projected, 2) + assert.equal(complete.fields.records_excluded, 2) + + // The usage spelling a real file uses lands under the gateway names too. + const rows = await materialize(items[0]) + assert.deepEqual(attributesOf(rows[1]).usage, { + input_tokens: 11, + output_tokens: 7, + cache_read_tokens: 3, + cache_write_tokens: 2, + }) + } finally { + await env.cleanup() + } +}) + test('an unrecognized provider fails closed: excluded, reported, and not covered_by anything', async () => { const env = await stageEnv() try { @@ -565,7 +680,7 @@ test('a relocated install is found through OPENCLAW_HOME, the same way settlemen path.join(dir, 'sess-relocated.jsonl'), [ JSON.stringify({ type: 'session', id: 'sess-relocated', cwd: '/work/repo', timestamp: '2026-07-30T10:00:00.000Z' }), - JSON.stringify({ type: 'message', ...ASSISTANT_RECORD }), + JSON.stringify(messageLine(ASSISTANT_RECORD)), ].join('\n') + '\n', 'utf8' ) diff --git a/test/plugins/openclaw-session-file.test.js b/test/plugins/openclaw-session-file.test.js index dfb40b3c..62801c44 100644 --- a/test/plugins/openclaw-session-file.test.js +++ b/test/plugins/openclaw-session-file.test.js @@ -181,6 +181,15 @@ test('a session file with no trailing newline is still one whole first line', () /* The full-transcript iteration: message records only */ /* ------------------------------------------------------------------ */ +// The record shape is the real one, verified against a live +// `~/.openclaw/agents/main/sessions/.jsonl` (#543): the record line +// carries `id`, `parentId`, `timestamp`, `type`, and every message field +// (`role`, `content`, `model`, `provider`, `api`, `stopReason`, `usage`) +// lives one level down under `message`. The suite used to assert against a +// flat envelope OpenClaw never writes, which is why a reader that read flat +// looked correct while every real session projected zero rows. +// +// @ref LLP 0158#decision [tests]: the message-envelope read rule test('readOpenclawSessionMessages returns only type:"message" records, in file order', async () => { const file = tempSessionFile([ headerLine({ id: 'session-abc', cwd: '/repo/here' }), @@ -188,22 +197,31 @@ test('readOpenclawSessionMessages returns only type:"message" records, in file o JSON.stringify({ type: 'message', id: 'msg-1', + parentId: null, timestamp: '2026-07-30T00:00:00.000Z', - role: 'user', - content: [{ type: 'text', text: 'hi' }], + message: { + role: 'user', + content: [{ type: 'text', text: 'hi' }], + timestamp: '2026-07-30T00:00:00.000Z', + }, }), JSON.stringify({ type: 'thinking_level_change', level: 'high' }), JSON.stringify({ type: 'message', id: 'msg-2', + parentId: 'msg-1', timestamp: '2026-07-30T00:00:05.000Z', - role: 'assistant', - model: 'claude-x', - provider: 'anthropic', - api: 'anthropic-messages', - stopReason: 'end_turn', - usage: { input_tokens: 10, output_tokens: 20 }, - content: [{ type: 'text', text: 'hello' }], + message: { + role: 'assistant', + model: 'claude-x', + provider: 'anthropic', + api: 'anthropic-messages', + stopReason: 'end_turn', + idempotencyKey: 'idem-1', + usage: { input: 10, output: 20, cacheRead: 0, cacheWrite: 0, totalTokens: 30, cost: 0.01 }, + content: [{ type: 'text', text: 'hello' }], + timestamp: '2026-07-30T00:00:05.000Z', + }, }), JSON.stringify({ type: 'custom', note: 'irrelevant' }), ].join('\n') + '\n') @@ -213,17 +231,192 @@ test('readOpenclawSessionMessages returns only type:"message" records, in file o assert.equal(messages[0].id, 'msg-1') assert.equal(messages[0].timestampMs, Date.parse('2026-07-30T00:00:00.000Z')) + assert.equal(messages[0].role, 'user') assert.equal(messages[0].model, undefined) assert.equal(messages[0].usage, undefined) - assert.deepEqual(messages[0].record.content, [{ type: 'text', text: 'hi' }]) + assert.deepEqual(messages[0].content, [{ type: 'text', text: 'hi' }]) assert.equal(messages[1].id, 'msg-2') + assert.equal(messages[1].role, 'assistant') assert.equal(messages[1].model, 'claude-x') assert.equal(messages[1].provider, 'anthropic') assert.equal(messages[1].api, 'anthropic-messages') assert.equal(messages[1].stopReason, 'end_turn') - assert.deepEqual(messages[1].usage, { input_tokens: 10, output_tokens: 20 }) - assert.deepEqual(messages[1].record.content, [{ type: 'text', text: 'hello' }]) + assert.deepEqual(messages[1].usage, { + input: 10, output: 20, cacheRead: 0, cacheWrite: 0, totalTokens: 30, cost: 0.01, + }) + assert.deepEqual(messages[1].content, [{ type: 'text', text: 'hello' }]) + // The untouched record line stays reachable for anything not normalized. + assert.equal(messages[1].record.parentId, 'msg-1') +}) + +test('a message field the nested envelope states is never read off the record line', async () => { + // Both levels state `provider`; the envelope owns the message's fields, so + // its value is the one that decides the backfill allowlist. A record line + // that happens to carry a same-named field cannot override it. + const file = tempSessionFile(JSON.stringify({ + type: 'message', + id: 'msg-1', + provider: 'record-line-value', + message: { role: 'assistant', provider: 'anthropic', content: [{ type: 'text', text: 'x' }] }, + }) + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].provider, 'anthropic') +}) + +test('a record with no nested envelope reads its fields off the record line', async () => { + // The envelope is where OpenClaw v3 writes them, but the record line is the + // envelope's own fallback: a record that nests nothing states whatever it + // states, rather than reading as a message with no role and no content. + const file = tempSessionFile(JSON.stringify({ + type: 'message', + id: 'msg-1', + timestamp: '2026-07-30T00:00:00.000Z', + role: 'assistant', + provider: 'anthropic', + content: [{ type: 'text', text: 'hello' }], + }) + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].role, 'assistant') + assert.equal(messages[0].provider, 'anthropic') + assert.deepEqual(messages[0].content, [{ type: 'text', text: 'hello' }]) +}) + +test('the record line supplies the timestamp when the nested envelope states none', async () => { + const file = tempSessionFile(JSON.stringify({ + type: 'message', + id: 'msg-1', + timestamp: '2026-07-30T00:00:00.000Z', + message: { role: 'user', content: 'hi' }, + }) + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].timestampMs, Date.parse('2026-07-30T00:00:00.000Z')) + assert.equal(messages[0].content, 'hi') +}) + +// The two fields a real record states at BOTH levels, and they resolve in +// OPPOSITE directions. `timestamp` is an ordinary message field, so the +// envelope owns it; it decides `message_created_at` on a backfilled row, the +// `--since` window, and the window the settlement ordinal match is bounded +// to, and a well-formed record's two values differ only by append latency. +// `id` is identity: the record line owns it, because a future OpenClaw that +// copied the provider's own id into the envelope would otherwise repoint +// every `message_id`/`part_id` R11's dedupe rests on, silently doubling the +// history. Pinning both directions is the point of this test. +// +// @ref LLP 0158#decision [tests]: envelope-first for a message's own fields, +// record line first for the identity the line is verified to state +test('the envelope wins for timestamp, the record line wins for id', async () => { + const file = tempSessionFile(JSON.stringify({ + type: 'message', + id: 'record-line-id', + timestamp: '2026-07-30T00:00:00.000Z', + message: { + id: 'nested-id', + timestamp: '2026-07-30T00:00:09.000Z', + role: 'user', + content: 'hi', + }, + }) + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].id, 'record-line-id') + assert.equal(messages[0].timestampMs, Date.parse('2026-07-30T00:00:09.000Z')) +}) + +test('a record that states its id only in the envelope still resolves an identity', async () => { + // The line owns `id`, but owning it does not mean refusing the envelope's: + // with nothing on the line there is no identity to protect, and reading the + // nested one beats emitting a hash `message_id` the settled row cannot + // match. + const file = tempSessionFile(JSON.stringify({ + type: 'message', + message: { id: 'nested-only-id', role: 'user', content: 'hi' }, + }) + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].id, 'nested-only-id') +}) + +test('a blank or wrong-typed nested field reads as absent, so the record line still supplies it', async () => { + // Rule 3 applies at BOTH levels before the fallback decides. A nested value + // that reads as absent cannot also be the value that suppresses the line: + // that would resolve this record to `unknown` and the backfill allowlist + // would exclude it fail-closed, the same silent drop as #543. + const file = tempSessionFile(JSON.stringify({ + type: 'message', + id: 'msg-1', + provider: 'anthropic', + model: 'claude-x', + usage: { input: 10, output: 20 }, + message: { role: 'assistant', content: 'x', provider: ' ', model: 42, usage: null }, + }) + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].provider, 'anthropic') + assert.equal(messages[0].model, 'claude-x') + assert.deepEqual(messages[0].usage, { input: 10, output: 20 }) +}) + +test('a blank nested field with nothing on the record line is absent, not a substitute', async () => { + // The other half of rule 3 at the nested level: with no line-level value to + // fall back to, an unconfirmable envelope field stays unresolved rather + // than becoming an empty string or a coerced number. + const file = tempSessionFile(JSON.stringify({ + type: 'message', + message: { id: ' ', timestamp: '', role: 'assistant', model: 42, stopReason: null, content: 'x' }, + }) + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].id, undefined) + assert.equal(messages[0].timestampMs, undefined) + assert.equal(messages[0].model, undefined) + assert.equal(messages[0].stopReason, undefined) + assert.equal(messages[0].role, 'assistant') +}) + +test('a nulled-out nested content is unstated, so the record line still supplies it', async () => { + // `content` is the one normalized field with no single shape to test (a + // string on some turns, a block array on others), so its present-value test + // refuses an explicit `null` and nothing else. That refusal is load-bearing, + // not cosmetic: without it a nulled-out envelope field would suppress the + // line and land `content: null` on the message, which the backfill projector + // drops and the settlement content index hashes into a key no live row can + // match, the same silent drop as #543. + const file = tempSessionFile([ + JSON.stringify({ + type: 'message', + id: 'msg-1', + content: 'from the record line', + message: { role: 'user', content: null }, + }), + JSON.stringify({ type: 'message', id: 'msg-2', message: { role: 'user', content: null } }), + ].join('\n') + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].content, 'from the record line') + assert.equal( + Object.hasOwn(messages[1], 'content'), + false, + 'with nothing on the line either, content is absent, never a null the consumers must re-check' + ) +}) + +test('a `message` key that is not an object leaves the record line as the only address', async () => { + // `openclawMessageEnvelope` requires a plain object. A record whose + // `message` is a string, an array or `null` is not an envelope, so the + // reader does not treat it as one; the line is read instead, and a line that + // states no role yields a message the consumers drop rather than a half-read + // one. The `null` case is why the guard is a plain-object test rather than a + // truthiness one: indexing a field off `null` would throw out of the whole + // file read, losing every later record with it. + const file = tempSessionFile([ + JSON.stringify({ type: 'message', id: 'msg-1', message: 'not an envelope', role: 'user', content: 'hi' }), + JSON.stringify({ type: 'message', id: 'msg-2', message: [{ type: 'text', text: 'x' }] }), + JSON.stringify({ type: 'message', id: 'msg-3', message: null, role: 'assistant', content: 'still read' }), + ].join('\n') + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].role, 'user') + assert.equal(messages[0].content, 'hi') + assert.equal(messages[1].role, undefined) + assert.equal(messages[1].content, undefined) + assert.equal(messages[2].role, 'assistant') + assert.equal(messages[2].content, 'still read') }) test('readOpenclawSessionMessages skips blank and unparseable lines without aborting the rest', async () => { diff --git a/test/plugins/openclaw-settlement.test.js b/test/plugins/openclaw-settlement.test.js index cf426520..0c48b50a 100644 --- a/test/plugins/openclaw-settlement.test.js +++ b/test/plugins/openclaw-settlement.test.js @@ -130,6 +130,30 @@ function iso(ms) { return new Date(ms).toISOString() } +/** + * Put a `type: "message"` record's message fields where OpenClaw puts them: + * `id`, `parentId`, and `timestamp` stay on the record line, `role`, + * `content`, `model`, `provider`, `api`, `stopReason`, and `usage` nest under + * `message` (verified against a live install, #543). Tests above author the + * records flat and this is the one place that nests them, so the enricher is + * always measured against the shape it will meet on disk rather than against + * the flat envelope the suite used to invent. + * + * @param {Record} record + * @returns {Record} + */ +function sessionFileLine(record) { + if (record.type !== 'message') return record + const { type, id, timestamp, parentId, ...message } = record + return { + type, + ...(id !== undefined ? { id } : {}), + ...(timestamp !== undefined ? { timestamp } : {}), + parentId: parentId ?? null, + message: { ...message, ...(timestamp !== undefined ? { timestamp } : {}) }, + } +} + /** * Write a session file under a throwaway `agents//sessions/` root * and return the root. @@ -144,7 +168,7 @@ function writeSessionFile(records, opts = {}) { fs.mkdirSync(sessionsDir, { recursive: true }) fs.writeFileSync( path.join(sessionsDir, opts.fileName ?? `${NATIVE_SESSION_ID}.jsonl`), - records.map((record) => JSON.stringify(record)).join('\n') + '\n' + records.map((record) => JSON.stringify(sessionFileLine(record))).join('\n') + '\n' ) return agentsDir }