From 44c708014a333516c5f65b858b490d46ef1cc767 Mon Sep 17 00:00:00 2001 From: test Date: Fri, 31 Jul 2026 22:00:43 +0000 Subject: [PATCH 1/3] OpenClaw session records are read one level too high (#543) A real OpenClaw v3 `type: "message"` record states only `id`, `parentId`, `timestamp`, and `type` on the record line: `role`, `content`, `model`, `provider`, `api`, `stopReason`, and `usage` are all nested under `message`. The LLP 0158 reader read them off the record line, so every field came back absent, every record resolved to `provider: unknown`, the backfill allowlist excluded all of them, and `hyp status` reported `backfill @hypaware/openclaw [done] (0 rows)` for a session it had failed to read. The settlement enricher was broken the same way one seam later (`record.role`/`record.content`), so a real session settled nothing either. The reader now owns the envelope address: fields are read from the nested `message` object, falling back to the record line for a record that nests none, and `role`/`content` are normalized fields rather than something each consumer picks out of the raw record. Both consumers read them off the normalized message, so neither can drift a level again. Fixtures across the three OpenClaw suites now write the real two-level shape through one helper each; the old flat fixtures asserted an envelope OpenClaw never writes, which is why the suite stayed green through the bug. LLP 0158 records the verified record shape, the envelope read rule, and the path-faithful-fixture consequence. Co-Authored-By: Claude --- .../openclaw/src/backfill.js | 18 +-- .../openclaw/src/session_file.js | 80 ++++++++++-- .../plugins-workspace/openclaw/src/settle.js | 9 +- .../plugins-workspace/openclaw/src/types.d.ts | 26 ++-- ...der-for-openclaw-session-jsonl.decision.md | 36 +++++- test/plugins/openclaw-backfill.test.js | 116 +++++++++++++++++- test/plugins/openclaw-session-file.test.js | 92 ++++++++++++-- test/plugins/openclaw-settlement.test.js | 26 +++- 8 files changed, 355 insertions(+), 48 deletions(-) diff --git a/hypaware-core/plugins-workspace/openclaw/src/backfill.js b/hypaware-core/plugins-workspace/openclaw/src/backfill.js index 10b1b728..ac542521 100644 --- a/hypaware-core/plugins-workspace/openclaw/src/backfill.js +++ b/hypaware-core/plugins-workspace/openclaw/src/backfill.js @@ -34,10 +34,11 @@ import { isPlainObject, sha256Hex, stringValue } from 'hypaware/core/util' * neither consumer may hold its own parse). * * The session file is the authoritative record of a turn: it already carries - * one flattened per-message record with the message's own native id, its - * timestamp, and for an assistant message the model, provider, api, stop - * reason, and usage. So this module builds an `AiGatewayProjectedExchange` - * straight off those fields. It deliberately does NOT route through + * one per-message record with the message's own native id and timestamp on + * the record line, and its role, content and (for an assistant message) the + * model, provider, api, stop reason and usage in the nested `message` + * envelope the LLP 0158 reader normalizes. So this module builds an + * `AiGatewayProjectedExchange` straight off those fields. It deliberately does NOT route through * `anthropicMessages()` (projector.js): those parse a wire request/response * pair, and there is no wire pair here to reconstruct. * @@ -492,10 +493,13 @@ function projectedExchangeFromSession(args) { * @returns {AiGatewayProjectedMessage | undefined} */ function projectedMessageFromRecord(message) { - const record = message.record - const rawRole = stringValue(record.role) + // Through the LLP 0158 reader's normalized fields, never `message.record`: + // OpenClaw nests `role`/`content` under the record line's `message` + // envelope, so the top-level read this used to do found neither and dropped + // every record of every real session (#543). + const rawRole = stringValue(message.role) if (!rawRole) return undefined - const content = record.content + const content = message.content // The gateway drops a message whose content normalizes to no blocks; doing it // here keeps `messages_projected` honest about what was actually emitted. if (typeof content === 'string' ? content.length === 0 : !Array.isArray(content) || content.length === 0) { diff --git a/hypaware-core/plugins-workspace/openclaw/src/session_file.js b/hypaware-core/plugins-workspace/openclaw/src/session_file.js index 6064c59c..cee22ec4 100644 --- a/hypaware-core/plugins-workspace/openclaw/src/session_file.js +++ b/hypaware-core/plugins-workspace/openclaw/src/session_file.js @@ -248,12 +248,20 @@ export async function readOpenclawSessionMessages(filePath) { * `undefined` when the line is not a `type: "message"` record. Guards the * same way the header parse does (rules 1-3 above), applied to the fields * LLP 0158's Context names as present on a message envelope: `id` and a - * timestamp on every message, `model`/`provider`/`api`/`stopReason`/`usage` - * on an assistant one. Every field this function does not normalize (role, - * content blocks, and anything else OpenClaw writes) is still reachable - * through `record`, the untouched envelope, so a caller that needs one is + * timestamp on every message, `role` and `content` on every message, + * `model`/`provider`/`api`/`stopReason`/`usage` on an assistant one. Every + * field this function does not normalize (`parentId`, `idempotencyKey`, + * `toolCallId`, and anything else OpenClaw writes) is still reachable + * through `record`, the untouched record line, so a caller that needs one is * not blocked on this reader growing a field for it. * + * `role` and `content` are normalized here rather than left to `record` + * precisely because their location is the non-obvious part + * ({@link openclawMessageEnvelope}): both consumers used to reach into + * `record.role`/`record.content` themselves, so both read one level too high + * and both dropped every real session (#543). A field whose *address* is the + * thing that is easy to get wrong belongs to the one reader. + * * @param {string} line * @returns {OpenclawSessionMessage | undefined} */ @@ -261,24 +269,74 @@ function parseOpenclawSessionMessage(line) { const row = parseMaybeJson(line) if (!isPlainObject(row)) return undefined if (row.type !== 'message') return undefined + const envelope = openclawMessageEnvelope(row) /** @type {OpenclawSessionMessage} */ const message = { record: row } - const id = nonBlankString(row.id) + const id = nonBlankString(messageField(envelope, row, 'id')) if (id !== undefined) message.id = id - const timestampMs = parseTimestampMs(row.timestamp) + const timestampMs = parseTimestampMs(messageField(envelope, row, 'timestamp')) if (timestampMs !== undefined) message.timestampMs = timestampMs - const model = nonBlankString(row.model) + const role = nonBlankString(messageField(envelope, row, 'role')) + if (role !== undefined) message.role = role + const content = messageField(envelope, row, 'content') + if (content !== undefined) message.content = content + const model = nonBlankString(messageField(envelope, row, 'model')) if (model !== undefined) message.model = model - const provider = nonBlankString(row.provider) + const provider = nonBlankString(messageField(envelope, row, 'provider')) if (provider !== undefined) message.provider = provider - const api = nonBlankString(row.api) + const api = nonBlankString(messageField(envelope, row, 'api')) if (api !== undefined) message.api = api - const stopReason = nonBlankString(row.stopReason) + const stopReason = nonBlankString(messageField(envelope, row, 'stopReason')) if (stopReason !== undefined) message.stopReason = stopReason - if (isPlainObject(row.usage)) message.usage = row.usage + const usage = messageField(envelope, row, 'usage') + if (isPlainObject(usage)) message.usage = usage return message } +/** + * The message envelope of a `type: "message"` record: the nested `message` + * object, not the record line. + * + * A record line states only what identifies and positions the message + * (`id`, `parentId`, `timestamp`, `type`); the message itself - `role`, + * `content`, and for an assistant turn `model`, `provider`, `api`, + * `stopReason`, `usage` - is one level down under `message`. Read at the top + * level every one of those fields is absent, which is not a loud failure: + * `provider` reads `undefined`, the backfill allowlist resolves the record to + * `unknown`, and the run reports a clean "0 rows" for a session it simply + * failed to read (#543). Fail-closed exclusion and a parse miss are + * indistinguishable at that seam, so the address has to be right here. + * + * The record line is the fallback, not a second address to prefer: a record + * that nests no `message` object states its fields on the line itself, and + * reading them there is better than reading a message with no role and no + * content. A field the envelope does state is never overridden by a + * same-named field on the line. + * + * @ref LLP 0158#decision [implements]: where a message's fields live is part + * of the one reader's knowledge, not something each consumer re-derives + * @param {Record} row + * @returns {Record} + */ +function openclawMessageEnvelope(row) { + return isPlainObject(row.message) ? row.message : row +} + +/** + * One message field, read from the envelope first and the record line + * second. Not a substitution across fields (rule 1): it is the same field + * name, looked for at the two levels one record can state it at. + * + * @param {Record} envelope + * @param {Record} row + * @param {string} key + * @returns {unknown} + */ +function messageField(envelope, row, key) { + const value = envelope[key] + return value === undefined ? row[key] : value +} + /** * A non-blank string, byte-identical, else `undefined`. Stricter than * `stringValue` (core util) on purpose: a whitespace-only value matches no diff --git a/hypaware-core/plugins-workspace/openclaw/src/settle.js b/hypaware-core/plugins-workspace/openclaw/src/settle.js index 7c1c1de4..dbfb7808 100644 --- a/hypaware-core/plugins-workspace/openclaw/src/settle.js +++ b/hypaware-core/plugins-workspace/openclaw/src/settle.js @@ -385,7 +385,7 @@ function buildOpenclawSessionIndex(candidate, header, messages) { // there is no identity to upgrade a row to, so it never enters the // content index. if (!message.id) continue - const key = sessionMatchKey(roles[i], message.record.content) + const key = sessionMatchKey(roles[i], message.content) if (ambiguous.has(key)) continue if (byContentKey.has(key)) { byContentKey.delete(key) @@ -540,13 +540,16 @@ function readMatchKey(attributes) { /** * A session message record's own role, as written in the file (the raw * value `sessionMatchKey` needs, since it owns the `toolResult` - * reconciliation itself). + * reconciliation itself). Read through the LLP 0158 reader's normalized + * field, never `message.record`: OpenClaw nests `role` under the record + * line's `message` envelope, and reading it a level too high made every + * record of every real session settle as `unknown` (#543). * * @param {OpenclawSessionMessage} message * @returns {string} */ function rawRole(message) { - return stringValue(message.record.role) ?? 'unknown' + return stringValue(message.role) ?? 'unknown' } /** diff --git a/hypaware-core/plugins-workspace/openclaw/src/types.d.ts b/hypaware-core/plugins-workspace/openclaw/src/types.d.ts index 6a076d37..9953612a 100644 --- a/hypaware-core/plugins-workspace/openclaw/src/types.d.ts +++ b/hypaware-core/plugins-workspace/openclaw/src/types.d.ts @@ -15,20 +15,26 @@ export interface OpenclawSessionHeader { /** * One `type: "message"` record recovered from an OpenClaw session * transcript. The normalized fields are the ones LLP 0158's Context names as - * present on every message envelope (`id`, a timestamp) or on an assistant - * envelope specifically (`model`, `provider`, `api`, `stopReason`, `usage`); - * each is a present-value read off the envelope, absent when the field is - * missing, non-string (for the string fields), or blank, following the same - * "unconfirmable is unresolvable" rule the header applies. `record` is the - * full raw envelope, for a caller that needs a field this reader does not - * normalize (e.g. message content/blocks); it is an untyped bag rather than - * `JsonObject` on purpose, the same choice `CodexRolloutItem.payload` makes - * for the same reason (an arbitrary parsed line, not a value this reader - * constructs and can vouch for the shape of). + * present on every message envelope (`id`, a timestamp, `role`, `content`) + * or on an assistant envelope specifically (`model`, `provider`, `api`, + * `stopReason`, `usage`); each is read off the nested `message` envelope, + * falling back to the record line, and is absent when the field is missing, + * non-string (for the string fields), or blank, following the same + * "unconfirmable is unresolvable" rule the header applies. `content` is + * whatever the envelope wrote (a string or a block array), passed through + * unnormalized. `record` is the full raw record line, for a caller that + * needs a field this reader does not normalize (`parentId`, `toolCallId`); + * it is an untyped bag rather than `JsonObject` on purpose, the same choice + * `CodexRolloutItem.payload` makes for the same reason (an arbitrary parsed + * line, not a value this reader constructs and can vouch for the shape of). + * Reaching into `record` for `role`/`content` is the #543 defect: those live + * under `message`, and the reader is the one place that knows it. */ export interface OpenclawSessionMessage { id?: string timestampMs?: number + role?: string + content?: unknown model?: string provider?: string api?: string diff --git a/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md b/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md index 1f790747..b719fdf8 100644 --- a/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md +++ b/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md @@ -22,9 +22,17 @@ The OpenClaw session file's first line is a header record: `{"type":"session","version":3,"id":"","timestamp":"...","cwd":"..."}` (verified against live files on this machine, 2026-07-30). Subsequent lines carry `model_change`, `thinking_level_change`, `custom`, and `message` -records; each `message` envelope has its own short id, a timestamp, and for -assistant messages the `model`, `provider`, `api`, `stopReason`, and full -`usage` (tokens and cost). +records. + +A `message` record is **two levels deep** (verified against a live +`~/.openclaw/agents/main/sessions/.jsonl`, 2026-07-31, issue #543). The +record line states only what identifies and positions the message, +`['id', 'message', 'parentId', 'timestamp', 'type']`; the message itself is +the nested `message` object, whose assistant-turn keys are +`['api', 'content', 'idempotencyKey', 'model', 'provider', 'role', +'stopReason', 'timestamp', 'usage']`. `usage` is OpenClaw's own +normalization: `input`, `output`, `cacheRead`, `cacheWrite`, `totalTokens`, +`cost`. Two HypAware consumers need this file: @@ -51,6 +59,15 @@ answers wrong: 4. The header read is a bounded prefix read of line 1, so the hot settle path never pays for a large transcript (LLP 0049 R6's affordability argument, applied as LLP 0150#bounded applies it). +5. A message's fields are read from the nested `message` envelope, with the + record line as the fallback for a record that nests none, never the other + way round. This is the rule that reads wrong most quietly: read one level + too high, every field is simply absent, and absence is a legal answer + everywhere downstream. `provider: undefined` resolves to `unknown`, the + backfill allowlist excludes it fail-closed, and the run reports a clean + `0 rows` for a session it never managed to read (#543). A parse miss and + an intended exclusion are indistinguishable at that seam, so the address + has to be right in the reader. LLP 0150 documented two shipped bugs (#453, #459) caused by exactly this shape: two modules holding copies of the same session-header rules for the @@ -95,12 +112,25 @@ slice two plugins needed was promoted to `src/core/codex/`. - The `cwd` predicate is shared with (or identical in behavior to) the Codex one, so "no usable cwd" means the same thing at every site that feeds the `.hypignore` gate. +- The reader normalizes every field whose *address* is non-obvious, `role` + and `content` included, rather than leaving them to callers to pick out of + the raw record. The raw record line stays exposed for genuinely + caller-specific fields, but "reach into `record` for a message field" is + not a supported read: both callers did exactly that for `role`/`content`, + both reached one level too high, and both dropped every real session + (#543). A field two callers must locate identically is the reader's. - The settlement enricher and the backfill provider both consume this reader. A future consumer that re-derived the fields for itself would be the same defect LLP 0150 removed, in a new place. - The invariant is tested once in core, with each caller keeping a short test pinning the rules at its own seam, following the LLP 0150 Consequences test shape. +- **Fixtures are path-faithful or they prove nothing.** Every test that + writes a session file writes the real two-level record shape, through one + helper per suite, so no fixture can quietly re-invent a flat envelope. + The suite that shipped #543 was green throughout: it asserted a reader + that read flat against a fixture that wrote flat, and neither was a + session file OpenClaw ever produced. ## Consequences diff --git a/test/plugins/openclaw-backfill.test.js b/test/plugins/openclaw-backfill.test.js index 43809465..7afd09b4 100644 --- a/test/plugins/openclaw-backfill.test.js +++ b/test/plugins/openclaw-backfill.test.js @@ -35,6 +35,35 @@ async function stageEnv() { } } +/** + * One `type: "message"` line in the shape OpenClaw actually appends: `id`, + * `parentId`, and `timestamp` on the record line, and every message field + * (`role`, `content`, `model`, `provider`, `api`, `stopReason`, `usage`) + * nested under `message`. Tests author records flat and this is the one + * place that puts them where OpenClaw puts them, so no fixture can drift + * back to the invented flat envelope the suite used to assert against + * (#543: a flat fixture made a reader that reads flat look correct while + * every real session projected zero rows). + * + * Verified against a live `~/.openclaw/agents/main/sessions/.jsonl`: + * record keys `['id', 'message', 'parentId', 'timestamp', 'type']`, assistant + * message keys `['api', 'content', 'idempotencyKey', 'model', 'provider', + * 'role', 'stopReason', 'timestamp', 'usage']`. + * + * @param {Record} fields + * @returns {Record} + */ +function messageLine(fields) { + const { id, timestamp, parentId, ...message } = fields + return { + type: 'message', + ...(id !== undefined ? { id } : {}), + ...(timestamp !== undefined ? { timestamp } : {}), + parentId: parentId ?? null, + message: { ...message, ...(timestamp !== undefined ? { timestamp } : {}) }, + } +} + /** * Write one `~/.openclaw/agents//sessions/.jsonl`. * @@ -64,7 +93,7 @@ async function writeSession(env, doc) { ...doc.header, })) } - for (const record of doc.records ?? []) lines.push(JSON.stringify({ type: 'message', ...record })) + for (const record of doc.records ?? []) lines.push(JSON.stringify(messageLine(record))) await fs.writeFile(filePath, lines.join('\n') + '\n', 'utf8') return filePath } @@ -192,6 +221,27 @@ function provider(env, opts = {}) { // Shape and native identity // --------------------------------------------------------------------------- +// @ref LLP 0158#decision [tests]: the message envelope is nested under +// `message`, so a fixture that states those fields flat is not a session file +// OpenClaw ever wrote and cannot prove the reader reads one (#543). +test('the fixture writes the record shape OpenClaw actually appends', async () => { + const env = await stageEnv() + try { + const filePath = await writeSession(env, { header: { cwd: '/work/repo' }, records: [ASSISTANT_RECORD] }) + const lines = (await fs.readFile(filePath, 'utf8')).trim().split('\n') + const record = JSON.parse(lines[1]) + assert.deepEqual(Object.keys(record).sort(), ['id', 'message', 'parentId', 'timestamp', 'type']) + assert.deepEqual( + Object.keys(record.message).sort(), + ['api', 'content', 'model', 'provider', 'role', 'stopReason', 'timestamp', 'usage'] + ) + assert.equal(record.provider, undefined, 'a real record states no provider at the top level') + assert.equal(record.message.provider, 'anthropic') + } finally { + await env.cleanup() + } +}) + test('projects one item per session file, with the header cwd and native session id', async () => { const env = await stageEnv() try { @@ -485,6 +535,70 @@ test('a claude-cli turn is excluded whole, prompt included, and reported as cove } }) +// The #543 regression case, both sides of the allowlist in one real-shape +// file: a session that mixes an `anthropic` turn with a `claude-cli` turn must +// PARTIALLY project. Before the envelope fix every record of every real +// session read `provider: undefined`, so the whole file resolved to `unknown` +// and the run reported `sessions_projected: 0` with an `excluded_backend` +// event naming a provider no session file ever stated. +test('a mixed real-shape session partially projects: anthropic turns land, claude-cli turns stay excluded', async () => { + const env = await stageEnv() + try { + await writeSession(env, { + header: { cwd: '/work/repo' }, + records: [ + USER_RECORD, + ASSISTANT_RECORD, + { + id: 'msg-user-2', + timestamp: '2026-07-30T10:01:00.000Z', + role: 'user', + content: [{ type: 'text', text: 'delegate this' }], + }, + { + id: 'msg-asst-2', + timestamp: '2026-07-30T10:01:05.000Z', + role: 'assistant', + content: [{ type: 'text', text: 'done' }], + model: 'claude-sonnet-4-6', + provider: 'claude-cli', + api: 'anthropic-messages', + stopReason: 'end_turn', + usage: { input: 4, output: 2, cacheRead: 0, cacheWrite: 0, totalTokens: 6, cost: 0 }, + }, + ], + }) + const { ctx, entries } = runContext() + const { items, events } = await collect(provider(env).run(ctx)) + + assert.equal(items.length, 1, 'the anthropic half of the session must project') + assert.deepEqual( + value(items[0]).messages.map((/** @type {any} */ m) => m.message_id), + ['msg-user-1', 'msg-asst-1'] + ) + const excluded = events.filter((e) => e.event === 'excluded_backend') + assert.deepEqual(excluded.map((e) => e.attributes?.provider), ['claude-cli']) + assert.equal(excluded[0].attributes?.record_count, 2) + assert.equal(excluded[0].attributes?.covered_by, 'claude_transcript') + + const complete = entries.find((e) => e.message === 'openclaw.backfill.scan_complete') + assert.equal(complete.fields.sessions_projected, 1) + assert.equal(complete.fields.messages_projected, 2) + assert.equal(complete.fields.records_excluded, 2) + + // The usage spelling a real file uses lands under the gateway names too. + const rows = await materialize(items[0]) + assert.deepEqual(attributesOf(rows[1]).usage, { + input_tokens: 11, + output_tokens: 7, + cache_read_tokens: 3, + cache_write_tokens: 2, + }) + } finally { + await env.cleanup() + } +}) + test('an unrecognized provider fails closed: excluded, reported, and not covered_by anything', async () => { const env = await stageEnv() try { diff --git a/test/plugins/openclaw-session-file.test.js b/test/plugins/openclaw-session-file.test.js index dfb40b3c..6f858820 100644 --- a/test/plugins/openclaw-session-file.test.js +++ b/test/plugins/openclaw-session-file.test.js @@ -181,6 +181,15 @@ test('a session file with no trailing newline is still one whole first line', () /* The full-transcript iteration: message records only */ /* ------------------------------------------------------------------ */ +// The record shape is the real one, verified against a live +// `~/.openclaw/agents/main/sessions/.jsonl` (#543): the record line +// carries `id`, `parentId`, `timestamp`, `type`, and every message field +// (`role`, `content`, `model`, `provider`, `api`, `stopReason`, `usage`) +// lives one level down under `message`. The suite used to assert against a +// flat envelope OpenClaw never writes, which is why a reader that read flat +// looked correct while every real session projected zero rows. +// +// @ref LLP 0158#decision [tests]: the message-envelope read rule test('readOpenclawSessionMessages returns only type:"message" records, in file order', async () => { const file = tempSessionFile([ headerLine({ id: 'session-abc', cwd: '/repo/here' }), @@ -188,22 +197,31 @@ test('readOpenclawSessionMessages returns only type:"message" records, in file o JSON.stringify({ type: 'message', id: 'msg-1', + parentId: null, timestamp: '2026-07-30T00:00:00.000Z', - role: 'user', - content: [{ type: 'text', text: 'hi' }], + message: { + role: 'user', + content: [{ type: 'text', text: 'hi' }], + timestamp: '2026-07-30T00:00:00.000Z', + }, }), JSON.stringify({ type: 'thinking_level_change', level: 'high' }), JSON.stringify({ type: 'message', id: 'msg-2', + parentId: 'msg-1', timestamp: '2026-07-30T00:00:05.000Z', - role: 'assistant', - model: 'claude-x', - provider: 'anthropic', - api: 'anthropic-messages', - stopReason: 'end_turn', - usage: { input_tokens: 10, output_tokens: 20 }, - content: [{ type: 'text', text: 'hello' }], + message: { + role: 'assistant', + model: 'claude-x', + provider: 'anthropic', + api: 'anthropic-messages', + stopReason: 'end_turn', + idempotencyKey: 'idem-1', + usage: { input: 10, output: 20, cacheRead: 0, cacheWrite: 0, totalTokens: 30, cost: 0.01 }, + content: [{ type: 'text', text: 'hello' }], + timestamp: '2026-07-30T00:00:05.000Z', + }, }), JSON.stringify({ type: 'custom', note: 'irrelevant' }), ].join('\n') + '\n') @@ -213,17 +231,67 @@ test('readOpenclawSessionMessages returns only type:"message" records, in file o assert.equal(messages[0].id, 'msg-1') assert.equal(messages[0].timestampMs, Date.parse('2026-07-30T00:00:00.000Z')) + assert.equal(messages[0].role, 'user') assert.equal(messages[0].model, undefined) assert.equal(messages[0].usage, undefined) - assert.deepEqual(messages[0].record.content, [{ type: 'text', text: 'hi' }]) + assert.deepEqual(messages[0].content, [{ type: 'text', text: 'hi' }]) assert.equal(messages[1].id, 'msg-2') + assert.equal(messages[1].role, 'assistant') assert.equal(messages[1].model, 'claude-x') assert.equal(messages[1].provider, 'anthropic') assert.equal(messages[1].api, 'anthropic-messages') assert.equal(messages[1].stopReason, 'end_turn') - assert.deepEqual(messages[1].usage, { input_tokens: 10, output_tokens: 20 }) - assert.deepEqual(messages[1].record.content, [{ type: 'text', text: 'hello' }]) + assert.deepEqual(messages[1].usage, { + input: 10, output: 20, cacheRead: 0, cacheWrite: 0, totalTokens: 30, cost: 0.01, + }) + assert.deepEqual(messages[1].content, [{ type: 'text', text: 'hello' }]) + // The untouched record line stays reachable for anything not normalized. + assert.equal(messages[1].record.parentId, 'msg-1') +}) + +test('a message field the nested envelope states is never read off the record line', async () => { + // Both levels state `provider`; the envelope owns the message's fields, so + // its value is the one that decides the backfill allowlist. A record line + // that happens to carry a same-named field cannot override it. + const file = tempSessionFile(JSON.stringify({ + type: 'message', + id: 'msg-1', + provider: 'record-line-value', + message: { role: 'assistant', provider: 'anthropic', content: [{ type: 'text', text: 'x' }] }, + }) + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].provider, 'anthropic') +}) + +test('a record with no nested envelope reads its fields off the record line', async () => { + // The envelope is where OpenClaw v3 writes them, but the record line is the + // envelope's own fallback: a record that nests nothing states whatever it + // states, rather than reading as a message with no role and no content. + const file = tempSessionFile(JSON.stringify({ + type: 'message', + id: 'msg-1', + timestamp: '2026-07-30T00:00:00.000Z', + role: 'assistant', + provider: 'anthropic', + content: [{ type: 'text', text: 'hello' }], + }) + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].role, 'assistant') + assert.equal(messages[0].provider, 'anthropic') + assert.deepEqual(messages[0].content, [{ type: 'text', text: 'hello' }]) +}) + +test('the record line supplies the timestamp when the nested envelope states none', async () => { + const file = tempSessionFile(JSON.stringify({ + type: 'message', + id: 'msg-1', + timestamp: '2026-07-30T00:00:00.000Z', + message: { role: 'user', content: 'hi' }, + }) + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].timestampMs, Date.parse('2026-07-30T00:00:00.000Z')) + assert.equal(messages[0].content, 'hi') }) test('readOpenclawSessionMessages skips blank and unparseable lines without aborting the rest', async () => { diff --git a/test/plugins/openclaw-settlement.test.js b/test/plugins/openclaw-settlement.test.js index cf426520..0c48b50a 100644 --- a/test/plugins/openclaw-settlement.test.js +++ b/test/plugins/openclaw-settlement.test.js @@ -130,6 +130,30 @@ function iso(ms) { return new Date(ms).toISOString() } +/** + * Put a `type: "message"` record's message fields where OpenClaw puts them: + * `id`, `parentId`, and `timestamp` stay on the record line, `role`, + * `content`, `model`, `provider`, `api`, `stopReason`, and `usage` nest under + * `message` (verified against a live install, #543). Tests above author the + * records flat and this is the one place that nests them, so the enricher is + * always measured against the shape it will meet on disk rather than against + * the flat envelope the suite used to invent. + * + * @param {Record} record + * @returns {Record} + */ +function sessionFileLine(record) { + if (record.type !== 'message') return record + const { type, id, timestamp, parentId, ...message } = record + return { + type, + ...(id !== undefined ? { id } : {}), + ...(timestamp !== undefined ? { timestamp } : {}), + parentId: parentId ?? null, + message: { ...message, ...(timestamp !== undefined ? { timestamp } : {}) }, + } +} + /** * Write a session file under a throwaway `agents//sessions/` root * and return the root. @@ -144,7 +168,7 @@ function writeSessionFile(records, opts = {}) { fs.mkdirSync(sessionsDir, { recursive: true }) fs.writeFileSync( path.join(sessionsDir, opts.fileName ?? `${NATIVE_SESSION_ID}.jsonl`), - records.map((record) => JSON.stringify(record)).join('\n') + '\n' + records.map((record) => JSON.stringify(sessionFileLine(record))).join('\n') + '\n' ) return agentsDir } From df7f11922288f7b3755da716cdd491eb1aa1ae4d Mon Sep 17 00:00:00 2001 From: test Date: Fri, 31 Jul 2026 22:17:57 +0000 Subject: [PATCH 2/3] Review fixes: the two-level read must guard both levels, and identity is the line's Follow-up to the #543 envelope fix, from review of PR #552. - `messageField` fell back on key *absence*, not value *usability*, so a present-but-unusable nested value permanently masked a good record-line one. A nested `provider: " "` beside a line-level `provider: "anthropic"` resolved the record to `unknown` and the allowlist excluded it fail-closed; a nested `timestamp` that did not parse dropped `message_created_at`, which re-dates the row to session start, defeats the `--since` window (a timestamp-less item is kept unconditionally) and puts the settlement ordinal match outside every window so the turn never dedupes. Rule 3's present-value test now runs at both levels before the fallback decides. - `id` is now read line-first, envelope-fallback. LLP 0158 verified message identity on the record line; envelope-first meant a future OpenClaw that copied the provider's own id into the nested message would silently repoint every `message_id` and `part_id`, so committed rows would stop deduping against new ones and the history would double with nothing raised. - The OPENCLAW_HOME relocation fixture still wrote the invented flat shape, bypassing `messageLine`, so it passed with the envelope read reverted. It now goes through the helper, and the shape pin carries `idempotencyKey` so it matches the live key list the same file documents. - Say which LEVEL `record` is: `parentId` is on the line, `idempotencyKey` and `toolCallId` are at `record.message`. The old wording invited the very read #543 was. - LLP 0158 gains rules 6 and 7 for the two behaviors above, and the stale "no live OpenClaw install was reachable" note on `usageAttributes` is reconciled with the spelling this work verified. Co-Authored-By: Claude --- .../openclaw/src/backfill.js | 17 ++-- .../openclaw/src/session_file.js | 93 +++++++++++++++---- .../plugins-workspace/openclaw/src/types.d.ts | 9 +- ...der-for-openclaw-session-jsonl.decision.md | 39 ++++++-- test/plugins/openclaw-backfill.test.js | 5 +- test/plugins/openclaw-session-file.test.js | 93 +++++++++++++++++++ 6 files changed, 215 insertions(+), 41 deletions(-) diff --git a/hypaware-core/plugins-workspace/openclaw/src/backfill.js b/hypaware-core/plugins-workspace/openclaw/src/backfill.js index ac542521..1c13ef24 100644 --- a/hypaware-core/plugins-workspace/openclaw/src/backfill.js +++ b/hypaware-core/plugins-workspace/openclaw/src/backfill.js @@ -38,9 +38,9 @@ import { isPlainObject, sha256Hex, stringValue } from 'hypaware/core/util' * the record line, and its role, content and (for an assistant message) the * model, provider, api, stop reason and usage in the nested `message` * envelope the LLP 0158 reader normalizes. So this module builds an - * `AiGatewayProjectedExchange` straight off those fields. It deliberately does NOT route through - * `anthropicMessages()` (projector.js): those parse a wire request/response - * pair, and there is no wire pair here to reconstruct. + * `AiGatewayProjectedExchange` straight off those fields. It deliberately + * does NOT route through `anthropicMessages()` (projector.js): those parse a + * wire request/response pair, and there is no wire pair here to reconstruct. * * That directness is what makes R11 true by construction rather than by * coincidence. A backfilled row carries the record's own `message_id`, so it @@ -567,11 +567,12 @@ function messageAttributes(message) { * * Each name is read through a small alias list rather than one spelling: the * session file's `usage` block is OpenClaw's own normalization of whatever its - * provider returned, and which spelling it settles on could not be verified at - * implementation time (no live OpenClaw install was reachable). Accepting the - * Anthropic wire spelling, the OpenAI wire spelling, and the camelCase - * normalized spelling costs a lookup and means an unverified guess cannot - * silently zero out a session's tokens. + * provider returned. A live install has since confirmed the camelCase + * spelling (`input`/`output`/`cacheRead`/`cacheWrite`, LLP 0158 Context), so + * the alias list is no longer a hedge against an unverified guess; it is kept + * because the block is still the client's normalization of a provider payload + * and a version that passed a wire spelling through would otherwise silently + * zero out a session's tokens. * * @ref LLP 0035#one-carrier [implements]: usage rides the assistant record * only, so it lands once per response diff --git a/hypaware-core/plugins-workspace/openclaw/src/session_file.js b/hypaware-core/plugins-workspace/openclaw/src/session_file.js index cee22ec4..11dac6cd 100644 --- a/hypaware-core/plugins-workspace/openclaw/src/session_file.js +++ b/hypaware-core/plugins-workspace/openclaw/src/session_file.js @@ -247,13 +247,16 @@ export async function readOpenclawSessionMessages(filePath) { * Parse one JSONL line into an {@link OpenclawSessionMessage}, or * `undefined` when the line is not a `type: "message"` record. Guards the * same way the header parse does (rules 1-3 above), applied to the fields - * LLP 0158's Context names as present on a message envelope: `id` and a - * timestamp on every message, `role` and `content` on every message, + * LLP 0158's Context names, at the level it names them at: `id` on the + * record line, a timestamp, `role` and `content` on every message, and * `model`/`provider`/`api`/`stopReason`/`usage` on an assistant one. Every - * field this function does not normalize (`parentId`, `idempotencyKey`, - * `toolCallId`, and anything else OpenClaw writes) is still reachable - * through `record`, the untouched record line, so a caller that needs one is - * not blocked on this reader growing a field for it. + * field this function does not normalize is still reachable through + * `record`, the untouched record line, so a caller that needs one is not + * blocked on this reader growing a field for it. `record` is the LINE, + * though, not the message: `parentId` is on it, but a message-level field + * this reader does not normalize (`idempotencyKey`, `toolCallId`) is at + * `record.message`. Naming the level is the whole point here, since reading + * a message field off the line is exactly #543. * * `role` and `content` are normalized here rather than left to `record` * precisely because their location is the non-obvious part @@ -272,24 +275,33 @@ function parseOpenclawSessionMessage(line) { const envelope = openclawMessageEnvelope(row) /** @type {OpenclawSessionMessage} */ const message = { record: row } - const id = nonBlankString(messageField(envelope, row, 'id')) + // `id` is the one field read the other way round, and deliberately so. + // LLP 0158's verified shape puts message IDENTITY on the record line; the + // nested envelope is OpenClaw's normalization of a provider response, and + // the day a version starts copying the provider's own `msg_...` id into it, + // envelope-first would silently repoint every `message_id` (and so every + // `part_id`) that backfill and settlement agree on. Already-committed rows + // would stop deduping against the new ones and the history would double, + // with nothing raised anywhere. The envelope stays the fallback, so a record + // that states identity only there is still read. + const id = nonBlankString(row.id) ?? nonBlankString(envelope.id) if (id !== undefined) message.id = id - const timestampMs = parseTimestampMs(messageField(envelope, row, 'timestamp')) + const timestampMs = messageField(envelope, row, 'timestamp', parseTimestampMs) if (timestampMs !== undefined) message.timestampMs = timestampMs - const role = nonBlankString(messageField(envelope, row, 'role')) + const role = messageField(envelope, row, 'role', nonBlankString) if (role !== undefined) message.role = role - const content = messageField(envelope, row, 'content') + const content = messageField(envelope, row, 'content', statedValue) if (content !== undefined) message.content = content - const model = nonBlankString(messageField(envelope, row, 'model')) + const model = messageField(envelope, row, 'model', nonBlankString) if (model !== undefined) message.model = model - const provider = nonBlankString(messageField(envelope, row, 'provider')) + const provider = messageField(envelope, row, 'provider', nonBlankString) if (provider !== undefined) message.provider = provider - const api = nonBlankString(messageField(envelope, row, 'api')) + const api = messageField(envelope, row, 'api', nonBlankString) if (api !== undefined) message.api = api - const stopReason = nonBlankString(messageField(envelope, row, 'stopReason')) + const stopReason = messageField(envelope, row, 'stopReason', nonBlankString) if (stopReason !== undefined) message.stopReason = stopReason - const usage = messageField(envelope, row, 'usage') - if (isPlainObject(usage)) message.usage = usage + const usage = messageField(envelope, row, 'usage', plainObject) + if (usage !== undefined) message.usage = usage return message } @@ -311,7 +323,12 @@ function parseOpenclawSessionMessage(line) { * that nests no `message` object states its fields on the line itself, and * reading them there is better than reading a message with no role and no * content. A field the envelope does state is never overridden by a - * same-named field on the line. + * same-named field on the line. The fallback is per FIELD, not per record + * ({@link messageField}), so a record that nests a partial envelope still + * recovers the rest of its fields from the line rather than reading as a + * message that is missing them. `id` is the single documented exception, + * read line-first because it is identity rather than content + * ({@link parseOpenclawSessionMessage}). * * @ref LLP 0158#decision [implements]: where a message's fields live is part * of the one reader's knowledge, not something each consumer re-derives @@ -327,14 +344,50 @@ function openclawMessageEnvelope(row) { * second. Not a substitution across fields (rule 1): it is the same field * name, looked for at the two levels one record can state it at. * + * `stated` is the field's own present-value test, and it runs at BOTH + * levels before the fallback decides. Running it only on the result would + * let a blank or wrong-typed envelope value shadow a usable one on the + * line, which would make that value absent (rule 3) and load-bearing at the + * same time: a nested `provider: " "` beside a line-level + * `provider: "anthropic"` would resolve the record to `unknown` and the + * backfill allowlist would exclude it fail-closed, the same silent drop + * #543 was. If a level does not state the field, it does not get a vote. + * + * @template T * @param {Record} envelope * @param {Record} row * @param {string} key + * @param {(value: unknown) => T | undefined} stated + * @returns {T | undefined} + */ +function messageField(envelope, row, key, stated) { + const fromEnvelope = stated(envelope[key]) + return fromEnvelope !== undefined ? fromEnvelope : stated(row[key]) +} + +/** + * `content`'s present-value test. Unlike every other normalized field it has + * no single shape to check: OpenClaw writes a string on some turns and a + * block array on others, and both consumers already accept either, so the + * value passes through as written. Only `null` is refused, so a nulled-out + * envelope field counts as unstated and the record line still gets its turn. + * + * @param {unknown} value * @returns {unknown} */ -function messageField(envelope, row, key) { - const value = envelope[key] - return value === undefined ? row[key] : value +function statedValue(value) { + return value === null ? undefined : value +} + +/** + * A plain object, else `undefined`: `usage`'s present-value test, so a + * non-object `usage` reads as absent at whichever level wrote it. + * + * @param {unknown} value + * @returns {Record | undefined} + */ +function plainObject(value) { + return isPlainObject(value) ? value : undefined } /** diff --git a/hypaware-core/plugins-workspace/openclaw/src/types.d.ts b/hypaware-core/plugins-workspace/openclaw/src/types.d.ts index 9953612a..80cad438 100644 --- a/hypaware-core/plugins-workspace/openclaw/src/types.d.ts +++ b/hypaware-core/plugins-workspace/openclaw/src/types.d.ts @@ -22,9 +22,12 @@ export interface OpenclawSessionHeader { * non-string (for the string fields), or blank, following the same * "unconfirmable is unresolvable" rule the header applies. `content` is * whatever the envelope wrote (a string or a block array), passed through - * unnormalized. `record` is the full raw record line, for a caller that - * needs a field this reader does not normalize (`parentId`, `toolCallId`); - * it is an untyped bag rather than `JsonObject` on purpose, the same choice + * unnormalized. `record` is the full raw record LINE, for a caller that + * needs a field this reader does not normalize: `parentId` is on the line + * itself, while a message-level field (`idempotencyKey`, `toolCallId`) is + * at `record.message`, one level down, and reading it off the line is the + * same mistake as #543. It is an untyped bag rather than `JsonObject` on + * purpose, the same choice * `CodexRolloutItem.payload` makes for the same reason (an arbitrary parsed * line, not a value this reader constructs and can vouch for the shape of). * Reaching into `record` for `role`/`content` is the #543 defect: those live diff --git a/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md b/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md index b719fdf8..65232e8d 100644 --- a/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md +++ b/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md @@ -60,14 +60,37 @@ answers wrong: path never pays for a large transcript (LLP 0049 R6's affordability argument, applied as LLP 0150#bounded applies it). 5. A message's fields are read from the nested `message` envelope, with the - record line as the fallback for a record that nests none, never the other - way round. This is the rule that reads wrong most quietly: read one level - too high, every field is simply absent, and absence is a legal answer - everywhere downstream. `provider: undefined` resolves to `unknown`, the - backfill allowlist excludes it fail-closed, and the run reports a clean - `0 rows` for a session it never managed to read (#543). A parse miss and - an intended exclusion are indistinguishable at that seam, so the address - has to be right in the reader. + record line as the fallback, never the other way round. This is the rule + that reads wrong most quietly: read one level too high, every field is + simply absent, and absence is a legal answer everywhere downstream. + `provider: undefined` resolves to `unknown`, the backfill allowlist + excludes it fail-closed, and the run reports a clean `0 rows` for a + session it never managed to read (#543). A parse miss and an intended + exclusion are indistinguishable at that seam, so the address has to be + right in the reader. +6. The fallback is per FIELD, not per record, and rule 3's present-value + test runs at BOTH levels before it decides. A record does not have to + nest nothing to read something off the line; it only has to state that + one field nowhere else. And a nested value that reads as absent (blank, + wrong-typed, `null`) cannot also be the value that suppresses the line: + that would make one value absent and load-bearing at once. A nested + `provider: " "` beside a line-level `provider: "anthropic"` would + otherwise resolve the record to `unknown` and lose it fail-closed, and a + nested `timestamp` that does not parse would drop `message_created_at`, + which re-dates the row to session start, defeats the `--since` window + (a timestamp-less item is kept unconditionally), and puts the settlement + ordinal match outside every window so the turn never dedupes. Same + silent-drop family as #543, one level down. +7. `id` is the one field read line-first, envelope-fallback, because it is + identity rather than content: the record line is where this document + verified message identity lives, and the nested envelope is OpenClaw's + normalization of a provider response. A version that started copying the + provider's own `msg_...` id into the envelope would, under rule 5, + silently repoint every `message_id` and therefore every `part_id` that + backfill and settlement agree on (LLP 0157 R11); already-committed rows + would stop deduping against new ones and the history would double with + nothing raised. The envelope stays the fallback, so a record that states + identity only there still resolves one. LLP 0150 documented two shipped bugs (#453, #459) caused by exactly this shape: two modules holding copies of the same session-header rules for the diff --git a/test/plugins/openclaw-backfill.test.js b/test/plugins/openclaw-backfill.test.js index 7afd09b4..5320eda8 100644 --- a/test/plugins/openclaw-backfill.test.js +++ b/test/plugins/openclaw-backfill.test.js @@ -206,6 +206,7 @@ const ASSISTANT_RECORD = { provider: 'anthropic', api: 'anthropic-messages', stopReason: 'end_turn', + idempotencyKey: 'idem-asst-1', usage: { input: 11, output: 7, cacheRead: 3, cacheWrite: 2 }, } @@ -233,7 +234,7 @@ test('the fixture writes the record shape OpenClaw actually appends', async () = assert.deepEqual(Object.keys(record).sort(), ['id', 'message', 'parentId', 'timestamp', 'type']) assert.deepEqual( Object.keys(record.message).sort(), - ['api', 'content', 'model', 'provider', 'role', 'stopReason', 'timestamp', 'usage'] + ['api', 'content', 'idempotencyKey', 'model', 'provider', 'role', 'stopReason', 'timestamp', 'usage'] ) assert.equal(record.provider, undefined, 'a real record states no provider at the top level') assert.equal(record.message.provider, 'anthropic') @@ -679,7 +680,7 @@ test('a relocated install is found through OPENCLAW_HOME, the same way settlemen path.join(dir, 'sess-relocated.jsonl'), [ JSON.stringify({ type: 'session', id: 'sess-relocated', cwd: '/work/repo', timestamp: '2026-07-30T10:00:00.000Z' }), - JSON.stringify({ type: 'message', ...ASSISTANT_RECORD }), + JSON.stringify(messageLine(ASSISTANT_RECORD)), ].join('\n') + '\n', 'utf8' ) diff --git a/test/plugins/openclaw-session-file.test.js b/test/plugins/openclaw-session-file.test.js index 6f858820..7a0e2126 100644 --- a/test/plugins/openclaw-session-file.test.js +++ b/test/plugins/openclaw-session-file.test.js @@ -294,6 +294,99 @@ test('the record line supplies the timestamp when the nested envelope states non assert.equal(messages[0].content, 'hi') }) +// The two fields a real record states at BOTH levels, and they resolve in +// OPPOSITE directions. `timestamp` is an ordinary message field, so the +// envelope owns it; it decides `message_created_at` on a backfilled row, the +// `--since` window, and the window the settlement ordinal match is bounded +// to, and a well-formed record's two values differ only by append latency. +// `id` is identity: the record line owns it, because a future OpenClaw that +// copied the provider's own id into the envelope would otherwise repoint +// every `message_id`/`part_id` R11's dedupe rests on, silently doubling the +// history. Pinning both directions is the point of this test. +// +// @ref LLP 0158#decision [tests]: envelope-first for a message's own fields, +// record line first for the identity the line is verified to state +test('the envelope wins for timestamp, the record line wins for id', async () => { + const file = tempSessionFile(JSON.stringify({ + type: 'message', + id: 'record-line-id', + timestamp: '2026-07-30T00:00:00.000Z', + message: { + id: 'nested-id', + timestamp: '2026-07-30T00:00:09.000Z', + role: 'user', + content: 'hi', + }, + }) + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].id, 'record-line-id') + assert.equal(messages[0].timestampMs, Date.parse('2026-07-30T00:00:09.000Z')) +}) + +test('a record that states its id only in the envelope still resolves an identity', async () => { + // The line owns `id`, but owning it does not mean refusing the envelope's: + // with nothing on the line there is no identity to protect, and reading the + // nested one beats emitting a hash `message_id` the settled row cannot + // match. + const file = tempSessionFile(JSON.stringify({ + type: 'message', + message: { id: 'nested-only-id', role: 'user', content: 'hi' }, + }) + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].id, 'nested-only-id') +}) + +test('a blank or wrong-typed nested field reads as absent, so the record line still supplies it', async () => { + // Rule 3 applies at BOTH levels before the fallback decides. A nested value + // that reads as absent cannot also be the value that suppresses the line: + // that would resolve this record to `unknown` and the backfill allowlist + // would exclude it fail-closed, the same silent drop as #543. + const file = tempSessionFile(JSON.stringify({ + type: 'message', + id: 'msg-1', + provider: 'anthropic', + model: 'claude-x', + usage: { input: 10, output: 20 }, + message: { role: 'assistant', content: 'x', provider: ' ', model: 42, usage: null }, + }) + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].provider, 'anthropic') + assert.equal(messages[0].model, 'claude-x') + assert.deepEqual(messages[0].usage, { input: 10, output: 20 }) +}) + +test('a blank nested field with nothing on the record line is absent, not a substitute', async () => { + // The other half of rule 3 at the nested level: with no line-level value to + // fall back to, an unconfirmable envelope field stays unresolved rather + // than becoming an empty string or a coerced number. + const file = tempSessionFile(JSON.stringify({ + type: 'message', + message: { id: ' ', timestamp: '', role: 'assistant', model: 42, stopReason: null, content: 'x' }, + }) + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].id, undefined) + assert.equal(messages[0].timestampMs, undefined) + assert.equal(messages[0].model, undefined) + assert.equal(messages[0].stopReason, undefined) + assert.equal(messages[0].role, 'assistant') +}) + +test('a `message` key that is not an object leaves the record line as the only address', async () => { + // `openclawMessageEnvelope` requires a plain object. A record whose + // `message` is a string or an array is not an envelope, so the reader does + // not treat it as one; the line is read instead, and a line that states no + // role yields a message the consumers drop rather than a half-read one. + const file = tempSessionFile([ + JSON.stringify({ type: 'message', id: 'msg-1', message: 'not an envelope', role: 'user', content: 'hi' }), + JSON.stringify({ type: 'message', id: 'msg-2', message: [{ type: 'text', text: 'x' }] }), + ].join('\n') + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].role, 'user') + assert.equal(messages[0].content, 'hi') + assert.equal(messages[1].role, undefined) + assert.equal(messages[1].content, undefined) +}) + test('readOpenclawSessionMessages skips blank and unparseable lines without aborting the rest', async () => { const file = tempSessionFile([ headerLine({ id: 'session-abc' }), From b60ca6ac1659ae3337202eee5fa3453e5be8891e Mon Sep 17 00:00:00 2001 From: test Date: Fri, 31 Jul 2026 22:46:05 +0000 Subject: [PATCH 3/3] Round-2 review fixes: pin content's present-value test, name the id exception Three follow-ups to the round-1 fixes, all in the same family the round-1 findings were: a new behavior that no test pins, and docs that state a rule the code does not follow. - `statedValue`, the `content` present-value test the guard-order fix introduced, was entirely unpinned: replacing it with the identity function left the whole suite green while a nulled-out nested `content` suppressed a usable record-line value and landed `content: null` on the message. Pinned in both directions (line supplies it; absent when neither level does). - `types.d.ts` still stated the blanket envelope-first rule over `id`, which the same commit made line-first (LLP 0158 rule 7), and over `content`, whose test refuses only `null`. The published declaration is what a package consumer reads, so it now names both exceptions. - LLP 0158 rule 6 claimed a blank or wrong-typed nested value can never suppress the line. True of the string fields, false of `content`: a nested `content: " "` or `content: 42` does suppress it. The rule now says what "reads as absent" is per field. Also extends the non-object-`message` test with the `null` case, the one input where the plain-object guard is the difference between reading the record line and throwing out of the whole file read. Co-Authored-By: Claude --- .../plugins-workspace/openclaw/src/types.d.ts | 10 +++-- ...der-for-openclaw-session-jsonl.decision.md | 6 ++- test/plugins/openclaw-session-file.test.js | 38 +++++++++++++++++-- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/hypaware-core/plugins-workspace/openclaw/src/types.d.ts b/hypaware-core/plugins-workspace/openclaw/src/types.d.ts index 80cad438..c8fee1ec 100644 --- a/hypaware-core/plugins-workspace/openclaw/src/types.d.ts +++ b/hypaware-core/plugins-workspace/openclaw/src/types.d.ts @@ -20,9 +20,13 @@ export interface OpenclawSessionHeader { * `stopReason`, `usage`); each is read off the nested `message` envelope, * falling back to the record line, and is absent when the field is missing, * non-string (for the string fields), or blank, following the same - * "unconfirmable is unresolvable" rule the header applies. `content` is - * whatever the envelope wrote (a string or a block array), passed through - * unnormalized. `record` is the full raw record LINE, for a caller that + * "unconfirmable is unresolvable" rule the header applies. `id` is the one + * field read the other way round, record line first and envelope second, + * because the line is where LLP 0158 rule 7 verified message identity + * lives. `content` is whatever the envelope wrote (a string or a block + * array), passed through unnormalized, so unlike the string fields it + * refuses only an explicit `null` rather than a blank or wrong-typed value. + * `record` is the full raw record LINE, for a caller that * needs a field this reader does not normalize: `parentId` is on the line * itself, while a message-level field (`idempotencyKey`, `toolCallId`) is * at `record.message`, one level down, and reading it off the line is the diff --git a/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md b/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md index 65232e8d..25649a14 100644 --- a/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md +++ b/llp/0158-one-reader-for-openclaw-session-jsonl.decision.md @@ -80,7 +80,11 @@ answers wrong: which re-dates the row to session start, defeats the `--since` window (a timestamp-less item is kept unconditionally), and puts the settlement ordinal match outside every window so the turn never dedupes. Same - silent-drop family as #543, one level down. + silent-drop family as #543, one level down. "Reads as absent" is each + field's own answer, not one predicate for all of them: the string fields + refuse blank and non-string alike, while `content` has no single shape to + check (a string on some turns, a block array on others) and so refuses + only an explicit `null`. 7. `id` is the one field read line-first, envelope-fallback, because it is identity rather than content: the record line is where this document verified message identity lives, and the nested envelope is OpenClaw's diff --git a/test/plugins/openclaw-session-file.test.js b/test/plugins/openclaw-session-file.test.js index 7a0e2126..62801c44 100644 --- a/test/plugins/openclaw-session-file.test.js +++ b/test/plugins/openclaw-session-file.test.js @@ -371,20 +371,52 @@ test('a blank nested field with nothing on the record line is absent, not a subs assert.equal(messages[0].role, 'assistant') }) +test('a nulled-out nested content is unstated, so the record line still supplies it', async () => { + // `content` is the one normalized field with no single shape to test (a + // string on some turns, a block array on others), so its present-value test + // refuses an explicit `null` and nothing else. That refusal is load-bearing, + // not cosmetic: without it a nulled-out envelope field would suppress the + // line and land `content: null` on the message, which the backfill projector + // drops and the settlement content index hashes into a key no live row can + // match, the same silent drop as #543. + const file = tempSessionFile([ + JSON.stringify({ + type: 'message', + id: 'msg-1', + content: 'from the record line', + message: { role: 'user', content: null }, + }), + JSON.stringify({ type: 'message', id: 'msg-2', message: { role: 'user', content: null } }), + ].join('\n') + '\n') + const messages = await readOpenclawSessionMessages(file) + assert.equal(messages[0].content, 'from the record line') + assert.equal( + Object.hasOwn(messages[1], 'content'), + false, + 'with nothing on the line either, content is absent, never a null the consumers must re-check' + ) +}) + test('a `message` key that is not an object leaves the record line as the only address', async () => { // `openclawMessageEnvelope` requires a plain object. A record whose - // `message` is a string or an array is not an envelope, so the reader does - // not treat it as one; the line is read instead, and a line that states no - // role yields a message the consumers drop rather than a half-read one. + // `message` is a string, an array or `null` is not an envelope, so the + // reader does not treat it as one; the line is read instead, and a line that + // states no role yields a message the consumers drop rather than a half-read + // one. The `null` case is why the guard is a plain-object test rather than a + // truthiness one: indexing a field off `null` would throw out of the whole + // file read, losing every later record with it. const file = tempSessionFile([ JSON.stringify({ type: 'message', id: 'msg-1', message: 'not an envelope', role: 'user', content: 'hi' }), JSON.stringify({ type: 'message', id: 'msg-2', message: [{ type: 'text', text: 'x' }] }), + JSON.stringify({ type: 'message', id: 'msg-3', message: null, role: 'assistant', content: 'still read' }), ].join('\n') + '\n') const messages = await readOpenclawSessionMessages(file) assert.equal(messages[0].role, 'user') assert.equal(messages[0].content, 'hi') assert.equal(messages[1].role, undefined) assert.equal(messages[1].content, undefined) + assert.equal(messages[2].role, 'assistant') + assert.equal(messages[2].content, 'still read') }) test('readOpenclawSessionMessages skips blank and unparseable lines without aborting the rest', async () => {