OpenClaw backfill reads message fields one level too high, projecting 0 rows

hypaware-core/plugins-workspace/openclaw/src/backfill.js

@@ -34,12 +34,13 @@ import { isPlainObject, sha256Hex, stringValue } from 'hypaware/core/util'
  * neither consumer may hold its own parse).
  *
  * The session file is the authoritative record of a turn: it already carries
- * one flattened per-message record with the message's own native id, its
- * timestamp, and for an assistant message the model, provider, api, stop
- * reason, and usage. So this module builds an `AiGatewayProjectedExchange`
- * straight off those fields. It deliberately does NOT route through
- * `anthropicMessages()` (projector.js): those parse a wire request/response
- * pair, and there is no wire pair here to reconstruct.
+ * one per-message record with the message's own native id and timestamp on
+ * the record line, and its role, content and (for an assistant message) the
+ * model, provider, api, stop reason and usage in the nested `message`
+ * envelope the LLP 0158 reader normalizes. So this module builds an
+ * `AiGatewayProjectedExchange` straight off those fields. It deliberately
+ * does NOT route through `anthropicMessages()` (projector.js): those parse a
+ * wire request/response pair, and there is no wire pair here to reconstruct.
  *
  * That directness is what makes R11 true by construction rather than by
  * coincidence. A backfilled row carries the record's own `message_id`, so it
@@ -492,10 +493,13 @@ function projectedExchangeFromSession(args) {
  * @returns {AiGatewayProjectedMessage | undefined}
  */
 function projectedMessageFromRecord(message) {
-  const record = message.record
-  const rawRole = stringValue(record.role)
+  // Through the LLP 0158 reader's normalized fields, never `message.record`:
+  // OpenClaw nests `role`/`content` under the record line's `message`
+  // envelope, so the top-level read this used to do found neither and dropped
+  // every record of every real session (#543).
+  const rawRole = stringValue(message.role)
   if (!rawRole) return undefined
-  const content = record.content
+  const content = message.content
   // The gateway drops a message whose content normalizes to no blocks; doing it
   // here keeps `messages_projected` honest about what was actually emitted.
   if (typeof content === 'string' ? content.length === 0 : !Array.isArray(content) || content.length === 0) {
@@ -563,11 +567,12 @@ function messageAttributes(message) {
  *
  * Each name is read through a small alias list rather than one spelling: the
  * session file's `usage` block is OpenClaw's own normalization of whatever its
- * provider returned, and which spelling it settles on could not be verified at
- * implementation time (no live OpenClaw install was reachable). Accepting the
- * Anthropic wire spelling, the OpenAI wire spelling, and the camelCase
- * normalized spelling costs a lookup and means an unverified guess cannot
- * silently zero out a session's tokens.
+ * provider returned. A live install has since confirmed the camelCase
+ * spelling (`input`/`output`/`cacheRead`/`cacheWrite`, LLP 0158 Context), so
+ * the alias list is no longer a hedge against an unverified guess; it is kept
+ * because the block is still the client's normalization of a provider payload
+ * and a version that passed a wire spelling through would otherwise silently
+ * zero out a session's tokens.
  *
  * @ref LLP 0035#one-carrier [implements]: usage rides the assistant record
  * only, so it lands once per response

hypaware-core/plugins-workspace/openclaw/src/session_file.js

@@ -247,12 +247,23 @@ export async function readOpenclawSessionMessages(filePath) {
  * Parse one JSONL line into an {@link OpenclawSessionMessage}, or
  * `undefined` when the line is not a `type: "message"` record. Guards the
  * same way the header parse does (rules 1-3 above), applied to the fields
- * LLP 0158's Context names as present on a message envelope: `id` and a
- * timestamp on every message, `model`/`provider`/`api`/`stopReason`/`usage`
- * on an assistant one. Every field this function does not normalize (role,
- * content blocks, and anything else OpenClaw writes) is still reachable
- * through `record`, the untouched envelope, so a caller that needs one is
- * not blocked on this reader growing a field for it.
+ * LLP 0158's Context names, at the level it names them at: `id` on the
+ * record line, a timestamp, `role` and `content` on every message, and
+ * `model`/`provider`/`api`/`stopReason`/`usage` on an assistant one. Every
+ * field this function does not normalize is still reachable through
+ * `record`, the untouched record line, so a caller that needs one is not
+ * blocked on this reader growing a field for it. `record` is the LINE,
+ * though, not the message: `parentId` is on it, but a message-level field
+ * this reader does not normalize (`idempotencyKey`, `toolCallId`) is at
+ * `record.message`. Naming the level is the whole point here, since reading
+ * a message field off the line is exactly #543.
+ *
+ * `role` and `content` are normalized here rather than left to `record`
+ * precisely because their location is the non-obvious part
+ * ({@link openclawMessageEnvelope}): both consumers used to reach into
+ * `record.role`/`record.content` themselves, so both read one level too high
+ * and both dropped every real session (#543). A field whose *address* is the
+ * thing that is easy to get wrong belongs to the one reader.
  *
  * @param {string} line
  * @returns {OpenclawSessionMessage | undefined}
@@ -261,24 +272,124 @@ function parseOpenclawSessionMessage(line) {
   const row = parseMaybeJson(line)
   if (!isPlainObject(row)) return undefined
   if (row.type !== 'message') return undefined
+  const envelope = openclawMessageEnvelope(row)
   /** @type {OpenclawSessionMessage} */
   const message = { record: row }
-  const id = nonBlankString(row.id)
+  // `id` is the one field read the other way round, and deliberately so.
+  // LLP 0158's verified shape puts message IDENTITY on the record line; the
+  // nested envelope is OpenClaw's normalization of a provider response, and
+  // the day a version starts copying the provider's own `msg_...` id into it,
+  // envelope-first would silently repoint every `message_id` (and so every
+  // `part_id`) that backfill and settlement agree on. Already-committed rows
+  // would stop deduping against the new ones and the history would double,
+  // with nothing raised anywhere. The envelope stays the fallback, so a record
+  // that states identity only there is still read.
+  const id = nonBlankString(row.id) ?? nonBlankString(envelope.id)
   if (id !== undefined) message.id = id
-  const timestampMs = parseTimestampMs(row.timestamp)
+  const timestampMs = messageField(envelope, row, 'timestamp', parseTimestampMs)
   if (timestampMs !== undefined) message.timestampMs = timestampMs
-  const model = nonBlankString(row.model)
+  const role = messageField(envelope, row, 'role', nonBlankString)
+  if (role !== undefined) message.role = role
+  const content = messageField(envelope, row, 'content', statedValue)
+  if (content !== undefined) message.content = content
+  const model = messageField(envelope, row, 'model', nonBlankString)
   if (model !== undefined) message.model = model
-  const provider = nonBlankString(row.provider)
+  const provider = messageField(envelope, row, 'provider', nonBlankString)
   if (provider !== undefined) message.provider = provider
-  const api = nonBlankString(row.api)
+  const api = messageField(envelope, row, 'api', nonBlankString)
   if (api !== undefined) message.api = api
-  const stopReason = nonBlankString(row.stopReason)
+  const stopReason = messageField(envelope, row, 'stopReason', nonBlankString)
   if (stopReason !== undefined) message.stopReason = stopReason
-  if (isPlainObject(row.usage)) message.usage = row.usage
+  const usage = messageField(envelope, row, 'usage', plainObject)
+  if (usage !== undefined) message.usage = usage
   return message
 }
 
+/**
+ * The message envelope of a `type: "message"` record: the nested `message`
+ * object, not the record line.
+ *
+ * A record line states only what identifies and positions the message
+ * (`id`, `parentId`, `timestamp`, `type`); the message itself - `role`,
+ * `content`, and for an assistant turn `model`, `provider`, `api`,
+ * `stopReason`, `usage` - is one level down under `message`. Read at the top
+ * level every one of those fields is absent, which is not a loud failure:
+ * `provider` reads `undefined`, the backfill allowlist resolves the record to
+ * `unknown`, and the run reports a clean "0 rows" for a session it simply
+ * failed to read (#543). Fail-closed exclusion and a parse miss are
+ * indistinguishable at that seam, so the address has to be right here.
+ *
+ * The record line is the fallback, not a second address to prefer: a record
+ * that nests no `message` object states its fields on the line itself, and
+ * reading them there is better than reading a message with no role and no
+ * content. A field the envelope does state is never overridden by a
+ * same-named field on the line. The fallback is per FIELD, not per record
+ * ({@link messageField}), so a record that nests a partial envelope still
+ * recovers the rest of its fields from the line rather than reading as a
+ * message that is missing them. `id` is the single documented exception,
+ * read line-first because it is identity rather than content
+ * ({@link parseOpenclawSessionMessage}).
+ *
+ * @ref LLP 0158#decision [implements]: where a message's fields live is part
+ * of the one reader's knowledge, not something each consumer re-derives
+ * @param {Record<string, unknown>} row
+ * @returns {Record<string, unknown>}
+ */
+function openclawMessageEnvelope(row) {
+  return isPlainObject(row.message) ? row.message : row
+}
+
+/**
+ * One message field, read from the envelope first and the record line
+ * second. Not a substitution across fields (rule 1): it is the same field
+ * name, looked for at the two levels one record can state it at.
+ *
+ * `stated` is the field's own present-value test, and it runs at BOTH
+ * levels before the fallback decides. Running it only on the result would
+ * let a blank or wrong-typed envelope value shadow a usable one on the
+ * line, which would make that value absent (rule 3) and load-bearing at the
+ * same time: a nested `provider: "  "` beside a line-level
+ * `provider: "anthropic"` would resolve the record to `unknown` and the
+ * backfill allowlist would exclude it fail-closed, the same silent drop
+ * #543 was. If a level does not state the field, it does not get a vote.
+ *
+ * @template T
+ * @param {Record<string, unknown>} envelope
+ * @param {Record<string, unknown>} row
+ * @param {string} key
+ * @param {(value: unknown) => T | undefined} stated
+ * @returns {T | undefined}
+ */
+function messageField(envelope, row, key, stated) {
+  const fromEnvelope = stated(envelope[key])
+  return fromEnvelope !== undefined ? fromEnvelope : stated(row[key])
+}
+
+/**
+ * `content`'s present-value test. Unlike every other normalized field it has
+ * no single shape to check: OpenClaw writes a string on some turns and a
+ * block array on others, and both consumers already accept either, so the
+ * value passes through as written. Only `null` is refused, so a nulled-out
+ * envelope field counts as unstated and the record line still gets its turn.
+ *
+ * @param {unknown} value
+ * @returns {unknown}
+ */
+function statedValue(value) {
+  return value === null ? undefined : value
+}
+
+/**
+ * A plain object, else `undefined`: `usage`'s present-value test, so a
+ * non-object `usage` reads as absent at whichever level wrote it.
+ *
+ * @param {unknown} value
+ * @returns {Record<string, unknown> | undefined}
+ */
+function plainObject(value) {
+  return isPlainObject(value) ? value : undefined
+}
+
 /**
  * A non-blank string, byte-identical, else `undefined`. Stricter than
  * `stringValue` (core util) on purpose: a whitespace-only value matches no

hypaware-core/plugins-workspace/openclaw/src/settle.js

@@ -385,7 +385,7 @@ function buildOpenclawSessionIndex(candidate, header, messages) {
     // there is no identity to upgrade a row to, so it never enters the
     // content index.
     if (!message.id) continue
-    const key = sessionMatchKey(roles[i], message.record.content)
+    const key = sessionMatchKey(roles[i], message.content)
     if (ambiguous.has(key)) continue
     if (byContentKey.has(key)) {
       byContentKey.delete(key)
@@ -540,13 +540,16 @@ function readMatchKey(attributes) {
 /**
  * A session message record's own role, as written in the file (the raw
  * value `sessionMatchKey` needs, since it owns the `toolResult`
- * reconciliation itself).
+ * reconciliation itself). Read through the LLP 0158 reader's normalized
+ * field, never `message.record`: OpenClaw nests `role` under the record
+ * line's `message` envelope, and reading it a level too high made every
+ * record of every real session settle as `unknown` (#543).
  *
  * @param {OpenclawSessionMessage} message
  * @returns {string}
  */
 function rawRole(message) {
-  return stringValue(message.record.role) ?? 'unknown'
+  return stringValue(message.role) ?? 'unknown'
 }
 
 /**

hypaware-core/plugins-workspace/openclaw/src/types.d.ts

@@ -15,20 +15,33 @@ export interface OpenclawSessionHeader {
 /**
  * One `type: "message"` record recovered from an OpenClaw session
  * transcript. The normalized fields are the ones LLP 0158's Context names as
- * present on every message envelope (`id`, a timestamp) or on an assistant
- * envelope specifically (`model`, `provider`, `api`, `stopReason`, `usage`);
- * each is a present-value read off the envelope, absent when the field is
- * missing, non-string (for the string fields), or blank, following the same
- * "unconfirmable is unresolvable" rule the header applies. `record` is the
- * full raw envelope, for a caller that needs a field this reader does not
- * normalize (e.g. message content/blocks); it is an untyped bag rather than
- * `JsonObject` on purpose, the same choice `CodexRolloutItem.payload` makes
- * for the same reason (an arbitrary parsed line, not a value this reader
- * constructs and can vouch for the shape of).
+ * present on every message envelope (`id`, a timestamp, `role`, `content`)
+ * or on an assistant envelope specifically (`model`, `provider`, `api`,
+ * `stopReason`, `usage`); each is read off the nested `message` envelope,
+ * falling back to the record line, and is absent when the field is missing,
+ * non-string (for the string fields), or blank, following the same
+ * "unconfirmable is unresolvable" rule the header applies. `id` is the one
+ * field read the other way round, record line first and envelope second,
+ * because the line is where LLP 0158 rule 7 verified message identity
+ * lives. `content` is whatever the envelope wrote (a string or a block
+ * array), passed through unnormalized, so unlike the string fields it
+ * refuses only an explicit `null` rather than a blank or wrong-typed value.
+ * `record` is the full raw record LINE, for a caller that
+ * needs a field this reader does not normalize: `parentId` is on the line
+ * itself, while a message-level field (`idempotencyKey`, `toolCallId`) is
+ * at `record.message`, one level down, and reading it off the line is the
+ * same mistake as #543. It is an untyped bag rather than `JsonObject` on
+ * purpose, the same choice
+ * `CodexRolloutItem.payload` makes for the same reason (an arbitrary parsed
+ * line, not a value this reader constructs and can vouch for the shape of).
+ * Reaching into `record` for `role`/`content` is the #543 defect: those live
+ * under `message`, and the reader is the one place that knows it.
  */
 export interface OpenclawSessionMessage {
   id?: string
   timestampMs?: number
+  role?: string
+  content?: unknown
   model?: string
   provider?: string
   api?: string