fix: fingerprint the fiber census page as SurrealDB stores it

CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- Merge no longer stops with "fiber census staged page is invalid" when a fiber has an
+  empty field. The SDK sends `None` as NONE and SurrealDB drops those keys, so a census
+  page fingerprinted with them never matched the page read back; pages are now encoded
+  exactly as they are stored.
+
 ## [3.12.0] — 2026-09-25 — consolidation can resume its committed work
 
 Large maintenance runs now make bounded progress and can continue from durable

src/surreal_memory/engine/consolidation.py

@@ -4244,34 +4244,52 @@ async def _iter_fiber_census_pages(
             cast("Any", self._storage)._query,
         )
 
+        def drop_none(value: Any) -> Any:
+            # The SDK sends None as NONE and SurrealDB drops NONE-valued keys (nested
+            # ones too), so a page fingerprinted with them never matches the page read
+            # back. Encode the page exactly as it will come back; decode_fiber already
+            # defaults every missing key.
+            if isinstance(value, dict):
+                return {k: drop_none(v) for k, v in value.items() if v is not None}
+            if isinstance(value, list):
+                return [drop_none(v) for v in value]
+            return value
+
         def encode_fiber(fiber: Fiber) -> dict[str, Any]:
-            return {
-                "id": fiber.id,
-                "neuron_ids": sorted(fiber.neuron_ids),
-                "synapse_ids": sorted(fiber.synapse_ids),
-                "anchor_neuron_id": fiber.anchor_neuron_id,
-                "pathway": list(fiber.pathway),
-                "conductivity": fiber.conductivity,
-                "last_conducted": fiber.last_conducted.isoformat()
-                if fiber.last_conducted
-                else None,
-                "time_start": fiber.time_start.isoformat() if fiber.time_start else None,
-                "time_end": fiber.time_end.isoformat() if fiber.time_end else None,
-                "coherence": fiber.coherence,
-                "salience": fiber.salience,
-                "frequency": fiber.frequency,
-                "summary": fiber.summary,
-                "essence": fiber.essence,
-                "last_ghost_shown_at": (
-                    fiber.last_ghost_shown_at.isoformat() if fiber.last_ghost_shown_at else None
+            return cast(
+                "dict[str, Any]",
+                drop_none(
+                    {
+                        "id": fiber.id,
+                        "neuron_ids": sorted(fiber.neuron_ids),
+                        "synapse_ids": sorted(fiber.synapse_ids),
+                        "anchor_neuron_id": fiber.anchor_neuron_id,
+                        "pathway": list(fiber.pathway),
+                        "conductivity": fiber.conductivity,
+                        "last_conducted": fiber.last_conducted.isoformat()
+                        if fiber.last_conducted
+                        else None,
+                        "time_start": fiber.time_start.isoformat() if fiber.time_start else None,
+                        "time_end": fiber.time_end.isoformat() if fiber.time_end else None,
+                        "coherence": fiber.coherence,
+                        "salience": fiber.salience,
+                        "frequency": fiber.frequency,
+                        "summary": fiber.summary,
+                        "essence": fiber.essence,
+                        "last_ghost_shown_at": (
+                            fiber.last_ghost_shown_at.isoformat()
+                            if fiber.last_ghost_shown_at
+                            else None
+                        ),
+                        "auto_tags": sorted(fiber.auto_tags),
+                        "agent_tags": sorted(fiber.agent_tags),
+                        "metadata": fiber.metadata,
+                        "compression_tier": fiber.compression_tier,
+                        "pinned": fiber.pinned,
+                        "created_at": fiber.created_at.isoformat(),
+                    }
                 ),
-                "auto_tags": sorted(fiber.auto_tags),
-                "agent_tags": sorted(fiber.agent_tags),
-                "metadata": fiber.metadata,
-                "compression_tier": fiber.compression_tier,
-                "pinned": fiber.pinned,
-                "created_at": fiber.created_at.isoformat(),
-            }
+            )
 
         def decode_fiber(row: dict[str, Any]) -> Fiber:
             def parse_time(key: str) -> datetime | None:

tests/integration/test_fiber_census_none_fields_db.py

@@ -0,0 +1,104 @@
+"""Live SurrealDB regression: the merge fiber census must survive fibers with empty fields.
+
+The Python SDK sends ``None`` as NONE, and SurrealDB drops NONE-valued keys from the
+stored record (nested ones inside ``metadata`` too). The census used to fingerprint the
+encoded page *with* those keys and re-fingerprint the page it read back *without* them,
+so every page holding a fiber with an empty field failed validation and merge stopped with
+"fiber census staged page is invalid" - on a real brain, where almost every fiber has an
+empty ``summary`` or ``last_ghost_shown_at``, merge never ran at all.
+
+Runs only against the explicitly opted-in loopback SMEM_TEST_SURREALDB_URL; the fixture
+creates a unique database and uses the disposable test credentials.
+"""
+
+from __future__ import annotations
+
+import os
+import uuid
+from collections.abc import AsyncIterator
+from urllib.parse import urlsplit
+
+import pytest
+import pytest_asyncio
+
+from surreal_memory.core.brain import Brain
+from surreal_memory.core.fiber import Fiber
+from surreal_memory.core.neuron import Neuron, NeuronType
+from surreal_memory.engine.consolidation import (
+    ConsolidationConfig,
+    ConsolidationEngine,
+    ConsolidationStrategy,
+)
+from surreal_memory.storage.surrealdb.store import SurrealDBStorage
+
+TEST_SURREALDB_URL = os.getenv("SMEM_TEST_SURREALDB_URL")
+TEST_AUTH = ("root", "root")
+
+
+def _is_loopback_test_url(url: str | None) -> bool:
+    if not url:
+        return False
+    try:
+        parsed = urlsplit(url)
+        return (
+            parsed.scheme in {"http", "https", "ws", "wss"}
+            and parsed.hostname in {"localhost", "127.0.0.1", "::1"}
+            and parsed.username is None
+            and parsed.password is None
+            and parsed.port is not None
+        )
+    except ValueError:
+        return False
+
+
+pytestmark = [
+    pytest.mark.integration,
+    pytest.mark.skipif(
+        not _is_loopback_test_url(TEST_SURREALDB_URL),
+        reason="requires an explicit loopback SMEM_TEST_SURREALDB_URL",
+    ),
+]
+
+
+@pytest_asyncio.fixture
+async def store() -> AsyncIterator[SurrealDBStorage]:
+    assert TEST_SURREALDB_URL is not None
+    storage = SurrealDBStorage(
+        url=TEST_SURREALDB_URL,
+        user=TEST_AUTH[0],
+        password=TEST_AUTH[1],
+        namespace="smem_ci",
+        database="it_" + uuid.uuid4().hex[:12],
+    )
+    await storage.initialize()
+    brain = Brain.create(name="fiber-census-none-it")
+    await storage.save_brain(brain)
+    storage.set_brain(brain.id)
+    try:
+        yield storage
+    finally:
+        await storage.close()
+
+
+async def test_merge_census_completes_with_fibers_that_have_empty_fields(
+    store: SurrealDBStorage,
+) -> None:
+    for i in range(3):
+        neurons = [Neuron.create(NeuronType.CONCEPT, f"census none {i} {s}") for s in ("a", "b")]
+        for neuron in neurons:
+            await store.add_neuron(neuron)
+        fiber = Fiber.create(
+            neuron_ids={n.id for n in neurons},
+            synapse_ids=set(),
+            anchor_neuron_id=neurons[0].id,
+            summary=None,  # empty field - dropped by the SDK/SurrealDB round trip
+        )
+        fiber.metadata["note"] = None  # nested empty value - dropped as well
+        await store.add_fiber(fiber)
+
+    report = await ConsolidationEngine(store, ConsolidationConfig()).run(
+        strategies=[ConsolidationStrategy.MERGE]
+    )
+
+    assert report.extra.get("failed_strategies") in (None, []), report.extra
+    assert report.extra.get("consolidation_status") == "completed", report.extra