From fb500e7f99c7758df21b759be9587e184043593e Mon Sep 17 00:00:00 2001 From: Robert Sigmundsson <230784065+RobertSigmundsson@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:36:46 +0200 Subject: [PATCH] fix: fingerprint the fiber census page as SurrealDB stores it The SDK sends None as NONE and SurrealDB drops NONE-valued keys, nested ones included, so a census page fingerprinted with its empty fields never matched the page read back. Every page holding a fiber with an empty summary, last_ghost_shown_at or metadata value failed validation and merge stopped with "fiber census staged page is invalid" - on a real brain merge never ran. Encode the page without None-valued keys; decode_fiber already defaults them. --- CHANGELOG.md | 7 ++ src/surreal_memory/engine/consolidation.py | 70 +++++++----- .../test_fiber_census_none_fields_db.py | 104 ++++++++++++++++++ 3 files changed, 155 insertions(+), 26 deletions(-) create mode 100644 tests/integration/test_fiber_census_none_fields_db.py diff --git a/CHANGELOG.md b/CHANGELOG.md index da001733..af71ab7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Merge no longer stops with "fiber census staged page is invalid" when a fiber has an + empty field. The SDK sends `None` as NONE and SurrealDB drops those keys, so a census + page fingerprinted with them never matched the page read back; pages are now encoded + exactly as they are stored. + ## [3.12.0] — 2026-09-25 — consolidation can resume its committed work Large maintenance runs now make bounded progress and can continue from durable diff --git a/src/surreal_memory/engine/consolidation.py b/src/surreal_memory/engine/consolidation.py index 3123e66d..ad923ade 100755 --- a/src/surreal_memory/engine/consolidation.py +++ b/src/surreal_memory/engine/consolidation.py @@ -4244,34 +4244,52 @@ async def _iter_fiber_census_pages( cast("Any", self._storage)._query, ) + def drop_none(value: Any) -> Any: + # The SDK sends None as NONE and SurrealDB drops NONE-valued keys (nested + # ones too), so a page fingerprinted with them never matches the page read + # back. Encode the page exactly as it will come back; decode_fiber already + # defaults every missing key. + if isinstance(value, dict): + return {k: drop_none(v) for k, v in value.items() if v is not None} + if isinstance(value, list): + return [drop_none(v) for v in value] + return value + def encode_fiber(fiber: Fiber) -> dict[str, Any]: - return { - "id": fiber.id, - "neuron_ids": sorted(fiber.neuron_ids), - "synapse_ids": sorted(fiber.synapse_ids), - "anchor_neuron_id": fiber.anchor_neuron_id, - "pathway": list(fiber.pathway), - "conductivity": fiber.conductivity, - "last_conducted": fiber.last_conducted.isoformat() - if fiber.last_conducted - else None, - "time_start": fiber.time_start.isoformat() if fiber.time_start else None, - "time_end": fiber.time_end.isoformat() if fiber.time_end else None, - "coherence": fiber.coherence, - "salience": fiber.salience, - "frequency": fiber.frequency, - "summary": fiber.summary, - "essence": fiber.essence, - "last_ghost_shown_at": ( - fiber.last_ghost_shown_at.isoformat() if fiber.last_ghost_shown_at else None + return cast( + "dict[str, Any]", + drop_none( + { + "id": fiber.id, + "neuron_ids": sorted(fiber.neuron_ids), + "synapse_ids": sorted(fiber.synapse_ids), + "anchor_neuron_id": fiber.anchor_neuron_id, + "pathway": list(fiber.pathway), + "conductivity": fiber.conductivity, + "last_conducted": fiber.last_conducted.isoformat() + if fiber.last_conducted + else None, + "time_start": fiber.time_start.isoformat() if fiber.time_start else None, + "time_end": fiber.time_end.isoformat() if fiber.time_end else None, + "coherence": fiber.coherence, + "salience": fiber.salience, + "frequency": fiber.frequency, + "summary": fiber.summary, + "essence": fiber.essence, + "last_ghost_shown_at": ( + fiber.last_ghost_shown_at.isoformat() + if fiber.last_ghost_shown_at + else None + ), + "auto_tags": sorted(fiber.auto_tags), + "agent_tags": sorted(fiber.agent_tags), + "metadata": fiber.metadata, + "compression_tier": fiber.compression_tier, + "pinned": fiber.pinned, + "created_at": fiber.created_at.isoformat(), + } ), - "auto_tags": sorted(fiber.auto_tags), - "agent_tags": sorted(fiber.agent_tags), - "metadata": fiber.metadata, - "compression_tier": fiber.compression_tier, - "pinned": fiber.pinned, - "created_at": fiber.created_at.isoformat(), - } + ) def decode_fiber(row: dict[str, Any]) -> Fiber: def parse_time(key: str) -> datetime | None: diff --git a/tests/integration/test_fiber_census_none_fields_db.py b/tests/integration/test_fiber_census_none_fields_db.py new file mode 100644 index 00000000..7d18cbbd --- /dev/null +++ b/tests/integration/test_fiber_census_none_fields_db.py @@ -0,0 +1,104 @@ +"""Live SurrealDB regression: the merge fiber census must survive fibers with empty fields. + +The Python SDK sends ``None`` as NONE, and SurrealDB drops NONE-valued keys from the +stored record (nested ones inside ``metadata`` too). The census used to fingerprint the +encoded page *with* those keys and re-fingerprint the page it read back *without* them, +so every page holding a fiber with an empty field failed validation and merge stopped with +"fiber census staged page is invalid" - on a real brain, where almost every fiber has an +empty ``summary`` or ``last_ghost_shown_at``, merge never ran at all. + +Runs only against the explicitly opted-in loopback SMEM_TEST_SURREALDB_URL; the fixture +creates a unique database and uses the disposable test credentials. +""" + +from __future__ import annotations + +import os +import uuid +from collections.abc import AsyncIterator +from urllib.parse import urlsplit + +import pytest +import pytest_asyncio + +from surreal_memory.core.brain import Brain +from surreal_memory.core.fiber import Fiber +from surreal_memory.core.neuron import Neuron, NeuronType +from surreal_memory.engine.consolidation import ( + ConsolidationConfig, + ConsolidationEngine, + ConsolidationStrategy, +) +from surreal_memory.storage.surrealdb.store import SurrealDBStorage + +TEST_SURREALDB_URL = os.getenv("SMEM_TEST_SURREALDB_URL") +TEST_AUTH = ("root", "root") + + +def _is_loopback_test_url(url: str | None) -> bool: + if not url: + return False + try: + parsed = urlsplit(url) + return ( + parsed.scheme in {"http", "https", "ws", "wss"} + and parsed.hostname in {"localhost", "127.0.0.1", "::1"} + and parsed.username is None + and parsed.password is None + and parsed.port is not None + ) + except ValueError: + return False + + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + not _is_loopback_test_url(TEST_SURREALDB_URL), + reason="requires an explicit loopback SMEM_TEST_SURREALDB_URL", + ), +] + + +@pytest_asyncio.fixture +async def store() -> AsyncIterator[SurrealDBStorage]: + assert TEST_SURREALDB_URL is not None + storage = SurrealDBStorage( + url=TEST_SURREALDB_URL, + user=TEST_AUTH[0], + password=TEST_AUTH[1], + namespace="smem_ci", + database="it_" + uuid.uuid4().hex[:12], + ) + await storage.initialize() + brain = Brain.create(name="fiber-census-none-it") + await storage.save_brain(brain) + storage.set_brain(brain.id) + try: + yield storage + finally: + await storage.close() + + +async def test_merge_census_completes_with_fibers_that_have_empty_fields( + store: SurrealDBStorage, +) -> None: + for i in range(3): + neurons = [Neuron.create(NeuronType.CONCEPT, f"census none {i} {s}") for s in ("a", "b")] + for neuron in neurons: + await store.add_neuron(neuron) + fiber = Fiber.create( + neuron_ids={n.id for n in neurons}, + synapse_ids=set(), + anchor_neuron_id=neurons[0].id, + summary=None, # empty field - dropped by the SDK/SurrealDB round trip + ) + fiber.metadata["note"] = None # nested empty value - dropped as well + await store.add_fiber(fiber) + + report = await ConsolidationEngine(store, ConsolidationConfig()).run( + strategies=[ConsolidationStrategy.MERGE] + ) + + assert report.extra.get("failed_strategies") in (None, []), report.extra + assert report.extra.get("consolidation_status") == "completed", report.extra