Merge FTS re-sync tags across import chunks

NEWS.md

@@ -15,6 +15,9 @@ border-radius: 128px;
     - Browser title shows series years and a Tags button.
     - Online tag match picker shows series and issue details to help
       disambiguate similar covers and titles.
+- Fixes
+    - Searches for a changed universe designation no longer miss comics after
+      large imports.
 
 ## v2.5.5
 

codex/librarian/scribe/importer/create/foreign_keys.py

@@ -223,6 +223,11 @@ def _series_summary_changed(obj: BaseModel, update_args: Mapping) -> bool:
             and update_args.get(SUMMARY_FIELD_NAME, obj.summary) != obj.summary
         )
 
+    def _add_fts_updated_pks(self, field_name: str, pks: set[int]) -> None:
+        """Queue comics to re-index, merged across the per-comic chunks."""
+        if pks:
+            self.metadata[FTS_UPDATED_M2MS].setdefault(field_name, set()).update(pks)
+
     def _bulk_update_models(self, model: type[BaseModel], status) -> int:
         count = 0
         update_tuples: list[tuple] | None = self.metadata[UPDATE_FKS].pop(model, None)
@@ -263,14 +268,8 @@ def _bulk_update_models(self, model: type[BaseModel], status) -> int:
             if model == Universe:
                 fts_update_pks.add(obj.pk)
 
-        if fts_update_pks:
-            field_name = model.__name__.lower() + "s"
-            self.metadata[FTS_UPDATED_M2MS][field_name] = fts_update_pks
-        if series_summary_pks:
-            # Merge: the per-comic phases run once per chunk.
-            self.metadata[FTS_UPDATED_M2MS].setdefault(
-                SERIES_SUMMARY_FTS_FIELD_NAME, set()
-            ).update(series_summary_pks)
+        self._add_fts_updated_pks(model.__name__.lower() + "s", fts_update_pks)
+        self._add_fts_updated_pks(SERIES_SUMMARY_FTS_FIELD_NAME, series_summary_pks)
 
         update_fields = tuple(update_args_map.keys())
         if model in COLLECTION_BASE_MODELS:

tests/importer/test_fts_chunked_resync.py

@@ -0,0 +1,123 @@
+"""
+An import's foreign-key search re-sync covers every per-comic chunk.
+
+``read → query → create_and_update → link`` runs once per chunk of paths,
+but the search index is synced once afterwards. The tags each chunk
+updated must accumulate until then, or only the last chunk's comics would
+be re-indexed.
+"""
+
+import shutil
+import zipfile
+from pathlib import Path
+from threading import Event, Lock
+from typing import Final, override
+from unittest.mock import patch
+
+from django.core.cache import cache
+from django.test import TestCase
+from loguru import logger
+
+from codex.librarian.mp_queue import LIBRARIAN_QUEUE
+from codex.librarian.scribe.importer.importer import ComicImporter
+from codex.librarian.scribe.importer.tasks import ImportTask
+from codex.models.comic import ComicFTS
+from codex.models.library import Library
+from codex.startup import codex_init
+from tests.importer.test_basic import run_full_import
+from tests.tmp_dirs import tmp_dir
+
+TMP_DIR: Final = tmp_dir(Path(__file__).stem)
+
+
+def _yaml(universe: str, designation: str) -> str:
+    return f"""comicbox:
+  publisher:
+    name: P
+  series:
+    name: S
+  universes:
+    {universe}:
+      designation: '{designation}'
+"""
+
+
+class TestFTSChunkedResync(TestCase):
+    """Two chunks each update a universe; both universes' comics re-index."""
+
+    @override
+    @classmethod
+    def setUpTestData(cls):
+        super().setUpTestData()
+        cache.clear()
+        codex_init()
+
+    @override
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            shutil.rmtree(TMP_DIR, ignore_errors=True)
+        finally:
+            super().tearDownClass()
+
+    @override
+    def setUp(self):
+        cache.clear()
+        TMP_DIR.mkdir(exist_ok=True, parents=True)
+        self.library = Library.objects.create(path=TMP_DIR)  # pyright: ignore[reportUninitializedInstanceVariable]
+
+    @override
+    def tearDown(self):
+        shutil.rmtree(TMP_DIR, ignore_errors=True)
+        cache.clear()
+
+    @staticmethod
+    def _write(name: str, yaml: str) -> str:
+        path = TMP_DIR / name
+        with zipfile.ZipFile(path, "w") as zf:
+            zf.writestr("comicbox.yaml", yaml)
+            zf.writestr("page1.jpg", b"\xff\xd8\xff\xe0not-a-real-jpeg")
+        return str(path)
+
+    def _importer(self, paths) -> ComicImporter:
+        # A same-size rewrite within the same second looks unchanged to the
+        # mtime prefilter, so always re-read.
+        task = ImportTask(
+            library_id=self.library.pk,
+            files_modified=frozenset(paths),
+            check_metadata_mtime=False,
+        )
+        return ComicImporter(task, logger, LIBRARIAN_QUEUE, Lock(), Event())
+
+    @staticmethod
+    def _universes_column(path: str) -> str:
+        return ComicFTS.objects.get(comic__path=path).universes
+
+    def test_every_chunk_reindexes_its_universes(self) -> None:
+        """Siblings outside the import pick up both chunks' designations."""
+        paths = {
+            name: self._write(f"{name}.cbz", _yaml(universe, designation))
+            for name, universe, designation in (
+                ("a1", "Alpha", "1"),
+                ("a2", "Alpha", "1"),
+                ("b1", "Beta", "2"),
+                ("b2", "Beta", "2"),
+            )
+        }
+        run_full_import(self._importer(paths.values()))
+
+        # Re-import one comic per universe with a new designation, one
+        # comic per chunk. a2 and b2 are re-indexed only by the re-sync.
+        self._write("a1.cbz", _yaml("Alpha", "11"))
+        self._write("b1.cbz", _yaml("Beta", "22"))
+        importer = self._importer((paths["a1"], paths["b1"]))
+        with patch.object(
+            ComicImporter, "_compute_chunk_size", staticmethod(lambda: 1)
+        ):
+            assert importer._run_per_comic_phases_chunked()  # noqa: SLF001
+        importer.fail_imports()
+        importer.delete()
+        importer.full_text_search()
+
+        assert self._universes_column(paths["a2"]) == "11,Alpha"
+        assert self._universes_column(paths["b2"]) == "22,Beta"