diff --git a/NEWS.md b/NEWS.md index f096d1be5..a941d63d5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -15,6 +15,9 @@ border-radius: 128px; - Browser title shows series years and a Tags button. - Online tag match picker shows series and issue details to help disambiguate similar covers and titles. +- Fixes + - Searches for a changed universe designation no longer miss comics after + large imports. ## v2.5.5 diff --git a/codex/librarian/scribe/importer/create/foreign_keys.py b/codex/librarian/scribe/importer/create/foreign_keys.py index 196cc517b..2c299be67 100644 --- a/codex/librarian/scribe/importer/create/foreign_keys.py +++ b/codex/librarian/scribe/importer/create/foreign_keys.py @@ -223,6 +223,11 @@ def _series_summary_changed(obj: BaseModel, update_args: Mapping) -> bool: and update_args.get(SUMMARY_FIELD_NAME, obj.summary) != obj.summary ) + def _add_fts_updated_pks(self, field_name: str, pks: set[int]) -> None: + """Queue comics to re-index, merged across the per-comic chunks.""" + if pks: + self.metadata[FTS_UPDATED_M2MS].setdefault(field_name, set()).update(pks) + def _bulk_update_models(self, model: type[BaseModel], status) -> int: count = 0 update_tuples: list[tuple] | None = self.metadata[UPDATE_FKS].pop(model, None) @@ -263,14 +268,8 @@ def _bulk_update_models(self, model: type[BaseModel], status) -> int: if model == Universe: fts_update_pks.add(obj.pk) - if fts_update_pks: - field_name = model.__name__.lower() + "s" - self.metadata[FTS_UPDATED_M2MS][field_name] = fts_update_pks - if series_summary_pks: - # Merge: the per-comic phases run once per chunk. - self.metadata[FTS_UPDATED_M2MS].setdefault( - SERIES_SUMMARY_FTS_FIELD_NAME, set() - ).update(series_summary_pks) + self._add_fts_updated_pks(model.__name__.lower() + "s", fts_update_pks) + self._add_fts_updated_pks(SERIES_SUMMARY_FTS_FIELD_NAME, series_summary_pks) update_fields = tuple(update_args_map.keys()) if model in COLLECTION_BASE_MODELS: diff --git a/tests/importer/test_fts_chunked_resync.py b/tests/importer/test_fts_chunked_resync.py new file mode 100644 index 000000000..6adbc9c28 --- /dev/null +++ b/tests/importer/test_fts_chunked_resync.py @@ -0,0 +1,123 @@ +""" +An import's foreign-key search re-sync covers every per-comic chunk. + +``read → query → create_and_update → link`` runs once per chunk of paths, +but the search index is synced once afterwards. The tags each chunk +updated must accumulate until then, or only the last chunk's comics would +be re-indexed. +""" + +import shutil +import zipfile +from pathlib import Path +from threading import Event, Lock +from typing import Final, override +from unittest.mock import patch + +from django.core.cache import cache +from django.test import TestCase +from loguru import logger + +from codex.librarian.mp_queue import LIBRARIAN_QUEUE +from codex.librarian.scribe.importer.importer import ComicImporter +from codex.librarian.scribe.importer.tasks import ImportTask +from codex.models.comic import ComicFTS +from codex.models.library import Library +from codex.startup import codex_init +from tests.importer.test_basic import run_full_import +from tests.tmp_dirs import tmp_dir + +TMP_DIR: Final = tmp_dir(Path(__file__).stem) + + +def _yaml(universe: str, designation: str) -> str: + return f"""comicbox: + publisher: + name: P + series: + name: S + universes: + {universe}: + designation: '{designation}' +""" + + +class TestFTSChunkedResync(TestCase): + """Two chunks each update a universe; both universes' comics re-index.""" + + @override + @classmethod + def setUpTestData(cls): + super().setUpTestData() + cache.clear() + codex_init() + + @override + @classmethod + def tearDownClass(cls): + try: + shutil.rmtree(TMP_DIR, ignore_errors=True) + finally: + super().tearDownClass() + + @override + def setUp(self): + cache.clear() + TMP_DIR.mkdir(exist_ok=True, parents=True) + self.library = Library.objects.create(path=TMP_DIR) # pyright: ignore[reportUninitializedInstanceVariable] + + @override + def tearDown(self): + shutil.rmtree(TMP_DIR, ignore_errors=True) + cache.clear() + + @staticmethod + def _write(name: str, yaml: str) -> str: + path = TMP_DIR / name + with zipfile.ZipFile(path, "w") as zf: + zf.writestr("comicbox.yaml", yaml) + zf.writestr("page1.jpg", b"\xff\xd8\xff\xe0not-a-real-jpeg") + return str(path) + + def _importer(self, paths) -> ComicImporter: + # A same-size rewrite within the same second looks unchanged to the + # mtime prefilter, so always re-read. + task = ImportTask( + library_id=self.library.pk, + files_modified=frozenset(paths), + check_metadata_mtime=False, + ) + return ComicImporter(task, logger, LIBRARIAN_QUEUE, Lock(), Event()) + + @staticmethod + def _universes_column(path: str) -> str: + return ComicFTS.objects.get(comic__path=path).universes + + def test_every_chunk_reindexes_its_universes(self) -> None: + """Siblings outside the import pick up both chunks' designations.""" + paths = { + name: self._write(f"{name}.cbz", _yaml(universe, designation)) + for name, universe, designation in ( + ("a1", "Alpha", "1"), + ("a2", "Alpha", "1"), + ("b1", "Beta", "2"), + ("b2", "Beta", "2"), + ) + } + run_full_import(self._importer(paths.values())) + + # Re-import one comic per universe with a new designation, one + # comic per chunk. a2 and b2 are re-indexed only by the re-sync. + self._write("a1.cbz", _yaml("Alpha", "11")) + self._write("b1.cbz", _yaml("Beta", "22")) + importer = self._importer((paths["a1"], paths["b1"])) + with patch.object( + ComicImporter, "_compute_chunk_size", staticmethod(lambda: 1) + ): + assert importer._run_per_comic_phases_chunked() # noqa: SLF001 + importer.fail_imports() + importer.delete() + importer.full_text_search() + + assert self._universes_column(paths["a2"]) == "11,Alpha" + assert self._universes_column(paths["b2"]) == "22,Beta"