From 2fdbdff5ea31aec0f1833a9db2862a9038ffbe51 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Sat, 29 Aug 2026 15:05:16 -0300 Subject: [PATCH 1/3] =?UTF-8?q?Habilita=20o=20cache=20de=20metadados=20par?= =?UTF-8?q?a=20as=20cole=C3=A7=C3=B5es=20previstas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/collections.py | 1 + config/settings/base.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/config/collections.py b/config/collections.py index 7249f5d..07b8afb 100644 --- a/config/collections.py +++ b/config/collections.py @@ -19,6 +19,7 @@ "preprints": "small", "pry": "small", "rve": "small", + "rvt": "small", "spa": "small", "sss": "small", "sza": "small", diff --git a/config/settings/base.py b/config/settings/base.py index 0b67885..acd89e8 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -488,3 +488,7 @@ # Collection size categories # ------------------------------------------------------------------------------ SUPPORTED_LOGFILE_EXTENSIONS = env.list("SUPPORTED_LOGFILE_EXTENSIONS", default=[".log", ".gz", ".zip"]) +PARSING_METADATA_CACHE_COLLECTIONS = env.list( + "PARSING_METADATA_CACHE_COLLECTIONS", + default=list(COLLECTION_ACRON3_SIZE_MAP), +) From 5cf89362e182acaeb0c7a3e84c1bb182fa27f894 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Sat, 29 Aug 2026 15:05:20 -0300 Subject: [PATCH 2/3] =?UTF-8?q?Separa=20o=20cache=20da=20prepara=C3=A7?= =?UTF-8?q?=C3=A3o=20de=20metadados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/services/parsing/metadata.py | 29 +++-- metrics/services/parsing/metadata_cache.py | 144 +++++++++++++++++++++ 2 files changed, 165 insertions(+), 8 deletions(-) create mode 100644 metrics/services/parsing/metadata_cache.py diff --git a/metrics/services/parsing/metadata.py b/metrics/services/parsing/metadata.py index c82984a..dc48dfa 100644 --- a/metrics/services/parsing/metadata.py +++ b/metrics/services/parsing/metadata.py @@ -11,6 +11,7 @@ from document.models import Document from log_manager_config.models import CollectionLogDirectory +from metrics.services.parsing import metadata_cache from source.models import Source TRANSLATOR_CLASSES = { @@ -30,20 +31,32 @@ def build_url_translation_manager(log_file): f"No URL translator class found for collection {log_file.collection}." ) - started = monotonic() - manager = url_translator.URLTranslationManager( - documents_metadata=Document.metadata(collection=log_file.collection), - sources_metadata=Source.metadata(collection=log_file.collection), - translator=translator_class, - ) + if metadata_cache.is_enabled(log_file.collection): + return metadata_cache.get_url_translation_manager( + log_file.collection, + translator_class, + _build_manager, + ) + + manager, elapsed = _build_manager(log_file.collection, translator_class) logging.info( - "Prepared parsing metadata for %s in %.3f seconds.", + "Prepared parsing metadata for %s without cache in %.3f seconds.", log_file.collection.acron3, - monotonic() - started, + elapsed, ) return manager +def _build_manager(collection, translator_class): + started = monotonic() + manager = url_translator.URLTranslationManager( + documents_metadata=Document.metadata(collection=collection), + sources_metadata=Source.metadata(collection=collection), + translator=translator_class, + ) + return manager, monotonic() - started + + def _get_log_file_translator_class(log_file): for directory in CollectionLogDirectory.objects.filter( config__collection=log_file.collection, diff --git a/metrics/services/parsing/metadata_cache.py b/metrics/services/parsing/metadata_cache.py new file mode 100644 index 0000000..ac1eb80 --- /dev/null +++ b/metrics/services/parsing/metadata_cache.py @@ -0,0 +1,144 @@ +import logging +from copy import copy +from threading import Lock +from time import monotonic + +from django.conf import settings +from django.db import connection, transaction +from django.db.models import Count, Max + +from config.collections import get_collection_size +from document.models import Document +from source.models import Source + +_CACHE_ENTRY = None +_CACHE_LOCK = Lock() + + +def is_enabled(collection): + enabled_collections = { + value.lower() + for value in getattr(settings, "PARSING_METADATA_CACHE_COLLECTIONS", []) + } + return collection.acron3.lower() in enabled_collections + + +def get_url_translation_manager(collection, translator_class, build_manager): + global _CACHE_ENTRY + + acronym = collection.acron3 + size = get_collection_size(acronym) + translator_name = translator_class.__name__ + + with _CACHE_LOCK: + entry = _CACHE_ENTRY + reason = _get_rebuild_reason(entry, collection, translator_name) + + if reason is None: + signature = _read_signature(collection) + if signature == entry["signature"]: + logging.info( + "Parsing metadata cache hit for %s (size=%s, signature=%s).", + acronym, + size, + signature, + ) + return _fresh_manager(entry) + reason = "signature_changed" + + started = monotonic() + new_entry = _build_cache_entry( + collection, + translator_class, + build_manager, + ) + elapsed = monotonic() - started + _CACHE_ENTRY = new_entry + + logging.info( + "Parsing metadata cache %s for %s " + "(size=%s, reason=%s, signature=%s, build_seconds=%.3f).", + "miss" if entry is None else "rebuild", + acronym, + size, + reason, + new_entry["signature"], + elapsed, + ) + return _fresh_manager(new_entry) + + +def clear(): + global _CACHE_ENTRY + + with _CACHE_LOCK: + _CACHE_ENTRY = None + + +def _get_rebuild_reason(entry, collection, translator_name): + if entry is None: + return "empty" + if entry["collection_id"] != collection.pk: + return "collection_changed" + if entry["translator_name"] != translator_name: + return "translator_changed" + return None + + +def _build_cache_entry(collection, translator_class, build_manager): + if connection.in_atomic_block: + return _load_cache_entry(collection, translator_class, build_manager) + + with transaction.atomic(): + with connection.cursor() as cursor: + cursor.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY") + return _load_cache_entry(collection, translator_class, build_manager) + + +def _load_cache_entry(collection, translator_class, build_manager): + signature = _get_collection_signature(collection) + manager, _ = build_manager(collection, translator_class) + return { + "collection_id": collection.pk, + "translator_class": translator_class, + "translator_name": translator_class.__name__, + "signature": signature, + "manager": manager, + } + + +def _read_signature(collection): + if connection.in_atomic_block: + return _get_collection_signature(collection) + + with transaction.atomic(): + with connection.cursor() as cursor: + cursor.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY") + return _get_collection_signature(collection) + + +def _get_collection_signature(collection): + document_signature = Document.objects.filter(collection=collection).aggregate( + count=Count("pk"), + max_updated=Max("updated"), + ) + source_signature = Source.objects.filter(collection=collection).aggregate( + count=Count("pk"), + max_updated=Max("updated"), + ) + return ( + document_signature["count"], + document_signature["max_updated"], + source_signature["count"], + source_signature["max_updated"], + ) + + +def _fresh_manager(entry): + manager = copy(entry["manager"]) + manager.translator = entry["translator_class"]( + manager.sources_metadata, + manager.documents_metadata, + ) + manager.is_translator_forced = True + return manager From f390b81a96146966fb61de0edd2ce7bb30402f0d Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Sat, 29 Aug 2026 15:05:25 -0300 Subject: [PATCH 3/3] =?UTF-8?q?Testa=20invalida=C3=A7=C3=A3o=20e=20isolame?= =?UTF-8?q?nto=20do=20cache=20de=20metadados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/tests/parsing/test_metadata.py | 3 +- metrics/tests/parsing/test_metadata_cache.py | 224 +++++++++++++++++++ 2 files changed, 226 insertions(+), 1 deletion(-) create mode 100644 metrics/tests/parsing/test_metadata_cache.py diff --git a/metrics/tests/parsing/test_metadata.py b/metrics/tests/parsing/test_metadata.py index 2c34269..22e76ba 100644 --- a/metrics/tests/parsing/test_metadata.py +++ b/metrics/tests/parsing/test_metadata.py @@ -1,7 +1,7 @@ from types import SimpleNamespace from unittest.mock import patch -from django.test import TestCase +from django.test import TestCase, override_settings from collection.models import Collection from log_manager_config.models import CollectionLogDirectory, LogManagerCollectionConfig @@ -57,6 +57,7 @@ def setUp(self): translator_class="books", ) + @override_settings(PARSING_METADATA_CACHE_COLLECTIONS=[]) @patch("metrics.services.parsing.metadata.url_translator.URLTranslationManager") @patch("metrics.services.parsing.metadata.Source.metadata") @patch("metrics.services.parsing.metadata.Document.metadata") diff --git a/metrics/tests/parsing/test_metadata_cache.py b/metrics/tests/parsing/test_metadata_cache.py new file mode 100644 index 0000000..a982968 --- /dev/null +++ b/metrics/tests/parsing/test_metadata_cache.py @@ -0,0 +1,224 @@ +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from collection.models import Collection +from config.collections import COLLECTION_ACRON3_SIZE_MAP, LOG_MANAGER_SEED_DATA +from document.models import Document +from log_manager_config.models import ( + CollectionLogDirectory, + LogManagerCollectionConfig, +) +from metrics.services.parsing import metadata, metadata_cache +from source.models import Source + + +@pytest.fixture(autouse=True) +def empty_metadata_cache(): + metadata_cache.clear() + yield + metadata_cache.clear() + + +@pytest.fixture +def parsing_collection(db, settings): + settings.PARSING_METADATA_CACHE_COLLECTIONS = ["prt"] + collection = Collection.objects.create(acron3="prt", acron2="pt") + config = LogManagerCollectionConfig.objects.create(collection=collection) + CollectionLogDirectory.objects.create( + config=config, + path="/app/logs/prt", + translator_class="classic", + ) + source = Source.objects.create( + collection=collection, + source_type=Source.SOURCE_TYPE_JOURNAL, + source_id="1234-5678", + scielo_issn="1234-5678", + title="Journal", + ) + document = Document.objects.create( + collection=collection, + source=source, + document_type=Document.DOCUMENT_TYPE_ARTICLE, + document_id="S123456782026000100001", + pid_v2="S123456782026000100001", + publication_year="2026", + ) + log_file = SimpleNamespace( + collection=collection, + path="/app/logs/prt/2026-07-07_scielo.pt.log.gz", + ) + return collection, source, document, log_file + + +def test_all_planned_collections_are_enabled_by_default(settings): + planned_collections = {item["acronym"] for item in LOG_MANAGER_SEED_DATA} + + assert set(COLLECTION_ACRON3_SIZE_MAP) == planned_collections + assert set(settings.PARSING_METADATA_CACHE_COLLECTIONS) == planned_collections + + +@pytest.mark.django_db +def test_cache_hit_reuses_metadata_but_isolates_mutable_translator( + parsing_collection, + django_assert_num_queries, +): + _, _, _, log_file = parsing_collection + + with ( + patch.object(Document, "metadata", wraps=Document.metadata) as documents, + patch.object(Source, "metadata", wraps=Source.metadata) as sources, + ): + first = metadata.build_url_translation_manager(log_file) + with django_assert_num_queries(3): + second = metadata.build_url_translation_manager(log_file) + + assert documents.call_count == 1 + assert sources.call_count == 1 + assert first is not second + assert first.translator is not second.translator + assert first.documents_metadata is second.documents_metadata + assert first.sources_metadata is second.sources_metadata + + first.translator.url_params = {"pid": "first"} + + assert not hasattr(second.translator, "url_params") + + +@pytest.mark.django_db +def test_cache_is_disabled_for_collection_outside_allowlist( + parsing_collection, + settings, +): + _, _, _, log_file = parsing_collection + settings.PARSING_METADATA_CACHE_COLLECTIONS = [] + + with ( + patch.object(Document, "metadata", wraps=Document.metadata) as documents, + patch.object(Source, "metadata", wraps=Source.metadata) as sources, + ): + metadata.build_url_translation_manager(log_file) + metadata.build_url_translation_manager(log_file) + + assert documents.call_count == 2 + assert sources.call_count == 2 + + +@pytest.mark.django_db +def test_document_update_invalidates_cache(parsing_collection): + _, _, document, log_file = parsing_collection + + with patch.object(Document, "metadata", wraps=Document.metadata) as documents: + first = metadata.build_url_translation_manager(log_file) + document.title = "Updated title" + document.save(update_fields=["title", "updated"]) + second = metadata.build_url_translation_manager(log_file) + + assert documents.call_count == 2 + assert first.documents_metadata is not second.documents_metadata + + +@pytest.mark.django_db +def test_document_creation_invalidates_cache(parsing_collection): + collection, source, _, log_file = parsing_collection + + with patch.object(Document, "metadata", wraps=Document.metadata) as documents: + first = metadata.build_url_translation_manager(log_file) + Document.objects.create( + collection=collection, + source=source, + document_type=Document.DOCUMENT_TYPE_ARTICLE, + document_id="S123456782026000100002", + pid_v2="S123456782026000100002", + ) + second = metadata.build_url_translation_manager(log_file) + + assert documents.call_count == 2 + assert first.documents_metadata is not second.documents_metadata + + +@pytest.mark.django_db +def test_source_deletion_invalidates_cache(parsing_collection): + _, source, document, log_file = parsing_collection + document.source = None + document.save(update_fields=["source", "updated"]) + + with patch.object(Source, "metadata", wraps=Source.metadata) as sources: + first = metadata.build_url_translation_manager(log_file) + source.delete() + second = metadata.build_url_translation_manager(log_file) + + assert sources.call_count == 2 + assert first.sources_metadata is not second.sources_metadata + + +@pytest.mark.django_db +def test_translator_change_rebuilds_cache(parsing_collection): + _, _, _, log_file = parsing_collection + directory = CollectionLogDirectory.objects.get() + + first = metadata.build_url_translation_manager(log_file) + directory.translator_class = "opac" + directory.save(update_fields=["translator_class", "updated"]) + second = metadata.build_url_translation_manager(log_file) + + assert first.translator.__class__.__name__ == "URLTranslatorClassicSite" + assert second.translator.__class__.__name__ == "URLTranslatorOPACSite" + assert first.documents_metadata is not second.documents_metadata + + +@pytest.mark.django_db +def test_new_collection_replaces_process_cache(parsing_collection, settings): + _, _, _, first_log_file = parsing_collection + settings.PARSING_METADATA_CACHE_COLLECTIONS = ["prt", "arg"] + other_collection = Collection.objects.create(acron3="arg", acron2="ar") + config = LogManagerCollectionConfig.objects.create(collection=other_collection) + CollectionLogDirectory.objects.create( + config=config, + path="/app/logs/arg", + translator_class="classic", + ) + other_log_file = SimpleNamespace( + collection=other_collection, + path="/app/logs/arg/2026-07-07_scielo.ar.log.gz", + ) + + with patch.object(Document, "metadata", wraps=Document.metadata) as documents: + metadata.build_url_translation_manager(first_log_file) + metadata.build_url_translation_manager(other_log_file) + metadata.build_url_translation_manager(first_log_file) + + assert documents.call_count == 3 + + +@pytest.mark.django_db +def test_failed_rebuild_preserves_previous_entry(parsing_collection): + _, _, document, log_file = parsing_collection + metadata.build_url_translation_manager(log_file) + previous_entry = metadata_cache._CACHE_ENTRY + document.title = "Changed" + document.save(update_fields=["title", "updated"]) + + with patch.object( + metadata_cache, + "_build_cache_entry", + side_effect=RuntimeError("metadata build failed"), + ): + with pytest.raises(RuntimeError, match="metadata build failed"): + metadata.build_url_translation_manager(log_file) + + assert metadata_cache._CACHE_ENTRY is previous_entry + + +@pytest.mark.django_db +def test_explicit_clear_forces_rebuild(parsing_collection): + _, _, _, log_file = parsing_collection + + with patch.object(Document, "metadata", wraps=Document.metadata) as documents: + metadata.build_url_translation_manager(log_file) + metadata_cache.clear() + metadata.build_url_translation_manager(log_file) + + assert documents.call_count == 2