Separa a preparação dos metadados do ambiente de processamento

metrics/services/parsing/environment.py

@@ -1,14 +1,6 @@
-from scielo_usage_counter.translator.books import URLTranslatorBooksSite
-from scielo_usage_counter.translator.classic import URLTranslatorClassicSite
-from scielo_usage_counter.translator.dataverse import URLTranslatorDataverseSite
-from scielo_usage_counter.translator.opac import URLTranslatorOPACSite
-from scielo_usage_counter.translator.opac_alpha import URLTranslatorOPACAlphaSite
-from scielo_usage_counter.translator.preprints import URLTranslatorPreprintsSite
+from scielo_usage_counter import log_handler
 
-from document.models import Document
-from log_manager_config.models import CollectionLogDirectory
-from scielo_usage_counter import log_handler, url_translator
-from source.models import Source
+from metrics.services.parsing.metadata import build_url_translation_manager
 
 
 def setup_parsing_environment(log_file, robots_list, mmdb):
@@ -19,40 +11,5 @@ def setup_parsing_environment(log_file, robots_list, mmdb):
     )
     log_parser.logfile = log_file.path
 
-    translator_class = _get_log_file_translator_class(log_file)
-    if not translator_class:
-        raise Exception(
-            f"No URL translator class found for collection {log_file.collection}."
-        )
-
-    url_translator_manager = url_translator.URLTranslationManager(
-        documents_metadata=Document.metadata(collection=log_file.collection),
-        sources_metadata=Source.metadata(collection=log_file.collection),
-        translator=translator_class,
-    )
+    url_translator_manager = build_url_translation_manager(log_file)
     return log_parser, url_translator_manager
-
-
-def _get_log_file_translator_class(log_file):
-    for directory in CollectionLogDirectory.objects.filter(
-        config__collection=log_file.collection,
-    ):
-        if directory.path in log_file.path and directory.translator_class:
-            return _get_translator_class(directory.translator_class)
-
-    return None
-
-
-def _get_translator_class(name):
-    if not name or not isinstance(name, str):
-        return None
-
-    translator_classes = {
-        "books": URLTranslatorBooksSite,
-        "classic": URLTranslatorClassicSite,
-        "dataverse": URLTranslatorDataverseSite,
-        "opac": URLTranslatorOPACSite,
-        "opac_alpha": URLTranslatorOPACAlphaSite,
-        "preprints": URLTranslatorPreprintsSite,
-    }
-    return translator_classes.get(name.lower())

metrics/services/parsing/metadata.py

@@ -0,0 +1,61 @@
+import logging
+from time import monotonic
+
+from scielo_usage_counter import url_translator
+from scielo_usage_counter.translator.books import URLTranslatorBooksSite
+from scielo_usage_counter.translator.classic import URLTranslatorClassicSite
+from scielo_usage_counter.translator.dataverse import URLTranslatorDataverseSite
+from scielo_usage_counter.translator.opac import URLTranslatorOPACSite
+from scielo_usage_counter.translator.opac_alpha import URLTranslatorOPACAlphaSite
+from scielo_usage_counter.translator.preprints import URLTranslatorPreprintsSite
+
+from document.models import Document
+from log_manager_config.models import CollectionLogDirectory
+from source.models import Source
+
+TRANSLATOR_CLASSES = {
+    "books": URLTranslatorBooksSite,
+    "classic": URLTranslatorClassicSite,
+    "dataverse": URLTranslatorDataverseSite,
+    "opac": URLTranslatorOPACSite,
+    "opac_alpha": URLTranslatorOPACAlphaSite,
+    "preprints": URLTranslatorPreprintsSite,
+}
+
+
+def build_url_translation_manager(log_file):
+    translator_class = _get_log_file_translator_class(log_file)
+    if not translator_class:
+        raise Exception(
+            f"No URL translator class found for collection {log_file.collection}."
+        )
+
+    started = monotonic()
+    manager = url_translator.URLTranslationManager(
+        documents_metadata=Document.metadata(collection=log_file.collection),
+        sources_metadata=Source.metadata(collection=log_file.collection),
+        translator=translator_class,
+    )
+    logging.info(
+        "Prepared parsing metadata for %s in %.3f seconds.",
+        log_file.collection.acron3,
+        monotonic() - started,
+    )
+    return manager
+
+
+def _get_log_file_translator_class(log_file):
+    for directory in CollectionLogDirectory.objects.filter(
+        config__collection=log_file.collection,
+    ):
+        if directory.path in log_file.path and directory.translator_class:
+            return _get_translator_class(directory.translator_class)
+
+    return None
+
+
+def _get_translator_class(name):
+    if not name or not isinstance(name, str):
+        return None
+
+    return TRANSLATOR_CLASSES.get(name.lower())

metrics/tests/parsing/test_environment.py

@@ -1,39 +1,35 @@
-from django.test import TestCase
-
-from metrics.services.parsing.environment import _get_translator_class
-
-
-class TranslatorClassTests(TestCase):
-    def test_books_maps_to_books_translator(self):
-        cls = _get_translator_class("books")
-        self.assertEqual(cls.__name__, "URLTranslatorBooksSite")
-
-    def test_classic_maps_to_classic_translator(self):
-        cls = _get_translator_class("classic")
-        self.assertEqual(cls.__name__, "URLTranslatorClassicSite")
-
-    def test_opac_maps_to_opac_translator(self):
-        cls = _get_translator_class("opac")
-        self.assertEqual(cls.__name__, "URLTranslatorOPACSite")
+from types import SimpleNamespace
+from unittest.mock import patch
 
-    def test_opac_alpha_maps_to_opac_alpha_translator(self):
-        cls = _get_translator_class("opac_alpha")
-        self.assertEqual(cls.__name__, "URLTranslatorOPACAlphaSite")
-
-    def test_preprints_maps_to_preprints_translator(self):
-        cls = _get_translator_class("preprints")
-        self.assertEqual(cls.__name__, "URLTranslatorPreprintsSite")
-
-    def test_dataverse_maps_to_dataverse_translator(self):
-        cls = _get_translator_class("dataverse")
-        self.assertEqual(cls.__name__, "URLTranslatorDataverseSite")
-
-    def test_unknown_name_returns_none(self):
-        self.assertIsNone(_get_translator_class("unknown"))
-
-    def test_none_returns_none(self):
-        self.assertIsNone(_get_translator_class(None))
+from django.test import TestCase
 
-    def test_case_insensitive(self):
-        cls = _get_translator_class("Books")
-        self.assertEqual(cls.__name__, "URLTranslatorBooksSite")
+from metrics.services.parsing.environment import setup_parsing_environment
+
+
+class ParsingEnvironmentTests(TestCase):
+    @patch("metrics.services.parsing.environment.build_url_translation_manager")
+    @patch("metrics.services.parsing.environment.log_handler.LogParser")
+    def test_setup_builds_parser_and_delegates_metadata_preparation(
+        self,
+        log_parser_class,
+        build_url_translation_manager,
+    ):
+        log_file = SimpleNamespace(path="/app/logs/books/2026-08-01.log.gz")
+        mmdb = SimpleNamespace(data=b"mmdb")
+        log_parser = log_parser_class.return_value
+        translation_manager = build_url_translation_manager.return_value
+
+        result = setup_parsing_environment(
+            log_file=log_file,
+            robots_list=["robot"],
+            mmdb=mmdb,
+        )
+
+        log_parser_class.assert_called_once_with(
+            mmdb_data=b"mmdb",
+            robots_list=["robot"],
+            output_mode="dict",
+        )
+        self.assertEqual(log_parser.logfile, log_file.path)
+        build_url_translation_manager.assert_called_once_with(log_file)
+        self.assertEqual(result, (log_parser, translation_manager))

metrics/tests/parsing/test_metadata.py

@@ -0,0 +1,99 @@
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from django.test import TestCase
+
+from collection.models import Collection
+from log_manager_config.models import CollectionLogDirectory, LogManagerCollectionConfig
+from metrics.services.parsing.metadata import (
+    _get_translator_class,
+    build_url_translation_manager,
+)
+
+
+class TranslatorClassTests(TestCase):
+    def test_books_maps_to_books_translator(self):
+        cls = _get_translator_class("books")
+        self.assertEqual(cls.__name__, "URLTranslatorBooksSite")
+
+    def test_classic_maps_to_classic_translator(self):
+        cls = _get_translator_class("classic")
+        self.assertEqual(cls.__name__, "URLTranslatorClassicSite")
+
+    def test_opac_maps_to_opac_translator(self):
+        cls = _get_translator_class("opac")
+        self.assertEqual(cls.__name__, "URLTranslatorOPACSite")
+
+    def test_opac_alpha_maps_to_opac_alpha_translator(self):
+        cls = _get_translator_class("opac_alpha")
+        self.assertEqual(cls.__name__, "URLTranslatorOPACAlphaSite")
+
+    def test_preprints_maps_to_preprints_translator(self):
+        cls = _get_translator_class("preprints")
+        self.assertEqual(cls.__name__, "URLTranslatorPreprintsSite")
+
+    def test_dataverse_maps_to_dataverse_translator(self):
+        cls = _get_translator_class("dataverse")
+        self.assertEqual(cls.__name__, "URLTranslatorDataverseSite")
+
+    def test_unknown_name_returns_none(self):
+        self.assertIsNone(_get_translator_class("unknown"))
+
+    def test_none_returns_none(self):
+        self.assertIsNone(_get_translator_class(None))
+
+    def test_case_insensitive(self):
+        cls = _get_translator_class("Books")
+        self.assertEqual(cls.__name__, "URLTranslatorBooksSite")
+
+
+class ParsingMetadataTests(TestCase):
+    def setUp(self):
+        self.collection = Collection.objects.create(acron3="books", acron2="bk")
+        config = LogManagerCollectionConfig.objects.create(collection=self.collection)
+        CollectionLogDirectory.objects.create(
+            config=config,
+            path="/app/logs/books",
+            translator_class="books",
+        )
+
+    @patch("metrics.services.parsing.metadata.url_translator.URLTranslationManager")
+    @patch("metrics.services.parsing.metadata.Source.metadata")
+    @patch("metrics.services.parsing.metadata.Document.metadata")
+    def test_builds_manager_with_collection_metadata_and_configured_translator(
+        self,
+        document_metadata,
+        source_metadata,
+        manager_class,
+    ):
+        log_file = SimpleNamespace(
+            collection=self.collection,
+            path="/app/logs/books/2026-08-01.log.gz",
+        )
+        documents = iter([{"document_id": "book:1"}])
+        sources = iter([{"source_id": "1"}])
+        document_metadata.return_value = documents
+        source_metadata.return_value = sources
+
+        manager = build_url_translation_manager(log_file)
+
+        document_metadata.assert_called_once_with(collection=self.collection)
+        source_metadata.assert_called_once_with(collection=self.collection)
+        manager_class.assert_called_once()
+        call_kwargs = manager_class.call_args.kwargs
+        self.assertIs(call_kwargs["documents_metadata"], documents)
+        self.assertIs(call_kwargs["sources_metadata"], sources)
+        self.assertEqual(call_kwargs["translator"].__name__, "URLTranslatorBooksSite")
+        self.assertIs(manager, manager_class.return_value)
+
+    def test_rejects_log_outside_configured_directories(self):
+        log_file = SimpleNamespace(
+            collection=self.collection,
+            path="/other/logs/2026-08-01.log.gz",
+        )
+
+        with self.assertRaisesRegex(
+            Exception,
+            "No URL translator class found for collection",
+        ):
+            build_url_translation_manager(log_file)