diff --git a/metrics/services/parsing/environment.py b/metrics/services/parsing/environment.py index dba5567..acab89f 100644 --- a/metrics/services/parsing/environment.py +++ b/metrics/services/parsing/environment.py @@ -1,14 +1,6 @@ -from scielo_usage_counter.translator.books import URLTranslatorBooksSite -from scielo_usage_counter.translator.classic import URLTranslatorClassicSite -from scielo_usage_counter.translator.dataverse import URLTranslatorDataverseSite -from scielo_usage_counter.translator.opac import URLTranslatorOPACSite -from scielo_usage_counter.translator.opac_alpha import URLTranslatorOPACAlphaSite -from scielo_usage_counter.translator.preprints import URLTranslatorPreprintsSite +from scielo_usage_counter import log_handler -from document.models import Document -from log_manager_config.models import CollectionLogDirectory -from scielo_usage_counter import log_handler, url_translator -from source.models import Source +from metrics.services.parsing.metadata import build_url_translation_manager def setup_parsing_environment(log_file, robots_list, mmdb): @@ -19,40 +11,5 @@ def setup_parsing_environment(log_file, robots_list, mmdb): ) log_parser.logfile = log_file.path - translator_class = _get_log_file_translator_class(log_file) - if not translator_class: - raise Exception( - f"No URL translator class found for collection {log_file.collection}." - ) - - url_translator_manager = url_translator.URLTranslationManager( - documents_metadata=Document.metadata(collection=log_file.collection), - sources_metadata=Source.metadata(collection=log_file.collection), - translator=translator_class, - ) + url_translator_manager = build_url_translation_manager(log_file) return log_parser, url_translator_manager - - -def _get_log_file_translator_class(log_file): - for directory in CollectionLogDirectory.objects.filter( - config__collection=log_file.collection, - ): - if directory.path in log_file.path and directory.translator_class: - return _get_translator_class(directory.translator_class) - - return None - - -def _get_translator_class(name): - if not name or not isinstance(name, str): - return None - - translator_classes = { - "books": URLTranslatorBooksSite, - "classic": URLTranslatorClassicSite, - "dataverse": URLTranslatorDataverseSite, - "opac": URLTranslatorOPACSite, - "opac_alpha": URLTranslatorOPACAlphaSite, - "preprints": URLTranslatorPreprintsSite, - } - return translator_classes.get(name.lower()) diff --git a/metrics/services/parsing/metadata.py b/metrics/services/parsing/metadata.py new file mode 100644 index 0000000..c82984a --- /dev/null +++ b/metrics/services/parsing/metadata.py @@ -0,0 +1,61 @@ +import logging +from time import monotonic + +from scielo_usage_counter import url_translator +from scielo_usage_counter.translator.books import URLTranslatorBooksSite +from scielo_usage_counter.translator.classic import URLTranslatorClassicSite +from scielo_usage_counter.translator.dataverse import URLTranslatorDataverseSite +from scielo_usage_counter.translator.opac import URLTranslatorOPACSite +from scielo_usage_counter.translator.opac_alpha import URLTranslatorOPACAlphaSite +from scielo_usage_counter.translator.preprints import URLTranslatorPreprintsSite + +from document.models import Document +from log_manager_config.models import CollectionLogDirectory +from source.models import Source + +TRANSLATOR_CLASSES = { + "books": URLTranslatorBooksSite, + "classic": URLTranslatorClassicSite, + "dataverse": URLTranslatorDataverseSite, + "opac": URLTranslatorOPACSite, + "opac_alpha": URLTranslatorOPACAlphaSite, + "preprints": URLTranslatorPreprintsSite, +} + + +def build_url_translation_manager(log_file): + translator_class = _get_log_file_translator_class(log_file) + if not translator_class: + raise Exception( + f"No URL translator class found for collection {log_file.collection}." + ) + + started = monotonic() + manager = url_translator.URLTranslationManager( + documents_metadata=Document.metadata(collection=log_file.collection), + sources_metadata=Source.metadata(collection=log_file.collection), + translator=translator_class, + ) + logging.info( + "Prepared parsing metadata for %s in %.3f seconds.", + log_file.collection.acron3, + monotonic() - started, + ) + return manager + + +def _get_log_file_translator_class(log_file): + for directory in CollectionLogDirectory.objects.filter( + config__collection=log_file.collection, + ): + if directory.path in log_file.path and directory.translator_class: + return _get_translator_class(directory.translator_class) + + return None + + +def _get_translator_class(name): + if not name or not isinstance(name, str): + return None + + return TRANSLATOR_CLASSES.get(name.lower()) diff --git a/metrics/tests/parsing/test_environment.py b/metrics/tests/parsing/test_environment.py index b496779..521de1d 100644 --- a/metrics/tests/parsing/test_environment.py +++ b/metrics/tests/parsing/test_environment.py @@ -1,39 +1,35 @@ -from django.test import TestCase - -from metrics.services.parsing.environment import _get_translator_class - - -class TranslatorClassTests(TestCase): - def test_books_maps_to_books_translator(self): - cls = _get_translator_class("books") - self.assertEqual(cls.__name__, "URLTranslatorBooksSite") - - def test_classic_maps_to_classic_translator(self): - cls = _get_translator_class("classic") - self.assertEqual(cls.__name__, "URLTranslatorClassicSite") - - def test_opac_maps_to_opac_translator(self): - cls = _get_translator_class("opac") - self.assertEqual(cls.__name__, "URLTranslatorOPACSite") +from types import SimpleNamespace +from unittest.mock import patch - def test_opac_alpha_maps_to_opac_alpha_translator(self): - cls = _get_translator_class("opac_alpha") - self.assertEqual(cls.__name__, "URLTranslatorOPACAlphaSite") - - def test_preprints_maps_to_preprints_translator(self): - cls = _get_translator_class("preprints") - self.assertEqual(cls.__name__, "URLTranslatorPreprintsSite") - - def test_dataverse_maps_to_dataverse_translator(self): - cls = _get_translator_class("dataverse") - self.assertEqual(cls.__name__, "URLTranslatorDataverseSite") - - def test_unknown_name_returns_none(self): - self.assertIsNone(_get_translator_class("unknown")) - - def test_none_returns_none(self): - self.assertIsNone(_get_translator_class(None)) +from django.test import TestCase - def test_case_insensitive(self): - cls = _get_translator_class("Books") - self.assertEqual(cls.__name__, "URLTranslatorBooksSite") +from metrics.services.parsing.environment import setup_parsing_environment + + +class ParsingEnvironmentTests(TestCase): + @patch("metrics.services.parsing.environment.build_url_translation_manager") + @patch("metrics.services.parsing.environment.log_handler.LogParser") + def test_setup_builds_parser_and_delegates_metadata_preparation( + self, + log_parser_class, + build_url_translation_manager, + ): + log_file = SimpleNamespace(path="/app/logs/books/2026-08-01.log.gz") + mmdb = SimpleNamespace(data=b"mmdb") + log_parser = log_parser_class.return_value + translation_manager = build_url_translation_manager.return_value + + result = setup_parsing_environment( + log_file=log_file, + robots_list=["robot"], + mmdb=mmdb, + ) + + log_parser_class.assert_called_once_with( + mmdb_data=b"mmdb", + robots_list=["robot"], + output_mode="dict", + ) + self.assertEqual(log_parser.logfile, log_file.path) + build_url_translation_manager.assert_called_once_with(log_file) + self.assertEqual(result, (log_parser, translation_manager)) diff --git a/metrics/tests/parsing/test_metadata.py b/metrics/tests/parsing/test_metadata.py new file mode 100644 index 0000000..2c34269 --- /dev/null +++ b/metrics/tests/parsing/test_metadata.py @@ -0,0 +1,99 @@ +from types import SimpleNamespace +from unittest.mock import patch + +from django.test import TestCase + +from collection.models import Collection +from log_manager_config.models import CollectionLogDirectory, LogManagerCollectionConfig +from metrics.services.parsing.metadata import ( + _get_translator_class, + build_url_translation_manager, +) + + +class TranslatorClassTests(TestCase): + def test_books_maps_to_books_translator(self): + cls = _get_translator_class("books") + self.assertEqual(cls.__name__, "URLTranslatorBooksSite") + + def test_classic_maps_to_classic_translator(self): + cls = _get_translator_class("classic") + self.assertEqual(cls.__name__, "URLTranslatorClassicSite") + + def test_opac_maps_to_opac_translator(self): + cls = _get_translator_class("opac") + self.assertEqual(cls.__name__, "URLTranslatorOPACSite") + + def test_opac_alpha_maps_to_opac_alpha_translator(self): + cls = _get_translator_class("opac_alpha") + self.assertEqual(cls.__name__, "URLTranslatorOPACAlphaSite") + + def test_preprints_maps_to_preprints_translator(self): + cls = _get_translator_class("preprints") + self.assertEqual(cls.__name__, "URLTranslatorPreprintsSite") + + def test_dataverse_maps_to_dataverse_translator(self): + cls = _get_translator_class("dataverse") + self.assertEqual(cls.__name__, "URLTranslatorDataverseSite") + + def test_unknown_name_returns_none(self): + self.assertIsNone(_get_translator_class("unknown")) + + def test_none_returns_none(self): + self.assertIsNone(_get_translator_class(None)) + + def test_case_insensitive(self): + cls = _get_translator_class("Books") + self.assertEqual(cls.__name__, "URLTranslatorBooksSite") + + +class ParsingMetadataTests(TestCase): + def setUp(self): + self.collection = Collection.objects.create(acron3="books", acron2="bk") + config = LogManagerCollectionConfig.objects.create(collection=self.collection) + CollectionLogDirectory.objects.create( + config=config, + path="/app/logs/books", + translator_class="books", + ) + + @patch("metrics.services.parsing.metadata.url_translator.URLTranslationManager") + @patch("metrics.services.parsing.metadata.Source.metadata") + @patch("metrics.services.parsing.metadata.Document.metadata") + def test_builds_manager_with_collection_metadata_and_configured_translator( + self, + document_metadata, + source_metadata, + manager_class, + ): + log_file = SimpleNamespace( + collection=self.collection, + path="/app/logs/books/2026-08-01.log.gz", + ) + documents = iter([{"document_id": "book:1"}]) + sources = iter([{"source_id": "1"}]) + document_metadata.return_value = documents + source_metadata.return_value = sources + + manager = build_url_translation_manager(log_file) + + document_metadata.assert_called_once_with(collection=self.collection) + source_metadata.assert_called_once_with(collection=self.collection) + manager_class.assert_called_once() + call_kwargs = manager_class.call_args.kwargs + self.assertIs(call_kwargs["documents_metadata"], documents) + self.assertIs(call_kwargs["sources_metadata"], sources) + self.assertEqual(call_kwargs["translator"].__name__, "URLTranslatorBooksSite") + self.assertIs(manager, manager_class.return_value) + + def test_rejects_log_outside_configured_directories(self): + log_file = SimpleNamespace( + collection=self.collection, + path="/other/logs/2026-08-01.log.gz", + ) + + with self.assertRaisesRegex( + Exception, + "No URL translator class found for collection", + ): + build_url_translation_manager(log_file)