diff --git a/VERSION b/VERSION index 3e3c2f1..ccbccc3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.1 +2.2.0 diff --git a/document/models.py b/document/models.py index a7a5d67..c1fa66f 100644 --- a/document/models.py +++ b/document/models.py @@ -5,6 +5,8 @@ from core.models import CommonControlField from source.models import Source +METADATA_ITERATOR_CHUNK_SIZE = 2000 + class Document(CommonControlField): DOCUMENT_TYPE_ARTICLE = "article" @@ -222,12 +224,11 @@ def delete_documents_by_raw_id(cls, collection, raw_id): @classmethod def metadata(cls, collection=None): - queryset = cls.objects.select_related( - "collection", - "source", - "parent_document", - ).only( - "collection__acron3", + queryset = cls.objects.all() + if collection: + queryset = queryset.filter(collection=collection) + + queryset = queryset.values( "default_lang", "default_media_format", "document_id", @@ -235,7 +236,6 @@ def metadata(cls, collection=None): "extra_data", "files", "identifiers", - "parent_document__document_id", "pid_generic", "pid_v2", "pid_v3", @@ -243,44 +243,38 @@ def metadata(cls, collection=None): "publication_date", "publication_year", "scielo_issn", - "source__scielo_issn", - "source__source_id", - "source__source_type", "text_langs", "title", + collection_acron3=models.F("collection__acron3"), + parent_document_identifier=models.F("parent_document__document_id"), + source_identifier=models.F("source__source_id"), + source_scielo_issn=models.F("source__scielo_issn"), + source_type_value=models.F("source__source_type"), ) - if collection: - queryset = queryset.filter(collection=collection) - - for document in queryset.iterator(): - source = document.source + for document in queryset.iterator(chunk_size=METADATA_ITERATOR_CHUNK_SIZE): yield { - "collection": document.collection.acron3, - "default_lang": document.default_lang, - "default_media_format": document.default_media_format, - "document_id": document.document_id, - "document_type": document.document_type, - "extra_data": document.extra_data or {}, - "files": document.files or {}, - "identifiers": document.identifiers or {}, - "parent_document_id": ( - document.parent_document.document_id - if document.parent_document - else None - ), - "pid_generic": document.pid_generic, - "pid_v2": document.pid_v2, - "pid_v3": document.pid_v3, - "processing_date": document.processing_date, - "publication_date": document.publication_date, - "publication_year": document.publication_year, - "scielo_issn": document.scielo_issn - or (source.scielo_issn if source else None), - "source_id": source.source_id if source else None, - "source_type": source.source_type if source else None, - "text_langs": document.text_langs or [], - "title": document.title, + "collection": document["collection_acron3"], + "default_lang": document["default_lang"], + "default_media_format": document["default_media_format"], + "document_id": document["document_id"], + "document_type": document["document_type"], + "extra_data": document["extra_data"] or {}, + "files": document["files"] or {}, + "identifiers": document["identifiers"] or {}, + "parent_document_id": document["parent_document_identifier"], + "pid_generic": document["pid_generic"], + "pid_v2": document["pid_v2"], + "pid_v3": document["pid_v3"], + "processing_date": document["processing_date"], + "publication_date": document["publication_date"], + "publication_year": document["publication_year"], + "scielo_issn": document["scielo_issn"] + or document["source_scielo_issn"], + "source_id": document["source_identifier"], + "source_type": document["source_type_value"], + "text_langs": document["text_langs"] or [], + "title": document["title"], } class Meta: diff --git a/source/models.py b/source/models.py index c88e9f0..c0cce65 100644 --- a/source/models.py +++ b/source/models.py @@ -5,6 +5,8 @@ from collection.models import Collection from core.models import CommonControlField +METADATA_ITERATOR_CHUNK_SIZE = 2000 + class Source(CommonControlField): SOURCE_TYPE_JOURNAL = "journal" @@ -198,9 +200,12 @@ def _extract_issns(identifiers): @classmethod def metadata(cls, collection=None): - queryset = cls.objects.select_related("collection").only( + queryset = cls.objects.all() + if collection: + queryset = queryset.filter(collection=collection) + + queryset = queryset.values( "acronym", - "collection__acron3", "default_lang", "extra_data", "identifiers", @@ -214,30 +219,28 @@ def metadata(cls, collection=None): "subject_areas", "title", "wos_subject_areas", + collection_acron3=models.F("collection__acron3"), ) - if collection: - queryset = queryset.filter(collection=collection) - - for source in queryset.iterator(): - identifiers = source.identifiers or {} + for source in queryset.iterator(chunk_size=METADATA_ITERATOR_CHUNK_SIZE): + identifiers = source["identifiers"] or {} yield { - "acronym": source.acronym, - "collection": source.collection.acron3, - "default_lang": source.default_lang, - "extra_data": source.extra_data or {}, + "acronym": source["acronym"], + "collection": source["collection_acron3"], + "default_lang": source["default_lang"], + "extra_data": source["extra_data"] or {}, "identifiers": identifiers, "issns": cls._extract_issns(identifiers), - "publication_date": source.publication_date, - "publication_year": source.publication_year, - "access_type": source.access_type, - "publisher_name": source.publisher_name or [], - "scielo_issn": source.scielo_issn, - "source_id": source.source_id, - "source_type": source.source_type, - "subject_areas": source.subject_areas or [], - "title": source.title, - "wos_subject_areas": source.wos_subject_areas or [], + "publication_date": source["publication_date"], + "publication_year": source["publication_year"], + "access_type": source["access_type"], + "publisher_name": source["publisher_name"] or [], + "scielo_issn": source["scielo_issn"], + "source_id": source["source_id"], + "source_type": source["source_type"], + "subject_areas": source["subject_areas"] or [], + "title": source["title"], + "wos_subject_areas": source["wos_subject_areas"] or [], } class Meta: diff --git a/source/tests/test_services.py b/source/tests/test_services.py index 4b72a08..1e3df9f 100644 --- a/source/tests/test_services.py +++ b/source/tests/test_services.py @@ -30,14 +30,36 @@ def test_metadata_exposes_generic_and_journal_fields(self): extra_data={"country": "BR"}, ) - metadata = list(Source.metadata(collection=collection)) + with self.assertNumQueries(1): + metadata = list(Source.metadata(collection=collection)) - self.assertEqual(len(metadata), 1) - self.assertEqual(metadata[0]["source_type"], Source.SOURCE_TYPE_JOURNAL) - self.assertEqual(metadata[0]["source_id"], "1234-5678") - self.assertEqual(metadata[0]["scielo_issn"], "1234-5678") - self.assertEqual(metadata[0]["issns"], {"1234-5678", "8765-4321"}) - self.assertEqual(metadata[0]["title"], "Test Journal") + self.assertEqual( + metadata, + [ + { + "access_type": None, + "acronym": "testjou", + "collection": "scl", + "default_lang": "en", + "extra_data": {"country": "BR"}, + "identifiers": { + "electronic_issn": "1234-5678", + "print_issn": "8765-4321", + "doi": "10.1590/example", + }, + "issns": {"1234-5678", "8765-4321"}, + "publication_date": "2024-01-15", + "publication_year": "2024", + "publisher_name": ["SciELO"], + "scielo_issn": "1234-5678", + "source_id": "1234-5678", + "source_type": Source.SOURCE_TYPE_JOURNAL, + "subject_areas": ["Health Sciences"], + "title": "Test Journal", + "wos_subject_areas": ["Medicine"], + } + ], + ) class BookSourceServiceTests(TestCase):