Skip to content

Reduz o custo de geração dos metadados de documentos e fontes #130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pitangainnovare merged 2 commits into from
Aug 29, 2026
Merged

Reduz o custo de geração dos metadados de documentos e fontes #130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d31b381
Reduz o custo de geração dos metadados de documentos e fontes
pitangainnovare Aug 29, 2026
9d4d2e5
Atualiza a versão da aplicação para 2.2.0
pitangainnovare Aug 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion VERSION
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
2.1.1
2.2.0
74 changes: 34 additions & 40 deletions document/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
from core.models import CommonControlField
from source.models import Source

METADATA_ITERATOR_CHUNK_SIZE = 2000


class Document(CommonControlField):
DOCUMENT_TYPE_ARTICLE = "article"
Expand Down Expand Up @@ -222,65 +224,57 @@ def delete_documents_by_raw_id(cls, collection, raw_id):

@classmethod
def metadata(cls, collection=None):
queryset = cls.objects.select_related(
"collection",
"source",
"parent_document",
).only(
"collection__acron3",
queryset = cls.objects.all()
if collection:
queryset = queryset.filter(collection=collection)

queryset = queryset.values(
"default_lang",
"default_media_format",
"document_id",
"document_type",
"extra_data",
"files",
"identifiers",
"parent_document__document_id",
"pid_generic",
"pid_v2",
"pid_v3",
"processing_date",
"publication_date",
"publication_year",
"scielo_issn",
"source__scielo_issn",
"source__source_id",
"source__source_type",
"text_langs",
"title",
collection_acron3=models.F("collection__acron3"),
parent_document_identifier=models.F("parent_document__document_id"),
source_identifier=models.F("source__source_id"),
source_scielo_issn=models.F("source__scielo_issn"),
source_type_value=models.F("source__source_type"),
)

if collection:
queryset = queryset.filter(collection=collection)

for document in queryset.iterator():
source = document.source
for document in queryset.iterator(chunk_size=METADATA_ITERATOR_CHUNK_SIZE):
yield {
"collection": document.collection.acron3,
"default_lang": document.default_lang,
"default_media_format": document.default_media_format,
"document_id": document.document_id,
"document_type": document.document_type,
"extra_data": document.extra_data or {},
"files": document.files or {},
"identifiers": document.identifiers or {},
"parent_document_id": (
document.parent_document.document_id
if document.parent_document
else None
),
"pid_generic": document.pid_generic,
"pid_v2": document.pid_v2,
"pid_v3": document.pid_v3,
"processing_date": document.processing_date,
"publication_date": document.publication_date,
"publication_year": document.publication_year,
"scielo_issn": document.scielo_issn
or (source.scielo_issn if source else None),
"source_id": source.source_id if source else None,
"source_type": source.source_type if source else None,
"text_langs": document.text_langs or [],
"title": document.title,
"collection": document["collection_acron3"],
"default_lang": document["default_lang"],
"default_media_format": document["default_media_format"],
"document_id": document["document_id"],
"document_type": document["document_type"],
"extra_data": document["extra_data"] or {},
"files": document["files"] or {},
"identifiers": document["identifiers"] or {},
"parent_document_id": document["parent_document_identifier"],
"pid_generic": document["pid_generic"],
"pid_v2": document["pid_v2"],
"pid_v3": document["pid_v3"],
"processing_date": document["processing_date"],
"publication_date": document["publication_date"],
"publication_year": document["publication_year"],
"scielo_issn": document["scielo_issn"]
or document["source_scielo_issn"],
"source_id": document["source_identifier"],
"source_type": document["source_type_value"],
"text_langs": document["text_langs"] or [],
"title": document["title"],
}

class Meta:
Expand Down
45 changes: 24 additions & 21 deletions source/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
from collection.models import Collection
from core.models import CommonControlField

METADATA_ITERATOR_CHUNK_SIZE = 2000


class Source(CommonControlField):
SOURCE_TYPE_JOURNAL = "journal"
Expand Down Expand Up @@ -198,9 +200,12 @@ def _extract_issns(identifiers):

@classmethod
def metadata(cls, collection=None):
queryset = cls.objects.select_related("collection").only(
queryset = cls.objects.all()
if collection:
queryset = queryset.filter(collection=collection)

queryset = queryset.values(
"acronym",
"collection__acron3",
"default_lang",
"extra_data",
"identifiers",
Expand All @@ -214,30 +219,28 @@ def metadata(cls, collection=None):
"subject_areas",
"title",
"wos_subject_areas",
collection_acron3=models.F("collection__acron3"),
)

if collection:
queryset = queryset.filter(collection=collection)

for source in queryset.iterator():
identifiers = source.identifiers or {}
for source in queryset.iterator(chunk_size=METADATA_ITERATOR_CHUNK_SIZE):
identifiers = source["identifiers"] or {}
yield {
"acronym": source.acronym,
"collection": source.collection.acron3,
"default_lang": source.default_lang,
"extra_data": source.extra_data or {},
"acronym": source["acronym"],
"collection": source["collection_acron3"],
"default_lang": source["default_lang"],
"extra_data": source["extra_data"] or {},
"identifiers": identifiers,
"issns": cls._extract_issns(identifiers),
"publication_date": source.publication_date,
"publication_year": source.publication_year,
"access_type": source.access_type,
"publisher_name": source.publisher_name or [],
"scielo_issn": source.scielo_issn,
"source_id": source.source_id,
"source_type": source.source_type,
"subject_areas": source.subject_areas or [],
"title": source.title,
"wos_subject_areas": source.wos_subject_areas or [],
"publication_date": source["publication_date"],
"publication_year": source["publication_year"],
"access_type": source["access_type"],
"publisher_name": source["publisher_name"] or [],
"scielo_issn": source["scielo_issn"],
"source_id": source["source_id"],
"source_type": source["source_type"],
"subject_areas": source["subject_areas"] or [],
"title": source["title"],
"wos_subject_areas": source["wos_subject_areas"] or [],
}

class Meta:
Expand Down
36 changes: 29 additions & 7 deletions source/tests/test_services.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,36 @@ def test_metadata_exposes_generic_and_journal_fields(self):
extra_data={"country": "BR"},
)

metadata = list(Source.metadata(collection=collection))
with self.assertNumQueries(1):
metadata = list(Source.metadata(collection=collection))

self.assertEqual(len(metadata), 1)
self.assertEqual(metadata[0]["source_type"], Source.SOURCE_TYPE_JOURNAL)
self.assertEqual(metadata[0]["source_id"], "1234-5678")
self.assertEqual(metadata[0]["scielo_issn"], "1234-5678")
self.assertEqual(metadata[0]["issns"], {"1234-5678", "8765-4321"})
self.assertEqual(metadata[0]["title"], "Test Journal")
self.assertEqual(
metadata,
[
{
"access_type": None,
"acronym": "testjou",
"collection": "scl",
"default_lang": "en",
"extra_data": {"country": "BR"},
"identifiers": {
"electronic_issn": "1234-5678",
"print_issn": "8765-4321",
"doi": "10.1590/example",
},
"issns": {"1234-5678", "8765-4321"},
"publication_date": "2024-01-15",
"publication_year": "2024",
"publisher_name": ["SciELO"],
"scielo_issn": "1234-5678",
"source_id": "1234-5678",
"source_type": Source.SOURCE_TYPE_JOURNAL,
"subject_areas": ["Health Sciences"],
"title": "Test Journal",
"wos_subject_areas": ["Medicine"],
}
],
)


class BookSourceServiceTests(TestCase):
Expand Down
Loading