From b0d5ef81a2680efd7351934faff25d67273915f4 Mon Sep 17 00:00:00 2001 From: Valentina Date: Mon, 2 Nov 2026 18:25:40 +0100 Subject: [PATCH 1/4] fix: fix modules partial_import + precedence --- lode/reader/loader.py | 13 +++++--- lode/reader/modules.py | 32 ++++++++++--------- tests/data/provenance/main.ttl | 7 ++-- tests/test_modules.py | 58 ++++++++++++++++++++++++++++------ 4 files changed, 77 insertions(+), 33 deletions(-) diff --git a/lode/reader/loader.py b/lode/reader/loader.py index 25c02e5..fc62f3d 100644 --- a/lode/reader/loader.py +++ b/lode/reader/loader.py @@ -91,15 +91,18 @@ def load(self, source: str) -> None: def _apply_modules(self) -> None: """Apply the requested import module to the loaded graph. - Precedence: partial_import, then imported, then closure. Also records the - IRIs that were imported, so the viewer can keep the original model metadata. + Formal imports come first (imported, else closure); partial_import then runs on + top of them, so it only dereferences terms the imports did not already bring in. + Also records the IRIs that were imported, so the viewer can keep the original + model metadata. """ - if self._partial_import: - self.graph, self.imported_uris = modules.apply_partial_import(self.graph) - elif self._imported: + if self._imported: self.graph, self.imported_uris = modules.apply_imported(self.graph) elif self._closure: self.graph, self.imported_uris = modules.apply_closure(self.graph) + if self._partial_import: + self.graph, visited = modules.apply_partial_import(self.graph) + self.imported_uris = set(self.imported_uris or ()) | visited ## ---------------------------------------------------------- # CONTENT NEGOTIATION FOR URLS diff --git a/lode/reader/modules.py b/lode/reader/modules.py index 737a07b..c3d6d22 100644 --- a/lode/reader/modules.py +++ b/lode/reader/modules.py @@ -2,12 +2,12 @@ """Enrichment modules: load external documents and add them to the graph of the documented ontology. -Three modules, activated by the Loader (one at a time; partial_import takes precedence): +Three modules, activated by the Loader (one at a time; partial_import has the lowest precedence): imported direct owl:imports (depth 1) closure transitive closure of owl:imports (depth N) - partial_import only the definitions of the external terms mentioned, taken from - their "home" ontology (1-hop dereferencing) + partial_import only the definitions of the external terms declared with + rdfs:isDefinedBy, taken from their "home" ontology (1-hop dereferencing) Two ways of loading an external document, shared by the modules: @@ -124,7 +124,9 @@ def apply_closure(graph: ProvenanceGraph): # ---------------------------------------------------------- # PARTIAL IMPORT MODULE (informal imports) -# ---------------------------------------------------------- +# +# Only terms declared with rdfs:isDefinedBy are considered. +#---------------------------------------------------------- # # Boundary: an external term receives # - what the documented ontology says about it, in the main graph, and @@ -171,8 +173,12 @@ def _namespace_failed(term, failed: set) -> bool: def apply_partial_import(graph: ProvenanceGraph): """Dereference (1 hop) the external IRIs described by the documented ontology and - add ONLY their definition, taken from their home ontology (not the redefinitions - in other ontologies). Terms that are merely used stay plain external links.""" + declared with rdfs:isDefinedBy, and add ONLY their definition, taken from their + home ontology (not the redefinitions in other ontologies). + + Without rdfs:isDefinedBy a term stays a plain external link and nothing is + downloaded for it. owl:imports documents are not loaded here: that is the job + of the imported/closure modules.""" # 1. External terms: IRIs mentioned in the graph, excluding the import IRIs, the # core vocabularies and the ontology's own terms. import_iris = set(graph.objects(None, OWL.imports)) @@ -181,18 +187,14 @@ def apply_partial_import(graph: ProvenanceGraph): external = {t for t in mentioned if not str(t).startswith(_SKIP_NS) and not (own and str(t).startswith(own))} - # 2. Keep only terms the documented ontology describes (subject of at least one of - # its triples): terms it merely uses stay plain external links. - described = {s for s, _, _ in graph if s in external} + # 2. Keep only terms declared with rdfs:isDefinedBy: the explicit signal that the + # term has a home worth downloading; other terms stay plain external links. + described = set(graph.subjects(RDFS.isDefinedBy, None)) & external - # 3. Load into a separate graph (one named graph per document). The owl:imports - # documents are loaded first, as informal imports (no recursion): they are often - # the home of the described terms, so they are not downloaded again. Then one - # attempt per namespace not covered yet. + # 3. Load into a separate graph (one named graph per document): one attempt per + # namespace not covered yet. fetched = ProvenanceGraph() visited, failed, homes = set(), set(), {} # homes: source -> loaded ontology IRI - for iri in (import_iris if described else ()): - _load_informal_import(str(iri), fetched, visited, failed, homes) for term in described: if _home(term, homes) is None and not _namespace_failed(term, failed): doc, frag = urldefrag(str(term)) diff --git a/tests/data/provenance/main.ttl b/tests/data/provenance/main.ttl index d980735..bb3c99f 100644 --- a/tests/data/provenance/main.ttl +++ b/tests/data/provenance/main.ttl @@ -1,13 +1,14 @@ # MAIN: the documented ontology. Imports imp; mentions home#T (home reachable), # ver#V (home with a versioned IRI) and missing#X (home unreachable); # restates the label of home#T, which must win over the home ontology. +# The three external terms are declared with rdfs:isDefinedBy (required by partial_import). @prefix owl: . @prefix rdfs: . a owl:Ontology ; owl:imports . a owl:Class ; rdfs:subClassOf , , . - rdfs:label "T from main"@en . + rdfs:label "T from main"@en ; rdfs:isDefinedBy . - rdfs:comment "V described by main" . - rdfs:comment "X described by main" . + rdfs:comment "V described by main" ; rdfs:isDefinedBy . + rdfs:comment "X described by main" ; rdfs:isDefinedBy . diff --git a/tests/test_modules.py b/tests/test_modules.py index 99faac6..532126c 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -242,8 +242,8 @@ def ds(self, main_path): # -- which graphs, which content ------------------------------------- def test_graphs_are_main_and_homes_only(self, ds): - """imp is formally imported but is nobody's home: no graph for it; missing is - unreachable: no graph either.""" + """imp is not loaded by partial_import (formal imports are for imported/closure): + no graph for it; missing is unreachable: no graph either.""" assert graphs(ds) == {MAIN, HOME, VER} @pytest.mark.parametrize("name,ctx", [("home", HOME), ("ver", VER)]) @@ -299,7 +299,8 @@ def test_home_by_dereferenced_document_when_iri_differs(self, ds): def test_unreachable_home_gets_no_external_definition(self, ds, offline_fetch): """missing is not served: only main's own triple, imp's redefinition must not fill the gap.""" assert "http://ex.org/missing" in offline_fetch - assert set(ds.predicate_objects(X.X)) == {(RDFS.comment, Literal("X described by main"))} + assert set(ds.predicate_objects(X.X)) == {(RDFS.comment, Literal("X described by main")), + (RDFS.isDefinedBy, URIRef("http://ex.org/missing"))} assert contexts(ds, (X.X, RDFS.comment, Literal("X described by main"))) == {MAIN} # -- fetching --------------------------------------------------------- @@ -320,8 +321,8 @@ def test_vocabulary_without_ontology_fetched_once(self, tmp_path, offline_fetch, path = write_ttl(tmp_path, "m.ttl", """ a owl:Ontology . rdfs:seeAlso , . - rdfs:comment "a described by m2" . - rdfs:comment "b described by m2" . + rdfs:comment "a described by m2" ; rdfs:isDefinedBy . + rdfs:comment "b described by m2" ; rdfs:isDefinedBy . """) ds = Loader(path, partial_import=True).get_graph() assert len([u for u in offline_fetch if u.startswith("http://ex.org/voc/")]) == 1 @@ -332,13 +333,32 @@ def test_failed_namespace_not_retried_per_term(self, tmp_path, offline_fetch): path = write_ttl(tmp_path, "m.ttl", """ a owl:Ontology . rdfs:seeAlso , , . - rdfs:comment "a" . - rdfs:comment "b" . - rdfs:comment "c" . + rdfs:comment "a" ; rdfs:isDefinedBy . + rdfs:comment "b" ; rdfs:isDefinedBy . + rdfs:comment "c" ; rdfs:isDefinedBy . """) Loader(path, partial_import=True) assert len([u for u in offline_fetch if u.startswith("http://ex.org/dead/")]) == 1 + # -- isDefinedBy gate ------------------------------------------------- + + def test_term_without_is_defined_by_is_not_fetched(self, tmp_path, offline_fetch): + """A described external term without rdfs:isDefinedBy triggers no download.""" + path = write_ttl(tmp_path, "m.ttl", """ + a owl:Ontology . + rdfs:seeAlso . + rdfs:comment "T described by m4" . +""") + ds = Loader(path, partial_import=True).get_graph() + assert not [u for u in offline_fetch if u.startswith("http://ex.org/home")] + assert (H.T, RDF.type, OWL.Class) not in ds + + def test_partial_import_runs_on_top_of_imported(self, main_path): + """With both flags the formal import comes first, then the homes of the + isDefinedBy terms are added.""" + ds = Loader(main_path, imported=True, partial_import=True).get_graph() + assert graphs(ds) == {MAIN, IMP, HOME, VER} + # ========================================================================== # 5. URL CACHE @@ -442,6 +462,25 @@ def test_no_blank_graph_names(self, main_path, flag): if line.rstrip().endswith("{"): assert line.startswith("<"), line + @pytest.mark.parametrize("flag", ["imported", "closure"]) + def test_header_is_the_loaded_ontology_not_an_imported_one(self, tmp_path, monkeypatch, flag): + """The page header describes the loaded ontology, even when an import declares a + different ontology IRI than the one it is imported by (e.g. versioned vs. base IRI, + as PROV-O does), so the filter on imported IRIs alone cannot recognise it.""" + foreign_doc = "http://ex.org/foreign-20130430" # the IRI owl:imports points to + monkeypatch.setitem(DOCS, foreign_doc, PREFIXES + """ + a owl:Ontology ; rdfs:label "Foreign title" . + a owl:Class . + """) + path = write_ttl(tmp_path, "own.ttl", f""" + a owl:Ontology ; rdfs:label "Own title" ; owl:imports <{foreign_doc}> . + a owl:Class . + """) + reader = Reader() + reader.load_instances(path, "owl", **{flag: True}) + label = str(reader.get_viewer().get_view_data()["metadata"]["label"]) + assert "Own title" in label + assert "Foreign title" not in label # ========================================================================== # 7. HTTP API (end-to-end) @@ -452,11 +491,10 @@ def test_no_blank_graph_names(self, main_path, flag): # Uploads go through the spool (spool.save) and are requested with # GET /extract?upload_id=..., the same path as _resolve_reader. -TRIPLE_FORMATS = {"ttl": "turtle", "nt": "nt", "rdf": "xml", "jsonld": "json-ld"} +TRIPLE_FORMATS = {"ttl": "turtle", "nt": "nt", "xml": "xml", "jsonld": "json-ld"} QUAD_EXTS = ["trig", "nq"] ALL_FLAGS = [None, *MODULES] - @pytest.fixture(scope="module") def client(): return TestClient(app) From 8ec24dd61c97dd3651d1547e5c15b4eaf140706d Mon Sep 17 00:00:00 2001 From: Valentina Date: Mon, 2 Nov 2026 18:26:28 +0100 Subject: [PATCH 2/4] refactor: centralise safe_export --- lode/api.py | 3 ++- lode/builder.py | 6 ++--- lode/viewer/base_viewer.py | 54 +++++++++++++++++++++++++++----------- tests/test_owl_viewer.py | 1 + 4 files changed, 43 insertions(+), 21 deletions(-) diff --git a/lode/api.py b/lode/api.py index dbb58aa..77c47ca 100644 --- a/lode/api.py +++ b/lode/api.py @@ -456,8 +456,9 @@ def extract_get( serialized = reader.get_viewer().export_resource(resource, serial["fmt"]) filename = iri_local_name(resource) or "resource" else: - serialized = reader.get_main_graph().serialize(format=serial["fmt"]) + serialized = reader.get_viewer().export_main(serial["fmt"]) filename = (iri_local_name(url) if url else "") or "graph" + return Response(content=serialized, media_type=serial["mime"], headers={"Content-Disposition": f'inline; filename="{filename}.{serial["ext"]}"'}) diff --git a/lode/builder.py b/lode/builder.py index 1a8a5c9..f568f48 100644 --- a/lode/builder.py +++ b/lode/builder.py @@ -214,10 +214,8 @@ def resource_url_index(uri, section=None): if reader is not None: for fmt, ext in _SER: - out = reader.get_main_graph().serialize(format=fmt) - if isinstance(out, bytes): - out = out.decode("utf-8") - _write_in(out_dir, f"{artefact_name}.{ext}", out) + _write_in(out_dir, f"{artefact_name}.{ext}", viewer.export_main(fmt)) + template_index = env_index.get_template("viewer.html") template_resource = env_resource.get_template("viewer.html") diff --git a/lode/viewer/base_viewer.py b/lode/viewer/base_viewer.py index 18f1fbb..ba968d7 100644 --- a/lode/viewer/base_viewer.py +++ b/lode/viewer/base_viewer.py @@ -73,6 +73,26 @@ def _clean_name(self, name: str) -> str: """Turn an attribute or getter name into a display label, using this viewer's vocabulary.""" return clean_name(name, self._config["vocabulary"]) + def _get_main_model(self) -> Optional[Model]: + """Return the Model of the documented ontology, or None when there is none. + + The documented ontology is the one that names the loaded graph + (`reader._ontology_iri`), so the choice does not depend on the order of the + cache: it also holds Models for imports, version IRIs and imported + ontologies. Artefacts that declare no ontology IRI fall back to the first + Model that does not come from an import. + """ + main_iri = str(self.reader._ontology_iri) if self.reader._ontology_iri else None + imported = getattr(self.reader, "_imported_uris", set()) or set() + models = [i for i in self.get_all_instances() if isinstance(i, Model)] + by_iri = next((m for m in models if str(m.get_has_identifier()) == main_iri), None) + if by_iri is not None: + return by_iri + return next( + (m for m in models if str(m.get_has_identifier()) not in imported), + models[0] if models else None, + ) + def _get_ontology_ns(self) -> str: """Return the namespace of the documented ontology (its IRI ending in '/' or '#'), or ''. @@ -80,12 +100,10 @@ def _get_ontology_ns(self) -> str: """ if self._ontology_ns is None: self._ontology_ns = '' - for inst in self.get_all_instances(): - if isinstance(inst, Model): - nid = inst.get_has_identifier() - if nid: - self._ontology_ns = nid if nid.endswith(('/', '#')) else nid + '/' - break + model = self._get_main_model() + nid = model.get_has_identifier() if model else None + if nid: + self._ontology_ns = nid if nid.endswith(('/', '#')) else nid + '/' return self._ontology_ns def _has_local_triples(self, uri) -> bool: @@ -409,17 +427,12 @@ def _find_and_format_metadata(self, all_instances: List[Resource], language=None Searches for the Model and its Statements and formatting them for the template. """ - ontology_model = None - - # 1. Find the Model (skip imported/closure ontologies) - imported = getattr(self.reader, "_imported_uris", set()) or set() - models = [i for i in all_instances if isinstance(i, Model)] - ontology_model = next( - (m for m in models if str(m.get_has_identifier()) not in imported), - models[0] if models else None, - ) + + # 1. Find the Model of the documented ontology (not the imported ones) + ontology_model = self._get_main_model() if not ontology_model: + return {} # 2. Prepare Output @@ -772,4 +785,13 @@ def export_resource(self, resource_uri: str, fmt: str = "turtle") -> str: for t in self.reader.get_provenance_subgraph(inst): if t in main: g.add(t) - return safe_serialize(g, fmt) \ No newline at end of file + return safe_serialize(g, fmt) + + def export_main(self, fmt: str = "turtle") -> str: + """Serialize the main graph, i.e. the triples of the loaded artefact only. + + Imported modules are left out: they belong to the provenance, not to the + export. A format that fails yields an error comment instead of raising, so + one bad format cannot break the whole export or build. + """ + return safe_serialize(self.reader.get_main_graph(), fmt) \ No newline at end of file diff --git a/tests/test_owl_viewer.py b/tests/test_owl_viewer.py index 1580cce..8ac898c 100644 --- a/tests/test_owl_viewer.py +++ b/tests/test_owl_viewer.py @@ -41,6 +41,7 @@ class _FakeReader: def __init__(self, cache): self._instance_cache = cache self._graph = _FakeGraph() + self._ontology_iri = None def get_provenance_subgraph(self, instance): return _FakeGraph() From 389f51b70e686339f871bb4e2eb76657b2d30458 Mon Sep 17 00:00:00 2001 From: Valentina Date: Mon, 2 Nov 2026 18:26:47 +0100 Subject: [PATCH 3/4] fix: change ext for rdf export into xml --- lode/viewer/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lode/viewer/helpers.py b/lode/viewer/helpers.py index 21a81d7..dd46824 100644 --- a/lode/viewer/helpers.py +++ b/lode/viewer/helpers.py @@ -20,7 +20,7 @@ # Export formats: triples of the loaded artefact only. SERIALIZATION_FORMATS = [ {"label": "Turtle", "fmt": "turtle", "mime": "text/turtle", "ext": "ttl", "icon": "bi-filetype-raw"}, - {"label": "RDF/XML", "fmt": "xml", "mime": "application/rdf+xml", "ext": "rdf", "icon": "bi-file-code"}, + {"label": "RDF/XML", "fmt": "xml", "mime": "application/rdf+xml", "ext": "xml", "icon": "bi-file-code"}, {"label": "JSON-LD", "fmt": "json-ld", "mime": "application/ld+json", "ext": "jsonld", "icon": "bi-filetype-json"}, {"label": "N-Triples", "fmt": "nt", "mime": "application/n-triples", "ext": "nt", "icon": "bi-file-text"}, ] From 3ff88b436af6c90657f1f75fd99e9f99ea836890 Mon Sep 17 00:00:00 2001 From: Valentina Date: Mon, 2 Nov 2026 18:35:14 +0100 Subject: [PATCH 4/4] chore: added missing author email --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c84d8b6..2bbde2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ version = "2.1.0" description = "New reengineered version of LODE, maintained by OpenCitations" authors = [ {name = "Valentina Pasqual", email = "valentina.pasqual2@unibo.it"}, - {name = "Ahmadreza Nazari"}, {name = "Silvio Peroni", email = "silvio.peroni@unibo.it"}, + {name = "Ahmadreza Nazari", email = "ahmadreza.nazari@studio.unibo.it"}, ] readme = "README.md" license = "ISC"