Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion lode/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -456,8 +456,9 @@ def extract_get(
serialized = reader.get_viewer().export_resource(resource, serial["fmt"])
filename = iri_local_name(resource) or "resource"
else:
serialized = reader.get_main_graph().serialize(format=serial["fmt"])
serialized = reader.get_viewer().export_main(serial["fmt"])
filename = (iri_local_name(url) if url else "") or "graph"

return Response(content=serialized, media_type=serial["mime"],
headers={"Content-Disposition": f'inline; filename="{filename}.{serial["ext"]}"'})

Expand Down
6 changes: 2 additions & 4 deletions lode/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -214,10 +214,8 @@ def resource_url_index(uri, section=None):

if reader is not None:
for fmt, ext in _SER:
out = reader.get_main_graph().serialize(format=fmt)
if isinstance(out, bytes):
out = out.decode("utf-8")
_write_in(out_dir, f"{artefact_name}.{ext}", out)
_write_in(out_dir, f"{artefact_name}.{ext}", viewer.export_main(fmt))


template_index = env_index.get_template("viewer.html")
template_resource = env_resource.get_template("viewer.html")
Expand Down
13 changes: 8 additions & 5 deletions lode/reader/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,15 +91,18 @@ def load(self, source: str) -> None:
def _apply_modules(self) -> None:
"""Apply the requested import module to the loaded graph.

Precedence: partial_import, then imported, then closure. Also records the
IRIs that were imported, so the viewer can keep the original model metadata.
Formal imports come first (imported, else closure); partial_import then runs on
top of them, so it only dereferences terms the imports did not already bring in.
Also records the IRIs that were imported, so the viewer can keep the original
model metadata.
"""
if self._partial_import:
self.graph, self.imported_uris = modules.apply_partial_import(self.graph)
elif self._imported:
if self._imported:
self.graph, self.imported_uris = modules.apply_imported(self.graph)
elif self._closure:
self.graph, self.imported_uris = modules.apply_closure(self.graph)
if self._partial_import:
self.graph, visited = modules.apply_partial_import(self.graph)
self.imported_uris = set(self.imported_uris or ()) | visited

## ----------------------------------------------------------
# CONTENT NEGOTIATION FOR URLS
Expand Down
32 changes: 17 additions & 15 deletions lode/reader/modules.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@
"""Enrichment modules: load external documents and add them to the graph of the
documented ontology.

Three modules, activated by the Loader (one at a time; partial_import takes precedence):
Three modules, activated by the Loader (one at a time; partial_import has the lowest precedence):

imported direct owl:imports (depth 1)
closure transitive closure of owl:imports (depth N)
partial_import only the definitions of the external terms mentioned, taken from
their "home" ontology (1-hop dereferencing)
partial_import only the definitions of the external terms declared with
rdfs:isDefinedBy, taken from their "home" ontology (1-hop dereferencing)

Two ways of loading an external document, shared by the modules:

Expand Down Expand Up @@ -124,7 +124,9 @@ def apply_closure(graph: ProvenanceGraph):

# ----------------------------------------------------------
# PARTIAL IMPORT MODULE (informal imports)
# ----------------------------------------------------------
#
# Only terms declared with rdfs:isDefinedBy are considered.
#----------------------------------------------------------
#
# Boundary: an external term receives
# - what the documented ontology says about it, in the main graph, and
Expand Down Expand Up @@ -171,8 +173,12 @@ def _namespace_failed(term, failed: set) -> bool:

def apply_partial_import(graph: ProvenanceGraph):
"""Dereference (1 hop) the external IRIs described by the documented ontology and
add ONLY their definition, taken from their home ontology (not the redefinitions
in other ontologies). Terms that are merely used stay plain external links."""
declared with rdfs:isDefinedBy, and add ONLY their definition, taken from their
home ontology (not the redefinitions in other ontologies).

Without rdfs:isDefinedBy a term stays a plain external link and nothing is
downloaded for it. owl:imports documents are not loaded here: that is the job
of the imported/closure modules."""
# 1. External terms: IRIs mentioned in the graph, excluding the import IRIs, the
# core vocabularies and the ontology's own terms.
import_iris = set(graph.objects(None, OWL.imports))
Expand All @@ -181,18 +187,14 @@ def apply_partial_import(graph: ProvenanceGraph):
external = {t for t in mentioned
if not str(t).startswith(_SKIP_NS) and not (own and str(t).startswith(own))}

# 2. Keep only terms the documented ontology describes (subject of at least one of
# its triples): terms it merely uses stay plain external links.
described = {s for s, _, _ in graph if s in external}
# 2. Keep only terms declared with rdfs:isDefinedBy: the explicit signal that the
# term has a home worth downloading; other terms stay plain external links.
described = set(graph.subjects(RDFS.isDefinedBy, None)) & external

# 3. Load into a separate graph (one named graph per document). The owl:imports
# documents are loaded first, as informal imports (no recursion): they are often
# the home of the described terms, so they are not downloaded again. Then one
# attempt per namespace not covered yet.
# 3. Load into a separate graph (one named graph per document): one attempt per
# namespace not covered yet.
fetched = ProvenanceGraph()
visited, failed, homes = set(), set(), {} # homes: source -> loaded ontology IRI
for iri in (import_iris if described else ()):
_load_informal_import(str(iri), fetched, visited, failed, homes)
for term in described:
if _home(term, homes) is None and not _namespace_failed(term, failed):
doc, frag = urldefrag(str(term))
Expand Down
54 changes: 38 additions & 16 deletions lode/viewer/base_viewer.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,19 +73,37 @@ def _clean_name(self, name: str) -> str:
"""Turn an attribute or getter name into a display label, using this viewer's vocabulary."""
return clean_name(name, self._config["vocabulary"])

def _get_main_model(self) -> Optional[Model]:
"""Return the Model of the documented ontology, or None when there is none.

The documented ontology is the one that names the loaded graph
(`reader._ontology_iri`), so the choice does not depend on the order of the
cache: it also holds Models for imports, version IRIs and imported
ontologies. Artefacts that declare no ontology IRI fall back to the first
Model that does not come from an import.
"""
main_iri = str(self.reader._ontology_iri) if self.reader._ontology_iri else None
imported = getattr(self.reader, "_imported_uris", set()) or set()
models = [i for i in self.get_all_instances() if isinstance(i, Model)]
by_iri = next((m for m in models if str(m.get_has_identifier()) == main_iri), None)
if by_iri is not None:
return by_iri
return next(
(m for m in models if str(m.get_has_identifier()) not in imported),
models[0] if models else None,
)

def _get_ontology_ns(self) -> str:
"""Return the namespace of the documented ontology (its IRI ending in '/' or '#'), or ''.

Computed once from the first Model instance and then cached.
"""
if self._ontology_ns is None:
self._ontology_ns = ''
for inst in self.get_all_instances():
if isinstance(inst, Model):
nid = inst.get_has_identifier()
if nid:
self._ontology_ns = nid if nid.endswith(('/', '#')) else nid + '/'
break
model = self._get_main_model()
nid = model.get_has_identifier() if model else None
if nid:
self._ontology_ns = nid if nid.endswith(('/', '#')) else nid + '/'
return self._ontology_ns

def _has_local_triples(self, uri) -> bool:
Expand Down Expand Up @@ -409,17 +427,12 @@ def _find_and_format_metadata(self, all_instances: List[Resource], language=None
Searches for the Model and its Statements and formatting them
for the template.
"""
ontology_model = None

# 1. Find the Model (skip imported/closure ontologies)
imported = getattr(self.reader, "_imported_uris", set()) or set()
models = [i for i in all_instances if isinstance(i, Model)]
ontology_model = next(
(m for m in models if str(m.get_has_identifier()) not in imported),
models[0] if models else None,
)

# 1. Find the Model of the documented ontology (not the imported ones)
ontology_model = self._get_main_model()

if not ontology_model:

return {}

# 2. Prepare Output
Expand Down Expand Up @@ -772,4 +785,13 @@ def export_resource(self, resource_uri: str, fmt: str = "turtle") -> str:
for t in self.reader.get_provenance_subgraph(inst):
if t in main:
g.add(t)
return safe_serialize(g, fmt)
return safe_serialize(g, fmt)

def export_main(self, fmt: str = "turtle") -> str:
"""Serialize the main graph, i.e. the triples of the loaded artefact only.

Imported modules are left out: they belong to the provenance, not to the
export. A format that fails yields an error comment instead of raising, so
one bad format cannot break the whole export or build.
"""
return safe_serialize(self.reader.get_main_graph(), fmt)
2 changes: 1 addition & 1 deletion lode/viewer/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
# Export formats: triples of the loaded artefact only.
SERIALIZATION_FORMATS = [
{"label": "Turtle", "fmt": "turtle", "mime": "text/turtle", "ext": "ttl", "icon": "bi-filetype-raw"},
{"label": "RDF/XML", "fmt": "xml", "mime": "application/rdf+xml", "ext": "rdf", "icon": "bi-file-code"},
{"label": "RDF/XML", "fmt": "xml", "mime": "application/rdf+xml", "ext": "xml", "icon": "bi-file-code"},
{"label": "JSON-LD", "fmt": "json-ld", "mime": "application/ld+json", "ext": "jsonld", "icon": "bi-filetype-json"},
{"label": "N-Triples", "fmt": "nt", "mime": "application/n-triples", "ext": "nt", "icon": "bi-file-text"},
]
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ version = "2.1.0"
description = "New reengineered version of LODE, maintained by OpenCitations"
authors = [
{name = "Valentina Pasqual", email = "valentina.pasqual2@unibo.it"},
{name = "Ahmadreza Nazari"},
{name = "Silvio Peroni", email = "silvio.peroni@unibo.it"},
{name = "Ahmadreza Nazari", email = "ahmadreza.nazari@studio.unibo.it"},
]
readme = "README.md"
license = "ISC"
Expand Down
7 changes: 4 additions & 3 deletions tests/data/provenance/main.ttl
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
# MAIN: the documented ontology. Imports imp; mentions home#T (home reachable),
# ver#V (home with a versioned IRI) and missing#X (home unreachable);
# restates the label of home#T, which must win over the home ontology.
# The three external terms are declared with rdfs:isDefinedBy (required by partial_import).
@prefix owl: <http://www.w3.org/2002/07/owl#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .

<http://ex.org/main> a owl:Ontology ; owl:imports <http://ex.org/imp> .
<http://ex.org/main#M> a owl:Class ;
rdfs:subClassOf <http://ex.org/home#T>, <http://ex.org/ver#V>, <http://ex.org/missing#X> .
<http://ex.org/home#T> rdfs:label "T from main"@en .
<http://ex.org/home#T> rdfs:label "T from main"@en ; rdfs:isDefinedBy <http://ex.org/home> .

<http://ex.org/ver#V> rdfs:comment "V described by main" .
<http://ex.org/missing#X> rdfs:comment "X described by main" .
<http://ex.org/ver#V> rdfs:comment "V described by main" ; rdfs:isDefinedBy <http://ex.org/ver/1.0> .
<http://ex.org/missing#X> rdfs:comment "X described by main" ; rdfs:isDefinedBy <http://ex.org/missing> .
58 changes: 48 additions & 10 deletions tests/test_modules.py
Original file line number Diff line number Diff line change
Expand Up @@ -242,8 +242,8 @@ def ds(self, main_path):
# -- which graphs, which content -------------------------------------

def test_graphs_are_main_and_homes_only(self, ds):
"""imp is formally imported but is nobody's home: no graph for it; missing is
unreachable: no graph either."""
"""imp is not loaded by partial_import (formal imports are for imported/closure):
no graph for it; missing is unreachable: no graph either."""
assert graphs(ds) == {MAIN, HOME, VER}

@pytest.mark.parametrize("name,ctx", [("home", HOME), ("ver", VER)])
Expand Down Expand Up @@ -299,7 +299,8 @@ def test_home_by_dereferenced_document_when_iri_differs(self, ds):
def test_unreachable_home_gets_no_external_definition(self, ds, offline_fetch):
"""missing is not served: only main's own triple, imp's redefinition must not fill the gap."""
assert "http://ex.org/missing" in offline_fetch
assert set(ds.predicate_objects(X.X)) == {(RDFS.comment, Literal("X described by main"))}
assert set(ds.predicate_objects(X.X)) == {(RDFS.comment, Literal("X described by main")),
(RDFS.isDefinedBy, URIRef("http://ex.org/missing"))}
assert contexts(ds, (X.X, RDFS.comment, Literal("X described by main"))) == {MAIN}

# -- fetching ---------------------------------------------------------
Expand All @@ -320,8 +321,8 @@ def test_vocabulary_without_ontology_fetched_once(self, tmp_path, offline_fetch,
path = write_ttl(tmp_path, "m.ttl", """
<http://ex.org/m2> a owl:Ontology .
<http://ex.org/m2#X> rdfs:seeAlso <http://ex.org/voc/a>, <http://ex.org/voc/b> .
<http://ex.org/voc/a> rdfs:comment "a described by m2" .
<http://ex.org/voc/b> rdfs:comment "b described by m2" .
<http://ex.org/voc/a> rdfs:comment "a described by m2" ; rdfs:isDefinedBy <http://ex.org/voc/a> .
<http://ex.org/voc/b> rdfs:comment "b described by m2" ; rdfs:isDefinedBy <http://ex.org/voc/b> .
""")
ds = Loader(path, partial_import=True).get_graph()
assert len([u for u in offline_fetch if u.startswith("http://ex.org/voc/")]) == 1
Expand All @@ -332,13 +333,32 @@ def test_failed_namespace_not_retried_per_term(self, tmp_path, offline_fetch):
path = write_ttl(tmp_path, "m.ttl", """
<http://ex.org/m3> a owl:Ontology .
<http://ex.org/m3#X> rdfs:seeAlso <http://ex.org/dead/a>, <http://ex.org/dead/b>, <http://ex.org/dead/c> .
<http://ex.org/dead/a> rdfs:comment "a" .
<http://ex.org/dead/b> rdfs:comment "b" .
<http://ex.org/dead/c> rdfs:comment "c" .
<http://ex.org/dead/a> rdfs:comment "a" ; rdfs:isDefinedBy <http://ex.org/dead/a> .
<http://ex.org/dead/b> rdfs:comment "b" ; rdfs:isDefinedBy <http://ex.org/dead/b> .
<http://ex.org/dead/c> rdfs:comment "c" ; rdfs:isDefinedBy <http://ex.org/dead/c> .
""")
Loader(path, partial_import=True)
assert len([u for u in offline_fetch if u.startswith("http://ex.org/dead/")]) == 1

# -- isDefinedBy gate -------------------------------------------------

def test_term_without_is_defined_by_is_not_fetched(self, tmp_path, offline_fetch):
"""A described external term without rdfs:isDefinedBy triggers no download."""
path = write_ttl(tmp_path, "m.ttl", """
<http://ex.org/m4> a owl:Ontology .
<http://ex.org/m4#X> rdfs:seeAlso <http://ex.org/home#T> .
<http://ex.org/home#T> rdfs:comment "T described by m4" .
""")
ds = Loader(path, partial_import=True).get_graph()
assert not [u for u in offline_fetch if u.startswith("http://ex.org/home")]
assert (H.T, RDF.type, OWL.Class) not in ds

def test_partial_import_runs_on_top_of_imported(self, main_path):
"""With both flags the formal import comes first, then the homes of the
isDefinedBy terms are added."""
ds = Loader(main_path, imported=True, partial_import=True).get_graph()
assert graphs(ds) == {MAIN, IMP, HOME, VER}


# ==========================================================================
# 5. URL CACHE
Expand Down Expand Up @@ -442,6 +462,25 @@ def test_no_blank_graph_names(self, main_path, flag):
if line.rstrip().endswith("{"):
assert line.startswith("<"), line

@pytest.mark.parametrize("flag", ["imported", "closure"])
def test_header_is_the_loaded_ontology_not_an_imported_one(self, tmp_path, monkeypatch, flag):
"""The page header describes the loaded ontology, even when an import declares a
different ontology IRI than the one it is imported by (e.g. versioned vs. base IRI,
as PROV-O does), so the filter on imported IRIs alone cannot recognise it."""
foreign_doc = "http://ex.org/foreign-20130430" # the IRI owl:imports points to
monkeypatch.setitem(DOCS, foreign_doc, PREFIXES + """
<http://ex.org/foreign#> a owl:Ontology ; rdfs:label "Foreign title" .
<http://ex.org/foreign#F> a owl:Class .
""")
path = write_ttl(tmp_path, "own.ttl", f"""
<http://ex.org/own> a owl:Ontology ; rdfs:label "Own title" ; owl:imports <{foreign_doc}> .
<http://ex.org/own#O> a owl:Class .
""")
reader = Reader()
reader.load_instances(path, "owl", **{flag: True})
label = str(reader.get_viewer().get_view_data()["metadata"]["label"])
assert "Own title" in label
assert "Foreign title" not in label

# ==========================================================================
# 7. HTTP API (end-to-end)
Expand All @@ -452,11 +491,10 @@ def test_no_blank_graph_names(self, main_path, flag):
# Uploads go through the spool (spool.save) and are requested with
# GET /extract?upload_id=..., the same path as _resolve_reader.

TRIPLE_FORMATS = {"ttl": "turtle", "nt": "nt", "rdf": "xml", "jsonld": "json-ld"}
TRIPLE_FORMATS = {"ttl": "turtle", "nt": "nt", "xml": "xml", "jsonld": "json-ld"}
QUAD_EXTS = ["trig", "nq"]
ALL_FLAGS = [None, *MODULES]


@pytest.fixture(scope="module")
def client():
return TestClient(app)
Expand Down
1 change: 1 addition & 0 deletions tests/test_owl_viewer.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ class _FakeReader:
def __init__(self, cache):
self._instance_cache = cache
self._graph = _FakeGraph()
self._ontology_iri = None

def get_provenance_subgraph(self, instance):
return _FakeGraph()
Expand Down
Loading