From 02e440f47e05481b3042962e7fd088aa027cca4b Mon Sep 17 00:00:00 2001 From: Mathias Vanden Auweele Date: Wed, 18 Nov 2026 21:06:33 +0000 Subject: [PATCH 1/4] perf(paths): evaluate transitive paths on term ids Part of #11 (phase 4). `?s

+ ?o`, `?s ^

+ ?o`, and `

*` with a constant end are now in the id tables: the predicate's edges are read once, sorted, and searched from each start, each end once per start as SPARQL's paths give them. Counts, grouped counts and row parts with such a path use them; a part with a path is always taken, since the evaluator finds a closure one start at a time. On a tree of 200,000 nodes (1.48 million pairs), release build: - GROUP BY ?o COUNT over ?s

+ ?o: 1.27 -> 0.49 s - the path joined with a class, counted: 1.89 -> 0.47 s - the path's rows alone: 1.63 -> 1.20 s Answers identical (sorted outputs compared). Left to the evaluator: the same variable at both ends, `*` between two variables, a constant the store doesn't have, and other path forms. --- CHANGELOG.md | 1 + crates/tripletrail/src/idtree.rs | 162 ++++++++++++++++++++++++++- crates/tripletrail/src/rows.rs | 4 +- crates/tripletrail/tests/id_paths.rs | 75 +++++++++++++ docs/store.md | 9 +- 5 files changed, 248 insertions(+), 3 deletions(-) create mode 100644 crates/tripletrail/tests/id_paths.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 60d4d36..8fe8956 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ versions may change the store format and the API; the release notes say how to u ### Changed +- Transitive property paths run on term ids (#11): `?s

+ ?o`, `?s ^

+ ?o`, and `

*` with a constant end, alone or joined, filtered, counted or grouped with other patterns, are found by a search over the predicate's edges on ids instead of one start at a time in the evaluator. Same answers. On a tree of 200,000 nodes (1.48 million pairs), a `GROUP BY` count over `?s

+ ?o` takes 0.49 s instead of 1.27 s, and the path joined with a class and counted 0.47 s instead of 1.89 s. `#pragma tt.id-rewrites off` turns it off. See [the store](docs/store.md#design). - `ORDER BY DESC(tt:cosine(?v, "[…]")) LIMIT k`, or `ORDER BY DESC(?score)` with `?score` bound to that cosine, is answered from the copy of the vectors that `tt:nearest` reads (#236): the query is evaluated again with `?v` kept to the vectors most similar, each one compared, so the answer stays exact and other patterns and filters still apply. It works on the union default graph and for a caller limited to some graphs. Among 100,000 vectors of 768 numbers: 40 ms instead of 6.2 s; at most two rounds, so a filter that keeps one row in a hundred costs about 130 ms instead of 93 ms. A query with its own dataset or under row-level rules, with an `OFFSET` and `LIMIT` past a sixteenth of the vectors, or over a property that also has plain strings in the form of a vector, is evaluated as before. See [vector search](docs/vector-search.md#functions). ## 0.1.0-rc.1 - 2026-11-17 diff --git a/crates/tripletrail/src/idtree.rs b/crates/tripletrail/src/idtree.rs index 6343028..6ed43ea 100644 --- a/crates/tripletrail/src/idtree.rs +++ b/crates/tripletrail/src/idtree.rs @@ -11,8 +11,9 @@ //! evaluated as usual. use oxigraph::sparql::Variable; +use spargebra::algebra::PropertyPathExpression; use spargebra::algebra::{Expression, Function, GraphPattern}; -use spargebra::term::NamedNodePattern; +use spargebra::term::{NamedNodePattern, TermPattern}; use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; use tripletrail_store::{GraphSelector, IdSlot, IdTriple, Kind, Snapshot, TermId}; @@ -28,6 +29,44 @@ const LOOKUP: f64 = 40.0; /// A key found by walking a sorted scan forward costs about as much as scanning this many /// quads. const WALK: f64 = 4.0; +/// The predicate of `

+`, `^

+` (also `(^

)+`) and `

*` forms, whether it is +/// followed backwards, and whether a zero-length path counts. +pub(crate) fn closure_path( + path: &PropertyPathExpression, +) -> Option<(&spargebra::term::NamedNode, bool, bool)> { + use PropertyPathExpression as P; + let (inner, zero, outer_reverse) = match path { + P::OneOrMore(p) => (&**p, false, false), + P::ZeroOrMore(p) => (&**p, true, false), + P::Reverse(p) => match &**p { + P::OneOrMore(p) => (&**p, false, true), + P::ZeroOrMore(p) => (&**p, true, true), + _ => return None, + }, + _ => return None, + }; + match inner { + P::NamedNode(n) => Some((n, outer_reverse, zero)), + P::Reverse(p) => match &**p { + P::NamedNode(n) => Some((n, !outer_reverse, zero)), + _ => None, + }, + _ => None, + } +} + +fn has_path(p: &GraphPattern) -> bool { + match p { + GraphPattern::Path { .. } => true, + GraphPattern::Join { left, right } + | GraphPattern::LeftJoin { left, right, .. } + | GraphPattern::Minus { left, right } + | GraphPattern::Union { left, right } => has_path(left) || has_path(right), + GraphPattern::Filter { inner, .. } => has_path(inner), + _ => false, + } +} + /// The id of an unbound value: 0 is the default graph, never a subject, predicate or object. const UNBOUND: TermId = TermId(0); @@ -113,11 +152,129 @@ impl Eval<'_> { self.filter(expr, t)? } } + GraphPattern::Path { + subject, + path, + object, + } => self.path(subject, path, object)?, _ => return None, }; (t.ids.len() <= max_ids()).then_some(t) } + /// `?s

+ ?o`, `?s ^

+ ?o`, and with a constant end `

*` too (#11): the pairs of + /// the transitive closure of the predicate's edges, found by a search on the ids from each + /// start, each end once per start (as SPARQL's paths give them). + fn path( + &self, + subject: &TermPattern, + path: &PropertyPathExpression, + object: &TermPattern, + ) -> Option { + let (predicate, reverse, zero) = closure_path(path)?; + let end = |t: &TermPattern| match t { + TermPattern::Variable(v) => Some(Err(v.clone())), + TermPattern::NamedNode(n) => Some(Ok(self.snapshot.term_id(n.as_ref().into())?)), + TermPattern::Literal(l) => Some(Ok(self.snapshot.term_id(l.as_ref().into())?)), + _ => None, + }; + // The direction the edges are followed in: from the subject, unless the subject is + // the variable and the object the constant. + let (mut from, mut to) = (end(subject)?, end(object)?); + let mut forward = !reverse; + let swapped = from.is_err() && to.is_ok(); + if swapped { + std::mem::swap(&mut from, &mut to); + forward = !forward; + } + let to = match (to, &from) { + (Err(v), Err(u)) if v != *u && !zero => v, + (Err(v), Ok(_)) => v, + // The same variable at both ends, both ends constant, or a zero-length path + // between two variables (every term of the graph): the evaluator's. + _ => return None, + }; + // The edges, as (from, to) in the direction followed, sorted to be found by node. + let (a, b) = ( + Variable::new_unchecked("__path_a"), + Variable::new_unchecked("__path_b"), + ); + let edges = self.bgp(&[spargebra::term::TriplePattern { + subject: a.clone().into(), + predicate: NamedNodePattern::NamedNode(predicate.clone()), + object: b.clone().into(), + }])?; + let (ca, cb) = (edges.col(&a)?, edges.col(&b)?); + let mut pairs: Vec<(TermId, TermId)> = (0..edges.len()) + .map(|r| { + let row = edges.row(r); + if forward { + (row[ca], row[cb]) + } else { + (row[cb], row[ca]) + } + }) + .collect(); + drop(edges); + pairs.sort_unstable(); + pairs.dedup(); + let next = |n: TermId| { + let start = pairs.partition_point(|&(x, _)| x < n); + pairs[start..] + .iter() + .take_while(move |&&(x, _)| x == n) + .map(|&(_, y)| y) + }; + let starts: Vec = match &from { + Ok(id) => vec![*id], + Err(_) => { + let mut s: Vec = pairs.iter().map(|&(x, _)| x).collect(); + s.dedup(); + s + } + }; + let limit = max_ids(); + let mut ids: Vec = Vec::new(); + let mut seen: HashMap = HashMap::new(); + let mut stack: Vec = Vec::new(); + for (i, &start) in starts.iter().enumerate() { + if i % 4096 == 0 && (self.cancelled)() { + return None; + } + let emit = |end: TermId, ids: &mut Vec| match &from { + Ok(_) => ids.push(end), + Err(_) => ids.extend([start, end]), + }; + if zero { + seen.insert(start, i); + emit(start, &mut ids); + } + stack.clear(); + stack.push(start); + while let Some(n) = stack.pop() { + for m in next(n) { + if seen.insert(m, i) != Some(i) { + emit(m, &mut ids); + stack.push(m); + } + } + if ids.len() > limit { + return None; + } + } + } + let vars = match &from { + Ok(_) => vec![to], + Err(v) => vec![v.clone(), to], + }; + Some(Table { + ones: ids.len() / vars.len(), + maybe: vec![false; vars.len()], + vars, + ids, + }) + } + /// The solutions of `p`, the right side of an operator over `t`'s rows: those of a basic /// graph pattern that share a value with a row of `t` only (a semi-join), so a few rows /// look its triples up by their values instead of scanning them. @@ -131,6 +288,9 @@ impl Eval<'_> { /// The quads all the triple patterns of `p` match, each on its own: about what the /// evaluator reads for it. pub(crate) fn input_estimate(&self, p: &GraphPattern) -> f64 { + if has_path(p) { + return f64::INFINITY; + } let mut total = 0.0; let mut add = |patterns: &[spargebra::term::TriplePattern]| { if let Some(compiled) = self.snapshot.compile_bgp(patterns, &[]) { diff --git a/crates/tripletrail/src/rows.rs b/crates/tripletrail/src/rows.rs index 5fdb93d..70cfa76 100644 --- a/crates/tripletrail/src/rows.rs +++ b/crates/tripletrail/src/rows.rs @@ -407,6 +407,7 @@ fn covered(p: &GraphPattern) -> bool { expression: None, } => covered(left) && covered(right), G::Filter { expr, inner } => covered_filter(expr) && covered(inner), + G::Path { path, .. } => crate::idtree::closure_path(path).is_some(), _ => false, } } @@ -420,7 +421,8 @@ fn covered_filter(e: &Expression) -> bool { fn worth(p: &GraphPattern) -> bool { use GraphPattern as G; match p { - G::Minus { .. } | G::Filter { .. } => true, + // A closure the evaluator would find one start at a time. + G::Minus { .. } | G::Filter { .. } | G::Path { .. } => true, G::Join { left, right } | G::LeftJoin { left, right, .. } | G::Union { left, right } => { worth(left) || worth(right) } diff --git a/crates/tripletrail/tests/id_paths.rs b/crates/tripletrail/tests/id_paths.rs new file mode 100644 index 0000000..37bf11f --- /dev/null +++ b/crates/tripletrail/tests/id_paths.rs @@ -0,0 +1,75 @@ +//! Transitive paths evaluated on term ids (#11): the same rows as the evaluator's. + +use tripletrail::Engine; +use tripletrail::oxigraph::io::RdfFormat; +use tripletrail::oxigraph::sparql::QueryResults; + +fn data() -> String { + let mut ttl = String::from("@prefix ex: .\n"); + // A tree, large enough for the rewrite, with a cycle and a self-loop at its edge. + for i in 1..3000 { + ttl.push_str(&format!("ex:n{i} ex:up ex:n{} .\n", (i - 1) / 4)); + ttl.push_str(&format!("ex:n{i} a ex:C{} .\n", i % 5)); + } + ttl.push_str("ex:n0 ex:up ex:loop1 . ex:loop1 ex:up ex:loop2 . ex:loop2 ex:up ex:loop1 .\n"); + ttl.push_str("ex:loop2 ex:up ex:loop2 .\n"); + ttl.push_str("ex:n5 ex:up \"a literal\" .\n"); + ttl.push_str("GRAPH ex:g { ex:n7 ex:up ex:elsewhere . }\n"); + ttl +} + +fn rows(e: &Engine, q: &str) -> Vec { + let QueryResults::Solutions(s) = e.query(q).unwrap() else { + panic!() + }; + let mut out: Vec = s + .map(|r| { + r.unwrap() + .iter() + .map(|(v, t)| format!("{v}={t}")) + .collect::>() + .join(" ") + }) + .collect(); + out.sort(); + out +} + +#[test] +fn closures_on_ids_give_the_evaluators_rows() { + let e = Engine::new_tripletrail(); + e.load(RdfFormat::TriG, data().as_bytes()).unwrap(); + for body in [ + "?s ex:up+ ?o", + "?s ^ex:up+ ?o", + "?s (^ex:up)+ ?o", + "?s ex:up+ ?o . ?o a ex:C2", + "?s a ex:C1 . ?s ex:up+ ?o FILTER NOT EXISTS { ?o a ex:C3 }", + "?s ex:up+ ?o MINUS { ?s a ex:C4 }", + "?s ex:up+ ex:n3 . ?s a ex:C1 FILTER(?s != ex:n13)", + "ex:n2999 ex:up+ ?o FILTER(isIRI(?o))", + "ex:n2999 ex:up* ?o FILTER(isIRI(?o))", + "?s ex:up* ex:n12 FILTER(isIRI(?s))", + "ex:nowhere ex:up* ?o FILTER(isIRI(?o))", + "?s ex:up+ \"a literal\" FILTER(isIRI(?s))", + "?s ex:up+ ?s FILTER(isIRI(?s))", + "?s ex:nothing+ ?o FILTER(isIRI(?s))", + ] { + let q = format!("PREFIX ex: SELECT * WHERE {{ {body} }}"); + let off = format!("#pragma tt.id-rewrites off\n{q}"); + let (got, want) = (rows(&e, &q), rows(&e, &off)); + assert!( + !want.is_empty() || body.contains("nothing") || body.contains("nowhere"), + "{body}" + ); + assert_eq!(got, want, "{body}"); + // Counted and grouped the same too. + let counted = format!( + "PREFIX ex: SELECT ?o (COUNT(*) AS ?n) WHERE {{ {body} }} GROUP BY ?o" + ); + if body.contains("?o") { + let off = format!("#pragma tt.id-rewrites off\n{counted}"); + assert_eq!(rows(&e, &counted), rows(&e, &off), "{body}"); + } + } +} diff --git a/docs/store.md b/docs/store.md index bf7fdde..2ce0d3c 100644 --- a/docs/store.md +++ b/docs/store.md @@ -58,7 +58,7 @@ there: `Engine::new_oxigraph()` or `TRIPLETRAIL_STORE=oxigraph`, `tripletrail se - **Counts on term ids.** Counts the stored counts can't answer are computed on term ids without building SPARQL solutions (#167): a basic graph pattern is joined in the store (merge, hash or index nested-loop joins, `Snapshot::count_ids`), and joins, `OPTIONAL`, - `MINUS`, `UNION`, `FILTER (NOT) EXISTS` and filters over them run on tables of ids (`crates/tripletrail/src/idtree.rs`). A small left side passes its values to the right + `MINUS`, `UNION`, `FILTER (NOT) EXISTS`, transitive paths and filters over them run on tables of ids (`crates/tripletrail/src/idtree.rs`). A small left side passes its values to the right side, which then looks them up instead of scanning (`Snapshot::join_ids_from`). Shapes whose result on ids isn't certain (a shared variable that may be unbound, `=` between different literals, a filtered `OPTIONAL`) are evaluated as usual. Filters on ids: `BOUND`, `!`, @@ -102,6 +102,13 @@ there: `Engine::new_oxigraph()` or `TRIPLETRAIL_STORE=oxigraph`, `tripletrail se BSBM 1M, the Sparqloscope MINUS / EXISTS / UNION queries returning their rows instead of a count: geometric mean 70 → 39 ms (`minus-join-small-large` 55 → 2 ms, `exists-join-3-star-2` 84 → 17 ms). + Transitive paths are in the tables too (#11): `?s

+ ?o`, `?s ^

+ ?o`, and `

*` + with one end constant, are the pairs of a search over the predicate's edges on ids, each + end once per start, as SPARQL's paths give them. A part with such a path is always taken, + since the evaluator finds a closure one start at a time. On a tree of 200,000 nodes + (1.48 million pairs): `GROUP BY ?o` with `COUNT` over `?s

+ ?o` 1.27 → 0.49 s, and the + path joined with a class and counted 1.89 → 0.47 s. A path with the same variable at both + ends, `*` between two variables, or other path expressions are left to the evaluator. `ORDER BY ?a DESC(?b) ... LIMIT n` (with `OFFSET`, `DISTINCT` or `REDUCED`) over a pattern the tables cover is ordered on ids too: IRIs and simple strings by their string ranks (#171), integers and booleans by value, unbound values first, and only the first From dc6d7db92720c24f634316c8604f96cd0708d499 Mon Sep 17 00:00:00 2001 From: Mathias Vanden Auweele Date: Wed, 18 Nov 2026 21:57:15 +0000 Subject: [PATCH 2/4] perf(query): search joined transitive paths from the values given A path joined with other patterns starts from the values they bind (backwards when they bind the object), looking edges up level by level for few starts, instead of building the whole closure. Parts with a path joined with patterns outside them are left to the evaluator, and keyed row counts leave paths to the seeded join. --- CHANGELOG.md | 2 +- crates/tripletrail/src/idtree.rs | 248 ++++++++++++++++++++++----- crates/tripletrail/src/rows.rs | 47 ++--- crates/tripletrail/tests/id_paths.rs | 51 ++++++ docs/store.md | 15 +- 5 files changed, 290 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fe8956..fa3ba58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ versions may change the store format and the API; the release notes say how to u ### Changed -- Transitive property paths run on term ids (#11): `?s

+ ?o`, `?s ^

+ ?o`, and `

*` with a constant end, alone or joined, filtered, counted or grouped with other patterns, are found by a search over the predicate's edges on ids instead of one start at a time in the evaluator. Same answers. On a tree of 200,000 nodes (1.48 million pairs), a `GROUP BY` count over `?s

+ ?o` takes 0.49 s instead of 1.27 s, and the path joined with a class and counted 0.47 s instead of 1.89 s. `#pragma tt.id-rewrites off` turns it off. See [the store](docs/store.md#design). +- Transitive property paths run on term ids (#11): `?s

+ ?o`, `?s ^

+ ?o`, and `

*` with a constant end, alone or joined, filtered, counted or grouped with other patterns, are found by a search over the predicate's edges on ids instead of one start at a time in the evaluator: from the values the other patterns give when they bind an end (either end), else over all the edges. Same answers. On a tree of 200,000 nodes (1.48 million pairs), a `GROUP BY` count over `?s

+ ?o` takes 0.19 s instead of 1.05 s, and the path joined with a class and counted 0.05 s instead of 1.61 s; a path from a few bound starts stays at about 1 ms. `#pragma tt.id-rewrites off` turns it off. See [the store](docs/store.md#design). - `ORDER BY DESC(tt:cosine(?v, "[…]")) LIMIT k`, or `ORDER BY DESC(?score)` with `?score` bound to that cosine, is answered from the copy of the vectors that `tt:nearest` reads (#236): the query is evaluated again with `?v` kept to the vectors most similar, each one compared, so the answer stays exact and other patterns and filters still apply. It works on the union default graph and for a caller limited to some graphs. Among 100,000 vectors of 768 numbers: 40 ms instead of 6.2 s; at most two rounds, so a filter that keeps one row in a hundred costs about 130 ms instead of 93 ms. A query with its own dataset or under row-level rules, with an `OFFSET` and `LIMIT` past a sixteenth of the vectors, or over a property that also has plain strings in the form of a vector, is evaluated as before. See [vector search](docs/vector-search.md#functions). ## 0.1.0-rc.1 - 2026-11-17 diff --git a/crates/tripletrail/src/idtree.rs b/crates/tripletrail/src/idtree.rs index 6ed43ea..4ff8685 100644 --- a/crates/tripletrail/src/idtree.rs +++ b/crates/tripletrail/src/idtree.rs @@ -55,7 +55,19 @@ pub(crate) fn closure_path( } } -fn has_path(p: &GraphPattern) -> bool { +/// The operands of a tree of joins, in order. +fn joined_parts<'p>(p: &'p GraphPattern, out: &mut Vec<&'p GraphPattern>) { + match p { + GraphPattern::Join { left, right } => { + joined_parts(left, out); + joined_parts(right, out); + } + other => out.push(other), + } +} + +/// Whether `p` has a path, outside an `EXISTS`. +pub(crate) fn has_path(p: &GraphPattern) -> bool { match p { GraphPattern::Path { .. } => true, GraphPattern::Join { left, right } @@ -67,6 +79,10 @@ fn has_path(p: &GraphPattern) -> bool { } } +/// A start searched by looking its edges up costs about as much as reading this many of +/// the predicate's edges: fewer starts than the edges over this are searched level by level. +const LEVEL_COST: f64 = 64.0; + /// The id of an unbound value: 0 is the default graph, never a subject, predicate or object. const UNBOUND: TermId = TermId(0); @@ -115,10 +131,17 @@ impl Eval<'_> { } let t = match p { GraphPattern::Bgp { patterns } => self.bgp(patterns)?, - GraphPattern::Join { left, right } => { - let l = self.eval(left)?; - let r = self.right(&l, right)?; - join(&l, &r, self.cancelled)? + GraphPattern::Join { .. } => { + // The parts joined, paths last: they start from the values the others give. + let mut parts = Vec::new(); + joined_parts(p, &mut parts); + parts.sort_by_key(|p| matches!(p, GraphPattern::Path { .. })); + let mut t = self.eval(parts[0])?; + for part in &parts[1..] { + let r = self.right(&t, part)?; + t = join(&t, &r, self.cancelled)?; + } + t } GraphPattern::LeftJoin { left, @@ -156,20 +179,24 @@ impl Eval<'_> { subject, path, object, - } => self.path(subject, path, object)?, + } => self.path(subject, path, object, None)?, _ => return None, }; (t.ids.len() <= max_ids()).then_some(t) } /// `?s

+ ?o`, `?s ^

+ ?o`, and with a constant end `

*` too (#11): the pairs of - /// the transitive closure of the predicate's edges, found by a search on the ids from each - /// start, each end once per start (as SPARQL's paths give them). + /// the transitive closure of the predicate's edges, each end once per start (as SPARQL's + /// paths give them). The starts are a constant end, else the values `seed` (the rows the + /// path is joined with) gives one end, searched from that end, else every node with an + /// edge. A few starts are searched by looking their edges up level by level; many, by + /// reading all the predicate's edges once. fn path( &self, subject: &TermPattern, path: &PropertyPathExpression, object: &TermPattern, + seed: Option<&Table>, ) -> Option

{ let (predicate, reverse, zero) = closure_path(path)?; let end = |t: &TermPattern| match t { @@ -182,37 +209,165 @@ impl Eval<'_> { // the variable and the object the constant. let (mut from, mut to) = (end(subject)?, end(object)?); let mut forward = !reverse; - let swapped = from.is_err() && to.is_ok(); - if swapped { + if from.is_err() && to.is_ok() { std::mem::swap(&mut from, &mut to); forward = !forward; } - let to = match (to, &from) { - (Err(v), Err(u)) if v != *u && !zero => v, + let mut to = match (to, &from) { + (Err(v), Err(u)) if v != *u => v, (Err(v), Ok(_)) => v, - // The same variable at both ends, both ends constant, or a zero-length path - // between two variables (every term of the graph): the evaluator's. + // The same variable at both ends, or both ends constant: the evaluator's. _ => return None, }; - // The edges, as (from, to) in the direction followed, sorted to be found by node. + // A seed that binds an end in every row gives the starts, from that end. + let bound = |v: &Variable| { + let t = seed?; + let c = t.col(v).filter(|&c| !t.maybe[c])?; + let mut values: Vec = (0..t.len()).map(|r| t.row(r)[c]).collect(); + values.sort_unstable(); + values.dedup(); + Some(values) + }; + let starts: Option> = match &mut from { + Ok(id) => Some(vec![*id]), + Err(u) => match bound(u) { + Some(values) => Some(values), + None => match bound(&to) { + Some(values) => { + std::mem::swap(u, &mut to); + forward = !forward; + Some(values) + } + None => None, + }, + }, + }; + if zero && starts.is_none() { + // A zero-length path between two free variables: every term of the graph. + return None; + } + let ids = match starts { + Some(starts) if (starts.len() as f64) * LEVEL_COST < self.edge_estimate(predicate) => { + self.closure_by_levels(predicate, forward, zero, &starts, from.is_err())? + } + starts => self.closure_of_all(predicate, forward, zero, starts, from.is_err())?, + }; + let vars = match from { + Ok(_) => vec![to], + Err(v) => vec![v, to], + }; + Some(Table { + ones: ids.len() / vars.len(), + maybe: vec![false; vars.len()], + vars, + ids, + }) + } + + fn edge_estimate(&self, predicate: &spargebra::term::NamedNode) -> f64 { + match self.snapshot.term_id(predicate.as_ref().into()) { + Some(p) => self.snapshot.estimate_ids(None, Some(p), None, self.graph) as f64, + None => 0.0, + } + } + + /// The pattern of an edge followed from `__path_a` to `__path_b`. + fn edge( + predicate: &spargebra::term::NamedNode, + forward: bool, + ) -> (spargebra::term::TriplePattern, Variable, Variable) { let (a, b) = ( Variable::new_unchecked("__path_a"), Variable::new_unchecked("__path_b"), ); - let edges = self.bgp(&[spargebra::term::TriplePattern { - subject: a.clone().into(), + let (s, o) = if forward { (&a, &b) } else { (&b, &a) }; + let t = spargebra::term::TriplePattern { + subject: s.clone().into(), predicate: NamedNodePattern::NamedNode(predicate.clone()), - object: b.clone().into(), - }])?; + object: o.clone().into(), + }; + (t, a, b) + } + + /// The closure from a few starts: the edges of each level's nodes looked up by their + /// values. Rows are (start, end), or the ends alone without `pairs`. + fn closure_by_levels( + &self, + predicate: &spargebra::term::NamedNode, + forward: bool, + zero: bool, + starts: &[TermId], + pairs: bool, + ) -> Option> { + let (pattern, a, b) = Self::edge(predicate, forward); + let limit = max_ids(); + let mut ids = Vec::new(); + let mut seen: std::collections::HashSet<(TermId, TermId)> = Default::default(); + let emit = |start: TermId, end: TermId, ids: &mut Vec| { + if pairs { + ids.push(start); + } + ids.push(end); + }; + let mut frontier: Vec<(TermId, TermId)> = starts.iter().map(|&s| (s, s)).collect(); + if zero { + for &s in starts { + seen.insert((s, s)); + emit(s, s, &mut ids); + } + } + while !frontier.is_empty() { + // The pairs seen take two ids each, as the rows out take theirs. + if (self.cancelled)() || ids.len() + 2 * seen.len() > limit { + return None; + } + let mut nodes: Vec = frontier.iter().map(|&(_, n)| n).collect(); + nodes.sort_unstable(); + nodes.dedup(); + let level = Table { + vars: vec![a.clone()], + ones: nodes.len(), + ids: nodes, + maybe: vec![false], + }; + let edges = self.bgp_from(std::slice::from_ref(&pattern), Some(&level))?; + let (ca, cb) = (edges.col(&a)?, edges.col(&b)?); + let mut next_of: HashMap> = HashMap::new(); + for r in 0..edges.len() { + let row = edges.row(r); + next_of.entry(row[ca]).or_default().push(row[cb]); + } + let mut next = Vec::new(); + for (start, n) in frontier { + for &m in next_of.get(&n).into_iter().flatten() { + if seen.insert((start, m)) { + emit(start, m, &mut ids); + next.push((start, m)); + } + } + } + frontier = next; + } + Some(ids) + } + + /// The closure from `starts`, or from every node with an edge, over all the predicate's + /// edges read once. + fn closure_of_all( + &self, + predicate: &spargebra::term::NamedNode, + forward: bool, + zero: bool, + starts: Option>, + pairs_out: bool, + ) -> Option> { + let (pattern, a, b) = Self::edge(predicate, forward); + let edges = self.bgp(std::slice::from_ref(&pattern))?; let (ca, cb) = (edges.col(&a)?, edges.col(&b)?); let mut pairs: Vec<(TermId, TermId)> = (0..edges.len()) .map(|r| { let row = edges.row(r); - if forward { - (row[ca], row[cb]) - } else { - (row[cb], row[ca]) - } + (row[ca], row[cb]) }) .collect(); drop(edges); @@ -225,14 +380,11 @@ impl Eval<'_> { .take_while(move |&&(x, _)| x == n) .map(|&(_, y)| y) }; - let starts: Vec = match &from { - Ok(id) => vec![*id], - Err(_) => { - let mut s: Vec = pairs.iter().map(|&(x, _)| x).collect(); - s.dedup(); - s - } - }; + let starts: Vec = starts.unwrap_or_else(|| { + let mut s: Vec = pairs.iter().map(|&(x, _)| x).collect(); + s.dedup(); + s + }); let limit = max_ids(); let mut ids: Vec = Vec::new(); let mut seen: HashMap = HashMap::new(); @@ -241,9 +393,11 @@ impl Eval<'_> { if i % 4096 == 0 && (self.cancelled)() { return None; } - let emit = |end: TermId, ids: &mut Vec| match &from { - Ok(_) => ids.push(end), - Err(_) => ids.extend([start, end]), + let emit = |end: TermId, ids: &mut Vec| { + if pairs_out { + ids.push(start); + } + ids.push(end); }; if zero { seen.insert(start, i); @@ -263,16 +417,7 @@ impl Eval<'_> { } } } - let vars = match &from { - Ok(_) => vec![to], - Err(v) => vec![v.clone(), to], - }; - Some(Table { - ones: ids.len() / vars.len(), - maybe: vec![false; vars.len()], - vars, - ids, - }) + Some(ids) } /// The solutions of `p`, the right side of an operator over `t`'s rows: those of a basic @@ -281,6 +426,14 @@ impl Eval<'_> { fn right(&self, t: &Table, p: &GraphPattern) -> Option
{ match p { GraphPattern::Bgp { patterns } => self.bgp_from(patterns, Some(t)), + GraphPattern::Path { + subject, + path, + object, + } => { + let r = self.path(subject, path, object, Some(t))?; + (r.ids.len() <= max_ids()).then_some(r) + } _ => self.eval(p), } } @@ -1894,6 +2047,11 @@ impl Eval<'_> { right_vars: &[Variable], how: Keyed, ) -> Option { + // A path is searched from the values the other side gives (`eval`), which may be few: + // counting each side on its own would search it from every node. + if has_path(left) || has_path(right) { + return None; + } let shared: Vec = in_scope(left) .into_iter() .filter(|v| right_vars.contains(v)) diff --git a/crates/tripletrail/src/rows.rs b/crates/tripletrail/src/rows.rs index 70cfa76..c617c0f 100644 --- a/crates/tripletrail/src/rows.rs +++ b/crates/tripletrail/src/rows.rs @@ -60,7 +60,7 @@ pub(crate) fn rewrite( base_iri, } => Query::Select { dataset: None, - pattern: pass.walk(pattern, false), + pattern: pass.walk(pattern, false, true), base_iri, }, Query::Construct { @@ -71,7 +71,7 @@ pub(crate) fn rewrite( } => Query::Construct { template, dataset: None, - pattern: pass.walk(pattern, false), + pattern: pass.walk(pattern, false, true), base_iri, }, other => other, @@ -86,10 +86,13 @@ struct Pass<'a> { impl Pass<'_> { /// `p` with its largest covered parts replaced; `limited`: a `LIMIT` without an order is - /// above, so the evaluator may stop before reading all rows. - fn walk(&mut self, p: GraphPattern, limited: bool) -> GraphPattern { + /// above, so the evaluator may stop before reading all rows; `alone`: no other part is + /// joined with `p`'s rows. A part with a path is taken only alone: joined with others, + /// the evaluator searches the path from the values they give it, which may be few. + fn walk(&mut self, p: GraphPattern, limited: bool, alone: bool) -> GraphPattern { use GraphPattern as G; if !limited + && (alone || !crate::idtree::has_path(&p)) && covered(&p) && worth(&p) && let Some(t) = self.eval.eval(&p) @@ -105,60 +108,60 @@ impl Pass<'_> { self.done += 1; return replaced; } - let mut w = |p: Box, limited: bool| Box::new(self.walk(*p, limited)); + let mut w = |p: Box, limited: bool, alone: bool| Box::new(self.walk(*p, limited, alone)); match p { G::Join { left, right } => G::Join { - left: w(left, limited), - right: w(right, limited), + left: w(left, limited, false), + right: w(right, limited, false), }, G::LeftJoin { left, right, expression, } => G::LeftJoin { - left: w(left, limited), - right: w(right, limited), + left: w(left, limited, alone), + right: w(right, limited, false), expression, }, G::Minus { left, right } => G::Minus { - left: w(left, limited), - right: w(right, limited), + left: w(left, limited, alone), + right: w(right, limited, false), }, G::Union { left, right } => G::Union { - left: w(left, limited), - right: w(right, limited), + left: w(left, limited, alone), + right: w(right, limited, alone), }, // The right side is evaluated once per left row, with its variables bound. G::Lateral { left, right } => G::Lateral { - left: w(left, limited), + left: w(left, limited, alone), right, }, G::Filter { expr, inner } => G::Filter { expr, - inner: w(inner, limited), + inner: w(inner, limited, alone), }, G::Extend { inner, variable, expression, } => G::Extend { - inner: w(inner, limited), + inner: w(inner, limited, alone), variable, expression, }, G::Project { inner, variables } => G::Project { - inner: w(inner, limited), + inner: w(inner, limited, alone), variables, }, G::Distinct { inner } => G::Distinct { - inner: w(inner, limited), + inner: w(inner, limited, alone), }, G::Reduced { inner } => G::Reduced { - inner: w(inner, limited), + inner: w(inner, limited, alone), }, // These read all the rows of their input. G::OrderBy { inner, expression } => G::OrderBy { - inner: w(inner, false), + inner: w(inner, false, alone), expression, }, G::Group { @@ -166,7 +169,7 @@ impl Pass<'_> { variables, aggregates, } => G::Group { - inner: w(inner, false), + inner: w(inner, false, alone), variables, aggregates, }, @@ -175,7 +178,7 @@ impl Pass<'_> { start, length, } => G::Slice { - inner: w(inner, limited || length.is_some()), + inner: w(inner, limited || length.is_some(), alone), start, length, }, diff --git a/crates/tripletrail/tests/id_paths.rs b/crates/tripletrail/tests/id_paths.rs index 37bf11f..1c18df9 100644 --- a/crates/tripletrail/tests/id_paths.rs +++ b/crates/tripletrail/tests/id_paths.rs @@ -15,6 +15,7 @@ fn data() -> String { ttl.push_str("ex:loop2 ex:up ex:loop2 .\n"); ttl.push_str("ex:n5 ex:up \"a literal\" .\n"); ttl.push_str("GRAPH ex:g { ex:n7 ex:up ex:elsewhere . }\n"); + ttl.push_str("ex:n2999 ex:id \"rare\" . ex:n1 ex:id \"top\" . ex:n3 ex:id \"top\" .\n"); ttl } @@ -54,6 +55,18 @@ fn closures_on_ids_give_the_evaluators_rows() { "?s ex:up+ \"a literal\" FILTER(isIRI(?s))", "?s ex:up+ ?s FILTER(isIRI(?s))", "?s ex:nothing+ ?o FILTER(isIRI(?s))", + // Joined with other parts: searched from the values they give, either way. + "?s ex:id \"rare\" . ?s ex:up+ ?o", + "?s ex:id \"rare\" . ?s ex:up* ?o", + "VALUES ?s { ex:n2999 ex:n77 } ?s ex:up+ ?o", + "?s ex:id \"rare\" OPTIONAL { ?s ex:up+ ?o }", + "?s ex:id \"rare\" . ?s ex:up+ ?m . ?m ex:up+ ?o", + "{ ?s ex:id \"rare\" } UNION { ?s ex:id \"top\" } ?s ex:up+ ?o", + "?o ex:id \"top\" . ?s ex:up+ ?o", + "?o ex:id \"top\" . ?s ^ex:up* ?o", + "?s a ex:C1 . ?s ex:up+ ?o", + "?o a ex:C1 . ?s ^ex:up+ ?o", + "?s ex:id ?x . ?s ex:up+ ?o MINUS { ?o ex:id \"top\" }", ] { let q = format!("PREFIX ex: SELECT * WHERE {{ {body} }}"); let off = format!("#pragma tt.id-rewrites off\n{q}"); @@ -73,3 +86,41 @@ fn closures_on_ids_give_the_evaluators_rows() { } } } + +/// A path joined with a part that gives few values searches from those values, not over the +/// whole closure: much faster than the closure on a larger tree. +#[test] +fn a_joined_path_searches_from_the_few_values_given() { + let mut ttl = String::from("@prefix ex: .\n"); + for i in 1..60_000 { + ttl.push_str(&format!("ex:n{i} ex:up ex:n{} .\n", (i - 1) / 4)); + } + ttl.push_str("ex:n59999 ex:id \"rare\" . ex:n5000 ex:id \"inner\" .\n"); + let e = Engine::new_tripletrail(); + e.load(RdfFormat::Turtle, ttl.as_bytes()).unwrap(); + let time = |body: &str| { + let q = + format!("PREFIX ex: SELECT (COUNT(*) AS ?n) WHERE {{ {body} }}"); + let start = std::time::Instant::now(); + let n = rows(&e, &q); + (start.elapsed(), n) + }; + let (all, _) = time("?s ex:up+ ?o"); + for body in [ + "?s ex:id \"rare\" . ?s ex:up+ ?o", + "?s ex:id \"rare\" OPTIONAL { ?s ex:up+ ?o }", + "VALUES ?s { ex:n59999 } ?s ex:up+ ?o", + "?o ex:id \"inner\" . ?s ex:up+ ?o", + "?o ex:id \"inner\" . ?o ^ex:up+ ?s", + ] { + let (few, n) = time(body); + assert!( + few * 10 < all, + "{body}: {few:?} against {all:?} for the whole closure" + ); + let off = format!( + "#pragma tt.id-rewrites off\nPREFIX ex: SELECT (COUNT(*) AS ?n) WHERE {{ {body} }}" + ); + assert_eq!(n, rows(&e, &off), "{body}"); + } +} diff --git a/docs/store.md b/docs/store.md index 2ce0d3c..0d3e513 100644 --- a/docs/store.md +++ b/docs/store.md @@ -104,11 +104,16 @@ there: `Engine::new_oxigraph()` or `TRIPLETRAIL_STORE=oxigraph`, `tripletrail se `exists-join-3-star-2` 84 → 17 ms). Transitive paths are in the tables too (#11): `?s

+ ?o`, `?s ^

+ ?o`, and `

*` with one end constant, are the pairs of a search over the predicate's edges on ids, each - end once per start, as SPARQL's paths give them. A part with such a path is always taken, - since the evaluator finds a closure one start at a time. On a tree of 200,000 nodes - (1.48 million pairs): `GROUP BY ?o` with `COUNT` over `?s

+ ?o` 1.27 → 0.49 s, and the - path joined with a class and counted 1.89 → 0.47 s. A path with the same variable at both - ends, `*` between two variables, or other path expressions are left to the evaluator. + end once per start, as SPARQL's paths give them. The search starts from a constant end; + else, joined with other patterns, from the values they give (the path comes after them in + the join), backwards when they bind the object; else from every node with an edge. A few + starts are searched by looking up their edges level by level, many by reading all the + predicate's edges once. A part with a path joined with patterns outside it (a `VALUES` + block, say) is left to the evaluator, which also starts from those values. On a tree of + 200,000 nodes (1.48 million pairs): `GROUP BY ?o` with `COUNT` over `?s

+ ?o` + 1.05 → 0.19 s, the path joined with a class and counted 1.61 → 0.05 s, and + `?s ex:id "rare" . ?s

+ ?o` stays at about 1 ms. A path with the same variable at both + ends, `*` between two free variables, or other path expressions are left to the evaluator. `ORDER BY ?a DESC(?b) ... LIMIT n` (with `OFFSET`, `DISTINCT` or `REDUCED`) over a pattern the tables cover is ordered on ids too: IRIs and simple strings by their string ranks (#171), integers and booleans by value, unbound values first, and only the first From 032fa8bd6f5cccf1bcfc3d769472c45156a8320e Mon Sep 17 00:00:00 2001 From: Mathias Vanden Auweele Date: Wed, 18 Nov 2026 23:15:51 +0000 Subject: [PATCH 3/4] fix(query): give up on a path level before it outgrows the query's memory The level search now counts the pairs a level can add (with the ids held in seen, next and the edges read) before making any, and leaves the path to the evaluator past the limit; cancellation is checked inside the level. The search over all edges checks the limit as each target is added, so a node with many edges cannot pass it either. --- crates/tripletrail/src/idtree.rs | 31 ++++++++++++++++++++++------ crates/tripletrail/tests/id_paths.rs | 26 +++++++++++++++++++++++ 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/crates/tripletrail/src/idtree.rs b/crates/tripletrail/src/idtree.rs index 4ff8685..f1a835a 100644 --- a/crates/tripletrail/src/idtree.rs +++ b/crates/tripletrail/src/idtree.rs @@ -316,9 +316,10 @@ impl Eval<'_> { emit(s, s, &mut ids); } } + // Each new pair takes its ids in the rows out, two in `seen` and two in `next`. + let per_pair = usize::from(pairs) + 5; while !frontier.is_empty() { - // The pairs seen take two ids each, as the rows out take theirs. - if (self.cancelled)() || ids.len() + 2 * seen.len() > limit { + if (self.cancelled)() { return None; } let mut nodes: Vec = frontier.iter().map(|&(_, n)| n).collect(); @@ -337,8 +338,22 @@ impl Eval<'_> { let row = edges.row(r); next_of.entry(row[ca]).or_default().push(row[cb]); } + // The most pairs this level can add, before any is made: past the limit, the + // evaluator's (it streams them). The edges read and `next_of` hold four ids each. + let most: usize = frontier + .iter() + .map(|(_, n)| next_of.get(n).map_or(0, Vec::len)) + .sum(); + let held = ids.len() + 2 * seen.len() + 4 * edges.len(); + if most.saturating_mul(per_pair).saturating_add(held) > limit { + return None; + } + drop(edges); let mut next = Vec::new(); - for (start, n) in frontier { + for (i, (start, n)) in frontier.into_iter().enumerate() { + if i % 4096 == 4095 && (self.cancelled)() { + return None; + } for &m in next_of.get(&n).into_iter().flatten() { if seen.insert((start, m)) { emit(start, m, &mut ids); @@ -410,11 +425,15 @@ impl Eval<'_> { if seen.insert(m, i) != Some(i) { emit(m, &mut ids); stack.push(m); + // A node with many edges: checked as its targets are added. + if ids.len() > limit { + return None; + } + if ids.len() % 65536 == 0 && (self.cancelled)() { + return None; + } } } - if ids.len() > limit { - return None; - } } } Some(ids) diff --git a/crates/tripletrail/tests/id_paths.rs b/crates/tripletrail/tests/id_paths.rs index 1c18df9..70fef4a 100644 --- a/crates/tripletrail/tests/id_paths.rs +++ b/crates/tripletrail/tests/id_paths.rs @@ -124,3 +124,29 @@ fn a_joined_path_searches_from_the_few_values_given() { assert_eq!(n, rows(&e, &off), "{body}"); } } + +/// A level that would make more pairs than the query's memory allows is not built: a hub +/// that every start reaches, with many leaves, is left to the evaluator. +#[test] +fn a_level_past_the_memory_limit_is_left_to_the_evaluator() { + let mut ttl = String::from("@prefix ex: .\n"); + for i in 0..50 { + ttl.push_str(&format!("ex:s{i} a ex:Src ; ex:e ex:hub .\n")); + } + for i in 0..4000 { + ttl.push_str(&format!("ex:hub ex:e ex:leaf{i} .\n")); + } + // Enough edges that 50 starts are searched level by level. + for i in 0..10_000 { + ttl.push_str(&format!("ex:x{i} ex:e ex:y{i} .\n")); + } + let e = Engine::new_tripletrail(); + e.load(RdfFormat::Turtle, ttl.as_bytes()).unwrap(); + let q = "PREFIX ex: SELECT (COUNT(*) AS ?n) WHERE { ?s a ex:Src . ?s ex:e+ ?o }"; + let n = rows(&e, q); + assert_eq!(n, rows(&e, &format!("#pragma tt.id-rewrites off\n{q}"))); + assert!(n[0].contains("200050"), "{n:?}"); + // 200,050 pairs take about a million ids in the search; under a limit of 1 MiB + // (131,072 ids) it gives up before the level and the evaluator counts them streaming. + assert_eq!(rows(&e.with_query_memory(1 << 20), q), n); +} From 065bbf4fff21c5d412f1b6e24dfa6bdf853324f7 Mon Sep 17 00:00:00 2001 From: Mathias Vanden Auweele Date: Wed, 18 Nov 2026 23:16:18 +0000 Subject: [PATCH 4/4] style(query): use is_multiple_of --- crates/tripletrail/src/idtree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tripletrail/src/idtree.rs b/crates/tripletrail/src/idtree.rs index f1a835a..91b1cb4 100644 --- a/crates/tripletrail/src/idtree.rs +++ b/crates/tripletrail/src/idtree.rs @@ -429,7 +429,7 @@ impl Eval<'_> { if ids.len() > limit { return None; } - if ids.len() % 65536 == 0 && (self.cancelled)() { + if ids.len().is_multiple_of(65536) && (self.cancelled)() { return None; } }