diff --git a/core/crates/core/src/sql_syntax/script/script_plan.rs b/core/crates/core/src/sql_syntax/script/script_plan.rs index dac24b57f6..c46ee00466 100644 --- a/core/crates/core/src/sql_syntax/script/script_plan.rs +++ b/core/crates/core/src/sql_syntax/script/script_plan.rs @@ -100,6 +100,24 @@ impl ScriptPlan { .collect() } + // Where the first statement's terminator starts, or the text's end when nothing ends it, lexed without keeping the + // tokens: SQLite reads only the first statement of a stored CREATE text, whatever follows it. A semicolon inside + // parentheses ends nothing, since SQLite reads it as one of a virtual table's module arguments, which the editor's + // split, like `sqlite3_complete`, does not know. + pub fn first_statement_end(text: &str, grammar: SqlGrammar) -> usize { + match rules_for(grammar) { + Some(rules) => Builder::new( + text, + rules, + LexicalSettings::default_for(grammar), + ServerReading::Client, + ";", + ) + .first_statement_end(), + None => text.len(), + } + } + pub fn batches(&self) -> &[ScriptBatch] { &self.batches } @@ -217,6 +235,35 @@ impl<'a> Builder<'a> { (plan, settled) } + fn first_statement_end(mut self) -> usize { + let mut depth = 0usize; + while self.lexer.position() < self.text.len() { + let start = self.lexer.position(); + let ended_at = if self.reads_directives && self.lexer.at_line_start() && self.accept_directive() { + start + } else { + let delimiter = (self.rules.boundary == Boundary::DelimiterDirectives).then_some(self.delimiter); + let Some(token) = self.lexer.next_token(delimiter) else { + break; + }; + match token.kind { + ScriptTokenKind::OpenParen => depth += 1, + ScriptTokenKind::CloseParen => depth = depth.saturating_sub(1), + ScriptTokenKind::Semicolon if depth > 0 => continue, + _ => {} + } + let token_start = token.range.start; + self.accept(token); + token_start + }; + self.tokens.clear(); + if !self.statements.is_empty() { + return ended_at; + } + } + self.text.len() + } + fn accept_directive(&mut self) -> bool { let line_start = self.lexer.position(); match self.rules.boundary { diff --git a/core/crates/core/tests/script_corpus.rs b/core/crates/core/tests/script_corpus.rs index c2d575258d..da06011b8e 100644 --- a/core/crates/core/tests/script_corpus.rs +++ b/core/crates/core/tests/script_corpus.rs @@ -30,6 +30,37 @@ fn batch_texts<'a>(plan: &ScriptPlan, text: &'a str) -> Vec<&'a str> { .collect() } +#[test] +fn the_first_statement_ends_where_its_terminator_starts() { + let cases = [ + ("CREATE TABLE t (a)", "CREATE TABLE t (a)"), + ("CREATE TABLE t (a); DROP TABLE t", "CREATE TABLE t (a)"), + ( + "CREATE TABLE t (a DEFAULT ';', \"b;\" /* ; */, [c;] -- ;\n); SELECT 1", + "CREATE TABLE t (a DEFAULT ';', \"b;\" /* ; */, [c;] -- ;\n)", + ), + ( + "CREATE TRIGGER r AFTER INSERT ON t BEGIN SELECT 1; SELECT 2; END; SELECT 3", + "CREATE TRIGGER r AFTER INSERT ON t BEGIN SELECT 1; SELECT 2; END", + ), + ("CREATE INDEX i ON t (a) /* ; */", "CREATE INDEX i ON t (a) /* ; */"), + ("CREATE VIEW v AS SELECT 1 -- note", "CREATE VIEW v AS SELECT 1 -- note"), + ( + "CREATE VIRTUAL TABLE t USING fts4(a;b, tokenize=simple (x;y)); DROP TABLE t", + "CREATE VIRTUAL TABLE t USING fts4(a;b, tokenize=simple (x;y))", + ), + ( + "CREATE VIRTUAL TABLE t USING fts4(a;b)", + "CREATE VIRTUAL TABLE t USING fts4(a;b)", + ), + ("", ""), + ]; + for (text, first) in cases { + let end = ScriptPlan::first_statement_end(text, SqlGrammar::Sqlite); + assert_eq!(text.get(..end), Some(first), "{text}"); + } +} + #[test] fn dollar_quoted_body_hides_semicolons() { let text = "CREATE FUNCTION f() RETURNS int AS $$ SELECT 1; SELECT 2; $$ LANGUAGE sql;\nSELECT 3;"; diff --git a/core/crates/drivers/sqlite/src/catalog/dependents.rs b/core/crates/drivers/sqlite/src/catalog/dependents.rs index e4abfb940c..ed8fdcdab4 100644 --- a/core/crates/drivers/sqlite/src/catalog/dependents.rs +++ b/core/crates/drivers/sqlite/src/catalog/dependents.rs @@ -1,4 +1,4 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use tablepro_core::ddl::{SchemaObjectKind, SchemaObjectSource}; use tablepro_core::sql_syntax::SqlGrammar; @@ -13,32 +13,40 @@ pub(crate) struct SchemaRow { pub sql: String, } -/// The objects a rebuild of `table` has to recreate, in schema order. -/// -/// Dropping a table drops its own indexes and triggers. A view, or a -/// trigger on another table, only names it in its SQL, and breaks when -/// the table is gone for a moment, so those are found by the name in -/// their token stream. A view that names a kept view is kept too. +// The objects a rebuild of `table` has to recreate, in schema order. Dropping a table drops its own indexes and +// triggers. A view, or a trigger on another table, only names it in its SQL and breaks while the table is gone, so those +// are found by the name in their tokens, and a view naming a kept view is kept too. Each kept name is looked up once +// among the rows that mention it, so a file's views in any order cost one pass over their words. pub(crate) fn select_dependents(table: &str, rows: Vec) -> Vec { - let mut names: Vec = vec![table.to_owned()]; let mut kept = vec![false; rows.len()]; - let mentions: Vec> = rows.iter().map(|row| words(&row.sql)).collect(); - - loop { - let mut grew = false; - for (index, row) in rows.iter().enumerate() { - if kept[index] || !keeps(row, &mentions[index], table, &names) { + let mut mentioned_by: HashMap> = HashMap::new(); + for (index, row) in rows.iter().enumerate() { + match row.kind { + SchemaObjectKind::Index | SchemaObjectKind::Trigger if row.table.eq_ignore_ascii_case(table) => { + kept[index] = true; + } + SchemaObjectKind::Index => {} + SchemaObjectKind::Trigger | SchemaObjectKind::View => { + for word in words(&row.sql) { + mentioned_by.entry(word).or_default().push(index); + } + } + } + } + let mut pending = vec![table.to_lowercase()]; + let mut named: HashSet = pending.iter().cloned().collect(); + while let Some(name) = pending.pop() { + for &index in mentioned_by.get(&name).map_or(&[][..], Vec::as_slice) { + if kept[index] { continue; } kept[index] = true; - if row.kind == SchemaObjectKind::View { - names.push(row.name.clone()); - grew = true; + let row = &rows[index]; + let folded = row.name.to_lowercase(); + if row.kind == SchemaObjectKind::View && named.insert(folded.clone()) { + pending.push(folded); } } - if !grew { - break; - } } rows.into_iter() @@ -53,17 +61,6 @@ pub(crate) fn select_dependents(table: &str, rows: Vec) -> Vec, table: &str, names: &[String]) -> bool { - let owned_by_table = row.table.eq_ignore_ascii_case(table); - match row.kind { - SchemaObjectKind::Index | SchemaObjectKind::Trigger if owned_by_table => true, - SchemaObjectKind::Index => false, - SchemaObjectKind::Trigger | SchemaObjectKind::View => { - names.iter().any(|name| mentions.contains(&name.to_lowercase())) - } - } -} - /// Every identifier the SQL names, lowercased and unquoted. A name /// inside a string literal or a comment is not a reference. fn words(sql: &str) -> HashSet { @@ -196,6 +193,39 @@ mod tests { assert!(select_dependents("orders", rows).is_empty()); } + #[test] + fn views_stored_before_the_views_they_name_are_kept() { + let mut rows: Vec = (1..2_000) + .rev() + .map(|index| { + row( + SchemaObjectKind::View, + &format!("v{index}"), + &format!("v{index}"), + &format!("CREATE VIEW v{index} AS SELECT * FROM v{}", index - 1), + ) + }) + .collect(); + rows.push(row( + SchemaObjectKind::View, + "v0", + "v0", + "CREATE VIEW v0 AS SELECT * FROM Orders", + )); + rows.push(row( + SchemaObjectKind::View, + "other", + "other", + "CREATE VIEW other AS SELECT 1", + )); + + let selected = select_dependents("orders", rows); + + assert_eq!(selected.len(), 2_000); + assert_eq!(selected.first().map(|object| object.name.as_str()), Some("v1999")); + assert_eq!(selected.last().map(|object| object.name.as_str()), Some("v0")); + } + #[test] fn a_quoted_name_reads_as_the_identifier_it_spells() { assert_eq!(unquote("\"a\"\"b\""), "a\"b"); diff --git a/core/crates/drivers/sqlite/src/catalog/mod.rs b/core/crates/drivers/sqlite/src/catalog/mod.rs index ebde8972e0..7bca991c64 100644 --- a/core/crates/drivers/sqlite/src/catalog/mod.rs +++ b/core/crates/drivers/sqlite/src/catalog/mod.rs @@ -17,6 +17,8 @@ use tablepro_core::column::{ use tablepro_core::sql_syntax::{LiteralToken, MAX_EXPRESSION_LEN, SqlGrammar, parse_expression}; use tablepro_core::value::{SqlDecimal, Value}; +use crate::statement_limit::{byte_length, capped}; + pub(crate) use dependents::{SchemaRow, select_dependents}; pub use runtime_version::{SQLITE_MIN_VERSION, SqliteVersion, check_runtime_version}; pub(crate) use sqlite_column_row::{SqliteColumnRow, TableFacts}; @@ -66,36 +68,47 @@ pub(crate) fn objects_sql(schema_sql: &str) -> String { /// By the table too, which the sidebar names a trigger by. pub(crate) fn trigger_sql(schema_sql: &str) -> String { format!( - "SELECT sql FROM {schema_sql}.sqlite_schema \ - WHERE type = 'trigger' AND name = ?1 AND tbl_name = ?2" + "SELECT {} FROM {schema_sql}.sqlite_schema \ + WHERE type = 'trigger' AND name = ?1 AND tbl_name = ?2", + capped("sql") ) } pub(crate) fn object_sql(schema_sql: &str) -> String { - format!("SELECT type, sql FROM {schema_sql}.sqlite_schema WHERE name = ?1 COLLATE NOCASE") + format!( + "SELECT type, {} FROM {schema_sql}.sqlite_schema WHERE name = ?1 COLLATE NOCASE", + capped("sql") + ) } pub(crate) fn index_sql(schema_sql: &str) -> String { - format!("SELECT sql FROM {schema_sql}.sqlite_schema WHERE type = 'index' AND name = ?1") + format!( + "SELECT {} FROM {schema_sql}.sqlite_schema WHERE type = 'index' AND name = ?1", + capped("sql") + ) } pub(crate) fn dependents_sql(schema_sql: &str) -> String { format!( - "SELECT type, name, tbl_name, sql FROM {schema_sql}.sqlite_schema \ - WHERE type IN ('index', 'trigger', 'view') AND sql IS NOT NULL ORDER BY rowid" + "SELECT type, name, tbl_name, {} FROM {schema_sql}.sqlite_schema \ + WHERE type IN ('index', 'trigger', 'view') AND sql IS NOT NULL ORDER BY rowid", + capped("sql") ) } // `small`: at most 64 KiB, with a key list SQLite cannot compile past ?3 bytes: no more key columns than REFERENCES -// and commas, each with three names no longer than the text and 512 bytes of opcodes. +// and commas, each with three names no longer than the text and 512 bytes of opcodes. Only a small one's text is read: +// counting a longer one would copy it whole, a stored tail with it. fn sized_tables(schema_sql: &str) -> String { + let bytes = byte_length("sql"); format!( - "WITH tables AS (SELECT rowid, name, sql, length(CAST(sql AS BLOB)) AS bytes, \ - (length(sql) - length(replace(lower(sql), 'references', ''))) / 10 \ - + length(sql) - length(replace(sql, ',', '')) AS key_columns \ + "WITH tables AS (SELECT rowid, name, {bytes} AS bytes, CASE WHEN {bytes} <= 65536 THEN sql END AS sql \ FROM {schema_sql}.sqlite_schema WHERE type = 'table' AND sql IS NOT NULL), \ sized AS (SELECT rowid, name, sql, \ - bytes <= 65536 AND (key_columns + 1) * (3 * bytes + 512) <= ?3 AS small FROM tables)" + CASE WHEN bytes <= 65536 THEN \ + ((length(sql) - length(replace(lower(sql), 'references', ''))) / 10 \ + + length(sql) - length(replace(sql, ',', '')) + 1) * (3 * bytes + 512) <= ?3 \ + ELSE 0 END AS small FROM tables)" ) } @@ -122,7 +135,10 @@ pub(crate) fn large_tables_sql(schema_sql: &str) -> String { // By rowid: a lookup by name scans the whole schema, once per child. pub(crate) fn schema_row_sql(schema_sql: &str) -> String { - format!("SELECT name, sql FROM {schema_sql}.sqlite_schema WHERE rowid = ?1") + format!( + "SELECT name, {} FROM {schema_sql}.sqlite_schema WHERE rowid = ?1", + capped("sql") + ) } /// `sqlite_stat1` exists only once ANALYZE has run, so its presence is diff --git a/core/crates/drivers/sqlite/src/catalog/runtime_version.rs b/core/crates/drivers/sqlite/src/catalog/runtime_version.rs index ea260b57ea..c48c4552d0 100644 --- a/core/crates/drivers/sqlite/src/catalog/runtime_version.rs +++ b/core/crates/drivers/sqlite/src/catalog/runtime_version.rs @@ -10,11 +10,10 @@ pub struct SqliteVersion { pub patch: u32, } -/// The oldest library the driver reads a catalogue from. 3.37.0 added -/// `PRAGMA table_list`, which is what says whether a table is STRICT. -/// Every system that ships GTK 4.22 has a newer one, so the floor turns -/// away no supported system. -pub const SQLITE_MIN_VERSION: SqliteVersion = SqliteVersion::new(3, 37, 0); +// The oldest library the driver reads a catalogue from. 3.43.0 added `octet_length`, which sizes a stored text without +// reading it, and 3.37.0 `PRAGMA table_list`, which says whether a table is STRICT. Every system that ships GTK 4.22 +// has a newer one, so the floor turns away no supported system. +pub const SQLITE_MIN_VERSION: SqliteVersion = SqliteVersion::new(3, 43, 0); impl SqliteVersion { pub const fn new(major: u32, minor: u32, patch: u32) -> Self { @@ -46,7 +45,7 @@ pub fn check_runtime_version(version: &SqliteVersion) -> Result<(), DriverError> if *version < SQLITE_MIN_VERSION { return Err(DriverError::Unsupported { feature: UnsupportedFeature::MinimumVersion { - required: "SQLite 3.37.0", + required: "SQLite 3.43.0", }, }); } @@ -58,18 +57,18 @@ mod tests { use super::*; #[test] - fn sqlite_runtime_version_below_3_37_is_unsupported() { - let refused = check_runtime_version(&SqliteVersion::new(3, 36, 0)); + fn sqlite_runtime_version_below_3_43_is_unsupported() { + let refused = check_runtime_version(&SqliteVersion::new(3, 42, 0)); assert_eq!( refused, Err(DriverError::Unsupported { feature: UnsupportedFeature::MinimumVersion { - required: "SQLite 3.37.0" + required: "SQLite 3.43.0" } }) ); - assert_eq!(check_runtime_version(&SqliteVersion::new(3, 37, 0)), Ok(())); + assert_eq!(check_runtime_version(&SqliteVersion::new(3, 43, 0)), Ok(())); assert_eq!(check_runtime_version(&SqliteVersion::new(3, 45, 1)), Ok(())); } diff --git a/core/crates/drivers/sqlite/src/catalog/table_definition.rs b/core/crates/drivers/sqlite/src/catalog/table_definition.rs index b4c5db6973..4b55d6e096 100644 --- a/core/crates/drivers/sqlite/src/catalog/table_definition.rs +++ b/core/crates/drivers/sqlite/src/catalog/table_definition.rs @@ -1,8 +1,9 @@ use std::collections::{HashMap, HashSet}; -use sqlparser::ast::{ColumnOption, Statement}; +use sqlparser::ast::Expr; use sqlparser::dialect::SQLiteDialect; use sqlparser::parser::Parser; +use sqlparser::tokenizer::Token; use tablepro_core::column::SqlExpression; use tablepro_core::ddl::InboundForeignKey; use tablepro_core::meta::TableRef; @@ -110,35 +111,35 @@ pub(crate) fn foreign_key_names( Ok(names) } -/// The expressions of the generated columns, read from the table's own -/// CREATE TABLE. The pragmas say a column is generated but not how. -/// The text is the parser's rendering of the expression: the table's -/// source keeps the original spelling for a rebuild. -pub(crate) fn generated_expressions(create_statement: &str) -> HashMap { - let Ok(statements) = Parser::parse_sql(&SQLiteDialect {}, create_statement) else { - tracing::debug!("the CREATE TABLE of a table with generated columns did not parse"); - return HashMap::new(); - }; - let Some(Statement::CreateTable(table)) = statements.into_iter().next() else { - return HashMap::new(); +// The pragmas say a column is generated but not how, so its expression comes from the CREATE TABLE. Each is parsed +// alone, since a parse holds many times the text it reads and a wide table's other columns need none. The text is the +// parser's rendering: the table's source keeps the spelling for a rebuild. +pub(crate) fn generated_expressions( + create_statement: &str, + budget: &mut CatalogBudget, +) -> Result, DriverError> { + let Some(texts) = crate::ddl::generated_expression_texts(create_statement) else { + tracing::debug!("the CREATE TABLE of a table with generated columns did not read"); + return Ok(HashMap::new()); }; - table - .columns - .into_iter() - .filter_map(|column| { - let expression = column.options.into_iter().find_map(|option| match option.option { - ColumnOption::Generated { - generation_expr: Some(expression), - .. - } => Some(expression), - _ => None, - })?; - Some(( - column.name.value, - SqlExpression::from_catalog_text(expression.to_string()), - )) - }) - .collect() + let mut expressions = HashMap::new(); + for (name, text) in texts { + crate::statement_limit::parse_fits(budget, text)?; + match parsed_expression(text) { + Some(expression) => { + let rendered = budget.keep(expression.to_string())?; + expressions.insert(name, SqlExpression::from_catalog_text(rendered)); + } + None => tracing::debug!("a generated column's expression did not parse"), + } + } + Ok(expressions) +} + +fn parsed_expression(text: &str) -> Option { + let mut parser = Parser::new(&SQLiteDialect {}).try_with_sql(text).ok()?; + let expression = parser.parse_expr().ok()?; + (parser.peek_token().token == Token::EOF).then_some(expression) } #[cfg(test)] @@ -255,11 +256,45 @@ mod tests { #[test] fn generated_expressions_come_from_the_create_statement() { let expressions = generated_expressions( - "CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED, c AS (a + 1))", - ); + "CREATE TABLE t (a INTEGER, b INTEGER GENERATED ALWAYS AS (a * 2) STORED, \"c d\" AS ((a + 1) * (a - 1)))", + &mut CatalogBudget::default(), + ) + .unwrap(); assert_eq!(expressions.get("b").map(SqlExpression::as_sql), Some("a * 2")); - assert_eq!(expressions.get("c").map(SqlExpression::as_sql), Some("a + 1")); + assert_eq!( + expressions.get("c d").map(SqlExpression::as_sql), + Some("(a + 1) * (a - 1)") + ); assert!(!expressions.contains_key("a")); } + + #[test] + fn a_wide_table_parses_only_its_generated_expressions() { + let columns: Vec = (0..2_000) + .map(|index| format!("c{index:04} {}", "x".repeat(200))) + .collect(); + let create = format!("CREATE TABLE t ({}, g AS (c0000 + 1))", columns.join(", ")); + let mut budget = CatalogBudget::new(tablepro_core::ByteLimit::new(1_000_000).unwrap()); + + let expressions = generated_expressions(&create, &mut budget).unwrap(); + + assert_eq!(expressions.get("g").map(SqlExpression::as_sql), Some("c0000 + 1")); + } + + #[test] + fn a_generated_expression_too_long_to_parse_stops_at_the_budget() { + let create = format!( + "CREATE TABLE t (a INTEGER, g AS (a IN ({})))", + vec!["0"; 2_000].join(",") + ); + let mut budget = CatalogBudget::new(tablepro_core::ByteLimit::new(1_000_000).unwrap()); + + let expressions = generated_expressions(&create, &mut budget); + + assert_eq!( + expressions, + Err(DriverError::CatalogTooLarge { limit_bytes: 1_000_000 }) + ); + } } diff --git a/core/crates/drivers/sqlite/src/connection.rs b/core/crates/drivers/sqlite/src/connection.rs index e54f07f244..2ae404648a 100644 --- a/core/crates/drivers/sqlite/src/connection.rs +++ b/core/crates/drivers/sqlite/src/connection.rs @@ -89,6 +89,9 @@ impl SqliteConnection { .acquire_timeout(liveness.pool_wait) .after_connect(|connection, _| { Box::pin(async move { + crate::statement_limit::install(connection) + .await + .map_err(|error| sqlx::Error::Protocol(error.to_string()))?; crate::attach_guard::install(connection) .await .map_err(|error| sqlx::Error::Protocol(error.to_string())) @@ -171,12 +174,28 @@ impl SqliteConnection { { let options = self.statement_bounded(options); let pooled = self.acquire(&options).await?; - self.run_on(options, pooled, work).await + self.run_on(options, pooled, ReadScope::Statements, work).await + } + + async fn read_catalog(&self, options: CallOptions, work: W) -> Result + where + T: Send + 'static, + W: for<'c> FnOnce(&'c mut SqlxConnection) -> BoxFuture<'c, Result> + Send + 'static, + { + let options = self.statement_bounded(options); + let pooled = self.acquire(&options).await?; + self.run_on(options, pooled, ReadScope::Snapshot, work).await } /// Run one call's work on a connection the call holds, under the /// call's own stop. - async fn run_on(&self, options: CallOptions, pooled: SqlitePooled, work: W) -> Result + async fn run_on( + &self, + options: CallOptions, + pooled: SqlitePooled, + scope: ReadScope, + work: W, + ) -> Result where T: Send + 'static, W: for<'c> FnOnce(&'c mut SqlxConnection) -> BoxFuture<'c, Result> + Send + 'static, @@ -186,9 +205,10 @@ impl SqliteConnection { let lock_timeout = self.settings.lock_timeout; self.calls .run_read(options, pooled, stop, move |mut pooled, _request| async move { - let result = match flag.install(pooled.connection(), lock_timeout).await { - Ok(()) => work(pooled.connection()).await, - Err(error) => Err(error), + let result = match (flag.install(pooled.connection(), lock_timeout).await, scope) { + (Ok(()), ReadScope::Statements) => work(pooled.connection()).await, + (Ok(()), ReadScope::Snapshot) => in_snapshot(&mut pooled, &flag, work).await, + (Err(error), _) => Err(error), }; (pooled, result) }) @@ -231,6 +251,7 @@ impl SqliteConnection { }), opened = SqlxConnection::connect_with(&read_only) => { let mut connection = opened.map_err(|error| map_open_error(error, &self.path))?; + crate::statement_limit::install(&mut connection).await?; crate::read_guard::install(&mut connection).await?; Ok(connection) } @@ -238,6 +259,38 @@ impl SqliteConnection { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ReadScope { + Statements, + Snapshot, +} + +// One read transaction: a schema change by another connection between the statements of one catalog read cannot make +// it miss a table. +async fn in_snapshot(pooled: &mut SqlitePooled, flag: &InterruptFlag, work: W) -> Result +where + W: for<'c> FnOnce(&'c mut SqlxConnection) -> BoxFuture<'c, Result>, +{ + pooled + .connection() + .execute("SAVEPOINT tablepro_catalog") + .await + .map_err(map_error)?; + let read = work(pooled.connection()).await; + // A stop that ended the read must not stop its release, which would leave the read's lock held, and has to stand + // again after it: the stop is what tells a lock wait it cut short from one that gave up. + let stopped = flag.is_raised(); + flag.lower(); + if let Err(error) = pooled.connection().execute("RELEASE tablepro_catalog").await { + tracing::warn!(%error, "a catalog read could not end its transaction; closing the connection"); + pooled.discard_on_release(); + } + if stopped { + flag.raise(); + } + read +} + /// User SQL on a read connection, run only once SQLite reads it as one /// statement. async fn read_user_sql( @@ -303,7 +356,7 @@ impl Connection for SqliteConnection { let probed = match self.pool.try_acquire() { Some(connection) => { let pooled = SqlitePooled::new(connection); - self.run_on(options, pooled, |connection| { + self.run_on(options, pooled, ReadScope::Statements, |connection| { Box::pin(async move { connection.execute("SELECT 1").await.map(|_| ()).map_err(map_error) }) }) .await @@ -319,13 +372,13 @@ impl Connection for SqliteConnection { } async fn list_schemas(&self, options: CallOptions) -> Result, DriverError> { - self.read(options, |connection| Box::pin(metadata::list_schemas(connection))) + self.read_catalog(options, |connection| Box::pin(metadata::list_schemas(connection))) .await } async fn list_objects(&self, scope: &ObjectScope, options: CallOptions) -> Result, DriverError> { let scope = scope.clone(); - self.read(options, move |connection| { + self.read_catalog(options, move |connection| { Box::pin(async move { metadata::list_objects(connection, &scope).await }) }) .await @@ -337,7 +390,7 @@ impl Connection for SqliteConnection { options: CallOptions, ) -> Result, DriverError> { let object = object.clone(); - self.read(options, move |connection| { + self.read_catalog(options, move |connection| { Box::pin(async move { metadata::fetch_object_definition(connection, &object).await }) }) .await @@ -345,7 +398,7 @@ impl Connection for SqliteConnection { async fn read_dump_catalog(&self, scope: &DumpScope, options: CallOptions) -> Result { let scope = scope.clone(); - self.read(options, move |connection| { + self.read_catalog(options, move |connection| { Box::pin(async move { crate::dump::read_dump_catalog(connection, &scope).await }) }) .await @@ -353,7 +406,7 @@ impl Connection for SqliteConnection { async fn fetch_columns(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { let table = table.clone(); - self.read(options, move |connection| { + self.read_catalog(options, move |connection| { Box::pin(async move { metadata::fetch_columns(connection, &table).await }) }) .await @@ -361,7 +414,7 @@ impl Connection for SqliteConnection { async fn fetch_indexes(&self, table: &TableRef, options: CallOptions) -> Result, DriverError> { let table = table.clone(); - self.read(options, move |connection| { + self.read_catalog(options, move |connection| { Box::pin(async move { metadata::fetch_indexes(connection, &table).await }) }) .await @@ -373,7 +426,7 @@ impl Connection for SqliteConnection { options: CallOptions, ) -> Result, DriverError> { let table = table.clone(); - self.read(options, move |connection| { + self.read_catalog(options, move |connection| { Box::pin(async move { metadata::fetch_foreign_keys(connection, &table).await }) }) .await @@ -385,7 +438,7 @@ impl Connection for SqliteConnection { options: CallOptions, ) -> Result, DriverError> { let table = table.clone(); - self.read(options, move |connection| { + self.read_catalog(options, move |connection| { Box::pin(async move { metadata::fetch_outbound_foreign_keys(connection, &table).await }) }) .await @@ -397,7 +450,7 @@ impl Connection for SqliteConnection { options: CallOptions, ) -> Result, DriverError> { let table = table.clone(); - self.read(options, move |connection| { + self.read_catalog(options, move |connection| { Box::pin(async move { metadata::fetch_inbound_foreign_keys(connection, &table).await }) }) .await @@ -409,7 +462,7 @@ impl Connection for SqliteConnection { options: CallOptions, ) -> Result { let table = table.clone(); - self.read(options, move |connection| { + self.read_catalog(options, move |connection| { Box::pin(async move { metadata::fetch_table_definition(connection, &table).await }) }) .await @@ -417,7 +470,7 @@ impl Connection for SqliteConnection { async fn row_identity(&self, table: &TableRef, options: CallOptions) -> Result { let table = table.clone(); - self.read(options, move |connection| { + self.read_catalog(options, move |connection| { Box::pin(async move { metadata::row_identity(connection, &table).await }) }) .await @@ -425,7 +478,7 @@ impl Connection for SqliteConnection { async fn table_estimate(&self, table: &TableRef, options: CallOptions) -> Result { let table = table.clone(); - self.read(options, move |connection| { + self.read_catalog(options, move |connection| { Box::pin(async move { metadata::table_estimate(connection, &table).await }) }) .await @@ -560,3 +613,43 @@ impl Connection for SqliteConnection { Ok(()) } } + +#[cfg(test)] +mod tests { + use sqlx::sqlite::SqliteConnectOptions; + + use super::*; + use crate::autocommit::is_autocommit; + + async fn in_transaction(connection: &mut SqlxConnection) -> Result { + let mut handle = connection.lock_handle().await.map_err(map_error)?; + Ok(!is_autocommit(&mut handle)) + } + + #[tokio::test] + async fn a_catalog_read_runs_in_one_transaction_and_ends_it() { + let directory = tempfile::tempdir().unwrap(); + let options = SqliteConnectOptions::new() + .filename(directory.path().join("snapshot.sqlite3")) + .create_if_missing(true); + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect_with(options) + .await + .unwrap(); + let mut pooled = SqlitePooled::new(pool.acquire().await.unwrap()); + let flag = InterruptFlag::new(); + + let inside = in_snapshot(&mut pooled, &flag, |connection| Box::pin(in_transaction(connection))).await; + let failed: Result<(), DriverError> = + in_snapshot(&mut pooled, &flag, |_| Box::pin(async { Err(DriverError::Cancelled) })).await; + + assert_eq!(inside, Ok(true), "every statement of the read sees one snapshot"); + assert_eq!(failed, Err(DriverError::Cancelled)); + assert_eq!( + in_transaction(pooled.connection()).await, + Ok(false), + "a read that failed still ends its transaction" + ); + } +} diff --git a/core/crates/drivers/sqlite/src/ddl/create_table_source.rs b/core/crates/drivers/sqlite/src/ddl/create_table_source.rs index 5cd00ffd8f..a607d5da64 100644 --- a/core/crates/drivers/sqlite/src/ddl/create_table_source.rs +++ b/core/crates/drivers/sqlite/src/ddl/create_table_source.rs @@ -289,18 +289,41 @@ impl<'a> CreateTableSource<'a> { users } - /// The columns the expression in the first parentheses of `range` - /// names. fn named_in(&self, range: Range) -> HashSet { + first_group(self.tokens_in(range)) + .map_or_else(HashSet::new, |group| named_columns(self.sql, group, Terms::Expression)) + } + + // The text inside the first parentheses of `range`, which for a generated column is its expression. + fn first_group_text(&self, range: Range) -> Option<&'a str> { + let group = first_group(self.tokens_in(range))?; + Some(self.text(group.first()?.range.start..group.last()?.range.end)) + } + + fn tokens_in(&self, range: Range) -> &[Token] { let start = self.tokens.partition_point(|token| token.range.start < range.start); let end = self.tokens.partition_point(|token| token.range.end <= range.end); - self.tokens - .get(start..end) - .and_then(first_group) - .map_or_else(HashSet::new, |group| named_columns(self.sql, group, Terms::Expression)) + self.tokens.get(start..end).unwrap_or_default() } } +// Each generated column's name and expression as written, or `None` when the text does not read. +pub(crate) fn generated_expression_texts(sql: &str) -> Option> { + let source = CreateTableSource::read(sql).ok()?; + Some( + source + .columns() + .filter_map(|(_, column)| { + let segment = column + .segments + .iter() + .find(|segment| segment.kind == ColumnConstraintKind::Generated)?; + Some((column.name.clone(), source.first_group_text(segment.range.clone())?)) + }) + .collect(), + ) +} + /// The names a CREATE TABLE gives its foreign keys, in the order /// `pragma_foreign_key_list` numbers them, which is from the last one /// declared. `None` for a key declared without a name, and no list when diff --git a/core/crates/drivers/sqlite/src/ddl/mod.rs b/core/crates/drivers/sqlite/src/ddl/mod.rs index 9ca9adc879..973124e0d0 100644 --- a/core/crates/drivers/sqlite/src/ddl/mod.rs +++ b/core/crates/drivers/sqlite/src/ddl/mod.rs @@ -26,6 +26,7 @@ use tablepro_core::{ForeignKeyInfo, Value}; use self::create_table_source::CreateTableSource; pub(crate) use self::create_table_source::{ DeclaredComparisons, declared_descending_key, declared_foreign_key_names, foreign_key_list_bytes, + generated_expression_texts, }; use self::declared_shape::readable; use crate::catalog::MAIN_SCHEMA; diff --git a/core/crates/drivers/sqlite/src/dump.rs b/core/crates/drivers/sqlite/src/dump.rs index fe1541c763..cfafbf9ecd 100644 --- a/core/crates/drivers/sqlite/src/dump.rs +++ b/core/crates/drivers/sqlite/src/dump.rs @@ -36,11 +36,16 @@ impl DumpDialect for SqliteDialect { /// `sqlite_schema` in the order the objects were made, each table with /// what `table_list` calls it, so a virtual table and its shadow tables /// can be told from the rest. An automatic index has no text. -const OBJECTS_SQL: &str = "SELECT m.type AS type, m.name AS name, m.sql AS sql, coalesce(l.type, '') AS kind \ - FROM \"main\".sqlite_schema AS m \ - LEFT JOIN pragma_table_list AS l ON m.type = 'table' AND l.schema = ?1 AND l.name = m.name \ - WHERE m.sql IS NOT NULL AND m.name NOT LIKE 'sqlite!_%' ESCAPE '!' \ - ORDER BY m.rowid"; +fn objects_sql() -> String { + format!( + "SELECT m.type AS type, m.name AS name, {}, coalesce(l.type, '') AS kind \ + FROM \"main\".sqlite_schema AS m \ + LEFT JOIN pragma_table_list AS l ON m.type = 'table' AND l.schema = ?1 AND l.name = m.name \ + WHERE m.sql IS NOT NULL AND m.name NOT LIKE 'sqlite!_%' ESCAPE '!' \ + ORDER BY m.rowid", + crate::statement_limit::capped("m.sql") + ) +} const SEQUENCE_SQL: &str = "SELECT name, CAST(seq AS INTEGER) AS seq FROM \"main\".sqlite_sequence WHERE name IS NOT NULL ORDER BY rowid"; @@ -65,7 +70,7 @@ pub(crate) async fn read_dump_catalog( feature: UnsupportedFeature::SqlDump, }); } - let entries = sqlx::query(OBJECTS_SQL) + let entries = sqlx::query(sqlx::AssertSqlSafe(objects_sql())) .bind(MAIN_SCHEMA) .fetch_all(&mut *connection) .await @@ -99,7 +104,7 @@ fn schema_entry(row: &SqliteRow) -> Result { Ok(SchemaEntry { kind: text(row, "type")?, name: text(row, "name")?, - sql: text(row, "sql")?, + sql: crate::statement_limit::stored_sql(row)?.unwrap_or_default(), table_kind: text(row, "kind")?, }) } diff --git a/core/crates/drivers/sqlite/src/index_sql.rs b/core/crates/drivers/sqlite/src/index_sql.rs index cf384bde75..20833d8d19 100644 --- a/core/crates/drivers/sqlite/src/index_sql.rs +++ b/core/crates/drivers/sqlite/src/index_sql.rs @@ -1,16 +1,7 @@ -//! The expression keys and the WHERE clause of an index, read from the -//! CREATE INDEX that SQLite stored for it. -//! -//! sqlparser parses the statement, which settles how many keys there -//! are and whether one has a sort order or the index a predicate. The -//! text of each comes from sqlparser's own token stream rather than -//! from the AST's spans: in sqlparser 0.63 a function call's span stops -//! before its closing parenthesis and a parenthesised expression's -//! span skips its parentheses, so a span slice would cut keys short. -//! The tokens carry exact positions, and quotes, brackets and comments -//! are already single tokens there, so the top-level commas are the -//! only places a key list can split. Where the tokens and the AST do -//! not line up, the AST's own rendering of the expression is used. +// Key texts come from sqlparser's tokens, not the AST's spans: in sqlparser 0.63 a function call's span stops before its +// closing parenthesis and a parenthesised expression's span skips its parentheses. Quotes, brackets and comments are +// single tokens, so the top-level commas are the only places a key list splits. Where the tokens and the AST do not +// line up, the AST's rendering is used. use std::ops::Range; @@ -20,16 +11,13 @@ use sqlparser::keywords::Keyword; use sqlparser::parser::Parser; use sqlparser::tokenizer::{Location, Token, Tokenizer}; -/// What an index's stored SQL says beyond the pragmas. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct IndexSql { - /// One entry per key, in key order: the key as written, without its - /// ASC or DESC. + // In key order, each as written without its ASC or DESC. pub keys: Vec, pub predicate: Option, } -/// Read a CREATE INDEX. `None` when it does not parse as exactly one. pub(crate) fn read_index_sql(sql: &str) -> Option { let dialect = SQLiteDialect {}; let mut statements = Parser::parse_sql(&dialect, sql).ok()?; @@ -64,15 +52,12 @@ fn texts(index: &CreateIndex, cut: Option) -> IndexSql { } } -/// The key texts and the predicate text the tokens gave. #[derive(Debug, Default)] struct Cut { keys: Vec, predicate: Option, } -/// The statement's tokens, each with the bytes of the text it came -/// from. struct SourceTokens<'a> { sql: &'a str, tokens: Vec<(Token, Range)>, @@ -80,14 +65,14 @@ struct SourceTokens<'a> { impl<'a> SourceTokens<'a> { fn read(dialect: &SQLiteDialect, sql: &'a str) -> Option { - let lines = LineStarts::new(sql); + let mut offsets = Offsets::new(sql); let tokens = Tokenizer::new(dialect, sql) .tokenize_with_location() .ok()? .into_iter() .map(|token| { - let start = lines.byte(token.span.start)?; - let end = lines.byte(token.span.end)?; + let start = offsets.byte(token.span.start)?; + let end = offsets.byte(token.span.end)?; Some((token.token, start..end)) }) .collect::>>()?; @@ -111,7 +96,6 @@ impl<'a> SourceTokens<'a> { }) } - /// The parenthesis after ON that opens the key list. fn key_list_open(&self) -> Option { let mut after_on = false; let mut depth = 0usize; @@ -127,8 +111,6 @@ impl<'a> SourceTokens<'a> { None } - /// The token ranges between the key list's top-level commas, and the - /// position of the parenthesis that closes it. fn segments(&self, open: usize) -> Option<(Vec>, usize)> { let mut segments = Vec::new(); let mut start = open + 1; @@ -166,8 +148,6 @@ impl<'a> SourceTokens<'a> { self.text(segment) } - /// Everything after a top-level WHERE that follows the key list, up - /// to a closing semicolon. fn predicate(&self, close: usize) -> Option { let start = self .tokens @@ -187,7 +167,6 @@ impl<'a> SourceTokens<'a> { self.text(self.trimmed(start..end)) } - /// The range without the whitespace and comments at either end. fn trimmed(&self, mut range: Range) -> Range { while range.start < range.end && self.is_trivia(range.start) { range.start += 1; @@ -212,31 +191,42 @@ impl<'a> SourceTokens<'a> { } } -/// Byte offsets for the tokenizer's line and column locations, which -/// count characters from 1. -struct LineStarts<'a> { +// The tokenizer's locations count lines and characters from 1 and come in text order, so one cursor walks the text +// once: counting from each line's start made a long one-line statement quadratic. +struct Offsets<'a> { text: &'a str, - starts: Vec, + line: u64, + column: u64, + byte: usize, } -impl<'a> LineStarts<'a> { +impl<'a> Offsets<'a> { fn new(text: &'a str) -> Self { - let starts = std::iter::once(0) - .chain(text.match_indices('\n').map(|(newline, _)| newline + 1)) - .collect(); - Self { text, starts } + Self { + text, + line: 1, + column: 1, + byte: 0, + } } - fn byte(&self, location: Location) -> Option { - let line = usize::try_from(location.line).ok()?.checked_sub(1)?; - let column = usize::try_from(location.column).ok()?.checked_sub(1)?; - let start = *self.starts.get(line)?; - let rest = self.text.get(start..)?; - match rest.char_indices().nth(column) { - Some((offset, _)) => Some(start + offset), - None if rest.chars().count() == column => Some(self.text.len()), - None => None, + fn byte(&mut self, location: Location) -> Option { + let target = (location.line, location.column); + if target < (self.line, self.column) { + return None; + } + while (self.line, self.column) < target { + let character = self.text.get(self.byte..)?.chars().next()?; + self.byte += character.len_utf8(); + match character { + '\n' => { + self.line += 1; + self.column = 1; + } + _ => self.column += 1, + } } + Some(self.byte) } } @@ -277,6 +267,16 @@ mod tests { assert_eq!(read.predicate.as_deref(), Some("id > 0")); } + #[test] + fn keys_after_wide_characters_on_their_line_keep_their_text() { + let sql = "CREATE INDEX \"ü\" ON \"té\" (lower(\"é,ß\"), \"ß\" DESC) WHERE \"é\" > '日本'"; + + let read = read_index_sql(sql).unwrap(); + + assert_eq!(read.keys, ["lower(\"é,ß\")", "\"ß\""]); + assert_eq!(read.predicate.as_deref(), Some("\"é\" > '日本'")); + } + #[test] fn keys_fall_back_to_the_parsed_expression_when_the_tokens_do_not_line_up() { let parsed = Parser::parse_sql(&SQLiteDialect {}, "CREATE INDEX i ON t (lower( a ), b) WHERE b > 1").unwrap(); @@ -294,6 +294,24 @@ mod tests { assert_eq!(read.predicate.as_deref(), Some("b > 1")); } + // Counted from each line's start, this took minutes. + #[test] + fn a_long_one_line_key_list_reads_in_one_pass() { + let terms = vec!["0"; 50_000].join(","); + let sql = format!("CREATE INDEX i ON t ((a IN ({terms})), b) WHERE a IN ({terms})"); + let started = std::time::Instant::now(); + + let read = read_index_sql(&sql).unwrap(); + + assert_eq!(read.keys, [format!("(a IN ({terms}))"), "b".to_owned()]); + assert_eq!(read.predicate, Some(format!("a IN ({terms})"))); + assert!( + started.elapsed() < std::time::Duration::from_secs(30), + "{:?}", + started.elapsed() + ); + } + #[test] fn something_other_than_create_index_is_not_read() { assert_eq!(read_index_sql("CREATE TABLE t (a)"), None); diff --git a/core/crates/drivers/sqlite/src/lib.rs b/core/crates/drivers/sqlite/src/lib.rs index f2468cb3a1..f893c3667e 100644 --- a/core/crates/drivers/sqlite/src/lib.rs +++ b/core/crates/drivers/sqlite/src/lib.rs @@ -27,6 +27,7 @@ mod read_guard; mod result_stream; mod sqlite_pooled; mod sqlite_standalone; +mod statement_limit; mod write_batch; pub use catalog::{SQLITE_MIN_VERSION, SqliteVersion, check_runtime_version}; diff --git a/core/crates/drivers/sqlite/src/metadata.rs b/core/crates/drivers/sqlite/src/metadata.rs index 11088a8a86..f775f754b4 100644 --- a/core/crates/drivers/sqlite/src/metadata.rs +++ b/core/crates/drivers/sqlite/src/metadata.rs @@ -29,6 +29,7 @@ use crate::catalog::{ use crate::ddl::{DeclaredComparisons, declared_descending_key, declared_foreign_key_names, foreign_key_list_bytes}; use crate::errors::{SQLITE_ERROR, map_error}; use crate::index_sql::read_index_sql; +use crate::statement_limit::{first_statement, parse_fits, stored_sql}; /// What `pragma_table_list` and `pragma_table_xinfo` say about one /// table, read together. @@ -103,14 +104,16 @@ pub(crate) async fn fetch_object_definition( }); }; let schema = object.schema.as_deref().filter(|schema| !schema.is_empty()); - let sql: Option> = sqlx::query_scalar(AssertSqlSafe(catalog::trigger_sql("e( + let sql: Option> = sqlx::query(AssertSqlSafe(catalog::trigger_sql("e( schema.unwrap_or(MAIN_SCHEMA), )))) .bind(object.name.as_str()) .bind(table) .fetch_optional(&mut *connection) .await - .map_err(map_error)?; + .map_err(map_error)? + .map(|row| stored_sql(&row)) + .transpose()?; let schema = quote(schema.unwrap_or(MAIN_SCHEMA)); Ok(sql.flatten().map(|sql| ObjectDefinition { sql: terminated(&qualify_created_name(&sql, SqlGrammar::Sqlite, &schema)), @@ -448,11 +451,13 @@ async fn inbound_keys( } // One child at a time: its CREATE TABLE is let go before the next is read. for (rowid, child) in large { - let row: Option<(String, Option)> = sqlx::query_as(AssertSqlSafe(catalog::schema_row_sql(&schema_sql))) + let row: Option<(String, Option)> = sqlx::query(AssertSqlSafe(catalog::schema_row_sql(&schema_sql))) .bind(rowid) .fetch_optional(&mut *connection) .await - .map_err(map_error)?; + .map_err(map_error)? + .map(|row| Ok::<_, DriverError>((text(&row, "name")?, stored_sql(&row)?))) + .transpose()?; // A row another connection gave to another table since is not this child's. let sql = budget.keep_optional(row.filter(|(name, _)| *name == child).and_then(|(_, sql)| sql))?; if !sql.as_deref().is_some_and(|sql| names_table(sql, &table.name)) { @@ -504,7 +509,8 @@ async fn small_children_keys( continue; } if read.sql.is_none() { - read.sql = budget.keep_optional(row.try_get("sql").map_err(map_error)?)?; + let sql: Option = row.try_get("sql").map_err(map_error)?; + read.sql = budget.keep_optional(sql.map(first_statement))?; } read.columns.push(InboundKeyColumn { id, @@ -757,7 +763,7 @@ pub(crate) async fn fetch_table_definition( &create_statement, shape.columns.iter().any(|c| c.generated_storage().is_some()), ) { - (Some(create), true) => generated_expressions(create), + (Some(create), true) => generated_expressions(create, &mut budget)?, _ => Default::default(), }; let columns = shape @@ -821,9 +827,8 @@ async fn create_statement( .fetch_optional(&mut *connection) .await .map_err(map_error)? - .map(|row| row.try_get::, _>("sql")) - .transpose() - .map_err(map_error)? + .map(|row| stored_sql(&row)) + .transpose()? .flatten(); budget.keep_optional(sql) } @@ -852,7 +857,7 @@ async fn dependents( kind, name: budget.keep(text(&row, "name")?)?, table: budget.keep(text(&row, "tbl_name")?)?, - sql: budget.keep(text(&row, "sql")?)?, + sql: budget.keep(stored_sql(&row)?.unwrap_or_default())?, }); } Ok(select_dependents(table, schema_rows)) @@ -915,24 +920,35 @@ async fn index( primary: bool, budget: &mut CatalogBudget, ) -> Result { - let stored_sql: Option = sqlx::query_scalar(AssertSqlSafe(catalog::index_sql("e(schema)))) + let read: Option = sqlx::query(AssertSqlSafe(catalog::index_sql("e(schema)))) .bind(name.as_str()) .fetch_optional(&mut *connection) .await .map_err(map_error)? + .map(|row| stored_sql(&row)) + .transpose()? .flatten(); - let stored_sql = budget.keep_optional(stored_sql)?; - let parsed = stored_sql.as_deref().and_then(read_index_sql); - let mut keys = Vec::new(); + let stored_sql = budget.keep_optional(read)?; + let mut listed = Vec::new(); let mut rows = sqlx::query(catalog::INDEX_XINFO_SQL) .bind(name.as_str()) .bind(schema) .fetch(&mut *connection); while let Some(key) = rows.try_next().await.map_err(map_error)? { - let position = keys.len(); let cid: i64 = key.try_get("cid").map_err(map_error)?; let descending = key.try_get::("desc").map_err(map_error)? != 0; let column: Option = budget.keep_optional(key.try_get("name").map_err(map_error)?)?; + listed.push((cid, descending, column)); + } + let parsed = match stored_sql.as_deref() { + Some(sql) if partial || listed.iter().any(|(cid, _, _)| *cid == -2) => { + parse_fits(budget, sql)?; + read_index_sql(sql) + } + _ => None, + }; + let mut keys = Vec::with_capacity(listed.len()); + for (position, (cid, descending, column)) in listed.into_iter().enumerate() { keys.push(match (cid, column) { (-2, _) => IndexKey::Expression { sql: budget.keep( diff --git a/core/crates/drivers/sqlite/src/statement_limit.rs b/core/crates/drivers/sqlite/src/statement_limit.rs new file mode 100644 index 0000000000..993206cad6 --- /dev/null +++ b/core/crates/drivers/sqlite/src/statement_limit.rs @@ -0,0 +1,106 @@ +use sqlx::Row; +use sqlx::sqlite::{SqliteConnection as SqlxConnection, SqliteRow}; +use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::sql_syntax::script::ScriptPlan; +use tablepro_core::{CatalogBudget, DriverError}; + +use crate::errors::map_error; + +// SQLite parses every CREATE statement of a file when it reads the schema, and the catalog reads parse each again, so +// a crafted file's statement is capped far past any real table's, below SQLite's 1 GB default (sqlite.org/limits.html). +// A schema with a longer one does not open: SQLite reports it malformed. +pub(crate) const MAX_STATEMENT_BYTES: i32 = 10_000_000; + +// SQLite's default, set because a build can raise it: sqlparser drops and renders an expression one call per level, and +// a stored statement's expressions are no deeper than SQLite let it load. +const MAX_EXPRESSION_DEPTH: i32 = 1_000; + +// SQLite reads a stored CREATE text only to its first statement's end and ignores the rest, which a crafted file can +// make as long as any value. Every read takes a stored text only up to the cap: `capped` selects it as `sql`, NULL past +// the cap, beside `sql_too_long`, and `stored_sql` reads the two and keeps the first statement. +pub(crate) fn capped(column: &str) -> String { + let bytes = byte_length(column); + format!( + "CASE WHEN {bytes} <= {MAX_STATEMENT_BYTES} THEN {column} END AS sql, \ + {bytes} > {MAX_STATEMENT_BYTES} AS sql_too_long" + ) +} + +// octet_length of a column takes the length from the record without reading the value. +pub(crate) fn byte_length(column: &str) -> String { + format!("octet_length({column})") +} + +pub(crate) fn stored_sql(row: &SqliteRow) -> Result, DriverError> { + if row.try_get::, _>("sql_too_long").map_err(map_error)? == Some(1) { + return Err(DriverError::CatalogTooLarge { + limit_bytes: u64::try_from(MAX_STATEMENT_BYTES).unwrap_or(u64::MAX), + }); + } + Ok(row + .try_get::, _>("sql") + .map_err(map_error)? + .map(first_statement)) +} + +// SQLite stores a statement without its terminator, so only a crafted text has one, and the rest stays as stored. +pub(crate) fn first_statement(mut sql: String) -> String { + let end = ScriptPlan::first_statement_end(&sql, SqlGrammar::Sqlite); + if end < sql.len() { + let kept = sql.get(..end).map_or(end, |statement| statement.trim_end().len()); + sql.truncate(kept); + sql.shrink_to_fit(); + } + sql +} + +// sqlparser holds up to about 420 bytes for each byte it parses, measured on an IN list of one-digit numbers, so a parse +// of stored text has to fit beside what the read holds. +const PARSED_BYTES_PER_BYTE: usize = 512; + +pub(crate) fn parse_fits(budget: &CatalogBudget, text: &str) -> Result<(), DriverError> { + budget.fits(text.len().saturating_mul(PARSED_BYTES_PER_BYTE)) +} + +pub(crate) async fn install(connection: &mut SqlxConnection) -> Result<(), DriverError> { + let mut handle = connection.lock_handle().await.map_err(map_error)?; + let raw = handle.as_raw_handle(); + // SAFETY: `raw` comes from the locked handle, so the connection is open and nothing runs on it meanwhile. + unsafe { + libsqlite3_sys::sqlite3_limit( + raw.as_ptr(), + libsqlite3_sys::SQLITE_LIMIT_SQL_LENGTH, + MAX_STATEMENT_BYTES, + ); + libsqlite3_sys::sqlite3_limit( + raw.as_ptr(), + libsqlite3_sys::SQLITE_LIMIT_EXPR_DEPTH, + MAX_EXPRESSION_DEPTH, + ); + }; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_stored_text_keeps_its_first_statement_and_a_module_argument_its_semicolon() { + let cut = |sql: &str| first_statement(sql.to_owned()); + + assert_eq!(cut("CREATE TABLE t (a) ; DROP TABLE t"), "CREATE TABLE t (a)"); + assert_eq!( + cut("CREATE VIRTUAL TABLE v USING fts4(a;b)"), + "CREATE VIRTUAL TABLE v USING fts4(a;b)" + ); + assert_eq!( + cut("CREATE VIRTUAL TABLE v USING fts4(a;b); SELECT 1"), + "CREATE VIRTUAL TABLE v USING fts4(a;b)" + ); + assert_eq!( + cut("CREATE VIEW v AS SELECT 1 -- a note"), + "CREATE VIEW v AS SELECT 1 -- a note" + ); + } +} diff --git a/core/crates/drivers/sqlite/src/write_batch.rs b/core/crates/drivers/sqlite/src/write_batch.rs index 40caef5aa7..9eafa2f9cb 100644 --- a/core/crates/drivers/sqlite/src/write_batch.rs +++ b/core/crates/drivers/sqlite/src/write_batch.rs @@ -398,16 +398,15 @@ async fn foreign_keys( ) -> Result, DriverError> { let mut budget = CatalogBudget::default(); let schema_sql = format!("\"{}\"", schema.replace('"', "\"\"")); - let create: Option<(String, Option)> = sqlx::query_as(AssertSqlSafe(object_sql(&schema_sql))) + let create: Option = sqlx::query(AssertSqlSafe(object_sql(&schema_sql))) .bind(table.name.as_str()) .fetch_optional(&mut *connection) .await - .map_err(map_error)?; - budget.fits( - create - .and_then(|(_, sql)| sql) - .map_or(0, |sql| foreign_key_list_bytes(&sql)), - )?; + .map_err(map_error)? + .map(|row| crate::statement_limit::stored_sql(&row)) + .transpose()? + .flatten(); + budget.fits(create.map_or(0, |sql| foreign_key_list_bytes(&sql)))?; let mut rows = sqlx::query(FOREIGN_KEY_LIST_SQL) .bind(table.name.as_str()) .bind(schema) diff --git a/core/crates/drivers/sqlite/tests/catalog_memory.rs b/core/crates/drivers/sqlite/tests/catalog_memory.rs index a599c9de4c..89bbb0acd5 100644 --- a/core/crates/drivers/sqlite/tests/catalog_memory.rs +++ b/core/crates/drivers/sqlite/tests/catalog_memory.rs @@ -1,6 +1,7 @@ // Catalog reads of a crafted file stop at the catalog budget. A test binary of its own because it counts every heap // allocation the process makes, which another test running beside it would add to. Add no other test here. +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -153,7 +154,7 @@ fn call() -> CallOptions { ) } -async fn open(path: &std::path::Path) -> Result, DriverError> { +async fn open(path: &Path) -> Result, DriverError> { let options = ConnectOptions { target: ConnectTarget::File(FileEndpoint::open_existing(path.to_path_buf())?), workspace_settings: WorkspaceSettings::workspace(false), @@ -172,19 +173,61 @@ fn too_large(read: &str, result: Result) { ); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn a_crafted_catalog_stops_at_the_budget() { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("crafted.sqlite3"); +type Outcome = Result<(), Box>; + +async fn file_with(directory: &Path, name: &str, schema: Vec) -> Result { + let path = directory.join(name); let options = SqliteConnectOptions::new() .filename(&path) .create_if_missing(true) .foreign_keys(false); - let mut writer = SqliteConnection::connect_with(&options).await.unwrap(); - for sql in crafted_schema() { - writer.execute(sqlx::AssertSqlSafe(sql)).await.unwrap(); + let mut writer = SqliteConnection::connect_with(&options).await?; + for sql in schema { + writer.execute(sqlx::AssertSqlSafe(sql)).await?; } - writer.close().await.unwrap(); + writer.close().await?; + Ok(path) +} + +fn zeros(terms: usize) -> String { + vec!["0"; terms].join(",") +} + +// One line of short terms, which sqlparser holds hundreds of times over and which an offset counted from the line's +// start made quadratic, and a stored text past SQLite's statement end, which the table sizing once copied whole. +fn parsed_schema() -> Vec { + vec![ + format!( + "CREATE TABLE generated_list (a INTEGER, g AS (a IN ({})))", + zeros(500_000) + ), + "CREATE TABLE indexed (a INTEGER)".to_owned(), + format!("CREATE INDEX listed ON indexed (a) WHERE a IN ({})", zeros(50_000)), + ] +} + +fn tailed_schema() -> Vec { + vec![ + "CREATE TABLE neighbour (id INTEGER PRIMARY KEY)".to_owned(), + "CREATE TABLE tailed (a INTEGER)".to_owned(), + "CREATE TABLE short_tail (a INTEGER REFERENCES neighbour)".to_owned(), + "PRAGMA writable_schema = ON".to_owned(), + "UPDATE sqlite_schema SET sql = sql || '; ' || replace(hex(zeroblob(5500000)), '00', 'x ') \ + WHERE name = 'tailed'" + .to_owned(), + "UPDATE sqlite_schema SET sql = sql || '; SELECT ' || replace(hex(zeroblob(2000000)), '00', '0,') || '0' \ + WHERE name = 'short_tail'" + .to_owned(), + "PRAGMA writable_schema = OFF".to_owned(), + ] +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_crafted_catalog_stops_at_the_budget() { + let directory = tempfile::tempdir().unwrap(); + let path = file_with(directory.path(), "crafted.sqlite3", crafted_schema()) + .await + .unwrap(); let connection = open(&path).await.unwrap(); let table = |name: &str| TableRef::unqualified(name); let child = table(&"n".repeat(LONG)); @@ -257,4 +300,61 @@ async fn a_crafted_catalog_stops_at_the_budget() { "keys into bulky_parent: {heap}, {native}" ); connection.close().await.unwrap(); + + parsed_texts_stay_bounded(directory.path()).await.unwrap(); + a_tail_is_not_read(directory.path()).await.unwrap(); +} + +async fn parsed_texts_stay_bounded(directory: &Path) -> Outcome { + let connection = open(&file_with(directory, "parsed.sqlite3", parsed_schema()).await?).await?; + let generated = TableRef::unqualified("generated_list"); + let (columns, _, _) = measured(connection.fetch_columns(&generated, call())).await; + assert_eq!(columns?.len(), 2); + let (read, heap, native) = measured(connection.fetch_table_definition(&generated, call())).await; + too_large("definition of generated_list", read); + assert!( + heap < BOUND && native < NATIVE_BOUND, + "definition of generated_list: {heap}, {native}" + ); + let (indexes, heap, native) = measured(connection.fetch_indexes(&TableRef::unqualified("indexed"), call())).await; + let predicate = format!("a IN ({})", zeros(50_000)); + assert_eq!( + indexes?.first().and_then(|index| index.predicate.as_deref()), + Some(predicate.as_str()) + ); + assert!( + heap < BOUND && native < NATIVE_BOUND, + "indexes of indexed: {heap}, {native}" + ); + connection.close().await?; + Ok(()) +} + +async fn a_tail_is_not_read(directory: &Path) -> Outcome { + let connection = open(&file_with(directory, "tailed.sqlite3", tailed_schema()).await?).await?; + let (read, heap, native) = + measured(connection.fetch_table_definition(&TableRef::unqualified("neighbour"), call())).await; + assert!( + matches!( + read, + Err(DriverError::CatalogTooLarge { + limit_bytes: 10_000_000 + }) + ), + "definition beside a tailed table: {:?}", + read.err() + ); + assert!( + heap < BOUND / 8 && native < 8 << 20, + "definition beside a tailed table: {heap}, {native}" + ); + let (keys, heap, native) = + measured(connection.fetch_foreign_keys(&TableRef::unqualified("short_tail"), call())).await; + assert_eq!(keys?.len(), 1); + assert!( + heap < BOUND / 8 && native < NATIVE_BOUND, + "keys of a table with a tail under the cap: {heap}, {native}" + ); + connection.close().await?; + Ok(()) } diff --git a/core/crates/drivers/sqlite/tests/integration/catalog.rs b/core/crates/drivers/sqlite/tests/integration/catalog.rs index 8987257169..6e39c8f531 100644 --- a/core/crates/drivers/sqlite/tests/integration/catalog.rs +++ b/core/crates/drivers/sqlite/tests/integration/catalog.rs @@ -3,12 +3,12 @@ use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection}; use sqlx::{Connection as _, Executor as _}; -use tablepro_core::Value; use tablepro_core::column::{ColumnDefault, ColumnKind, IntegerKind, IntegerWidth}; use tablepro_core::ddl::{GeneratedStorage, IdentityStyle, SchemaObjectKind}; use tablepro_core::meta::{ObjectScope, TableRef}; +use tablepro_core::{DriverError, ServerCode, Value}; -use crate::support::{TestFile, apply, call}; +use crate::support::{TestFile, apply, call, connect, writable}; #[tokio::test] async fn strict_table_is_not_dynamic() { @@ -481,3 +481,127 @@ async fn dependents_and_source_are_read() { assert_eq!(generated.expression.as_sql(), "total * 2"); connection.close().await.unwrap(); } + +async fn create(file: &TestFile, sql: String) { + let mut writer = SqliteConnection::connect_with(&SqliteConnectOptions::new().filename(&file.path)) + .await + .unwrap(); + writer.execute(sqlx::AssertSqlSafe(sql)).await.unwrap(); + writer.close().await.unwrap(); +} + +// Past ten million bytes SQLite calls the schema malformed, so a crafted statement cannot reach the catalog reads. +#[tokio::test] +async fn a_create_statement_past_ten_million_bytes_does_not_open() { + let file = TestFile::new().await; + create( + &file, + format!("CREATE TABLE fits (a INTEGER /* {} */)", "x".repeat(9_000_000)), + ) + .await; + let connection = file.open().await; + let columns = connection.fetch_columns(&TableRef::unqualified("fits"), call()).await; + assert_eq!(columns.map(|columns| columns.len()), Ok(1)); + connection.close().await.unwrap(); + + create( + &file, + format!("CREATE TABLE past (a INTEGER /* {} */)", "x".repeat(10_000_000)), + ) + .await; + let opened = connect(file.endpoint(), writable()).await.map(|_| ()); + + assert!( + matches!(&opened, Err(DriverError::Server(diagnostics)) + if matches!(diagnostics.code, Some(ServerCode::Sqlite { primary: 11, .. }))), + "{opened:?}" + ); +} + +// SQLite parses a stored CREATE text only to its statement's end, so a crafted file can carry any tail after it; the +// catalog reads that parse the text refuse it past the same cap, before parsing it. +#[tokio::test] +async fn a_stored_create_text_past_the_cap_is_refused_before_it_is_parsed() { + let file = TestFile::new().await; + create(&file, "CREATE TABLE t (a INTEGER)".to_owned()).await; + let mut writer = SqliteConnection::connect_with(&SqliteConnectOptions::new().filename(&file.path)) + .await + .unwrap(); + writer.execute("PRAGMA writable_schema = ON").await.unwrap(); + writer + .execute( + "UPDATE sqlite_schema SET sql = sql || '; ' || replace(hex(zeroblob(5500000)), '00', 'x ') \ + WHERE name = 't'", + ) + .await + .unwrap(); + writer.close().await.unwrap(); + let connection = file.open().await; + let table = TableRef::unqualified("t"); + + let objects = connection.list_objects(&ObjectScope::DefaultSchema, call()).await; + let columns = connection.fetch_columns(&table, call()).await; + let keys = connection.fetch_foreign_keys(&table, call()).await.map(|_| ()); + let definition = connection.fetch_table_definition(&table, call()).await.map(|_| ()); + + assert_eq!( + objects.map(|objects| objects.len()), + Ok(1), + "SQLite read the statement and left the tail" + ); + assert_eq!( + columns.map(|columns| columns.len()), + Ok(1), + "a read that parses no text reads as before" + ); + let refused = Err(DriverError::CatalogTooLarge { + limit_bytes: 10_000_000, + }); + assert_eq!(keys, refused); + assert_eq!(definition, refused); + connection.close().await.unwrap(); +} + +// A tail under the cap is cut where SQLite stops reading: an index's tail of 50,000 additions, which sqlparser would +// drop one call per term, and a table's tail, which would reach a rebuild's source and a dump. +#[tokio::test] +async fn a_stored_text_is_read_to_its_first_statement() { + let file = TestFile::new().await; + create(&file, "CREATE TABLE p (id INTEGER PRIMARY KEY)".to_owned()).await; + create(&file, "CREATE TABLE c (a INTEGER REFERENCES p)".to_owned()).await; + create(&file, "CREATE INDEX ci ON c (a) WHERE a > 0".to_owned()).await; + let mut writer = SqliteConnection::connect_with(&SqliteConnectOptions::new().filename(&file.path)) + .await + .unwrap(); + writer.execute("PRAGMA writable_schema = ON").await.unwrap(); + writer + .execute("UPDATE sqlite_schema SET sql = sql || '; DROP TABLE p' WHERE name = 'c'") + .await + .unwrap(); + writer + .execute(sqlx::AssertSqlSafe(format!( + "UPDATE sqlite_schema SET sql = sql || '; SELECT {}' WHERE name = 'ci'", + vec!["1"; 50_000].join("+") + ))) + .await + .unwrap(); + writer.close().await.unwrap(); + let connection = file.open().await; + let table = TableRef::unqualified("c"); + + let definition = connection.fetch_table_definition(&table, call()).await.unwrap(); + let indexes = connection.fetch_indexes(&table, call()).await.unwrap(); + + assert_eq!( + definition.source.map(|source| source.create_statement), + Some("CREATE TABLE c (a INTEGER REFERENCES p)".to_owned()) + ); + assert_eq!(definition.foreign_keys.len(), 1); + assert_eq!(indexes.len(), 1); + assert_eq!(indexes[0].predicate.as_deref(), Some("a > 0")); + assert_eq!( + indexes[0].definition.as_deref(), + Some("CREATE INDEX \"main\".ci ON c (a) WHERE a > 0") + ); + connection.close().await.unwrap(); +} diff --git a/core/crates/drivers/sqlite/tests/integration/lock_wait.rs b/core/crates/drivers/sqlite/tests/integration/lock_wait.rs index caf2758980..f056e8216c 100644 --- a/core/crates/drivers/sqlite/tests/integration/lock_wait.rs +++ b/core/crates/drivers/sqlite/tests/integration/lock_wait.rs @@ -5,6 +5,7 @@ use std::time::Duration; use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection}; use sqlx::{Connection as _, Executor as _}; +use tablepro_core::meta::TableRef; use tablepro_core::{ CallOptions, DriverError, EditorSessionOptions, EditorSessionState, RunEnd, RunOptions, Value, WriteKind, }; @@ -102,3 +103,42 @@ async fn an_editor_run_waiting_for_a_lock_stops_and_the_next_run_starts() { session.close().await; connection.close().await.unwrap(); } + +// A catalog read runs in one transaction; the stop that cuts its lock wait short still reads as the stop. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_catalog_read_waiting_for_a_lock_stops_as_the_stop_it_was() { + let file = TestFile::new().await; + let connection = file.open().await; + apply(connection.as_ref(), &["CREATE TABLE t (a)"]).await.unwrap(); + let mut holder = SqliteConnection::connect_with(&SqliteConnectOptions::new().filename(&file.path)) + .await + .unwrap(); + holder.execute("BEGIN EXCLUSIVE").await.unwrap(); + let table = TableRef::unqualified("t"); + + let cancelled = connection + .fetch_columns(&table, CallOptions::new(None, cancel_after(Duration::from_millis(200)))) + .await + .map(|_| ()); + let deadline = Instant::now() + Duration::from_millis(200); + let timed_out = connection + .fetch_columns(&table, CallOptions::new(Some(deadline), CancellationToken::new())) + .await + .map(|_| ()); + holder.execute("ROLLBACK").await.unwrap(); + holder.close().await.unwrap(); + + assert_eq!(cancelled, Err(DriverError::Cancelled)); + assert!(matches!(timed_out, Err(DriverError::Timeout { .. })), "{timed_out:?}"); + let columns = connection + .fetch_columns(&table, CallOptions::new(None, CancellationToken::new())) + .await; + assert_eq!( + columns.map(|columns| columns.len()), + Ok(1), + "the next read starts clean" + ); + let written = apply(connection.as_ref(), &["INSERT INTO t VALUES (1)"]).await; + assert!(written.is_ok(), "the read left no lock behind: {written:?}"); + connection.close().await.unwrap(); +} diff --git a/core/docs/decisions/0029-a-catalog-read-keeps-a-byte-budget.md b/core/docs/decisions/0029-a-catalog-read-keeps-a-byte-budget.md index dc0a2b6645..8bdcfc1622 100644 --- a/core/docs/decisions/0029-a-catalog-read-keeps-a-byte-budget.md +++ b/core/docs/decisions/0029-a-catalog-read-keeps-a-byte-budget.md @@ -13,6 +13,10 @@ A file engine reads its catalog from the file the user opened, and the file choo - SQLite and DuckDB charge each text a read keeps as it arrives, and each copy made per key before it is made. A text read and dropped before the next one, as each child's CREATE TABLE, is released. - Each text of the file is read once per call, and SQLite's catalog rows stream, unsorted where a sort would hold every row first. +- A SQLite catalog read runs in one read transaction, so a schema change by another connection between its statements cannot make it miss a table. +- The workspace's SQLite connections set `SQLITE_LIMIT_SQL_LENGTH` to 10,000,000 bytes, the per-connection limit sqlite.org/security.html names for untrusted files. SQLite applies it to each CREATE statement as it reads the schema, so a file with a longer one does not open: SQLite reports the schema malformed (`driver.server`, SQLite error 11). SQLite parses a stored text only to its first statement's end, so a text can carry any tail after it; each catalog read takes a stored text only up to the same cap, fails with `CatalogTooLarge` past it, and keeps only the first statement, cut where its terminator starts, so neither a parse, a rebuild's source nor a dump sees the tail. A semicolon inside parentheses is a virtual table's module argument to SQLite, so it ends nothing there. The size comes from `octet_length`, which reads it from the record without reading the text, so the driver needs SQLite 3.43, which every system with GTK 4.22 has. Only a table of at most 64 KiB has its text counted for the keys into a table. +- The workspace connections also set `SQLITE_LIMIT_EXPR_DEPTH` to SQLite's default 1,000, since a build can raise it: sqlparser drops and renders an expression one call per level. +- A parse with sqlparser runs only once the text, at 512 bytes for each byte, fits the budget. An index's statement is parsed only for an expression key or a WHERE, and a generated column's expression alone, not its table's statement. - SQLite runs `pragma_foreign_key_list` on a table only once the key list's cost, worked out from its CREATE TABLE (`foreign_key_list_bytes`), fits the budget. The keys into a table ask every table of at most 64 KiB whose key list cannot cost more than the budget in one read; each other table is read alone, searched in linear time for the parent's name, and costed first. - A save's foreign key check holds each checked table's name and each broken key once, and each broken row as a key index and a rowid, all charged to one budget for the checks before and after the save. Past it the save rolls back with `CatalogTooLarge`: a check by count alone could let the save move a broken key to other rows. - DuckDB builds every table's constraint rows, a chunk at a time, before a query filters them, and holds the whole result before the first row. A constraint read runs only once both fit, bounded from `duckdb_tables()` and `duckdb_dependencies()`. It reads each child's and each parent's CREATE TABLE once, and charges the column names it keeps from them. @@ -22,13 +26,22 @@ A file engine reads its catalog from the file the user opened, and the file choo - **An error, not a cut list.** A table definition plans structure changes, and a rebuild planned from a list cut short would drop what was left out. - **64 MB.** A schema of 5,000 tables keyed to one table reads its keys in a few megabytes, and a million table names fit under it. - **In the driver.** The driver knows which texts it copies per key and what SQLite compiles for them, and no app has to pass a limit. +- **SQLite's limits, and where they stop.** The statement limit bounds what SQLite and the catalog reads parse, but not what `pragma_foreign_key_list` compiles: one copy of a column's name for each REFERENCES on it, so a statement of S bytes can cost about S²/52 bytes, 331 MB from a 117 KB table. `SQLITE_LIMIT_VDBE_OP` stops that compile only after its op count is spent, 153 MB at 25,000, and would refuse real queries well before it helps. `sqlite3_hard_heap_limit64` is process-wide and would cap the app's own databases. So the key list keeps its cost check, and the budget keeps bounding the copies a read makes per key, which no limit counts. +- **Parses beside SQLite's own.** SQLite holds about 54 bytes for each byte of a statement it loads with the schema, and keeps it for the connection's life. The CREATE TABLE reader holds about as much while it runs, so it reads within the statement cap. sqlparser holds up to about 420, measured on an IN list of one-digit numbers, so its parses count against the budget, and it parses only what a read needs from it. +- **10 MB.** A real CREATE TABLE is a few kilobytes, two thousand columns about a hundred, and the 5,000-table catalog reads as before. `SQLITE_LIMIT_LENGTH` stays at its default: it bounds every value, the user's own data among them. Editor sessions keep SQLite's defaults, since a script can hold a longer statement; the workspace connection has already refused such a file. ## Consequences - Large real schemas read as before, and the keys into a table read faster: 5,000 children in 0.2 s, against 1.4 s. - One value is read before it is charged, and sqlx queues up to 50 rows ahead of the reader. SQLite bounds a value at `SQLITE_MAX_LENGTH`, and a CREATE TABLE over 64 KiB is read alone. - In a DuckDB file with a crafted table, every read that touches constraints fails, the columns of any table among them, since DuckDB builds the crafted table's rows for each. The table list still reads. +- DuckDB has no per-connection limit on a statement's or a schema's length; its `memory_limit` bounds the buffer manager, not what its catalog functions build. DuckDB keeps the bounds above. - DuckDB's own catalog functions, `duckdb_columns()` and `duckdb_constraints()`, build a row for every column and constraint of every table before a filter, each with its table's name, and a column can repeat a constraint. The bound counts table names and the text, not a long column name repeated in its constraints; that work is DuckDB's. - A save on a table with millions of rows that already break its keys rolls back rather than compare them. Its error lists the first ten tables and parents. - Not budgeted: the schema list, reads of one value, and the SQL dumps, which read each object once. - The apps word the code: Linux in `linux/crates/app/src/ui/error_text/driver.rs`, Windows in `Resources.resw`. + +## Known limits + +- SQLite's own foreign key check on a structure save can compile far more than the file holds: it expands a virtual generated column's expression once per key, 217 MB from a 12.7 KB file, measured. `foreign_key_list_bytes` does not count it. A real cap needs a counting allocator armed per read (`SQLITE_CONFIG_MALLOC`) or a reader in a process of its own. It takes a crafted file the user opens and a structure change the user saves. +- DuckDB's counterpart, expressions nested past the stacks DuckDB runs on, is ADR 0031. diff --git a/linux/changelog.d/4291.fixed.md b/linux/changelog.d/4291.fixed.md new file mode 100644 index 0000000000..d277740e0e --- /dev/null +++ b/linux/changelog.d/4291.fixed.md @@ -0,0 +1 @@ +The keys pointing at a table could miss a table while another program changed the SQLite file's schema. diff --git a/linux/changelog.d/4291.security.md b/linux/changelog.d/4291.security.md new file mode 100644 index 0000000000..6ca5f69639 --- /dev/null +++ b/linux/changelog.d/4291.security.md @@ -0,0 +1 @@ +A crafted SQLite file could make reading its structure take gigabytes of memory or minutes of CPU. diff --git a/linux/docs/drivers/sqlite.md b/linux/docs/drivers/sqlite.md index a2eee8d2e1..3d3edcb8aa 100644 --- a/linux/docs/drivers/sqlite.md +++ b/linux/docs/drivers/sqlite.md @@ -1,6 +1,6 @@ # SQLite -Driver id `sqlite`, crate `core/crates/drivers/sqlite`. It runs on sqlx 0.9 with `sqlite-unbundled` on Linux, so it links the system's libsqlite3 rather than a second copy (the other targets build the copy sqlx ships), and calls `libsqlite3-sys` directly twice: `sqlite3_get_autocommit`, until sqlx exposes a connection's transaction state, and `sqlite3_busy_handler`, so a stop reaches a statement waiting for a lock. The library must be SQLite 3.37.0 or newer (`SQLITE_MIN_VERSION`), which the driver checks when it opens the file. The contract it implements is in [adding-drivers.md](../adding-drivers.md). +Driver id `sqlite`, crate `core/crates/drivers/sqlite`. It runs on sqlx 0.9 with `sqlite-unbundled` on Linux, so it links the system's libsqlite3 rather than a second copy (the other targets build the copy sqlx ships), and calls `libsqlite3-sys` directly twice: `sqlite3_get_autocommit`, until sqlx exposes a connection's transaction state, and `sqlite3_busy_handler`, so a stop reaches a statement waiting for a lock. The library must be SQLite 3.43.0 or newer (`SQLITE_MIN_VERSION`), which the driver checks when it opens the file. The contract it implements is in [adding-drivers.md](../adding-drivers.md). There is no network, login or TLS: the target is a `FileEndpoint`, an absolute path with a `FileOpenMode`. @@ -17,7 +17,7 @@ Every connection to the file opens with the same options: - sqlx's statement logging is off. - A file DuckDB has open in the app is `Busy(FileLocked)`, and an `ATTACH` of one is refused by an authorizer every connection installs: SQLite would close the file again on finding no database in it, which drops the POSIX locks DuckDB holds on it (`tablepro_core::claimed_by_another`). The authorizer sees a file name the `ATTACH` spells as a string. -The connect reads `sqlite_version()` and counts `sqlite_schema` once, so a file that is not a database fails the connect (`NotADatabase`) rather than the first page. Either statement can be the one that finds out: SQLite 3.51 reads the schema before it compiles a connection's first SELECT, while 3.46 waits for a statement that names a table. SQLite only says it could not open a file; the driver tells a moved file (`FileNotFound`) from one the user may not read (`FileAccessDenied`) by looking at the path. +The connect reads `sqlite_version()` and counts `sqlite_schema` once, so a file that is not a database fails the connect (`NotADatabase`) rather than the first page. Either statement can be the one that finds out: SQLite 3.51 reads the schema before it compiles a connection's first SELECT, while 3.46 waits for a statement that names a table. SQLite only says it could not open a file; the driver tells a moved file (`FileNotFound`) from one the user may not read (`FileAccessDenied`) by looking at the path. Each workspace connection caps a statement at 10,000,000 bytes (`SQLITE_LIMIT_SQL_LENGTH`), and SQLite applies it to the schema it reads, so a file with a longer CREATE statement fails the connect as a malformed schema, and an expression's depth at SQLite's default 1,000 (`SQLITE_LIMIT_EXPR_DEPTH`), which a build can raise (core ADR 0029). ### New databases