diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 790c1af78b..0dc784efa3 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -99,6 +99,9 @@ jobs: # The sign-in's loopback listener and the token's file on each platform. - name: Account unit tests run: cargo test --locked --target "$TARGET" -p tablepro-account --lib + # DuckDB recurses on threads whose stack each platform sizes (core/docs/decisions/0031-duckdb-expression-depth.md). + - name: DuckDB's deepest expressions + run: cargo test --locked --target "$TARGET" -p tablepro-driver-duckdb --test deep_expressions # The Arm64 image has no OpenSSH Server, and installing it takes about 20 minutes, so Arm64 runs these nightly. - name: Start Windows' OpenSSH Server id: sshd diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift index 5f9519a15e..a07bac083d 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift @@ -38,6 +38,9 @@ enum DuckDBReleaseOutcome: Equatable { actor DuckDBConnectionActor { private static let logger = Logger(subsystem: "com.TablePro", category: "DuckDBConnectionActor") + private let executor = DuckDBStackExecutor(name: "com.TablePro.duckdb") + nonisolated var unownedExecutor: UnownedSerialExecutor { executor.asUnownedSerialExecutor() } + private var database: duckdb_database? private var connection: duckdb_connection? // Renders the rows of a result the session's own connection is still producing. @@ -279,6 +282,15 @@ actor DuckDBConnectionActor { String(localized: "This build of DuckDB does not accept the storage version that keeps files readable by DuckDB 1.x") ) } + guard duckdb_set_config( + config, + DuckDBExpressionDepth.configurationOption, + DuckDBExpressionDepth.limit + ) != DuckDBError else { + throw DuckDBPluginError.connectionFailed( + String(localized: "This build of DuckDB does not accept a limit on how deep an expression nests") + ) + } if !spec.allowsPersistentSecrets { guard duckdb_set_config(config, "allow_persistent_secrets", "false") != DuckDBError else { throw DuckDBPluginError.connectionFailed( diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBExpressionDepth.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBExpressionDepth.swift new file mode 100644 index 0000000000..e78706c3d5 --- /dev/null +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBExpressionDepth.swift @@ -0,0 +1,117 @@ +// +// DuckDBExpressionDepth.swift +// DuckDBDriverPlugin +// + +import CDuckDB +import Foundation + +// DuckDB binds about 1 KiB of stack per level of an expression, also on threads of its own that get 512 KiB on macOS +// and iOS, where its default limit of 1,000 overflows them (core/docs/decisions/0031-duckdb-expression-depth.md). +nonisolated enum DuckDBExpressionDepth { + static let configurationOption = "max_expression_depth" + static let limit = "250" + + // DuckDB reads a file's expressions at open and ATTACH, prints them and writes them back with no limit of its + // own, about 0.84 KiB a level: the stack its command line runs on, where a cooperative thread has 512 KiB. + static let stackSize = 8 << 20 + + static func open( + _ path: String, + config: duckdb_config? + ) -> (state: duckdb_state, database: duckdb_database?, error: UnsafeMutablePointer?) { + nonisolated(unsafe) let config = config + return onStack { + var database: duckdb_database? + var error: UnsafeMutablePointer? + let state = duckdb_open_ext(path, &database, config, &error) + return (state: state, database: database, error: error) + } ?? (state: DuckDBError, database: nil, error: nil) + } + + static func close(_ database: inout duckdb_database?) { + nonisolated(unsafe) let closing = database + _ = onStack { + var closed = closing + duckdb_close(&closed) + } + database = nil + } + + static func onStack(_ body: @escaping @Sendable () -> Value) -> Value? { + nonisolated(unsafe) var value: Value? + let finished = DispatchSemaphore(value: 0) + let thread = Thread { + value = body() + finished.signal() + } + thread.stackSize = stackSize + thread.start() + finished.wait() + return value + } +} + +// An actor's jobs, in order, on one thread of its own with DuckDB's stack. +nonisolated final class DuckDBStackExecutor: SerialExecutor, @unchecked Sendable { + private let pending = DuckDBPendingJobs() + + init(name: String) { + let pending = pending + let thread = Thread { pending.run() } + thread.name = name + thread.stackSize = DuckDBExpressionDepth.stackSize + thread.qualityOfService = .userInitiated + thread.start() + } + + deinit { + pending.stop() + } + + // The ExecutorJob form needs macOS 14 and the app runs on 13. + func enqueue(_ job: UnownedJob) { + pending.add(job, on: asUnownedSerialExecutor()) + } + + func asUnownedSerialExecutor() -> UnownedSerialExecutor { + UnownedSerialExecutor(ordinary: self) + } +} + +private nonisolated final class DuckDBPendingJobs: @unchecked Sendable { + private let condition = NSCondition() + private var jobs: [(job: UnownedJob, executor: UnownedSerialExecutor)] = [] + private var stopped = false + + func add(_ job: UnownedJob, on executor: UnownedSerialExecutor) { + condition.lock() + jobs.append((job: job, executor: executor)) + condition.signal() + condition.unlock() + } + + // The executor goes with its actor, which has no job left by then. + func stop() { + condition.lock() + stopped = true + condition.signal() + condition.unlock() + } + + func run() { + while true { + condition.lock() + while jobs.isEmpty && !stopped { + condition.wait() + } + guard !jobs.isEmpty else { + condition.unlock() + return + } + let next = jobs.removeFirst() + condition.unlock() + next.job.runSynchronously(on: next.executor) + } + } +} diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBIsolatedImport.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBIsolatedImport.swift index 143ecacf7d..449a59d464 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBIsolatedImport.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBIsolatedImport.swift @@ -13,6 +13,7 @@ enum DuckDBIsolatedImport { ("autoinstall_known_extensions", "false"), ("autoload_known_extensions", "false"), ("allow_community_extensions", "false"), + (DuckDBExpressionDepth.configurationOption, DuckDBExpressionDepth.limit), ] // duckdb_set_config rejects allowed_directories, and DuckDB refuses to change it once external access is off. @@ -47,14 +48,14 @@ enum DuckDBIsolatedImport { throw DuckDBPluginError.connectionFailed("DuckDB rejected \(option.name) = \(option.value)") } - var database: duckdb_database? - var openError: UnsafeMutablePointer? - guard duckdb_open_ext(file, &database, config, &openError) != DuckDBError else { - let detail = openError.map { String(cString: $0) } ?? "unknown error" - duckdb_free(openError) + let opened = DuckDBExpressionDepth.open(file, config: config) + var database = opened.database + guard opened.state != DuckDBError else { + let detail = opened.error.map { String(cString: $0) } ?? "unknown error" + duckdb_free(opened.error) throw DuckDBPluginError.connectionFailed("Failed to open DuckDB database at '\(file)': \(detail)") } - defer { duckdb_close(&database) } + defer { DuckDBExpressionDepth.close(&database) } var handle: duckdb_connection? guard duckdb_connect(database, &handle) != DuckDBError, let connection = handle else { @@ -67,14 +68,22 @@ enum DuckDBIsolatedImport { } if Task.isCancelled { throw CancellationError() } - // IMPORT DATABASE expands into the folder's statements, which duckdb_prepare refuses. - for statement in lockdownStatements(allowing: folder) + [importStatement(folder: folder)] { - var result = duckdb_result() - defer { duckdb_destroy_result(&result) } - guard duckdb_query(connection, statement, &result) != DuckDBError else { - let message = duckdb_result_error(&result).map { String(cString: $0) } ?? "Unknown DuckDB error" - throw DuckDBPluginError.queryFailed(message) + // IMPORT DATABASE expands into the folder's statements, which duckdb_prepare refuses. An ATTACH among them + // reads that file's expressions too. + nonisolated(unsafe) let importing = connection + let statements = lockdownStatements(allowing: folder) + [importStatement(folder: folder)] + let failure = DuckDBExpressionDepth.onStack { () -> String? in + for statement in statements { + var result = duckdb_result() + defer { duckdb_destroy_result(&result) } + guard duckdb_query(importing, statement, &result) != DuckDBError else { + return duckdb_result_error(&result).map { String(cString: $0) } ?? "Unknown DuckDB error" + } } + return nil + } ?? "DuckDB did not run the import" + if let failure { + throw DuckDBPluginError.queryFailed(failure) } } } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index d5d1920a0e..d69b4216d5 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -201485,6 +201485,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "This build of DuckDB does not accept a limit on how deep an expression nests" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This build of DuckDB does not accept a read-only access mode" : { "extractionState" : "manual", "generatesSymbol" : false, diff --git a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver.swift index 4ddad41ca9..081f7c8430 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver.swift @@ -168,9 +168,9 @@ nonisolated final class DuckDBInterruptHandle: @unchecked Sendable { actor DuckDBActor { static let maxRows = 100_000 - private let queue = DispatchSerialQueue(label: "com.TablePro.duckdb") + private let executor = DuckDBStackExecutor(name: "com.TablePro.duckdb") - nonisolated var unownedExecutor: UnownedSerialExecutor { queue.asUnownedSerialExecutor() } + nonisolated var unownedExecutor: UnownedSerialExecutor { executor.asUnownedSerialExecutor() } nonisolated let interruptHandle = DuckDBInterruptHandle() private var database: duckdb_database? @@ -191,6 +191,7 @@ actor DuckDBActor { ("storage_compatibility_version", "v0.10.2"), ("autoinstall_known_extensions", "false"), ("autoload_known_extensions", "false"), + (DuckDBExpressionDepth.configurationOption, DuckDBExpressionDepth.limit), ] func open(path: String) throws { diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/DuckDBDriverTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/DuckDBDriverTests.swift index 92f843a957..80e9ee75d8 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/DuckDBDriverTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/DuckDBDriverTests.swift @@ -1,3 +1,5 @@ +import CDuckDB +import Foundation import TableProDatabase @testable import TableProMobile import TableProModels @@ -532,3 +534,72 @@ struct DuckDBNulTextTests { #expect(after.rows == [["k", "other"], [key, "x\u{0}y"]]) } } + +// Built past DuckDB's default limit on a stack that holds it, as another program could. +private actor DeepDuckDBFileWriter { + private let executor = DuckDBStackExecutor(name: "DeepDuckDBFileWriter") + nonisolated var unownedExecutor: UnownedSerialExecutor { executor.asUnownedSerialExecutor() } + + func create(_ path: String, check: String) -> Bool { + var config: duckdb_config? + var database: duckdb_database? + var connection: duckdb_connection? + defer { + duckdb_disconnect(&connection) + duckdb_close(&database) + duckdb_destroy_config(&config) + } + guard duckdb_create_config(&config) != DuckDBError, + duckdb_set_config(config, "max_expression_depth", "100000") != DuckDBError, + duckdb_open_ext(path, &database, config, nil) != DuckDBError, + duckdb_connect(database, &connection) != DuckDBError + else { return false } + var result = duckdb_result() + defer { duckdb_destroy_result(&result) } + return duckdb_query(connection, "CREATE TABLE t (a INTEGER CHECK (\(check) > 0)); CHECKPOINT", &result) + != DuckDBError + } +} + +struct DuckDBDeepExpressionTests { + private func deepFile(in directory: URL) async throws -> URL { + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let file = directory.appendingPathComponent("deep.duckdb") + let check = String(repeating: "(", count: 900) + "a" + String(repeating: " + 1)", count: 900) + #expect(await DeepDuckDBFileWriter().create(file.path, check: check)) + return file + } + + @Test("A CHECK nested 900 deep opens, and reading its table's keys fails at the depth limit instead of overflowing") + func deepCheckOpensAndItsKeysAreRefused() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("duckdb-deep-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: directory) } + let file = try await deepFile(in: directory) + + let driver = DuckDBDriver(source: .file(file)) + try await driver.connect() + let refusal = await #expect(throws: DuckDBDriverError.self) { + try await driver.fetchColumns(table: "t", schema: nil) + } + try await driver.disconnect() + + #expect(refusal?.localizedDescription.contains("Max expression depth limit of 250 exceeded") == true) + } + + @Test("An ATTACH of a file nested 900 deep reads its catalog without overflowing") + func deepFileAttaches() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("duckdb-deep-attach-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: directory) } + let file = try await deepFile(in: directory) + + let driver = DuckDBDriver(source: .inMemory) + try await driver.connect() + _ = try await driver.execute(query: "ATTACH '\(file.path)' AS deep (READ_ONLY)") + let tables = try await driver.execute(query: "SELECT table_name FROM duckdb_tables() WHERE database_name = 'deep'") + try await driver.disconnect() + + #expect(tables.rows == [["t"]]) + } +} diff --git a/apple/TableProMobile/project.yml b/apple/TableProMobile/project.yml index d1671a8228..da9cb32d25 100644 --- a/apple/TableProMobile/project.yml +++ b/apple/TableProMobile/project.yml @@ -77,6 +77,8 @@ targets: # The socket key a Mac connection stores, so iOS refuses exactly what the Mac driver dials # through a socket. - ../Plugins/MySQLDriverPlugin/MySQLLocalSocket.swift + # The expression depth limit and the open's stack both apps open a DuckDB file with. + - ../Plugins/DuckDBDriverPlugin/DuckDBExpressionDepth.swift - ../Plugins/Shared/PluginQueryTimeout.swift # The toolbar glyph rule both apps follow on and before the 26 releases. - ../TablePro/Core/Services/Infrastructure/Toolbar/ToolbarSymbols.swift diff --git a/apple/changelog.d/4293.security.md b/apple/changelog.d/4293.security.md new file mode 100644 index 0000000000..f699c16b18 --- /dev/null +++ b/apple/changelog.d/4293.security.md @@ -0,0 +1 @@ +Opening or browsing a DuckDB file with deeply nested constraints, defaults or views could crash the app. diff --git a/core/crates/drivers/duckdb/src/connection.rs b/core/crates/drivers/duckdb/src/connection.rs index cb4554ed26..ac7387de23 100644 --- a/core/crates/drivers/duckdb/src/connection.rs +++ b/core/crates/drivers/duckdb/src/connection.rs @@ -176,15 +176,36 @@ fn server_info(version: &str) -> ServerInfo { ServerInfo::new("DuckDB", version, part(0), part(1), part(2)) } +// DuckDB reads, prints and saves the file's expressions one call per level with no limit of its own; 8 MiB is the +// stack its CLI runs on, where a runtime's blocking thread has 2 (ADR 0030). +const DUCKDB_STACK_BYTES: usize = 8 << 20; + +pub(crate) fn on_duckdb_stack(work: W) -> Result +where + T: Send, + W: FnOnce() -> T + Send, +{ + std::thread::scope(|scope| { + std::thread::Builder::new() + .name("duckdb".to_owned()) + .stack_size(DUCKDB_STACK_BYTES) + .spawn_scoped(scope, work) + .map_err(|error| DriverError::Protocol(format!("could not start a thread for DuckDB: {error}")))? + .join() + .map_err(|_| DriverError::Protocol("DuckDB failed inside a call".to_owned())) + }) +} + /// Run DuckDB work that blocks, off the async threads. pub(crate) async fn blocking(work: W) -> Result where T: Send + 'static, W: FnOnce() -> Result + Send + 'static, { - tokio::task::spawn_blocking(work) + tokio::task::spawn_blocking(move || on_duckdb_stack(work)) .await .map_err(|error| DriverError::Protocol(format!("DuckDB work stopped: {error}")))? + .and_then(|result| result) } /// Hand a run's events to `sink` while `produce` makes them on the diff --git a/core/crates/drivers/duckdb/src/instance.rs b/core/crates/drivers/duckdb/src/instance.rs index 790b9f180e..89a3a8f4de 100644 --- a/core/crates/drivers/duckdb/src/instance.rs +++ b/core/crates/drivers/duckdb/src/instance.rs @@ -77,7 +77,11 @@ impl std::fmt::Debug for Instance { impl Drop for Instance { fn drop(&mut self) { let mut open = OPEN.lock().unwrap_or_else(PoisonError::into_inner); - drop(self.root.lock().unwrap_or_else(PoisonError::into_inner).take()); + let root = self.root.lock().unwrap_or_else(PoisonError::into_inner).take(); + // Closing checkpoints the file, which writes every expression in it back out, whichever thread drops this. + if let Err(error) = crate::connection::on_duckdb_stack(move || drop(root)) { + tracing::warn!(%error, "DuckDB did not close the file cleanly"); + } if open .get(&self.id) .is_some_and(|entry| entry.instance.strong_count() == 0) @@ -259,15 +263,13 @@ impl Instance { Ok(Link::new(connection, self.clone())) } - /// A connection for the app's own reads and saves. `SET - /// default_collation` in an editor tab reaches every connection to - /// the database, and `=` on a key would then find `'ABC'` and `'abc'` - /// alike, which DuckDB keeps as two rows: these compare byte for byte - /// whatever the tabs set. + // A tab's SET reaches every connection; the app's own compare keys byte for byte and keep a depth DuckDB fits. pub(crate) fn workspace_link(self: &Arc) -> Result { let link = self.link()?; link.connection - .execute_batch("SET SESSION default_collation = ''") + .execute_batch(&format!( + "SET SESSION default_collation = ''; SET SESSION max_expression_depth = {MAX_EXPRESSION_DEPTH}" + )) .map_err(map_error)?; Ok(link) } @@ -381,9 +383,10 @@ pub(crate) fn attach_unless_open( /// this keeps every file a connection creates or attaches readable there. const STORAGE_VERSION: &str = "v0.10.2"; -/// Every connection opens the same way. Extensions are never fetched: -/// a query that needs one DuckDB does not link in fails instead of -/// downloading it, as the app makes no requests of its own. +// DuckDB binds about 1 KiB of stack per level, also on threads of its own that get 512 KiB on macOS (ADR 0030). +const MAX_EXPRESSION_DEPTH: u32 = 250; + +// Extensions are never fetched: a query needing one DuckDB does not link in fails, as the app makes no requests. pub(crate) fn config(read_only: bool) -> Result { let mode = match read_only { true => AccessMode::ReadOnly, @@ -394,6 +397,7 @@ pub(crate) fn config(read_only: bool) -> Result { .and_then(|config| config.with("storage_compatibility_version", STORAGE_VERSION)) .and_then(|config| config.with("autoinstall_known_extensions", "false")) .and_then(|config| config.with("autoload_known_extensions", "false")) + .and_then(|config| config.with("max_expression_depth", MAX_EXPRESSION_DEPTH.to_string())) .map_err(map_error) } diff --git a/core/crates/drivers/duckdb/src/pool.rs b/core/crates/drivers/duckdb/src/pool.rs index c3d77c0c27..697c4c96be 100644 --- a/core/crates/drivers/duckdb/src/pool.rs +++ b/core/crates/drivers/duckdb/src/pool.rs @@ -1,4 +1,3 @@ -use std::panic::{AssertUnwindSafe, catch_unwind}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Weak}; use std::time::Duration; @@ -7,6 +6,7 @@ use deadpool::Runtime; use deadpool::managed::{Manager, Metrics, Object, Pool, PoolError, RecycleResult, Timeouts}; use tablepro_core::{BusyReason, CallResource, DriverError, LivenessPolicy, LossPhase, StopRequest}; +use crate::connection::on_duckdb_stack; use crate::instance::{Instance, Link, StopHandle}; /// The workspace pool's size. DuckDB runs the queries of one database @@ -175,8 +175,8 @@ impl DuckCall { let stopped = request.as_ref().is_some_and(StopRequest::is_requested); let outcome = match (&mut held, stopped) { (_, true) => Ok(Err(crate::errors::interrupted())), - (Held::Pooled(object), false) => catch_unwind(AssertUnwindSafe(|| work(object))), - (Held::Own(link), false) => catch_unwind(AssertUnwindSafe(|| work(link))), + (Held::Pooled(object), false) => on_duckdb_stack(|| work(object)), + (Held::Own(link), false) => on_duckdb_stack(|| work(link)), (Held::Empty, false) => Ok(Err(DriverError::Protocol("the call holds no connection".to_owned()))), }; stop.set_stoppable(false); @@ -185,10 +185,7 @@ impl DuckCall { // A connection whose work broke off, or whose caller gave // it up, holds a state nothing knows. Ok(result) => (detach(held), result), - Err(_) => ( - detach(held), - Err(DriverError::Protocol("DuckDB failed inside a call".to_owned())), - ), + Err(error) => (detach(held), Err(error)), } }) .await; diff --git a/core/crates/drivers/duckdb/tests/deep_expressions.rs b/core/crates/drivers/duckdb/tests/deep_expressions.rs new file mode 100644 index 0000000000..fd7eb14b93 --- /dev/null +++ b/core/crates/drivers/duckdb/tests/deep_expressions.rs @@ -0,0 +1,212 @@ +// DuckDB parses, binds, reads and prints an expression one call per level, partly on threads of its own, and a +// level past what a thread's stack holds ends the process. A target of its own, which CI also runs on Windows and +// macOS, where those threads get other stacks than on Linux. + +// cfg(test) is what lets the helpers fail a case with unwrap. +#[cfg(test)] +mod cases { + use std::path::{Path, PathBuf}; + use std::sync::Arc; + use std::time::Duration; + + use tablepro_core::meta::{ObjectScope, TableRef}; + use tablepro_core::statement::{GuardedStatement, Statement, StatementBuilder}; + use tablepro_core::{ + CallOptions, ConnectEnvironment, ConnectOptions, ConnectTarget, Connection, DatabaseDriver, DriverError, + EditorSessionOptions, FileEndpoint, ForeignKeyMode, RowLimit, RunEnd, RunOptions, ServerCode, SinkFlow, + UserSql, WorkspaceSettings, WriteBatch, WriteKind, + }; + use tablepro_driver_duckdb::DuckDbDriver; + use tablepro_test_support::conformance::test_liveness; + use tempfile::TempDir; + use tokio::time::Instant; + use tokio_util::sync::CancellationToken; + + // The deepest a CHECK, DEFAULT, generated column, view or macro nests under DuckDB's own default limit of 1,000. + const DUCKDB_DEFAULT_DEPTH: usize = 992; + + fn nested(levels: usize, inner: &str) -> String { + format!("{}{inner}{}", "abs(".repeat(levels), ")".repeat(levels)) + } + + // DuckDB recurses while it writes these too, so the file is made on a stack that holds them, on that one thread: + // a CHECKPOINT statement runs on DuckDB's own threads otherwise. + fn create(directory: &TempDir, statements: Vec) -> PathBuf { + let path = directory.path().join("deep.duckdb"); + let target = path.clone(); + std::thread::Builder::new() + .stack_size(256 << 20) + .spawn(move || { + let writer = duckdb::Connection::open_with_flags(&target, duckdb::Config::default().threads(1)?)?; + statements.iter().try_for_each(|sql| writer.execute_batch(sql)) + }) + .unwrap() + .join() + .unwrap() + .unwrap(); + path + } + + async fn open(path: &Path) -> Arc { + let options = ConnectOptions { + target: ConnectTarget::File(FileEndpoint::open_existing(path.to_path_buf()).unwrap()), + workspace_settings: WorkspaceSettings::workspace(false), + liveness: test_liveness(), + environment: ConnectEnvironment::new("test"), + }; + DuckDbDriver.connect(options, CancellationToken::new()).await.unwrap() + } + + fn call() -> CallOptions { + CallOptions::new(Some(Instant::now() + Duration::from_secs(60)), CancellationToken::new()) + } + + fn statement(sql: &str) -> Statement { + let mut builder = StatementBuilder::new(); + builder.push_sql(sql); + builder.finish() + } + + async fn query(connection: &dyn Connection, sql: &str) -> Result { + connection + .query_bound(&statement(sql), RowLimit::EDITOR_DEFAULT, call()) + .await + } + + struct Discard; + + #[async_trait::async_trait] + impl tablepro_core::ResultSink for Discard { + async fn accept(&mut self, _event: tablepro_core::ResultEvent) -> SinkFlow { + SinkFlow::Continue + } + } + + #[track_caller] + fn refused_as_too_deep(read: &str, result: Result) { + let error = match result { + Err(DriverError::RolledBack { source, .. }) => *source, + Err(error) => error, + Ok(value) => panic!("{read} read a value: {value:?}"), + }; + let DriverError::Server(diagnostics) = &error else { + panic!("{read}: {error:?}"); + }; + assert!( + matches!(&diagnostics.code, Some(ServerCode::DuckDb { error_type }) if error_type == "Binder" || error_type == "Parser"), + "{read}: {error:?}" + ); + assert!( + diagnostics + .message + .starts_with("Max expression depth limit of 250 exceeded"), + "{read}: {error:?}" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_file_as_deep_as_duckdb_allows_is_read_or_refused() { + let directory = tempfile::tempdir().unwrap(); + let path = create( + &directory, + vec![format!( + "CREATE TABLE other (id INTEGER PRIMARY KEY); \ + CREATE TABLE t (a INTEGER CHECK ({check} > 0), b INTEGER DEFAULT ({default}), c INTEGER AS ({check})); \ + CREATE VIEW v AS SELECT {check} AS x FROM t; \ + CREATE MACRO m(x) AS {macro_body}; \ + CHECKPOINT;", + check = nested(DUCKDB_DEFAULT_DEPTH, "a"), + default = nested(DUCKDB_DEFAULT_DEPTH, "1"), + macro_body = nested(DUCKDB_DEFAULT_DEPTH, "x"), + )], + ); + let connection = open(&path).await; + let table = |name: &str| TableRef::new(Some("main".to_owned()), name); + + let objects = connection + .list_objects(&ObjectScope::DefaultSchema, call()) + .await + .unwrap(); + assert_eq!( + objects.iter().map(|object| object.name.as_str()).collect::>(), + ["other", "t", "v"] + ); + refused_as_too_deep("the columns of t", connection.fetch_columns(&table("t"), call()).await); + refused_as_too_deep( + "the definition of other", + connection.fetch_table_definition(&table("other"), call()).await, + ); + refused_as_too_deep( + "the keys of other", + connection.fetch_foreign_keys(&table("other"), call()).await, + ); + refused_as_too_deep("the view", query(connection.as_ref(), "SELECT * FROM v").await); + refused_as_too_deep("the macro", query(connection.as_ref(), "SELECT m(1)").await); + let insert = WriteBatch::new( + WriteKind::SchemaChanges, + TableRef::unqualified("t"), + vec![GuardedStatement::unguarded(statement("INSERT INTO t (a) VALUES (1)"))], + ForeignKeyMode::Enforced, + ); + refused_as_too_deep("an insert", connection.write_batch(&insert, call()).await); + connection.close().await.unwrap(); + } + + // Made by raising DuckDB's limit. DuckDB reads it back at open with no limit of its own, deeper than the 2 MiB a + // runtime's blocking thread has. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_file_deeper_than_a_runtime_thread_holds_opens() { + let directory = tempfile::tempdir().unwrap(); + let path = create( + &directory, + vec![ + "SET max_expression_depth = 100000".to_owned(), + format!( + "CREATE TABLE t (a INTEGER CHECK ({} > 0)); CHECKPOINT;", + nested(3_000, "a") + ), + ], + ); + + let connection = open(&path).await; + + assert_eq!(connection.server_info().product, "DuckDB"); + connection.close().await.unwrap(); + } + + // A tab's SET GLOBAL reaches every connection to the database; the app's own reads keep the limit. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn the_limit_holds_on_every_connection_after_a_tab_raises_it() { + let directory = tempfile::tempdir().unwrap(); + let path = create(&directory, vec!["CHECKPOINT".to_owned()]); + let connection = open(&path).await; + let editor = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await + .unwrap(); + let run = |sql: &str| { + let sql = UserSql::new(sql); + let editor = editor.clone(); + async move { + editor + .run(sql, RunOptions::new(CancellationToken::new()), Box::new(Discard)) + .await + .unwrap() + .end + } + }; + let select = |levels: usize| format!("SELECT {}", nested(levels, "1")); + + assert_eq!(run(&select(240)).await, RunEnd::Completed); + let RunEnd::Failed(refused) = run(&select(250)).await else { + panic!("the tab ran an expression past the limit"); + }; + refused_as_too_deep("the tab", Err::<(), _>(refused)); + + assert_eq!(run("SET GLOBAL max_expression_depth = 100000").await, RunEnd::Completed); + assert!(query(connection.as_ref(), &select(240)).await.is_ok()); + refused_as_too_deep("the workspace", query(connection.as_ref(), &select(250)).await); + editor.close().await; + connection.close().await.unwrap(); + } +} diff --git a/core/docs/decisions/0031-duckdb-expression-depth.md b/core/docs/decisions/0031-duckdb-expression-depth.md new file mode 100644 index 0000000000..b201792996 --- /dev/null +++ b/core/docs/decisions/0031-duckdb-expression-depth.md @@ -0,0 +1,32 @@ +# 0031: DuckDB's expression depth fits the stacks it runs on + +- **Status**: Accepted +- **Date**: 2026-11-15 + +## Context + +DuckDB parses, binds, reads back, prints and saves an expression one C++ call per level of nesting. A file chooses how deep its CHECK constraints, defaults, generated columns, views and macros nest. DuckDB guards the parser, the binder and the planner with `max_expression_depth`, 1,000 by default, and nothing else: reading the catalog at open, printing an expression and writing it at a checkpoint have no limit. + +Measured on the DuckDB 2.0.0 the core and the Apple apps link, clang `-O2` and `-O3` on arm64, per level: the binder about 0.95 KiB, reading the file at open about 0.84 KiB, printing and saving about 0.3 KiB, and a view in a chain about 2.9 KiB, two binder levels each. + +DuckDB runs part of a query on threads of its own, created with the platform's default stack: 512 KiB on macOS and iOS, the executable's reserve on Windows (1.5 MiB in the Native AOT app, 1 MiB in a Rust test), `RLIMIT_STACK` on Linux, usually 8 MiB. `duckdb_constraints()` binds every table's constraints there, so a CHECK about 575 levels deep, under DuckDB's own limit, ended the process on macOS when any table's keys were read. The driver ran DuckDB on the runtime's 2 MiB blocking threads, and the Swift apps run it on 512 KiB cooperative threads, where opening a file whose CHECK nests 900 deep ended the process too. + +## Decision + +- Every DuckDB database the apps open on a user's file sets `max_expression_depth` to 250: about 240 KiB on the binder, under half of the smallest stack DuckDB runs on. Past it DuckDB refuses the statement with its own Parser or Binder error, `DriverError::Server` with the DuckDB code, which every app words. The core's own connections set it again for their session, so an editor tab's `SET GLOBAL` does not reach them. +- The core runs every DuckDB call, and the close that checkpoints the file, on a thread of 8 MiB, the stack DuckDB's command line runs on, started for the call inside the runtime's blocking task (`on_duckdb_stack`). +- The Swift apps run their DuckDB actor on a thread of 8 MiB through a custom executor (`DuckDBStackExecutor`), so an open, an ATTACH and a close get it as every query does, and the macOS isolated import runs its open, its statements and its close on one. + +## Rationale + +- **The limit DuckDB documents.** Its description warns that a deeper limit can overflow the stack; 250 is the value that fits the 512 KiB DuckDB gives its own threads with room to spare. +- **Not DuckDB's threads.** DuckDB runs its tasks on threads an embedder starts when `external_threads` equals `threads` (`duckdb_execute_tasks_state`). Measured, `SET threads` below that count then fails, above it DuckDB starts threads of its own with the default stack again, and those took the wake-ups `duckdb_finish_execution` sends, so a close waited for good. +- **A thread per call in the core.** Starting one costs about 40 µs; the runtime's blocking task waits on it, so a shutdown still waits for DuckDB's work. +- **An executor in the Swift apps.** An actor's custom executor is how Swift runs isolated work on a thread of the program's choosing; the cooperative pool and a dispatch queue give 512 KiB. It needs only the `UnownedJob` form, as the macOS app runs on 13. + +## Consequences + +- An expression, subquery or view chain nested past 250 levels fails in every app where DuckDB's own command line allows 1,000: about 240 chained `+` or `||` terms. `AND`, `OR`, `IN` lists, `CASE` and `coalesce` stay flat and are not limited. A user can raise it with `SET max_expression_depth`, and DuckDB's warning applies; in the Swift apps the app's own reads share that session, so the raised limit reaches them too. +- In a file with a CHECK nested past 250, every read that touches constraints fails, the columns of any table among them, as with a crafted table under 0029. The table list still reads. +- A file whose expressions nest past DuckDB's own limit, made by raising it, can still overflow DuckDB's threads where DuckDB prints or saves them, about 1,700 levels on macOS, and the 8 MiB open past about 9,700: DuckDB has no limit there, and only a process of its own would contain it. The core reads such a file back as far as DuckDB's command line does. +- The tests build files at 992 levels, DuckDB's own deepest, and at 3,000, and run on Linux, Windows and macOS (`core.yml`). diff --git a/core/docs/decisions/README.md b/core/docs/decisions/README.md index 8d9e4c7e0e..99dba4484f 100644 --- a/core/docs/decisions/README.md +++ b/core/docs/decisions/README.md @@ -38,3 +38,4 @@ The format, the statuses and how a later ADR changes an earlier one follow [the | [0028](0028-data-masking.md) | A shared result masks its columns before it leaves the machine, with the macOS app's output | Accepted | `tablepro_core::masking` gives the macOS styles byte for byte; `SharePlan` applies the sheet's Keep, Mask or Remove per column for the snapshot, its preview and summary; the snapshot names each masked column's style and keeps to the macOS body budget; `masking::detect` suggests masks from names, values and the SQL by PLAN-MASKING 8.3. | | [0029](0029-a-catalog-read-keeps-a-byte-budget.md) | A catalog read keeps a byte budget | Accepted | `CatalogBudget` bounds what one SQLite or DuckDB catalog read holds at 64 MB, counting each text kept and each copy made per key, and fails `DriverError::CatalogTooLarge` past it; SQLite costs a key list before it compiles one and streams rows unsorted; DuckDB bounds a constraint read before it runs; a save's foreign key check rolls back past the budget. | | [0030](0030-storage-import-and-library-through-the-ffi.md) | Saved queries, favourites, history pins, CSV import and connection copies cross the FFI | Accepted | `QueryHistoryStore` gives `SavedQueryStore` and `FavouriteStore` over its database, pins and exports runs; `CsvImport` previews, suggests a mapping, plans a file of up to 64 MiB with row faults as values and runs it once under the row save's Safe Mode gate; `ConnectionLibrary` duplicates with every secret after the original and exports without secrets. | +| [0031](0031-duckdb-expression-depth.md) | DuckDB's expression depth fits the stacks it runs on | Accepted | Every DuckDB open of a user's file sets `max_expression_depth` to 250, which fits the 512 KiB DuckDB gives its own threads on macOS, past which DuckDB refuses the statement; the core runs each DuckDB call and the close on an 8 MiB thread, and the Swift apps run their DuckDB actor on one. | diff --git a/linux/changelog.d/4293.security.md b/linux/changelog.d/4293.security.md new file mode 100644 index 0000000000..f699c16b18 --- /dev/null +++ b/linux/changelog.d/4293.security.md @@ -0,0 +1 @@ +Opening or browsing a DuckDB file with deeply nested constraints, defaults or views could crash the app.