Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/core.yml
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,9 @@ jobs:
# The sign-in's loopback listener and the token's file on each platform.
- name: Account unit tests
run: cargo test --locked --target "$TARGET" -p tablepro-account --lib
# DuckDB recurses on threads whose stack each platform sizes (core/docs/decisions/0031-duckdb-expression-depth.md).
- name: DuckDB's deepest expressions
run: cargo test --locked --target "$TARGET" -p tablepro-driver-duckdb --test deep_expressions
# The Arm64 image has no OpenSSH Server, and installing it takes about 20 minutes, so Arm64 runs these nightly.
- name: Start Windows' OpenSSH Server
id: sshd
Expand Down
12 changes: 12 additions & 0 deletions apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ enum DuckDBReleaseOutcome: Equatable {
actor DuckDBConnectionActor {
private static let logger = Logger(subsystem: "com.TablePro", category: "DuckDBConnectionActor")

private let executor = DuckDBStackExecutor(name: "com.TablePro.duckdb")
nonisolated var unownedExecutor: UnownedSerialExecutor { executor.asUnownedSerialExecutor() }

private var database: duckdb_database?
private var connection: duckdb_connection?
// Renders the rows of a result the session's own connection is still producing.
Expand Down Expand Up @@ -279,6 +282,15 @@ actor DuckDBConnectionActor {
String(localized: "This build of DuckDB does not accept the storage version that keeps files readable by DuckDB 1.x")
)
}
guard duckdb_set_config(
config,
DuckDBExpressionDepth.configurationOption,
DuckDBExpressionDepth.limit
) != DuckDBError else {
throw DuckDBPluginError.connectionFailed(
String(localized: "This build of DuckDB does not accept a limit on how deep an expression nests")
)
}
if !spec.allowsPersistentSecrets {
guard duckdb_set_config(config, "allow_persistent_secrets", "false") != DuckDBError else {
throw DuckDBPluginError.connectionFailed(
Expand Down
117 changes: 117 additions & 0 deletions apple/Plugins/DuckDBDriverPlugin/DuckDBExpressionDepth.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
//
// DuckDBExpressionDepth.swift
// DuckDBDriverPlugin
//

import CDuckDB
import Foundation

// DuckDB binds about 1 KiB of stack per level of an expression, also on threads of its own that get 512 KiB on macOS
// and iOS, where its default limit of 1,000 overflows them (core/docs/decisions/0031-duckdb-expression-depth.md).
nonisolated enum DuckDBExpressionDepth {
static let configurationOption = "max_expression_depth"
static let limit = "250"

// DuckDB reads a file's expressions at open and ATTACH, prints them and writes them back with no limit of its
// own, about 0.84 KiB a level: the stack its command line runs on, where a cooperative thread has 512 KiB.
static let stackSize = 8 << 20

static func open(
_ path: String,
config: duckdb_config?
) -> (state: duckdb_state, database: duckdb_database?, error: UnsafeMutablePointer<CChar>?) {
nonisolated(unsafe) let config = config
return onStack {
var database: duckdb_database?
var error: UnsafeMutablePointer<CChar>?
let state = duckdb_open_ext(path, &database, config, &error)
return (state: state, database: database, error: error)
} ?? (state: DuckDBError, database: nil, error: nil)
}

static func close(_ database: inout duckdb_database?) {
nonisolated(unsafe) let closing = database
_ = onStack {
var closed = closing
duckdb_close(&closed)
}
database = nil
}

static func onStack<Value>(_ body: @escaping @Sendable () -> Value) -> Value? {
nonisolated(unsafe) var value: Value?
let finished = DispatchSemaphore(value: 0)
let thread = Thread {
value = body()
finished.signal()
}
thread.stackSize = stackSize
thread.start()
finished.wait()
return value
}
}

// An actor's jobs, in order, on one thread of its own with DuckDB's stack.
nonisolated final class DuckDBStackExecutor: SerialExecutor, @unchecked Sendable {
private let pending = DuckDBPendingJobs()

init(name: String) {
let pending = pending
let thread = Thread { pending.run() }
thread.name = name
thread.stackSize = DuckDBExpressionDepth.stackSize
thread.qualityOfService = .userInitiated
thread.start()
}

deinit {
pending.stop()
}

// The ExecutorJob form needs macOS 14 and the app runs on 13.
func enqueue(_ job: UnownedJob) {
pending.add(job, on: asUnownedSerialExecutor())
}

func asUnownedSerialExecutor() -> UnownedSerialExecutor {
UnownedSerialExecutor(ordinary: self)
}
}

private nonisolated final class DuckDBPendingJobs: @unchecked Sendable {
private let condition = NSCondition()
private var jobs: [(job: UnownedJob, executor: UnownedSerialExecutor)] = []
private var stopped = false

func add(_ job: UnownedJob, on executor: UnownedSerialExecutor) {
condition.lock()
jobs.append((job: job, executor: executor))
condition.signal()
condition.unlock()
}

// The executor goes with its actor, which has no job left by then.
func stop() {
condition.lock()
stopped = true
condition.signal()
condition.unlock()
}

func run() {
while true {
condition.lock()
while jobs.isEmpty && !stopped {
condition.wait()
}
guard !jobs.isEmpty else {
condition.unlock()
return
}
let next = jobs.removeFirst()
condition.unlock()
next.job.runSynchronously(on: next.executor)
}
}
}
35 changes: 22 additions & 13 deletions apple/Plugins/DuckDBDriverPlugin/DuckDBIsolatedImport.swift
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ enum DuckDBIsolatedImport {
("autoinstall_known_extensions", "false"),
("autoload_known_extensions", "false"),
("allow_community_extensions", "false"),
(DuckDBExpressionDepth.configurationOption, DuckDBExpressionDepth.limit),
]

// duckdb_set_config rejects allowed_directories, and DuckDB refuses to change it once external access is off.
Expand Down Expand Up @@ -47,14 +48,14 @@ enum DuckDBIsolatedImport {
throw DuckDBPluginError.connectionFailed("DuckDB rejected \(option.name) = \(option.value)")
}

var database: duckdb_database?
var openError: UnsafeMutablePointer<CChar>?
guard duckdb_open_ext(file, &database, config, &openError) != DuckDBError else {
let detail = openError.map { String(cString: $0) } ?? "unknown error"
duckdb_free(openError)
let opened = DuckDBExpressionDepth.open(file, config: config)
var database = opened.database
guard opened.state != DuckDBError else {
let detail = opened.error.map { String(cString: $0) } ?? "unknown error"
duckdb_free(opened.error)
throw DuckDBPluginError.connectionFailed("Failed to open DuckDB database at '\(file)': \(detail)")
}
defer { duckdb_close(&database) }
defer { DuckDBExpressionDepth.close(&database) }

var handle: duckdb_connection?
guard duckdb_connect(database, &handle) != DuckDBError, let connection = handle else {
Expand All @@ -67,14 +68,22 @@ enum DuckDBIsolatedImport {
}
if Task.isCancelled { throw CancellationError() }

// IMPORT DATABASE expands into the folder's statements, which duckdb_prepare refuses.
for statement in lockdownStatements(allowing: folder) + [importStatement(folder: folder)] {
var result = duckdb_result()
defer { duckdb_destroy_result(&result) }
guard duckdb_query(connection, statement, &result) != DuckDBError else {
let message = duckdb_result_error(&result).map { String(cString: $0) } ?? "Unknown DuckDB error"
throw DuckDBPluginError.queryFailed(message)
// IMPORT DATABASE expands into the folder's statements, which duckdb_prepare refuses. An ATTACH among them
// reads that file's expressions too.
nonisolated(unsafe) let importing = connection
let statements = lockdownStatements(allowing: folder) + [importStatement(folder: folder)]
let failure = DuckDBExpressionDepth.onStack { () -> String? in
for statement in statements {
var result = duckdb_result()
defer { duckdb_destroy_result(&result) }
guard duckdb_query(importing, statement, &result) != DuckDBError else {
return duckdb_result_error(&result).map { String(cString: $0) } ?? "Unknown DuckDB error"
}
}
return nil
} ?? "DuckDB did not run the import"
if let failure {
throw DuckDBPluginError.queryFailed(failure)
}
}
}
4 changes: 4 additions & 0 deletions apple/TablePro/Resources/Localizable.xcstrings
Original file line number Diff line number Diff line change
Expand Up @@ -201485,6 +201485,10 @@
"extractionState" : "manual",
"generatesSymbol" : false
},
"This build of DuckDB does not accept a limit on how deep an expression nests" : {
"extractionState" : "manual",
"generatesSymbol" : false
},
"This build of DuckDB does not accept a read-only access mode" : {
"extractionState" : "manual",
"generatesSymbol" : false,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -168,9 +168,9 @@ nonisolated final class DuckDBInterruptHandle: @unchecked Sendable {
actor DuckDBActor {
static let maxRows = 100_000

private let queue = DispatchSerialQueue(label: "com.TablePro.duckdb")
private let executor = DuckDBStackExecutor(name: "com.TablePro.duckdb")

nonisolated var unownedExecutor: UnownedSerialExecutor { queue.asUnownedSerialExecutor() }
nonisolated var unownedExecutor: UnownedSerialExecutor { executor.asUnownedSerialExecutor() }

nonisolated let interruptHandle = DuckDBInterruptHandle()
private var database: duckdb_database?
Expand All @@ -191,6 +191,7 @@ actor DuckDBActor {
("storage_compatibility_version", "v0.10.2"),
("autoinstall_known_extensions", "false"),
("autoload_known_extensions", "false"),
(DuckDBExpressionDepth.configurationOption, DuckDBExpressionDepth.limit),
]

func open(path: String) throws {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import CDuckDB
import Foundation
import TableProDatabase
@testable import TableProMobile
import TableProModels
Expand Down Expand Up @@ -532,3 +534,72 @@ struct DuckDBNulTextTests {
#expect(after.rows == [["k", "other"], [key, "x\u{0}y"]])
}
}

// Built past DuckDB's default limit on a stack that holds it, as another program could.
private actor DeepDuckDBFileWriter {
private let executor = DuckDBStackExecutor(name: "DeepDuckDBFileWriter")
nonisolated var unownedExecutor: UnownedSerialExecutor { executor.asUnownedSerialExecutor() }

func create(_ path: String, check: String) -> Bool {
var config: duckdb_config?
var database: duckdb_database?
var connection: duckdb_connection?
defer {
duckdb_disconnect(&connection)
duckdb_close(&database)
duckdb_destroy_config(&config)
}
guard duckdb_create_config(&config) != DuckDBError,
duckdb_set_config(config, "max_expression_depth", "100000") != DuckDBError,
duckdb_open_ext(path, &database, config, nil) != DuckDBError,
duckdb_connect(database, &connection) != DuckDBError
else { return false }
var result = duckdb_result()
defer { duckdb_destroy_result(&result) }
return duckdb_query(connection, "CREATE TABLE t (a INTEGER CHECK (\(check) > 0)); CHECKPOINT", &result)
!= DuckDBError
}
}

struct DuckDBDeepExpressionTests {
private func deepFile(in directory: URL) async throws -> URL {
try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
let file = directory.appendingPathComponent("deep.duckdb")
let check = String(repeating: "(", count: 900) + "a" + String(repeating: " + 1)", count: 900)
#expect(await DeepDuckDBFileWriter().create(file.path, check: check))
return file
}

@Test("A CHECK nested 900 deep opens, and reading its table's keys fails at the depth limit instead of overflowing")
func deepCheckOpensAndItsKeysAreRefused() async throws {
let directory = FileManager.default.temporaryDirectory
.appendingPathComponent("duckdb-deep-\(UUID().uuidString)", isDirectory: true)
defer { try? FileManager.default.removeItem(at: directory) }
let file = try await deepFile(in: directory)

let driver = DuckDBDriver(source: .file(file))
try await driver.connect()
let refusal = await #expect(throws: DuckDBDriverError.self) {
try await driver.fetchColumns(table: "t", schema: nil)
}
try await driver.disconnect()

#expect(refusal?.localizedDescription.contains("Max expression depth limit of 250 exceeded") == true)
}

@Test("An ATTACH of a file nested 900 deep reads its catalog without overflowing")
func deepFileAttaches() async throws {
let directory = FileManager.default.temporaryDirectory
.appendingPathComponent("duckdb-deep-attach-\(UUID().uuidString)", isDirectory: true)
defer { try? FileManager.default.removeItem(at: directory) }
let file = try await deepFile(in: directory)

let driver = DuckDBDriver(source: .inMemory)
try await driver.connect()
_ = try await driver.execute(query: "ATTACH '\(file.path)' AS deep (READ_ONLY)")
let tables = try await driver.execute(query: "SELECT table_name FROM duckdb_tables() WHERE database_name = 'deep'")
try await driver.disconnect()

#expect(tables.rows == [["t"]])
}
}
2 changes: 2 additions & 0 deletions apple/TableProMobile/project.yml
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ targets:
# The socket key a Mac connection stores, so iOS refuses exactly what the Mac driver dials
# through a socket.
- ../Plugins/MySQLDriverPlugin/MySQLLocalSocket.swift
# The expression depth limit and the open's stack both apps open a DuckDB file with.
- ../Plugins/DuckDBDriverPlugin/DuckDBExpressionDepth.swift
- ../Plugins/Shared/PluginQueryTimeout.swift
# The toolbar glyph rule both apps follow on and before the 26 releases.
- ../TablePro/Core/Services/Infrastructure/Toolbar/ToolbarSymbols.swift
Expand Down
1 change: 1 addition & 0 deletions apple/changelog.d/4293.security.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Opening or browsing a DuckDB file with deeply nested constraints, defaults or views could crash the app.
23 changes: 22 additions & 1 deletion core/crates/drivers/duckdb/src/connection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -176,15 +176,36 @@ fn server_info(version: &str) -> ServerInfo {
ServerInfo::new("DuckDB", version, part(0), part(1), part(2))
}

// DuckDB reads, prints and saves the file's expressions one call per level with no limit of its own; 8 MiB is the
// stack its CLI runs on, where a runtime's blocking thread has 2 (ADR 0030).
const DUCKDB_STACK_BYTES: usize = 8 << 20;

pub(crate) fn on_duckdb_stack<T, W>(work: W) -> Result<T, DriverError>
where
T: Send,
W: FnOnce() -> T + Send,
{
std::thread::scope(|scope| {
std::thread::Builder::new()
.name("duckdb".to_owned())
.stack_size(DUCKDB_STACK_BYTES)
.spawn_scoped(scope, work)
.map_err(|error| DriverError::Protocol(format!("could not start a thread for DuckDB: {error}")))?
.join()
.map_err(|_| DriverError::Protocol("DuckDB failed inside a call".to_owned()))
})
}

/// Run DuckDB work that blocks, off the async threads.
pub(crate) async fn blocking<T, W>(work: W) -> Result<T, DriverError>
where
T: Send + 'static,
W: FnOnce() -> Result<T, DriverError> + Send + 'static,
{
tokio::task::spawn_blocking(work)
tokio::task::spawn_blocking(move || on_duckdb_stack(work))
.await
.map_err(|error| DriverError::Protocol(format!("DuckDB work stopped: {error}")))?
.and_then(|result| result)
}

/// Hand a run's events to `sink` while `produce` makes them on the
Expand Down
Loading
Loading