From c827743b526e195a24600055819cd7d74f9e759f Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Fri, 13 Nov 2026 17:24:23 +0700 Subject: [PATCH 1/2] perf(sqlite): pace the Remote SQLite row stream and run one statement at a time on the agent connection --- .../SQLiteAgentBackend.swift | 122 +++++++-- .../SQLiteAgentProtocol.swift | 5 +- .../SQLiteDriverPlugin/SQLitePlugin.swift | 15 +- .../SQLiteAgentLoopback.swift | 197 ++++++++++++++ .../SQLiteAgentProtocolTests.swift | 22 ++ .../SQLiteAgentStreamPacingTests.swift | 246 ++++++++++++++++++ apple/project.yml | 5 +- .../probes/check-stream-backpressure.sh | 94 ++++++- 8 files changed, 659 insertions(+), 47 deletions(-) create mode 100644 apple/Plugins/SQLiteDriverTests/SQLiteAgentLoopback.swift create mode 100644 apple/Plugins/SQLiteDriverTests/SQLiteAgentStreamPacingTests.swift diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentBackend.swift b/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentBackend.swift index 624a52e18e..140b8615e9 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentBackend.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentBackend.swift @@ -24,6 +24,10 @@ actor SQLiteAgentBackend: SQLiteExecutionBackend { private let helloBudget: SQLiteAgentHelloBudget private var busyTimeoutMilliseconds: Int32 = 0 + // Every reply goes to the connection's one pending operation, and an await lets the next call in, so statements queue. + private var statementRunning = false + private var waitingStatements: [CheckedContinuation] = [] + nonisolated let canceller: SQLiteCanceller nonisolated var resolvedServerVersion: String? { connection.cachedServerVersion } @@ -32,9 +36,10 @@ actor SQLiteAgentBackend: SQLiteExecutionBackend { port: Int, path: String, token: String, - helloBudget: SQLiteAgentHelloBudget + helloBudget: SQLiteAgentHelloBudget, + heartbeatInterval: TimeInterval = SQLiteAgentConnection.defaultHeartbeatInterval ) { - let connection = SQLiteAgentConnection(host: host, port: port) + let connection = SQLiteAgentConnection(host: host, port: port, heartbeatInterval: heartbeatInterval) self.connection = connection self.path = path self.token = token @@ -98,6 +103,8 @@ actor SQLiteAgentBackend: SQLiteExecutionBackend { rules: SQLiteStatementRules, rowCap: Int ) async throws -> SQLiteRawResult { + await takeTurn() + defer { endTurn() } let startTime = Date() let accumulator = SQLiteAgentResultAccumulator() @@ -141,8 +148,13 @@ actor SQLiteAgentBackend: SQLiteExecutionBackend { rules: SQLiteStatementRules, continuation: AsyncThrowingStream.Continuation ) async throws { + await takeTurn() + defer { endTurn() } + try Task.checkCancellation() _ = try await withTaskCancellationHandler { - try await connection.execute( + // A cancel frame sent before the statement starts does nothing, so a reader that left by now ends it here. + try Task.checkCancellation() + return try await connection.execute( sql: query, parameters: [], scope: Self.agentScope(rules), @@ -162,7 +174,8 @@ actor SQLiteAgentBackend: SQLiteExecutionBackend { batch.append(values[index..<(index + columnCount)].map(Self.decode)) index += columnCount } - continuation.yield(.rows(batch)) + // Blocks the reader thread, so the channel and the agent wait too; heartbeats go out on their own timer. + _ = PluginRowStream.deliverBlocking(.rows(batch), to: continuation) } ) } onCancel: { @@ -171,6 +184,22 @@ actor SQLiteAgentBackend: SQLiteExecutionBackend { continuation.finish() } + private func takeTurn() async { + guard statementRunning else { + statementRunning = true + return + } + await withCheckedContinuation { waitingStatements.append($0) } + } + + private func endTurn() { + guard !waitingStatements.isEmpty else { + statementRunning = false + return + } + waitingStatements.removeFirst().resume() + } + // Nil runs the plain execute, so an unrestricted statement sends the frame it always sent. The agent loads no // extension, so a Safe Mode read loses nothing under its read-only scope. static func agentScope(_ rules: SQLiteStatementRules) -> SQLiteAgentStatementScope? { @@ -263,16 +292,18 @@ struct SQLiteAgentExecuteOutcome: Sendable { /// the reader thread's cancel and heartbeat frames never interleave with a partial request. final class SQLiteAgentConnection: @unchecked Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "SQLiteAgentConnection") - private static let heartbeatInterval: TimeInterval = 15 + static let defaultHeartbeatInterval: TimeInterval = 15 private let host: String private let port: Int + private let heartbeatInterval: TimeInterval private let stateLock = NSLock() private var socketFD: Int32 = -1 private var pending: SQLiteAgentOperation? private var terminalFailure: Error? private var closed = false + private var readerOwnsDescriptor = false private let writeLock = NSLock() private var reader = SQLiteAgentFrameReader() @@ -286,14 +317,18 @@ final class SQLiteAgentConnection: @unchecked Sendable { set { versionLock.lock(); _cachedServerVersion = newValue; versionLock.unlock() } } - init(host: String, port: Int) { + init(host: String, port: Int, heartbeatInterval: TimeInterval) { self.host = host self.port = port + self.heartbeatInterval = heartbeatInterval } func connect(token: String) throws { let fd = socket(AF_INET, SOCK_STREAM, 0) guard fd >= 0 else { throw SQLitePluginError.connectionFailed("could not create a local socket") } + // A heartbeat after the session went away gets EPIPE; the default SIGPIPE would end the app. + var noSignal: Int32 = 1 + setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &noSignal, socklen_t(MemoryLayout.size)) var addr = sockaddr_in() addr.sin_family = sa_family_t(AF_INET) @@ -313,11 +348,31 @@ final class SQLiteAgentConnection: @unchecked Sendable { stateLock.lock() socketFD = fd closed = false + readerOwnsDescriptor = false stateLock.unlock() - try writeAll(SQLiteAgentProtocol.admissionPreamble(token: token)) + do { + try writeAll(SQLiteAgentProtocol.admissionPreamble(token: token)) + } catch { + close() + throw error + } + + stateLock.lock() + guard !closed else { + stateLock.unlock() + throw SQLitePluginError.notConnected + } + readerOwnsDescriptor = true + stateLock.unlock() - let thread = Thread { [weak self] in self?.readLoop() } + let thread = Thread { [weak self] in + guard let self else { + Darwin.close(fd) + return + } + self.readLoop(fd: fd) + } thread.name = "com.TablePro.sqlite.agent-reader" thread.stackSize = 512 * 1_024 readerThread = thread @@ -374,19 +429,29 @@ final class SQLiteAgentConnection: @unchecked Sendable { closed = true let fd = socketFD socketFD = -1 + let readerOwnsDescriptor = self.readerOwnsDescriptor + // Under the lock, so a reader on its way out cannot close the number before this reaches it. + if fd >= 0 { shutdown(fd, SHUT_RDWR) } let inflight = pending pending = nil + let timer = heartbeatTimer + heartbeatTimer = nil stateLock.unlock() - heartbeatTimer?.cancel() - heartbeatTimer = nil - if fd >= 0 { - shutdown(fd, SHUT_RDWR) - Darwin.close(fd) + timer?.cancel() + if fd >= 0, !readerOwnsDescriptor { + closeDescriptor(fd) } inflight?.fail(SQLitePluginError.notConnected) } + // After any writer that read the number before it was cleared, so none writes to a reused descriptor. + private func closeDescriptor(_ fd: Int32) { + writeLock.lock() + Darwin.close(fd) + writeLock.unlock() + } + // MARK: - Operation lifecycle private func withOperation( @@ -408,21 +473,28 @@ final class SQLiteAgentConnection: @unchecked Sendable { } stateLock.lock() - if closed { - stateLock.unlock() - continuation.resume(throwing: SQLitePluginError.notConnected) - return - } /// A terminal reply that arrived before this operation registered, such as the launcher's /// "no python3" notice landing before the hello op was set, is delivered to this /// operation rather than lost. The reader thread stores it under the same lock, so the /// two orderings, store-then-register and register-then-route, both reach here correctly. + /// It comes before the closed check, since the reader closes the connection once it stops. if let stored = terminalFailure { terminalFailure = nil stateLock.unlock() operation.fail(stored) return } + if closed { + stateLock.unlock() + continuation.resume(throwing: SQLitePluginError.notConnected) + return + } + // The backend runs one statement at a time; replacing the pending one would hand it another's replies. + guard pending == nil else { + stateLock.unlock() + operation.fail(SQLitePluginError.queryFailed("another statement is still running on the remote SQLite session")) + return + } pending = operation stateLock.unlock() @@ -474,14 +546,14 @@ final class SQLiteAgentConnection: @unchecked Sendable { // MARK: - Reader - private func readLoop() { + // The reader closes its own descriptor on the way out, so it never reads a number something else reused. + private func readLoop(fd: Int32) { var buffer = [UInt8](repeating: 0, count: 32 * 1_024) while true { stateLock.lock() - let fd = socketFD let isClosed = closed stateLock.unlock() - guard !isClosed, fd >= 0 else { break } + guard !isClosed else { break } let count = read(fd, &buffer, buffer.count) if count <= 0 { break } @@ -497,6 +569,9 @@ final class SQLiteAgentConnection: @unchecked Sendable { } } completeOrStoreFailure(SQLitePluginError.notConnected) + // Nothing reads the socket from here on, so a later request would wait for a reply forever. + close() + closeDescriptor(fd) } private func route(_ reply: SQLiteAgentReply) { @@ -530,8 +605,11 @@ final class SQLiteAgentConnection: @unchecked Sendable { // MARK: - Heartbeat private func startHeartbeat() { + stateLock.lock() + defer { stateLock.unlock() } + guard !closed, heartbeatTimer == nil else { return } let timer = DispatchSource.makeTimerSource(queue: DispatchQueue.global()) - timer.schedule(deadline: .now() + Self.heartbeatInterval, repeating: Self.heartbeatInterval) + timer.schedule(deadline: .now() + heartbeatInterval, repeating: heartbeatInterval) timer.setEventHandler { @Sendable [weak self] in self?.sendHeartbeat() } diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentProtocol.swift b/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentProtocol.swift index d7195c7d59..f19c7f8cb8 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentProtocol.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentProtocol.swift @@ -212,9 +212,10 @@ struct SQLiteAgentFrameReader { private var readIndex = 0 private var expectsLauncherNotice = true + // A read rarely ends on a frame boundary, so waiting for an empty buffer kept every byte of a long result. mutating func append(_ data: Data) { - if readIndex > 0, readIndex == buffer.count { - buffer.removeAll(keepingCapacity: true) + if readIndex > 0, readIndex >= buffer.count - readIndex { + buffer.removeSubrange(0.. AsyncThrowingStream { let queryToRun = String(query) - let policy: AsyncThrowingStream.Continuation.BufferingPolicy = backend is SQLiteLocalBackend - ? .bufferingOldest(PluginRowStream.boundedBufferBatches) - : .unbounded - return AsyncThrowingStream(bufferingPolicy: policy) { continuation in + return AsyncThrowingStream(bufferingPolicy: .bufferingOldest(PluginRowStream.boundedBufferBatches)) { continuation in let streamTask = Task { do { try await self.backend.streamQuery(queryToRun, rules: rules, continuation: continuation) diff --git a/apple/Plugins/SQLiteDriverTests/SQLiteAgentLoopback.swift b/apple/Plugins/SQLiteDriverTests/SQLiteAgentLoopback.swift new file mode 100644 index 0000000000..557c9addf7 --- /dev/null +++ b/apple/Plugins/SQLiteDriverTests/SQLiteAgentLoopback.swift @@ -0,0 +1,197 @@ +// +// SQLiteAgentLoopback.swift +// SQLiteDriverTests +// + +import Darwin +import Foundation +import os + +// The SSH session's stand-in: a loopback listener that admits one client by its token line, runs the agent under +// the local python3, and relays each direction on its own thread, so the agent's output can wait on a reader that +// stopped while the client's frames keep reaching it, as over the session pump. When the agent exits, the client's +// socket closes, as the relay's does when its channel closes. +final class SQLiteAgentLoopback: @unchecked Sendable { + static var python3Path: String? { + ["/usr/bin/python3", "/opt/homebrew/bin/python3", "/usr/local/bin/python3"].first { + FileManager.default.isExecutableFile(atPath: $0) + } + } + + // How the launcher runs the shipped program. + static var shippedArguments: [String] { + ["-I", "-S", "-c", RemoteSQLiteAgentSource.interpreterBootstrap, RemoteSQLiteAgentSource.encodedProgram] + } + + let port: Int + let token = UUID().uuidString + + private struct Counters { + var agentBytes = 0 + var heartbeats = 0 + var requests = 0 + var clientClosed = false + } + + private let listenFD: Int32 + private let executable: String + private let arguments: [String] + private let showsErrors: Bool + private let counters = OSAllocatedUnfairLock(initialState: Counters()) + private let process = OSAllocatedUnfairLock(initialState: nil) + + // The local python3 by default; a probe passes ssh and the launcher command to run the agent on a real server, + // and keeps ssh's own errors on standard error. + init( + arguments: [String] = SQLiteAgentLoopback.shippedArguments, + executable: String? = nil, + showsErrors: Bool = false + ) throws { + guard let executable = executable ?? Self.python3Path else { throw LoopbackError.noPython } + self.executable = executable + self.arguments = arguments + self.showsErrors = showsErrors + + let fd = socket(AF_INET, SOCK_STREAM, 0) + guard fd >= 0 else { throw LoopbackError.socket } + var address = sockaddr_in() + address.sin_family = sa_family_t(AF_INET) + address.sin_addr.s_addr = inet_addr("127.0.0.1") + let bound = withUnsafePointer(to: &address) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + bind(fd, $0, socklen_t(MemoryLayout.size)) + } + } + var length = socklen_t(MemoryLayout.size) + let named = withUnsafeMutablePointer(to: &address) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { getsockname(fd, $0, &length) } + } + guard bound == 0, named == 0, listen(fd, 1) == 0 else { + Darwin.close(fd) + throw LoopbackError.socket + } + listenFD = fd + port = Int(UInt16(bigEndian: address.sin_port)) + + Thread.detachNewThread { [self] in acceptOne() } + } + + // Bytes the agent wrote toward the client. + var agentBytes: Int { counters.withLock { $0.agentBytes } } + var heartbeats: Int { counters.withLock { $0.heartbeats } } + // Execute frames the client sent. + var requests: Int { counters.withLock { $0.requests } } + var agentIsRunning: Bool { process.withLock { $0?.isRunning ?? false } } + var clientClosed: Bool { counters.withLock { $0.clientClosed } } + + func killAgent() { + process.withLock { $0?.terminate() } + } + + func close() { + killAgent() + shutdown(listenFD, SHUT_RDWR) + Darwin.close(listenFD) + } + + enum LoopbackError: Error { case noPython, socket } + + private func acceptOne() { + let client = accept(listenFD, nil, nil) + guard client >= 0 else { return } + var noSignal: Int32 = 1 + setsockopt(client, SOL_SOCKET, SO_NOSIGPIPE, &noSignal, socklen_t(MemoryLayout.size)) + guard readLine(client) == "TPRSQL1 \(token)" else { + Darwin.close(client) + return + } + + let toAgent = Pipe() + let fromAgent = Pipe() + let agent = Process() + agent.executableURL = URL(fileURLWithPath: executable) + agent.arguments = arguments + agent.standardInput = toAgent + agent.standardOutput = fromAgent + agent.standardError = showsErrors ? FileHandle.standardError : FileHandle.nullDevice + do { + try agent.run() + } catch { + Darwin.close(client) + return + } + process.withLock { $0 = agent } + let agentInput = toAgent.fileHandleForWriting.fileDescriptor + _ = fcntl(agentInput, F_SETNOSIGPIPE, 1) + let agentOutput = fromAgent.fileHandleForReading.fileDescriptor + + Thread.detachNewThread { [self] in + relayToAgent(from: client, to: agentInput) + toAgent.fileHandleForWriting.closeFile() + Darwin.close(client) + counters.withLock { $0.clientClosed = true } + } + Thread.detachNewThread { [self] in + relayToClient(from: agentOutput, to: client) + shutdown(client, SHUT_RDWR) + } + } + + private func readLine(_ fd: Int32) -> String? { + var line = Data() + var byte: UInt8 = 0 + while line.count < 256 { + guard recv(fd, &byte, 1, 0) == 1 else { return nil } + if byte == 0x0A { return String(data: line, encoding: .utf8) } + line.append(byte) + } + return nil + } + + // Counts each frame by its opcode on the way: 0x04 is a heartbeat, 0x02 and 0x06 an execute. + private func relayToAgent(from client: Int32, to agentInput: Int32) { + var buffer = [UInt8](repeating: 0, count: 32_768) + var frames: [UInt8] = [] + while true { + let count = read(client, &buffer, buffer.count) + guard count > 0 else { return } + frames.append(contentsOf: buffer[0..= 5 { + let length = frames[offset..<(offset + 4)].reduce(0) { ($0 << 8) | Int($1) } + guard frames.count - offset >= 4 + length else { break } + let opcode = frames[offset + 4] + counters.withLock { + if opcode == 0x04 { $0.heartbeats += 1 } + if opcode == 0x02 || opcode == 0x06 { $0.requests += 1 } + } + offset += 4 + length + } + frames.removeFirst(offset) + guard writeAll(buffer[0.. 0 else { return } + counters.withLock { $0.agentBytes += count } + guard writeAll(buffer[0.., to fd: Int32) -> Bool { + bytes.withUnsafeBytes { raw in + guard let base = raw.baseAddress else { return true } + var offset = 0 + while offset < raw.count { + let written = write(fd, base.advanced(by: offset), raw.count - offset) + guard written > 0 else { return false } + offset += written + } + return true + } + } +} diff --git a/apple/Plugins/SQLiteDriverTests/SQLiteAgentProtocolTests.swift b/apple/Plugins/SQLiteDriverTests/SQLiteAgentProtocolTests.swift index 626e1cfc00..de9d2dd4c0 100644 --- a/apple/Plugins/SQLiteDriverTests/SQLiteAgentProtocolTests.swift +++ b/apple/Plugins/SQLiteDriverTests/SQLiteAgentProtocolTests.swift @@ -87,6 +87,28 @@ struct SQLiteAgentProtocolTests { #expect(try reader.nextReply() == .done(changes: 3, truncated: false)) } + // The reader drops what it already decoded between reads, so a long stream must not lose or repeat a byte. + @Test func longStreamSplitAcrossReadsDecodesEveryFrame() throws { + let replies = (0..<300).map { index in + SQLiteAgentReply.rows(columnCount: 2, values: [.text(Data("\(index)".utf8)), .blob(Data(repeating: UInt8(index % 256), count: index * 37))]) + } + var stream = Data() + for reply in replies { stream.append(SQLiteAgentFrameEncoder.encode(reply)) } + + var reader = SQLiteAgentFrameReader() + var decoded: [SQLiteAgentReply] = [] + var offset = 0 + var chunk = 1 + while offset < stream.count { + let end = min(offset + chunk, stream.count) + reader.append(stream.subdata(in: offset.. [String] { + let deadline = "IDLE_DEADLINE_SECONDS = 60.0" + let poll = "time.sleep(5.0)" + let source = RemoteSQLiteAgentSource.python + try #require(source.contains(deadline) && source.contains(poll), "the agent's watchdog constants moved") + let scaled = source + .replacingOccurrences(of: deadline, with: "IDLE_DEADLINE_SECONDS = \(watchdogSeconds)") + .replacingOccurrences(of: poll, with: "time.sleep(0.1)") + return ["-I", "-S", "-c", scaled] + } + + private struct Fixture { + let loopback: SQLiteAgentLoopback + let driver: SQLitePluginDriver + let backend: SQLiteAgentBackend + let path: String + } + + private static func makeDatabase() async throws -> String { + let path = FileManager.default.temporaryDirectory.appendingPathComponent("agent-pacing-\(UUID().uuidString).db").path + let local = SQLitePluginDriver(config: DriverConnectionConfig( + host: "", port: 0, username: "", password: "", database: path + )) + try await local.connect() + _ = try await local.execute(query: """ + CREATE TABLE t AS WITH RECURSIVE g(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM g WHERE x < \(rowCount)) + SELECT x AS id, hex(randomblob(100)) AS payload FROM g + """) + local.disconnect() + return path + } + + // Closes the session, the agent and the file on every way out, so a failed case leaves no python3 running. + private static func withAgent( + arguments: [String] = SQLiteAgentLoopback.shippedArguments, + heartbeat: TimeInterval = heartbeatSeconds, + _ body: (Fixture) async throws -> Void + ) async throws { + let path = try await makeDatabase() + defer { try? FileManager.default.removeItem(atPath: path) } + let loopback = try SQLiteAgentLoopback(arguments: arguments) + defer { loopback.close() } + let fields = [SQLiteAgentProtocol.backendFieldKey: SQLiteAgentProtocol.agentBackendValue] + let backend = SQLiteAgentBackend( + host: "127.0.0.1", + port: loopback.port, + path: path, + token: loopback.token, + helloBudget: try #require(SQLiteAgentHelloBudget.forAgent(additionalFields: fields)), + heartbeatInterval: heartbeat + ) + let driver = SQLitePluginDriver( + config: DriverConnectionConfig( + host: "127.0.0.1", port: loopback.port, username: "", password: "", database: path, + additionalFields: fields + ), + backend: backend + ) + do { + try await driver.connect() + try await body(Fixture(loopback: loopback, driver: driver, backend: backend, path: path)) + } catch { + await backend.close() + throw error + } + await backend.close() + } + + // The agent waits once the bytes it sent stop growing for a while. + private static func waitUntilAgentWaits(_ loopback: SQLiteAgentLoopback) async throws { + var last = -1 + var steadySince = ContinuousClock.now + try await waitUntil(timeout: .seconds(30)) { + let now = loopback.agentBytes + if now != last { + last = now + steadySince = .now + return false + } + return now > 0 && ContinuousClock.now - steadySince > .milliseconds(500) + } + } + + private struct NoAnswer: Error {} + + // A statement queued behind a stream that never ends ignores the time limit's cancellation, so it gets its own. + private static func answer(_ statement: @escaping @Sendable () async throws -> PluginQueryResult) async throws -> String? { + let result = await BoundedCall.result(within: .seconds(30)) { + do { + return Result.success(try await statement()) + } catch { + return .failure(error) + } + } + guard let result else { throw NoAnswer() } + return try result.get().rows.first?.first?.asText + } + + private static func ids(_ batch: [PluginRow]) -> [Int] { + batch.compactMap { $0.first?.asText.flatMap(Int.init) } + } + + // Counts the rows taken and the ids that are not the next one expected. + private struct OrderedRead { + var expected = 1 + var mismatches = 0 + + mutating func take(_ element: PluginStreamElement) { + guard case .rows(let batch) = element else { return } + for id in SQLiteAgentStreamPacingTests.ids(batch) { + if id != expected { mismatches += 1 } + expected += 1 + } + } + } + + @Test("A reader held past the agent's watchdog still gets every row, in order, on a live session") + func heldReaderOutlastsTheWatchdog() async throws { + try await Self.withAgent(arguments: try Self.scaledProgram()) { fixture in + var iterator = fixture.driver.streamRows(query: Self.scan).makeAsyncIterator() + var read = OrderedRead() + while read.expected == 1, let element = try await iterator.next() { read.take(element) } + + try await Self.waitUntilAgentWaits(fixture.loopback) + // Long enough for the watchdog to have ended the session twice over without heartbeats. + let heartbeatsBeforeHold = fixture.loopback.heartbeats + let holdHeartbeats = Int(Self.watchdogSeconds * 2.5 / Self.heartbeatSeconds) + try await waitUntil(timeout: .seconds(60)) { + fixture.loopback.heartbeats - heartbeatsBeforeHold >= holdHeartbeats + } + #expect(fixture.loopback.agentIsRunning) + let heldBytes = fixture.loopback.agentBytes + + while let element = try await iterator.next() { read.take(element) } + #expect(read.expected - 1 == Self.rowCount) + #expect(read.mismatches == 0) + #expect(heldBytes < Self.bufferedBytesLimit, "the agent sent \(heldBytes) of \(fixture.loopback.agentBytes) bytes by the end of the hold") + + let driver = fixture.driver + #expect(try await Self.answer { try await driver.execute(query: "SELECT count(*) FROM t") } == String(Self.rowCount)) + } + } + + @Test("Without heartbeats the same hold outlasts the watchdog and the read fails") + func holdWithoutHeartbeatsLosesTheSession() async throws { + try await Self.withAgent(arguments: try Self.scaledProgram(), heartbeat: 3_600) { fixture in + var iterator = fixture.driver.streamRows(query: Self.scan).makeAsyncIterator() + var read = OrderedRead() + // On a loaded machine the watchdog can end the session before the first batch arrives. + await #expect(throws: (any Error).self) { + while read.expected == 1, let element = try await iterator.next() { read.take(element) } + try await waitUntil(timeout: .seconds(30)) { !fixture.loopback.agentIsRunning } + while let element = try await iterator.next() { read.take(element) } + } + #expect(!fixture.loopback.agentIsRunning) + #expect(read.expected - 1 < Self.rowCount) + } + } + + @Test("A reader that leaves stops the agent's read, and the next statement gets its own answer") + func abandonedReadStopsTheAgent() async throws { + try await Self.withAgent { fixture in + let bytesBeforeRead = fixture.loopback.agentBytes + var iterator: AsyncThrowingStream.Iterator? = + fixture.driver.streamRows(query: Self.scan).makeAsyncIterator() + var rows = 0 + while rows == 0, let element = try await iterator?.next() { + if case .rows(let batch) = element { rows += batch.count } + } + try await Self.waitUntilAgentWaits(fixture.loopback) + iterator = nil + + let driver = fixture.driver + #expect(try await Self.answer { try await driver.execute(query: "SELECT 42") } == "42") + #expect(fixture.loopback.agentIsRunning) + let readBytes = fixture.loopback.agentBytes - bytesBeforeRead + #expect(readBytes < Self.bufferedBytesLimit, "the agent sent \(readBytes) bytes") + } + } + + @Test("A statement sent while a stream is open waits for it instead of taking its rows") + func statementWaitsForTheOpenStream() async throws { + try await Self.withAgent { fixture in + var iterator = fixture.driver.streamRows(query: Self.scan).makeAsyncIterator() + var read = OrderedRead() + while read.expected == 1, let element = try await iterator.next() { read.take(element) } + + let driver = fixture.driver + let statement = Task { try await driver.execute(query: "SELECT 42") } + try await Self.waitUntilAgentWaits(fixture.loopback) + #expect(fixture.loopback.requests == 1, "the statement went out while the stream was open") + + while let element = try await iterator.next() { read.take(element) } + #expect(read.expected - 1 == Self.rowCount) + #expect(read.mismatches == 0) + #expect(try await Self.answer { try await statement.value } == "42") + } + } + + @Test("An agent that goes away fails the next statement at once instead of ending the app or hanging") + func agentGoneFailsTheNextStatement() async throws { + try await Self.withAgent(heartbeat: 0.05) { fixture in + _ = try await fixture.driver.execute(query: "SELECT 1") + + fixture.loopback.killAgent() + try await waitUntil { fixture.loopback.clientClosed } + + // The second write to a closed socket raised SIGPIPE, and a request nothing would read waited forever. + let driver = fixture.driver + for _ in 0..<3 { + let failed = await BoundedCall.result(within: .seconds(5)) { + do { + _ = try await driver.execute(query: "SELECT 1") + return false + } catch { + return true + } + } + #expect(failed == true) + } + } + } +} diff --git a/apple/project.yml b/apple/project.yml index c5848b3bd9..2a14130d72 100644 --- a/apple/project.yml +++ b/apple/project.yml @@ -839,9 +839,12 @@ targets: - path: Plugins/SQLExportPlugin excludes: - Info.plist + # The agent the app launches over SSH, so the agent backend's tests run the shipped program. + - TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift + - TablePro/Core/RemoteSQLite/RemoteSQLiteWire.swift dependencies: - package: TableProCore - products: [TableProProcess, TableProSQLGrammar] + products: [TableProProcess, TableProSQLGrammar, TableProTestSupport] ClickHouseDriverTests: templates: [PluginTests, ClickHouseLinkage] diff --git a/apple/scripts/probes/check-stream-backpressure.sh b/apple/scripts/probes/check-stream-backpressure.sh index c14dba646c..b97ad95e91 100755 --- a/apple/scripts/probes/check-stream-backpressure.sh +++ b/apple/scripts/probes/check-stream-backpressure.sh @@ -8,15 +8,20 @@ # without it holds every row the consumer has not taken yet. # # Usage: -# scripts/probes/check-stream-backpressure.sh [--rows N] [--delay-ms N] [--root DIR] +# scripts/probes/check-stream-backpressure.sh [--rows N] [--delay-ms N] [--hold-seconds N] [--root DIR] # -# engine: postgresql, mysql, sqlite (a scratch file, no server) +# engine: postgresql, mysql, sqlite (a scratch file, no server), sqlite-agent (the same file read through the +# remote SQLite agent, run by the local python3 behind a loopback relay that stands in for the SSH session) # --rows rows in the probe table, default 1000000 # --delay-ms the consumer's wait per batch, default 20 +# --hold-seconds how long the consumer waits after its first batch, default 0 # --root the apple/ folder whose sources to build, default this one; Libs comes from it too # # TP_STREAM_HOST, TP_STREAM_PORT, TP_STREAM_USER, TP_STREAM_PASSWORD and TP_STREAM_DATABASE name the server, by # default a local one on the engine's port as the current user. TP_STREAM_BUILD_DIR keeps the harness between runs. +# For sqlite-agent, TP_STREAM_SSH holds ssh's arguments up to the destination, separated by single spaces with no +# quoting: the agent then runs on that server through the launcher command the app sends. TP_STREAM_DATABASE must then +# name an existing scratch file there; the probe fills it with the table and the file keeps that size afterwards. # Exits 1 when the read fails, 3 when the probe cannot run. set -uo pipefail @@ -25,11 +30,14 @@ ENGINE="${1:-}" shift || true ROWS=1000000 DELAY_MS=20 +HOLD_SECONDS=0 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +HERE="$ROOT" while [ $# -gt 0 ]; do case "$1" in --rows) ROWS="${2:?}"; shift 2 ;; --delay-ms) DELAY_MS="${2:?}"; shift 2 ;; + --hold-seconds) HOLD_SECONDS="${2:?}"; shift 2 ;; --root) ROOT="$(cd "${2:?}" && pwd)"; shift 2 ;; *) echo "unknown argument $1" >&2; exit 3 ;; esac @@ -52,14 +60,14 @@ case "$ENGINE" in digits="(SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9)" CREATE="CREATE TABLE tp_stream_probe AS SELECT id, REPEAT(MD5(id), 6) AS payload FROM (SELECT d0.n + d1.n * 10 + d2.n * 100 + d3.n * 1000 + d4.n * 10000 + d5.n * 100000 + d6.n * 1000000 + 1 AS id FROM $digits d0, $digits d1, $digits d2, $digits d3, $digits d4, $digits d5, $digits d6) AS numbers WHERE id <= $ROWS" ;; - sqlite) + sqlite | sqlite-agent) FOLDER=SQLiteDriverPlugin CLASS=SQLitePlugin C_MODULE="" PRODUCTS="CSQLite TableProSQLiteCore TableProSQLQuoting TableProSQLVocabulary" LINK="$ROOT/Libs/libsqlite3_vendored.a" CREATE="CREATE TABLE tp_stream_probe AS WITH RECURSIVE g(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM g WHERE x < $ROWS) SELECT x AS id, hex(randomblob(96)) AS payload FROM g" ;; *) - echo "usage: $0 [--rows N] [--delay-ms N] [--root DIR]" >&2 + echo "usage: $0 [--rows N] [--delay-ms N] [--hold-seconds N] [--root DIR]" >&2 exit 3 ;; esac @@ -88,9 +96,15 @@ fi SOURCES="$WORK/Sources/Probe" rm -rf "$SOURCES" "$WORK/CModule" mkdir -p "$SOURCES" -if [ "$ENGINE" = sqlite ] && [ -z "$DATABASE" ]; then +if [ "$ENGINE" = sqlite-agent ] && [ -n "${TP_STREAM_SSH:-}" ] && [ -z "${TP_STREAM_DATABASE:-}" ]; then + echo "TP_STREAM_SSH needs TP_STREAM_DATABASE, a scratch file on that server" >&2 + exit 3 +fi +if [ "$FOLDER" = SQLiteDriverPlugin ] && [ -z "$DATABASE" ]; then DATABASE="$WORK/probe.sqlite" rm -f "$DATABASE" "$DATABASE-wal" "$DATABASE-shm" + # The agent opens a file that exists and never creates one. + if [ "$ENGINE" = sqlite-agent ]; then : > "$DATABASE"; fi fi while IFS= read -r source; do @@ -99,6 +113,13 @@ done < <(find "$PLUGINS/$FOLDER" -name '*.swift' -not -path '*/Tests/*') for shared in PluginConnectTimeout PluginQueryTimeout PluginURLSessionTaskBox; do [ -f "$PLUGINS/Shared/$shared.swift" ] && ln -s "$PLUGINS/Shared/$shared.swift" "$SOURCES/shared_$shared.swift" done +# The agent program and the relay come from this script's checkout, so --root can name a tree from before the relay. +if [ "$ENGINE" = sqlite-agent ]; then + for helper in TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift TablePro/Core/RemoteSQLite/RemoteSQLiteWire.swift \ + Plugins/SQLiteDriverTests/SQLiteAgentLoopback.swift; do + ln -s "$HERE/$helper" "$SOURCES/agent_$(basename "$helper")" + done +fi dependencies='.product(name: "TableProPluginKit", package: "TableProCore"), .product(name: "TableProSQLGrammar", package: "TableProCore"), .product(name: "TableProProcess", package: "TableProCore"),' @@ -149,9 +170,47 @@ let package = Package( ) MANIFEST -cat > "$SOURCES/probe_Engine.swift" << SWIFT +if [ "$ENGINE" = sqlite-agent ]; then + cat > "$SOURCES/probe_Engine.swift" << 'SWIFT' +import Foundation +import TableProPluginKit + +typealias EnginePlugin = SQLitePlugin + +enum ProbeEndpoint { + static func config(host: String, port: Int, username: String, password: String, database: String) throws -> DriverConnectionConfig { + let ssh = ProcessInfo.processInfo.environment["TP_STREAM_SSH"].map { $0.split(separator: " ").map(String.init) } + let loopback = try ssh.map { + // -T: no pty, as on the app's exec channel; a pty would rewrite the binary frames. + try SQLiteAgentLoopback( + arguments: ["-T"] + $0 + [RemoteSQLiteAgentSource.launcherCommand()], executable: "/usr/bin/ssh", showsErrors: true + ) + } ?? SQLiteAgentLoopback() + return DriverConnectionConfig( + host: "127.0.0.1", port: loopback.port, username: "", password: "", database: database, + additionalFields: [ + SQLiteAgentProtocol.backendFieldKey: SQLiteAgentProtocol.agentBackendValue, + SQLiteAgentProtocol.tokenFieldKey: loopback.token, + ] + ) + } +} +SWIFT +else + cat > "$SOURCES/probe_Engine.swift" << SWIFT +import TableProPluginKit + typealias EnginePlugin = $CLASS + +enum ProbeEndpoint { + static func config(host: String, port: Int, username: String, password: String, database: String) throws -> DriverConnectionConfig { + DriverConnectionConfig( + host: host, port: port, username: username, password: password, database: database, additionalFields: [:] + ) + } +} SWIFT +fi cat > "$SOURCES/probe_Probe.swift" << 'SWIFT' import Darwin @@ -162,14 +221,21 @@ import TableProPluginKit enum Probe { static func main() async { let arguments = CommandLine.arguments - guard arguments.count >= 8, let delay = Int(arguments[7]) else { - print("usage: Probe ") + guard arguments.count >= 9, let delay = Int(arguments[7]), let hold = Int(arguments[8]) else { + print("usage: Probe ") + exit(3) + } + let config: DriverConnectionConfig + do { + config = try ProbeEndpoint.config( + host: arguments[1], port: Int(arguments[2]) ?? 0, username: arguments[3], password: arguments[4], + database: arguments[5] + ) + } catch { + print("cannot reach the engine: \(error)") exit(3) } - let driver = EnginePlugin().createDriver(config: DriverConnectionConfig( - host: arguments[1], port: Int(arguments[2]) ?? 0, username: arguments[3], password: arguments[4], - database: arguments[5], additionalFields: [:] - )) + let driver = EnginePlugin().createDriver(config: config) do { try await driver.connect() try await driver.applyQueryTimeout(0) @@ -191,7 +257,7 @@ enum Probe { guard case .rows(let batch) = element else { continue } rows += batch.count batches += 1 - try await Task.sleep(for: .milliseconds(delay)) + try await Task.sleep(for: .milliseconds(batches == 1 ? delay + hold * 1_000 : delay)) } } catch { failure = error @@ -259,4 +325,4 @@ echo "building the $ENGINE harness from $ROOT (the first build takes a few minut grep -E "error:" "$WORK/build.log" | head -30 >&2 exit 3 } -"$WORK/.build/release/Probe" "$HOST" "$PORT" "$USER_NAME" "$PASSWORD" "$DATABASE" "$CREATE" "$DELAY_MS" +"$WORK/.build/release/Probe" "$HOST" "$PORT" "$USER_NAME" "$PASSWORD" "$DATABASE" "$CREATE" "$DELAY_MS" "$HOLD_SECONDS" From 0ddfa9296782ad0ba13f57cb11aed92963a58387 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Fri, 13 Nov 2026 18:16:49 +0700 Subject: [PATCH 2/2] docs(changelog): add the Remote SQLite pacing and session fixes --- apple/changelog.d/4265.fixed.1.md | 1 + apple/changelog.d/4265.fixed.2.md | 1 + apple/changelog.d/4265.fixed.md | 1 + 3 files changed, 3 insertions(+) create mode 100644 apple/changelog.d/4265.fixed.1.md create mode 100644 apple/changelog.d/4265.fixed.2.md create mode 100644 apple/changelog.d/4265.fixed.md diff --git a/apple/changelog.d/4265.fixed.1.md b/apple/changelog.d/4265.fixed.1.md new file mode 100644 index 0000000000..b70cf7fbee --- /dev/null +++ b/apple/changelog.d/4265.fixed.1.md @@ -0,0 +1 @@ +TablePro could quit when the SQLite session on an SSH server ended while its connection stayed open. diff --git a/apple/changelog.d/4265.fixed.2.md b/apple/changelog.d/4265.fixed.2.md new file mode 100644 index 0000000000..fde3bf2565 --- /dev/null +++ b/apple/changelog.d/4265.fixed.2.md @@ -0,0 +1 @@ +A query on a SQLite file on an SSH server could wait forever after the session on the server ended. diff --git a/apple/changelog.d/4265.fixed.md b/apple/changelog.d/4265.fixed.md new file mode 100644 index 0000000000..538172a465 --- /dev/null +++ b/apple/changelog.d/4265.fixed.md @@ -0,0 +1 @@ +Copying, comparing or exporting a large table from a SQLite file on an SSH server held the whole table in memory.