diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantScrollPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantScrollPager.swift index 28c0f493e8..32b5413c65 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantScrollPager.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantScrollPager.swift @@ -53,36 +53,17 @@ public extension QdrantClient { ).points } - // Without vectors this reads payloads only, which is how an export learns every key before its header. + // Without vectors this reads payloads only, which is how an export learns every key before its header. A page is + // read only when the reader asks for the next one. func pointPages( collection: String, pageSize: Int, withVector: Bool = true ) -> AsyncThrowingStream<[QdrantPoint], Error> { - AsyncThrowingStream { continuation in - let task = Task { - do { - var cursor: Any? - repeat { - try Task.checkCancellation() - let page = try await self.scroll( - collection: collection, - body: Self.scrollBody( - filter: nil, limit: max(pageSize, 1), cursor: cursor, withData: true, withVector: withVector - ) - ) - if !page.points.isEmpty { - continuation.yield(page.points) - } - cursor = page.nextCursor - } while cursor != nil - continuation.finish() - } catch { - continuation.finish(throwing: error) - } - } - continuation.onTermination = { @Sendable _ in task.cancel() } - } + let pager = QdrantPointPager( + client: self, collection: collection, pageSize: max(pageSize, 1), withVector: withVector + ) + return AsyncThrowingStream(unfolding: { try await pager.next() }) } static func scrollBody( @@ -106,3 +87,38 @@ public extension QdrantClient { return body } } + +// One reader pulls the pages, so one call at a time reaches the cursor. +private final class QdrantPointPager: @unchecked Sendable { + private let client: QdrantClient + private let collection: String + private let pageSize: Int + private let withVector: Bool + private var cursor: Any? + private var isDone = false + + init(client: QdrantClient, collection: String, pageSize: Int, withVector: Bool) { + self.client = client + self.collection = collection + self.pageSize = pageSize + self.withVector = withVector + } + + func next() async throws -> [QdrantPoint]? { + while !isDone { + try Task.checkCancellation() + let page = try await client.scroll( + collection: collection, + body: QdrantClient.scrollBody( + filter: nil, limit: pageSize, cursor: cursor, withData: true, withVector: withVector + ) + ) + cursor = page.nextCursor + isDone = cursor == nil + if !page.points.isEmpty { + return page.points + } + } + return nil + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginRowStreamTerminationTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginRowStreamTerminationTests.swift new file mode 100644 index 0000000000..8797f7a68b --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginRowStreamTerminationTests.swift @@ -0,0 +1,101 @@ +import Foundation +import TableProPluginKit +import Testing + +struct PluginRowStreamTerminationTests { + private static let header = PluginStreamHeader(columns: ["c"], columnTypeNames: ["TEXT"]) + + @Test("A stream its producer finished runs no abort action, even once the consumer lets it go") + func finishedStreamRunsNoAbortAction() async throws { + let probe = AbortProbe() + var stream: AsyncThrowingStream? = PluginRowStream.make { continuation, abort in + probe.watch(abort) + continuation.yield(.header(Self.header)) + continuation.yield(.rows([[.text("a")]])) + continuation.finish() + } + var elements = 0 + if let stream { + for try await _ in stream { elements += 1 } + } + stream = nil + + #expect(elements == 2) + #expect(probe.actions == 0) + #expect(probe.abort?.isAborted == false) + } + + @Test("A stream its producer finished with an error runs no abort action") + func failedStreamRunsNoAbortAction() async { + let probe = AbortProbe() + var stream: AsyncThrowingStream? = PluginRowStream.make { continuation, abort in + probe.watch(abort) + continuation.yield(.header(Self.header)) + continuation.finish(throwing: ProducerFailure()) + } + await #expect(throws: ProducerFailure.self) { + if let stream { + for try await _ in stream {} + } + } + stream = nil + + #expect(probe.actions == 0) + } + + @Test("A consumer that lets the stream go before its end runs the abort actions") + func droppedStreamRunsTheAbortActions() async throws { + let probe = AbortProbe() + var stream: AsyncThrowingStream? = PluginRowStream.make { continuation, abort in + probe.watch(abort) + continuation.yield(.header(Self.header)) + } + var iterator = stream?.makeAsyncIterator() + _ = try await iterator?.next() + iterator = nil + stream = nil + + #expect(probe.actions == 1) + #expect(probe.abort?.isAborted == true) + } + + @Test("A consumer task cancelled while it waits runs the abort actions") + func cancelledConsumerRunsTheAbortActions() async throws { + let probe = AbortProbe() + let stream = PluginRowStream.make { continuation, abort in + probe.watch(abort) + continuation.yield(.header(Self.header)) + } + let reader = Task { + for try await _ in stream {} + } + reader.cancel() + _ = try? await reader.value + + #expect(probe.actions == 1) + _ = stream + } +} + +private struct ProducerFailure: Error {} + +private final class AbortProbe: @unchecked Sendable { + private let lock = NSLock() + private var watched: PluginStreamAbort? + private var count = 0 + + var abort: PluginStreamAbort? { + lock.withLock { watched } + } + + var actions: Int { + lock.withLock { count } + } + + func watch(_ abort: PluginStreamAbort) { + lock.withLock { watched = abort } + abort.onAbort { [self] in + lock.withLock { count += 1 } + } + } +} diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift index 15758e58f6..fc53cd4a15 100644 --- a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift @@ -1004,6 +1004,16 @@ public final class OracleCoreConnection: @unchecked Sendable { _ query: String, batching: OracleRowBatcher, continuation: AsyncThrowingStream.Continuation + ) async throws { + try await streamQuery(query, batching: batching, onElement: { continuation.yield($0) }) + continuation.finish() + } + + // The next row is read only once `onElement` returns, so a caller that waits in it holds the statement open. + public func streamQuery( + _ query: String, + batching: OracleRowBatcher, + onElement: @escaping @Sendable (OracleStreamElement) async -> Void ) async throws { let role = OracleTransactionRole(of: query) await queryGate.acquire() @@ -1022,13 +1032,12 @@ public final class OracleCoreConnection: @unchecked Sendable { regions: regions, batching: batching, on: connection, - continuation: continuation + onElement: onElement ) } await recordSuccess(of: role, on: admitted.session) forgetSessionZonesIfChanged(by: query) await queryGate.release() - continuation.finish() } catch { let mapped = mapExecutionError(error) forgetSessionZonesIfChanged(by: query) @@ -1045,33 +1054,33 @@ public final class OracleCoreConnection: @unchecked Sendable { regions: OracleTimeZoneRegions, batching: OracleRowBatcher, on connection: OracleNIO.OracleConnection, - continuation: AsyncThrowingStream.Continuation + onElement: @escaping @Sendable (OracleStreamElement) async -> Void ) async throws { let statement = OracleStatement(unsafeSQL: query) let stream = try await connection.execute(statement, options: options, logger: nioLogger) let columns = stream.columns.map(OracleResultColumn.init) var decoder = CellDecoder(columns: columns, zones: zones, regions: regions) - continuation.yield(.header(columns: columns.map(\.descriptor))) + await onElement(.header(columns: columns.map(\.descriptor))) var batcher = batching do { for try await row in stream { try Task.checkCancellation() if let batch = batcher.append(decoder.decode(row)) { - continuation.yield(.rows(batch)) + await onElement(.rows(batch)) } } } catch { // The rows read before a failure still reach the reader ahead of the error, as they did one by one. - if let batch = batcher.remainder() { continuation.yield(.rows(batch)) } + if let batch = batcher.remainder() { await onElement(.rows(batch)) } throw error } - if let batch = batcher.remainder() { continuation.yield(.rows(batch)) } + if let batch = batcher.remainder() { await onElement(.rows(batch)) } await warnOnce(about: decoder.unsupportedTypeNames) guard columns.isEmpty else { return } - continuation.yield(.affectedRows(try await stream.affectedRows)) + await onElement(.affectedRows(try await stream.affectedRows)) } // MARK: - Session Time Zones diff --git a/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift b/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift index 661f20f618..20e586fa24 100644 --- a/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift +++ b/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift @@ -451,10 +451,11 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in - Task { + PluginRowStream.make { continuation, abort in + let task = Task { do { - let result = try await perform { [self] () -> PluginQueryResult in + try Task.checkCancellation() + let result = try await self.perform { [self] () -> PluginQueryResult in if let bql = Self.extractBQLQuery(from: query) { return try executeBQL(query: bql) } @@ -473,6 +474,7 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { continuation.finish(throwing: error) } } + abort.onAbort { task.cancel() } } } diff --git a/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver+Execution.swift b/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver+Execution.swift index 10e7bb644d..64979b93ea 100644 --- a/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver+Execution.swift +++ b/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver+Execution.swift @@ -83,7 +83,7 @@ extension BigQueryPluginDriver { } func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let streamTask = Task { do { try await self.runningStatements.run { @@ -93,9 +93,7 @@ extension BigQueryPluginDriver { continuation.finish(throwing: BigQueryError.wrap(error)) } } - continuation.onTermination = { @Sendable _ in - streamTask.cancel() - } + abort.onAbort { streamTask.cancel() } } } @@ -260,7 +258,7 @@ extension BigQueryPluginDriver { let firstRows = BigQueryTypeMapper.flattenRows(from: firstPage, schema: schema) if !firstRows.isEmpty { - continuation.yield(.rows(firstRows)) + _ = await PluginRowStream.deliver(.rows(firstRows), to: continuation) } var pageToken = firstPage.pageToken @@ -273,7 +271,7 @@ extension BigQueryPluginDriver { ) let nextRows = BigQueryTypeMapper.flattenRows(from: nextPage, schema: schema) if !nextRows.isEmpty { - continuation.yield(.rows(nextRows)) + _ = await PluginRowStream.deliver(.rows(nextRows), to: continuation) } pageToken = nextPage.pageToken } diff --git a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift index e362dfd30d..71d84b3060 100644 --- a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift +++ b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift @@ -149,7 +149,7 @@ extension BigtablePluginDriver { // MARK: - Export func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let task = Task { do { if case .browse(let spec)? = try? BigtableStatement.parse(query), spec.limit == 0 { @@ -168,7 +168,7 @@ extension BigtablePluginDriver { continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in task.cancel() } + abort.onAbort { task.cancel() } } } @@ -190,7 +190,7 @@ extension BigtablePluginDriver { ))) } if !rows.isEmpty { - continuation.yield(.rows(page.rows(alignedTo: columns).map { $0.map(Self.cell) })) + _ = await PluginRowStream.deliver(.rows(page.rows(alignedTo: columns).map { $0.map(Self.cell) }), to: continuation) } after = rows.count == Self.exportPageSize ? rows.last?.key : nil } while after != nil diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraConnection.swift b/apple/Plugins/CassandraDriverPlugin/CassandraConnection.swift index effc22404e..e3b4a7d825 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraConnection.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraConnection.swift @@ -959,10 +959,8 @@ actor CassandraConnectionActor { return "Unknown error" } - /// The whole loop is one non-suspending actor job holding the cooperative-pool thread, and - /// cass_future_wait blocks it, so nothing outside can interrupt a page. `abort` is the only - /// way the consumer's departure reaches this body, and it is read between pages and between - /// rows rather than mid-request. + // One non-suspending actor job, blocked in cass_future_wait and in `deliverBlocking`, so the consumer's departure + // reaches it only through `abort`, read between pages and between rows rather than mid-request. func streamQuery( _ cql: String, consistency: CassandraConsistency?, @@ -1040,6 +1038,7 @@ actor CassandraConnectionActor { let colCount = cass_result_column_count(result) let iterator = cass_iterator_from_result(result) + var page: [PluginRow] = [] if let iterator { var rowsThisPage = 0 @@ -1051,7 +1050,7 @@ actor CassandraConnectionActor { } let row = cass_iterator_get_row(iterator) guard let row else { continue } - continuation.yield(.rows([Self.decodeRow(row, columnCount: colCount)])) + page.append(Self.decodeRow(row, columnCount: colCount)) } cass_iterator_free(iterator) } @@ -1064,6 +1063,10 @@ actor CassandraConnectionActor { cass_result_free(result) + if !page.isEmpty, !aborted { + _ = PluginRowStream.deliverBlocking(.rows(page), to: continuation) + } + if !hasMore || aborted { break } } diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift b/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift index 00f92dc5b0..f11d9e7f63 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift @@ -483,7 +483,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen return streamBrowse(browse, consistency: consistency) } let cql = stripTrailingSemicolon(query) - return PluginRowStream.make { continuation, abort in + return PluginRowStream.makeBounded { continuation, abort in let streamTask = Task { do { try await self.connectionActor.streamQuery( @@ -494,7 +494,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen continuation.finish(throwing: error) } } - _ = streamTask + abort.onAbort { streamTask.cancel() } } } diff --git a/apple/Plugins/ClickHouseDriverPlugin/ClickHouseHTTPChunks.swift b/apple/Plugins/ClickHouseDriverPlugin/ClickHouseHTTPChunks.swift index 704497b82a..c2e1f90f3c 100644 --- a/apple/Plugins/ClickHouseDriverPlugin/ClickHouseHTTPChunks.swift +++ b/apple/Plugins/ClickHouseDriverPlugin/ClickHouseHTTPChunks.swift @@ -89,14 +89,14 @@ nonisolated internal final class ClickHouseHTTPChunks: NSObject, URLSessionDataD } guard !isFailure else { return } - continuation.yield(data) - let shouldSuspend = lock.withLock { () -> Bool in - guard !isTransferSuspended else { return false } + // Suspended before the reader sees the chunk: a reader done with a short chunk first would find nothing to + // resume, and the suspend after it would stop the transfer for good. + lock.withLock { + guard !isTransferSuspended else { return } isTransferSuspended = true - return true + dataTask.suspend() } - guard shouldSuspend else { return } - dataTask.suspend() + continuation.yield(data) } internal func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) { diff --git a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift index 72389cdf12..4fd1021ed5 100644 --- a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift +++ b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift @@ -340,14 +340,11 @@ extension DynamoDBPluginDriver { // MARK: - Streaming - /// Streams every item a read returns, for an export, a copy or a compare. - /// - /// A stream states its columns once, before its first row, and a DynamoDB item may carry an - /// attribute no earlier item had. The items are therefore written to a temporary file while the - /// union of their attributes is collected, and the rows are sent once the read has finished, so - /// an attribute first seen on the last page still gets its column. + // A stream states its columns before its first row, and an item may carry an attribute no earlier item had, so + // the items go to a temporary file while their attributes are collected and the rows follow once the read ends. + // The rows are read back from the file only as fast as the consumer takes them. func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let task = Task { do { try await self.run { session in @@ -358,7 +355,7 @@ extension DynamoDBPluginDriver { continuation.finish(throwing: error) } } - continuation.onTermination = { _ in task.cancel() } + abort.onAbort { task.cancel() } } } @@ -415,8 +412,9 @@ extension DynamoDBPluginDriver { } ?? sorted.count let slice = source.window.offset < end ? Array(sorted[source.window.offset.. AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let streamTask = Task { do { try await self.performStreamRows(query: query, context: context, continuation: continuation) @@ -194,9 +194,7 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } - continuation.onTermination = { @Sendable _ in - streamTask.cancel() - } + abort.onAbort { streamTask.cancel() } } } @@ -267,20 +265,32 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } continuation.yield(.header(header)) + // The scan hands each batch over synchronously, so a batch goes out just before the next is read, where the + // read can wait for the consumer. + var unsent: [PluginRow] = [] do { - try await scanKeyValues(in: range, client: client) { kvs in - let rows = kvs.map(Self.filterRow).filter(matcher.matches).map(\.cells) - if !rows.isEmpty { - continuation.yield(.rows(rows)) + try await scanKeyValues( + in: range, + client: client, + beforeNextBatch: { + guard !unsent.isEmpty else { return } + _ = await PluginRowStream.deliver(.rows(unsent), to: continuation) + unsent = [] + }, + body: { kvs in + unsent = kvs.map(Self.filterRow).filter(matcher.matches).map(\.cells) + return true } - return true - } + ) } catch let error where error.isEtcdRevisionCompacted { // Rows already went out, so the read cannot start over at a newer revision without repeating them. throw EtcdError.serverError(String( localized: "etcd compacted the revision this export started from, so it stopped part way. Run it again." )) } + if !unsent.isEmpty { + _ = await PluginRowStream.deliver(.rows(unsent), to: continuation) + } continuation.finish() } @@ -1001,6 +1011,7 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private func scanKeyValues( in range: EtcdKeyRange, client: EtcdHttpClient, + beforeNextBatch: () async -> Void = {}, body: ([EtcdKeyValue]) throws -> Bool ) async throws { try await EtcdRangeScan.scan( @@ -1009,6 +1020,7 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { maxItems: Self.maxKeys, key: { $0.key }, fetch: { range, limit, revision in + await beforeNextBatch() var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: limit) req.revision = revision let response = try await client.rangeRequest(req) diff --git a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Execution.swift b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Execution.swift index 682b0e8057..e5715c3b6a 100644 --- a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Execution.swift +++ b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Execution.swift @@ -235,7 +235,7 @@ extension FirebasePluginDriver { // MARK: - Export func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let task = Task { do { if case .browse(let spec)? = try? FirebaseStatement.parse(query), spec.limit == 0 { @@ -251,7 +251,7 @@ extension FirebasePluginDriver { continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in task.cancel() } + abort.onAbort { task.cancel() } } } @@ -265,7 +265,7 @@ extension FirebasePluginDriver { continuation.yield(.header(PluginStreamHeader(columns: header ?? [], columnTypeNames: types, estimatedRowCount: nil))) } let fixed = header ?? [] - continuation.yield(.rows(rows.map { row in Self.align(row, columns: columns, to: fixed) })) + _ = await PluginRowStream.deliver(.rows(rows.map { row in Self.align(row, columns: columns, to: fixed) }), to: continuation) } if header == nil { continuation.yield(.header(PluginStreamHeader(columns: [Self.extraColumn], columnTypeNames: ["map"], estimatedRowCount: nil))) @@ -283,7 +283,7 @@ extension FirebasePluginDriver { private func forEachExportPage( _ spec: FirebaseBrowseSpec, - page: ([String], [String], [[PluginCellValue]]) throws -> Void + page: ([String], [String], [[PluginCellValue]]) async throws -> Void ) async throws { switch variant { case .firestore: @@ -310,13 +310,13 @@ extension FirebasePluginDriver { operation: String(format: String(localized: "export %@"), spec.table)) guard !documents.isEmpty else { return } let rows = FirestoreRowSet(documents: documents, target: target) - try page(rows.columns, rows.columnTypes, rows.rows.map { $0.map(Self.cell) }) + try await page(rows.columns, rows.columnTypes, rows.rows.map { $0.map(Self.cell) }) cursor = documents.count == Self.exportPageSize ? documents.last?.path : nil } while cursor != nil case .realtimeDatabase: guard spec.filters.isEmpty else { let result = try await browse(spec) - try page(result.columns, result.columnTypeNames, result.rows) + try await page(result.columns, result.columnTypeNames, result.rows) return } let client = try requireRealtimeDatabase() @@ -331,7 +331,7 @@ extension FirebasePluginDriver { let children = try await client.children(node, plan: plan) guard !children.isEmpty else { return } let rows = RealtimeDatabaseRowSet(children: children, nodePath: node) - try page(rows.columns, rows.columnTypes, rows.rows.map { $0.map(PluginCellValue.fromOptional) }) + try await page(rows.columns, rows.columnTypes, rows.rows.map { $0.map(PluginCellValue.fromOptional) }) lastKey = children.count == Self.exportPageSize ? children.last?.key : nil } while lastKey != nil } diff --git a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift index 9e670008b7..673794c1e0 100644 --- a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift +++ b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift @@ -26,6 +26,9 @@ internal final class InfluxDBConnection: @unchecked Sendable { private let lock = NSLock() private var session: URLSession? + // A streamed read holds one response for as long as its consumer takes, and a suspended task still counts toward + // the resource limit, so streams run on a session that keeps the system default. + private var streamSession: URLSession? private var inFlight: [ObjectIdentifier: PluginURLSessionTaskBox] = [:] private let queryTimeout: HttpQueryTimeoutBox @@ -44,6 +47,9 @@ internal final class InfluxDBConnection: @unchecked Sendable { configuration.timeoutIntervalForResource = HttpQueryTimeout.sessionResourceTimeout let delegate = skipTLSVerify ? InfluxDBTrustingDelegate() : PluginRedirectRefusingDelegate() session = URLSession(configuration: configuration, delegate: delegate, delegateQueue: nil) + // The session made its own copy of the configuration above. + configuration.timeoutIntervalForResource = URLSessionConfiguration.default.timeoutIntervalForResource + streamSession = URLSession(configuration: configuration, delegate: delegate, delegateQueue: nil) } func close() { @@ -51,6 +57,8 @@ internal final class InfluxDBConnection: @unchecked Sendable { lock.withLock { session?.invalidateAndCancel() session = nil + streamSession?.invalidateAndCancel() + streamSession = nil } } @@ -115,7 +123,7 @@ internal final class InfluxDBConnection: @unchecked Sendable { lock.withLock { inFlight[key] = taskBox } defer { lock.withLock { _ = inFlight.removeValue(forKey: key) } } // Made under the lock `close()` invalidates with, as in `perform`. - guard let task = lock.withLock({ session?.dataTask(with: urlRequest) }) else { + guard let task = lock.withLock({ streamSession?.dataTask(with: urlRequest) }) else { throw InfluxError.notConnected } let chunks = InfluxDBHTTPChunks(task: task) diff --git a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver+Stream.swift b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver+Stream.swift index 9bf9af67a4..4524dde441 100644 --- a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver+Stream.swift +++ b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver+Stream.swift @@ -7,7 +7,7 @@ import TableProPluginKit extension InfluxDBPluginDriver { func streamRows(query: String) -> AsyncThrowingStream { guard let remembered = streamedGrid(query) else { return collectedStream(query) } - return PluginRowStream.make { continuation, abort in + return PluginRowStream.makeBounded { continuation, abort in let producer = Task { do { try await self.streamGrid(remembered, abort: abort, continuation: continuation) @@ -61,24 +61,24 @@ extension InfluxDBPluginDriver { schema: schema ) var headerSent = false - func send(_ batches: [InfluxResultTable]) { + func send(_ batches: [InfluxResultTable]) async { for batch in batches { if !headerSent { headerSent = true continuation.yield(.header(PluginStreamHeader(columns: batch.columns, columnTypeNames: batch.typeNames))) } if !batch.rows.isEmpty { - continuation.yield(.rows(Self.cells(batch.rows))) + _ = await PluginRowStream.deliver(.rows(Self.cells(batch.rows)), to: continuation) } } } try await connection.stream(request) { chunk in guard !abort.isAborted else { throw CancellationError() } let batches = try decoder.consume(chunk) - send(batches) + await send(batches) } let rest = try decoder.finish() - send(rest) + await send(rest) } // PluginKit's default stream, which the override above replaces for every statement. diff --git a/apple/Plugins/LibSQLDriverPlugin/LibSQLPluginDriver.swift b/apple/Plugins/LibSQLDriverPlugin/LibSQLPluginDriver.swift index 5c9478a233..2b0fbfb0bc 100644 --- a/apple/Plugins/LibSQLDriverPlugin/LibSQLPluginDriver.swift +++ b/apple/Plugins/LibSQLDriverPlugin/LibSQLPluginDriver.swift @@ -283,7 +283,7 @@ final class LibSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Streaming func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let streamTask = Task { do { try await self.performStreamRows(query: query, continuation: continuation) @@ -292,9 +292,7 @@ final class LibSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } - continuation.onTermination = { @Sendable _ in - streamTask.cancel() - } + abort.onAbort { streamTask.cancel() } } } diff --git a/apple/Plugins/LibSQLDriverPlugin/SQLiteLocalBackend.swift b/apple/Plugins/LibSQLDriverPlugin/SQLiteLocalBackend.swift index b0d9e348e9..8994f7d575 100644 --- a/apple/Plugins/LibSQLDriverPlugin/SQLiteLocalBackend.swift +++ b/apple/Plugins/LibSQLDriverPlugin/SQLiteLocalBackend.swift @@ -180,7 +180,7 @@ actor SQLiteLocalBackend { while stepResult == SQLITE_ROW { if Task.isCancelled { if !batch.isEmpty { - continuation.yield(.rows(batch)) + _ = PluginRowStream.deliverBlocking(.rows(batch), to: continuation) } continuation.finish(throwing: CancellationError()) return @@ -188,14 +188,15 @@ actor SQLiteLocalBackend { batch.append(rowValues(of: statement, count: columnCount)) if batch.count >= batchSize { - continuation.yield(.rows(batch)) + // Blocks the actor's thread: an await here would let another call reach the handle mid-statement. + _ = PluginRowStream.deliverBlocking(.rows(batch), to: continuation) batch.removeAll(keepingCapacity: true) } stepResult = sqlite3_step(statement) } if !batch.isEmpty { - continuation.yield(.rows(batch)) + _ = PluginRowStream.deliverBlocking(.rows(batch), to: continuation) } do { diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift index ab847e9f4c..c86b077cb8 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift @@ -334,7 +334,7 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { func streamRows(query: String) -> AsyncThrowingStream { let statement = query.trimmingCharacters(in: .whitespacesAndNewlines) let spec = MemcachedBrowseSpec.refusalMessage(statement) == nil ? MemcachedBrowseSpec.parse(statement) : nil - return PluginRowStream.make { continuation, abort in + return PluginRowStream.makeBounded { continuation, abort in let task = Task { do { if let spec { @@ -374,7 +374,7 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let batch = page.entries[start ..< end] let values = try await self.values(of: batch.map(\.key), on: connection) let rows = self.rows(of: batch, values: values, serverTime: page.serverTime) - if !rows.isEmpty { continuation.yield(.rows(rows)) } + if !rows.isEmpty { _ = await PluginRowStream.deliver(.rows(rows), to: continuation) } start = end } } diff --git a/apple/Plugins/Neo4jDriverPlugin/Neo4jConnection.swift b/apple/Plugins/Neo4jDriverPlugin/Neo4jConnection.swift index 970a601ae8..ce469d21a0 100644 --- a/apple/Plugins/Neo4jDriverPlugin/Neo4jConnection.swift +++ b/apple/Plugins/Neo4jDriverPlugin/Neo4jConnection.swift @@ -381,7 +381,7 @@ actor Neo4jConnection { _ statement: BoltRequest, batchSize: Int64, interruptible: Bool, - receive: @Sendable (_ keys: [String], _ records: [[PackStreamValue]], _ isFirst: Bool) -> Bool + receive: @Sendable (_ keys: [String], _ records: [[PackStreamValue]], _ isFirst: Bool) async -> Bool ) async throws { try await withTurn(interruptible: interruptible) { wire, since in let replies = try await Self.exchange([statement, .pull(count: batchSize)], on: wire, cancellableSince: since) @@ -391,7 +391,7 @@ actor Neo4jConnection { var batch = replies[1] var isFirst = true while true { - let wanted = receive(keys, batch.records, isFirst) + let wanted = await receive(keys, batch.records, isFirst) isFirst = false guard batch.hasMore else { return } guard wanted, !Task.isCancelled else { diff --git a/apple/Plugins/Neo4jDriverPlugin/Neo4jPluginDriver.swift b/apple/Plugins/Neo4jDriverPlugin/Neo4jPluginDriver.swift index 4bca23eae3..c07018f4d6 100644 --- a/apple/Plugins/Neo4jDriverPlugin/Neo4jPluginDriver.swift +++ b/apple/Plugins/Neo4jDriverPlugin/Neo4jPluginDriver.swift @@ -236,7 +236,7 @@ final class Neo4jPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // Pulls a batch at a time, so an export never holds the whole result. func streamRows(query: String, context: PluginStatementContext) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in + PluginRowStream.makeBounded { continuation, abort in let task = Task { do { try await self.stream(query, context: context, into: continuation) @@ -245,7 +245,7 @@ final class Neo4jPluginDriver: PluginDatabaseDriver, @unchecked Sendable { continuation.finish(throwing: error) } } - continuation.onTermination = { _ in task.cancel() } + abort.onAbort { task.cancel() } } } @@ -278,8 +278,7 @@ final class Neo4jPluginDriver: PluginDatabaseDriver, @unchecked Sendable { continuation.yield(.header(PluginStreamHeader(columns: table.columns, columnTypeNames: table.typeNames))) } guard !table.rows.isEmpty else { return true } - if case .terminated = continuation.yield(.rows(Self.cells(table.rows))) { return false } - return true + return await PluginRowStream.deliver(.rows(Self.cells(table.rows)), to: continuation) } } diff --git a/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift b/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift index c083766d07..c3764f36da 100644 --- a/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift +++ b/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift @@ -513,22 +513,11 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } let batching = OracleRowBatcher(rowsPerBatch: OracleRowBatcher.batchRows, rowBudget: rowBudget) - return PluginRowStream.make { continuation, abort in + return PluginRowStream.makeBounded { continuation, abort in + // A plain `Task {}` is not a child of this one, so only the explicit cancel on abort stops oracle-nio. let streamTask = Task { - // A plain `Task {}` is not a child of this one, so only the explicit cancel on abort stops oracle-nio. - let coreStream = AsyncThrowingStream { coreContinuation in - let coreTask = Task { - do { - try await core.streamQuery(query, batching: batching, continuation: coreContinuation) - } catch { - coreContinuation.finish(throwing: error) - } - } - abort.onAbort { coreTask.cancel() } - coreContinuation.onTermination = { @Sendable _ in coreTask.cancel() } - } do { - for try await element in coreStream { + try await core.streamQuery(query, batching: batching, onElement: { element in switch element { case .header(let columns): continuation.yield(.header(PluginStreamHeader( @@ -536,13 +525,13 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { columnTypeNames: columns.map(\.typeName) ))) case .rows(let batch): - continuation.yield(.rows(batch.map { $0.map(\.asPluginCell) })) + _ = await PluginRowStream.deliver(.rows(batch.map { $0.map(\.asPluginCell) }), to: continuation) case .affectedRows: // The plugin stream carries rows only. The app sends writes and DDL through // `execute`, which reports the count, and bounds only reads through this stream. break } - } + }) continuation.finish() } catch let error as OracleCoreError { continuation.finish(throwing: error.asPluginError) diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift index c1dcb1036b..acb8479308 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift @@ -151,7 +151,7 @@ extension QdrantPluginDriver { // The protocol's default `streamRows` runs `execute` once and buffers the whole collection. func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let task = Task { do { if let name = QdrantOperations.decodeExport(query) { @@ -170,7 +170,7 @@ extension QdrantPluginDriver { continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in task.cancel() } + abort.onAbort { task.cancel() } } } @@ -193,7 +193,7 @@ extension QdrantPluginDriver { estimatedRowCount: collection.pointsCount ))) for try await points in client.pointPages(collection: name, pageSize: Self.exportPageSize) { - continuation.yield(.rows(points.map(cells(columns: columns, typeNames: typeNames)))) + _ = await PluginRowStream.deliver(.rows(points.map(cells(columns: columns, typeNames: typeNames))), to: continuation) } } } diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift index 58d7650645..ff639ea0f0 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift @@ -521,7 +521,7 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let streamTask = Task { do { try await self.performStreamRows(query: query, continuation: continuation) @@ -529,9 +529,7 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in - streamTask.cancel() - } + abort.onAbort { streamTask.cancel() } } } @@ -610,7 +608,7 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let batchKeys = Array(page.keys[batchStart ..< batchEnd]) let rowBatch = try await buildKeySummaryRows(keys: batchKeys, connection: conn, stringByteLimit: nil) if !rowBatch.isEmpty { - continuation.yield(.rows(rowBatch)) + _ = await PluginRowStream.deliver(.rows(rowBatch), to: continuation) } batchStart = batchEnd } @@ -636,7 +634,7 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let batch = Array(keys[start ..< min(start + Self.keySummaryBatchSize, keys.count)]) let rows = try await buildKeySummaryRows(keys: batch, connection: conn, stringByteLimit: nil) if !rows.isEmpty { - continuation.yield(.rows(rows)) + _ = await PluginRowStream.deliver(.rows(rows), to: continuation) } } continuation.finish() diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLiteExecutionBackend.swift b/apple/Plugins/SQLiteDriverPlugin/SQLiteExecutionBackend.swift index 5906915c69..dadb6da54b 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLiteExecutionBackend.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLiteExecutionBackend.swift @@ -467,20 +467,21 @@ actor SQLiteLocalBackend: SQLiteExecutionBackend { var stepResult = firstStep.result while stepResult == SQLITE_ROW { if Task.isCancelled { - if !batch.isEmpty { continuation.yield(.rows(batch)) } + if !batch.isEmpty { _ = PluginRowStream.deliverBlocking(.rows(batch), to: continuation) } sqlite3_finalize(statement) continuation.finish(throwing: CancellationError()) return } batch.append(Self.readRow(statement, columnCount: columnCount)) if batch.count >= batchSize { - continuation.yield(.rows(batch)) + // Blocks the actor's thread: an await here would let another call reach the handle mid-statement. + _ = PluginRowStream.deliverBlocking(.rows(batch), to: continuation) batch.removeAll(keepingCapacity: true) } stepResult = sqlite3_step(statement) } - if !batch.isEmpty { continuation.yield(.rows(batch)) } + if !batch.isEmpty { _ = PluginRowStream.deliverBlocking(.rows(batch), to: continuation) } guard stepResult == SQLITE_DONE else { let message = String(cString: sqlite3_errmsg(db)) diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift b/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift index 30560e4b83..480aa448d8 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift @@ -690,9 +690,14 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { stream(query, scope: SQLiteStatementScope(context)) } + // Over the SSH agent the stream stays unbounded: a reader waiting for its consumer stops the tunnel relaying the + // heartbeats, and the agent's idle watchdog ends the session. private func stream(_ query: String, scope: SQLiteStatementScope) -> AsyncThrowingStream { let queryToRun = String(query) - return AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + let policy: AsyncThrowingStream.Continuation.BufferingPolicy = backend is SQLiteLocalBackend + ? .bufferingOldest(PluginRowStream.boundedBufferBatches) + : .unbounded + return AsyncThrowingStream(bufferingPolicy: policy) { continuation in let streamTask = Task { do { try await self.backend.streamQuery(queryToRun, scope: scope, continuation: continuation) diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift index 42a037a4b2..ba4a63384b 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift @@ -741,7 +741,8 @@ final class SnowflakeConnection: @unchecked Sendable { let columns: [SnowflakeColumnMeta] let inlineRows: [[PluginCellValueBox]] let estimatedRowCount: Int - let batches: AsyncThrowingStream<[[PluginCellValueBox]], Error> + let chunkURLs: [URL] + let chunkHeaders: [String: String] } func queryStreamed( @@ -758,58 +759,56 @@ final class SnowflakeConnection: @unchecked Sendable { let headers = Self.chunkRequestHeaders(from: data) let urls = try Self.chunkURLs(chunks) - let stream = AsyncThrowingStream<[[PluginCellValueBox]], Error> { continuation in - guard !urls.isEmpty else { - continuation.finish() - return + return StreamedResult( + columns: columns, + inlineRows: inlineRows, + estimatedRowCount: inlineRows.count + chunkRowCount, + chunkURLs: urls, + chunkHeaders: headers + ) + } + + // Chunks reach `body` in order. A download starts only as a chunk is handed on, so the chunks downloading or + // waiting for one that lags never number more than `chunkDownloadWorkers`. + func forEachChunk( + of result: StreamedResult, + _ body: ([[PluginCellValueBox]]) async throws -> Void + ) async throws { + let urls = result.chunkURLs + guard !urls.isEmpty else { return } + let headers = result.chunkHeaders + let columns = result.columns + let session = self.session + try await withThrowingTaskGroup(of: (Int, [[PluginCellValueBox]]).self) { group in + var nextIndex = 0 + while nextIndex < min(Self.chunkDownloadWorkers, urls.count) { + let index = nextIndex + group.addTask { + (index, try await Self.downloadChunk( + urls[index], headers: headers, session: session, columns: columns + )) + } + nextIndex += 1 } - let task = Task { [sessions] in - do { - var buffer: [Int: [[PluginCellValueBox]]] = [:] - var nextToYield = 0 - try await withThrowingTaskGroup(of: (Int, [[PluginCellValueBox]]).self) { group in - var nextIndex = 0 - while nextIndex < min(Self.chunkDownloadWorkers, urls.count) { - let index = nextIndex - group.addTask { - (index, try await Self.downloadChunk( - urls[index], headers: headers, session: sessions.query, columns: columns - )) - } - nextIndex += 1 - } - while let (index, rows) = try await group.next() { - buffer[index] = rows - while let ready = buffer[nextToYield] { - buffer[nextToYield] = nil - continuation.yield(ready) - nextToYield += 1 - } - if nextIndex < urls.count { - let next = nextIndex - group.addTask { - (next, try await Self.downloadChunk( - urls[next], headers: headers, session: sessions.query, columns: columns - )) - } - nextIndex += 1 - } + var held: [Int: [[PluginCellValueBox]]] = [:] + var nextToHandOn = 0 + while let (index, rows) = try await group.next() { + held[index] = rows + while let ready = held.removeValue(forKey: nextToHandOn) { + try await body(ready) + nextToHandOn += 1 + if nextIndex < urls.count { + let next = nextIndex + group.addTask { + (next, try await Self.downloadChunk( + urls[next], headers: headers, session: session, columns: columns + )) } + nextIndex += 1 } - continuation.finish() - } catch { - continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in task.cancel() } } - - return StreamedResult( - columns: columns, - inlineRows: inlineRows, - estimatedRowCount: inlineRows.count + chunkRowCount, - batches: stream - ) } private static let chunkDownloadWorkers = 4 diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift index aebb75e0fa..b47c91f2b6 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift @@ -627,29 +627,29 @@ final class SnowflakePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let task = Task { do { guard let conn = self.connection else { throw SnowflakeError.notConnected } let trimmed = query.replacingOccurrences(of: ";\\s*\\z", with: "", options: .regularExpression) - let streamed = try await conn.queryStreamed(trimmed, owner: queryOwner) + let streamed = try await conn.queryStreamed(trimmed, owner: self.queryOwner) continuation.yield(.header(PluginStreamHeader( columns: streamed.columns.map(\.name), columnTypeNames: streamed.columns.map(SnowflakeTypeMapper.displayType), estimatedRowCount: streamed.estimatedRowCount ))) if !streamed.inlineRows.isEmpty { - continuation.yield(.rows(streamed.inlineRows.map { row in row.map(Self.cellValue) })) + _ = await PluginRowStream.deliver(.rows(streamed.inlineRows.map { row in row.map(Self.cellValue) }), to: continuation) } - for try await batch in streamed.batches { - continuation.yield(.rows(batch.map { row in row.map(Self.cellValue) })) + try await conn.forEachChunk(of: streamed) { batch in + _ = await PluginRowStream.deliver(.rows(batch.map { row in row.map(Self.cellValue) }), to: continuation) } continuation.finish() } catch { continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in task.cancel() } + abort.onAbort { task.cancel() } } } diff --git a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Execution.swift b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Execution.swift index a017a93fa2..c8a212d16b 100644 --- a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Execution.swift +++ b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Execution.swift @@ -40,8 +40,10 @@ extension SpannerPluginDriver { } } let id = self.registerUserOperation { task.cancel() } - abort.onAbort { - task.cancel() + abort.onAbort { task.cancel() } + // A stream its producer finishes never aborts, so the entry goes when the task ends. + Task { + await task.value self.unregisterUserOperation(id) } } diff --git a/apple/Plugins/TableProPluginKit/PluginStreamAbort.swift b/apple/Plugins/TableProPluginKit/PluginStreamAbort.swift index 66f7dd8d42..26f1a7d138 100644 --- a/apple/Plugins/TableProPluginKit/PluginStreamAbort.swift +++ b/apple/Plugins/TableProPluginKit/PluginStreamAbort.swift @@ -48,16 +48,9 @@ public final class PluginStreamAbort: @unchecked Sendable { } public enum PluginRowStream { - /// Builds a row stream whose abort signal is already wired to its termination. - /// - /// The handler is installed before `build` runs and sets the flag synchronously on whichever - /// thread terminated the stream, so a producer that has not started yet still sees it and can - /// decline to send the query at all. A producer that hops the signal onto its own serial queue - /// instead sees it only after it has finished, which is the defect this exists to make - /// unrepeatable. - /// - /// Termination follows the lifetime of the stream, not the consumer's `break`: a consumer that - /// stops reading but holds the stream in scope has not terminated it. + // The handler is installed before `build` runs and sets the flag on whichever thread ended the stream, so a + // producer that has not started yet still sees it and can decline to send the query. Termination follows the + // stream's lifetime, not the consumer's `break`: a consumer that stops reading but holds the stream has not ended it. public static func make( bufferingPolicy: AsyncThrowingStream.Continuation.BufferingPolicy = .unbounded, _ build: @escaping @Sendable ( @@ -67,7 +60,11 @@ public enum PluginRowStream { ) -> AsyncThrowingStream { let abort = PluginStreamAbort() return AsyncThrowingStream(bufferingPolicy: bufferingPolicy) { continuation in - continuation.onTermination = { @Sendable _ in abort.abort() } + // Only the consumer ends a stream as `.cancelled`, by cancelling its task or letting the stream go. A + // producer's own `finish` is `.finished`, and aborting then ran its cancel actions after every whole read. + continuation.onTermination = { @Sendable termination in + if case .cancelled = termination { abort.abort() } + } build(continuation, abort) } } diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift b/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift index d285ae54af..f3c5c8b7ac 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift @@ -229,14 +229,14 @@ final class TeradataPluginDriver: PluginDatabaseDriver, @unchecked Sendable { func streamRows(query: String) -> AsyncThrowingStream { let connection = self.connection - return PluginRowStream.make { continuation, abort in + return PluginRowStream.makeBounded { continuation, abort in let task = Task { guard let connection else { continuation.finish(throwing: TeradataWireError.connectionFailed("not connected")) return } do { - // Each reply's rows go out as it arrives, before the next one is asked for. + // Each reply's rows go out as it arrives, and the next one is asked for once the reader has room. try await connection.stream( query, onColumns: { columns in @@ -247,8 +247,11 @@ final class TeradataPluginDriver: PluginDatabaseDriver, @unchecked Sendable { }, onRows: { rows in guard !abort.isAborted else { return false } - continuation.yield(.rows(rows.map { row in row.map { $0.asPluginCell } })) - return true + let delivered = PluginRowStream.deliverBlocking( + .rows(rows.map { row in row.map { $0.asPluginCell } }), + to: continuation + ) + return delivered && !abort.isAborted } ) continuation.finish() diff --git a/apple/Plugins/TypesenseDriverPlugin/TypesenseConnection.swift b/apple/Plugins/TypesenseDriverPlugin/TypesenseConnection.swift index c55ad2c08b..017488be85 100644 --- a/apple/Plugins/TypesenseDriverPlugin/TypesenseConnection.swift +++ b/apple/Plugins/TypesenseDriverPlugin/TypesenseConnection.swift @@ -50,6 +50,9 @@ internal struct TypesenseResponse { internal final class TypesenseConnection: NSObject, @unchecked Sendable { private let lock = NSLock() private var _session: URLSession? + // An export holds one response for as long as its consumer takes, and a paused read still counts toward the + // resource limit, so it runs on a session that keeps the system default. + private var _streamSession: URLSession? private var _currentTask: URLSessionDataTask? private var _serverVersion: String? private let queryTimeout = HttpQueryTimeoutBox() @@ -98,7 +101,13 @@ internal final class TypesenseConnection: NSObject, @unchecked Sendable { sessionConfig.timeoutIntervalForRequest = connectTimeout sessionConfig.timeoutIntervalForResource = HttpQueryTimeout.sessionResourceTimeout let session = URLSession(configuration: sessionConfig, delegate: self, delegateQueue: nil) - lock.withLock { _session = session } + // The session made its own copy of the configuration above. + sessionConfig.timeoutIntervalForResource = URLSessionConfiguration.default.timeoutIntervalForResource + let streamSession = URLSession(configuration: sessionConfig, delegate: self, delegateQueue: nil) + lock.withLock { + _session = session + _streamSession = streamSession + } let response = try await request( method: "GET", @@ -118,6 +127,8 @@ internal final class TypesenseConnection: NSObject, @unchecked Sendable { _currentTask = nil _session?.finishTasksAndInvalidate() _session = nil + _streamSession?.finishTasksAndInvalidate() + _streamSession = nil } } @@ -180,10 +191,10 @@ internal final class TypesenseConnection: NSObject, @unchecked Sendable { /// collection never holds it in memory. func streamExport( collection: String, - onDocument: @escaping ([String: Any]) -> Bool + onDocument: ([String: Any]) async -> Bool ) async throws { let session: URLSession = try lock.withLock { - guard let session = _session else { throw TypesenseError.notConnected } + guard let session = _streamSession else { throw TypesenseError.notConnected } return session } let path = TypesenseOperations.exportPath(collection: collection) @@ -224,7 +235,7 @@ internal final class TypesenseConnection: NSObject, @unchecked Sendable { let data = line.data(using: .utf8), let document = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { continue } - if !onDocument(document) { return } + if await !onDocument(document) { return } } } diff --git a/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver+Execution.swift b/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver+Execution.swift index 0e86a82cdf..b9f4c274a5 100644 --- a/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver+Execution.swift +++ b/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver+Execution.swift @@ -109,11 +109,10 @@ extension TypesensePluginDriver { // MARK: - Streaming Export - /// Export reads the collection's JSONL endpoint a line at a time, so a collection larger than - /// memory still exports and the 250-hit search ceiling never applies. Anything that is not an - /// export tag runs as an ordinary query and is handed over in one batch. + // Export reads the collection's JSONL endpoint a line at a time, so a collection larger than memory still exports + // and the 250-hit search ceiling never applies. Anything else runs as an ordinary query, handed over in one batch. func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let task = Task { do { try await self.performStreamRows(query: query, continuation: continuation) @@ -121,7 +120,7 @@ extension TypesensePluginDriver { continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in task.cancel() } + abort.onAbort { task.cancel() } } } @@ -154,12 +153,12 @@ extension TypesensePluginDriver { try await connection.streamExport(collection: collection) { document in batch.append(document) guard batch.count >= Self.exportBatchSize else { return true } - continuation.yield(.rows(TypesenseSchema.rows(for: batch, columns: columns, length: .whole))) + _ = await PluginRowStream.deliver(.rows(TypesenseSchema.rows(for: batch, columns: columns, length: .whole)), to: continuation) batch.removeAll(keepingCapacity: true) return true } if !batch.isEmpty { - continuation.yield(.rows(TypesenseSchema.rows(for: batch, columns: columns, length: .whole))) + _ = await PluginRowStream.deliver(.rows(TypesenseSchema.rows(for: batch, columns: columns, length: .whole)), to: continuation) } continuation.finish() } diff --git a/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Execution.swift b/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Execution.swift index cb9d6ef252..47d0a6572c 100644 --- a/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Execution.swift +++ b/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Execution.swift @@ -42,14 +42,13 @@ extension WeaviatePluginDriver { static let exportPageSize = 500 - /// Exports one collection by walking it a page at a time, yielding each page instead of - /// holding the whole collection. The protocol default runs `execute` once and buffers - /// everything, which is fine for a grid page and not for an export. + // An export walks the collection a page at a time instead of holding it whole. The protocol default runs + // `execute` once and buffers everything, fine for a grid page and not for an export. func streamRows(query: String) -> AsyncThrowingStream { guard let collectionName = WeaviateOperations.decodeExport(query) else { return defaultStreamRows(query: query) } - return AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + return PluginRowStream.makeBounded { continuation, abort in let task = Task { do { try await self.streamCollection(collectionName, continuation: continuation) @@ -58,7 +57,7 @@ extension WeaviatePluginDriver { continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in task.cancel() } + abort.onAbort { task.cancel() } } } @@ -76,11 +75,12 @@ extension WeaviatePluginDriver { ))) for try await objects in client.objectPages(collection: name, pageSize: Self.exportPageSize) { - continuation.yield(.rows(objects.map { object in + let rows: [PluginRow] = objects.map { object in WeaviateObjectCodec.row(for: object, columns: columns).map { value in value.map(PluginCellValue.text) ?? .null } - })) + } + _ = await PluginRowStream.deliver(.rows(rows), to: continuation) } } diff --git a/apple/TablePro/Core/Compare/CompareRowService.swift b/apple/TablePro/Core/Compare/CompareRowService.swift index fde3b89964..310d2d6a12 100644 --- a/apple/TablePro/Core/Compare/CompareRowService.swift +++ b/apple/TablePro/Core/Compare/CompareRowService.swift @@ -132,6 +132,18 @@ internal struct CompareRowService { // MARK: - Plumbing + // The sides are built inside the lift: a stream starts its statement when it is built, and a bounded one parks + // its connection once its buffer is full, so a lift sent after it would wait behind that statement for good. + internal static func readingBothSides( + liftingTimeoutOn drivers: [DatabaseDriver], + sides makeSides: () -> Sides, + _ body: (Sides) async throws -> T + ) async throws -> T { + try await BulkReadTimeout.lifted(on: drivers) { + try await body(makeSides()) + } + } + private func withBothSides( source: DatabaseEndpoint, sourceConnection: DatabaseConnection, @@ -188,16 +200,17 @@ internal struct CompareRowService { String(localized: "The target driver cannot stream rows for a comparison.") ) } - let sides = RowSides( - source: sourceRead.provider(driver: sourcePlugin), - target: targetRead.provider(driver: targetPlugin), - resolver: KeyLookupResolver( - source: (sourceRead, sourcePlugin), - target: (targetRead, targetPlugin) - ), - targetDriver: targetPlugin - ) - return try await BulkReadTimeout.lifted(on: [sourceDriver, targetDriver]) { + return try await Self.readingBothSides(liftingTimeoutOn: [sourceDriver, targetDriver]) { + RowSides( + source: sourceRead.provider(driver: sourcePlugin), + target: targetRead.provider(driver: targetPlugin), + resolver: KeyLookupResolver( + source: (sourceRead, sourcePlugin), + target: (targetRead, targetPlugin) + ), + targetDriver: targetPlugin + ) + } _: { sides in try await body(engine, sides) } } diff --git a/apple/TablePro/Core/Compare/StreamingRowProvider.swift b/apple/TablePro/Core/Compare/StreamingRowProvider.swift index 55c04befd1..b6fb6f44cf 100644 --- a/apple/TablePro/Core/Compare/StreamingRowProvider.swift +++ b/apple/TablePro/Core/Compare/StreamingRowProvider.swift @@ -5,11 +5,11 @@ // Feeds the merge join from a driver's row stream. It holds one batch of rows // at a time, and hands them out one row at a time. // -// It is not what bounds a comparison's memory. PostgreSQL, MySQL, SQL Server, -// SAP HANA and DuckDB hold a few batches and wait for the walk; the other -// drivers' streams run ahead of a side the walk consumes slowly, inside the -// stream's own buffer. The row limit, the filter, and the key order the two -// sides share are what keep a comparison's footprint down there. +// It is not what bounds a comparison's memory. Most drivers' streams hold a +// few batches and wait for the walk; ClickHouse, Elasticsearch, Trino, MongoDB, +// Spanner and SQLite over SSH run ahead of a side the walk consumes slowly, +// inside the stream's own buffer. The row limit, the filter, and the key order +// the two sides share are what keep a comparison's footprint down there. // import Foundation diff --git a/apple/TablePro/Core/Database/Access/AgentTransactionGuard.swift b/apple/TablePro/Core/Database/Access/AgentTransactionGuard.swift index 2b9ae43802..a69f121875 100644 --- a/apple/TablePro/Core/Database/Access/AgentTransactionGuard.swift +++ b/apple/TablePro/Core/Database/Access/AgentTransactionGuard.swift @@ -44,7 +44,8 @@ internal struct AgentTransactionGuard: Sendable { ) case .oracle: return Savepoint(open: savepointStatement, release: nil, rollback: rollbackStatement) - case .redshift, .duckdb, .redis, .other: + // Teradata has no savepoint. + case .redshift, .duckdb, .redis, .teradata, .other: return nil } } diff --git a/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift b/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift index 2eb427e9b1..342cb3a70f 100644 --- a/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift +++ b/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift @@ -12,11 +12,11 @@ // // The read side is only as incremental as the driver is. `streamRows` has a // protocol default that calls `execute` and yields the whole result at once, -// which Dameng and Teradata still inherit, so on those two a large table is -// materialised before the first batch is written and Stop cannot land until -// it is. Every other SQL driver implements it for real. Export has the same -// property through the same call; a capability that lets this refuse rather -// than inherit would have to be declared by all 32 plugins. +// which Dameng still inherits, so there a large table is materialised before +// the first batch is written and Stop cannot land until it is. Every other SQL +// driver implements it for real. Export has the same property through the same +// call; a capability that lets this refuse rather than inherit would have to be +// declared by every plugin. // // This is the write path CSV and JSON import already use, through the same // `SQLStatementGenerator` and its per-engine bind-parameter ceiling. The one diff --git a/apple/TablePro/Core/Plugins/QueryResultExportDataSource.swift b/apple/TablePro/Core/Plugins/QueryResultExportDataSource.swift index d3a8ecfef7..8247482ecf 100644 --- a/apple/TablePro/Core/Plugins/QueryResultExportDataSource.swift +++ b/apple/TablePro/Core/Plugins/QueryResultExportDataSource.swift @@ -172,32 +172,10 @@ final class QueryResultExportDataSource: PluginExportDataSource, @unchecked Send streamExportRows(table: object.name, databaseName: object.databaseName) } + // Pulled rather than pumped by a task of its own, which kept every filled chunk the writer had not taken yet. func streamExportRows(table: String, databaseName: String) -> AsyncThrowingStream { - let stream = streamRows(table: table, databaseName: databaseName) - let documents = exportDocuments - return AsyncThrowingStream { continuation in - let task = Task { - var offset = 0 - do { - for try await element in stream { - switch element { - case .header(let header): - continuation.yield(.header(header)) - case .rows(let rows): - let end = offset + rows.count - let aligned = end <= documents.count ? Array(documents[offset...AsyncIterator + private let documents: [PluginExportDocument?] + private var offset = 0 + + init(_ stream: AsyncThrowingStream, documents: [PluginExportDocument?]) { + iterator = stream.makeAsyncIterator() + self.documents = documents + } + + func next() async throws -> PluginExportStreamElement? { + guard let element = try await iterator.next() else { return nil } + switch element { + case .header(let header): + return .header(header) + case .rows(let rows): + let end = offset + rows.count + let aligned = end <= documents.count ? Array(documents[offset.. Void)? + func applyQueryTimeout(_ seconds: Int) async throws { + await applyQueryTimeoutGate?() applyQueryTimeoutValues.append(seconds) } diff --git a/apple/TableProTests/Core/Compare/CompareTimeoutLiftOrderTests.swift b/apple/TableProTests/Core/Compare/CompareTimeoutLiftOrderTests.swift new file mode 100644 index 0000000000..eb6dcd9028 --- /dev/null +++ b/apple/TableProTests/Core/Compare/CompareTimeoutLiftOrderTests.swift @@ -0,0 +1,53 @@ +// +// CompareTimeoutLiftOrderTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +@MainActor +struct CompareTimeoutLiftOrderTests { + // One queue runs the connection's statements in turn, as a libpq session's does, so a lift sent while a bounded + // stream waits at a full buffer would wait behind it, and the walk that would drain the buffer never starts. + @Test( + "A comparison lifts the query timeout before its streams start their statements", + .timeLimit(.minutes(1)) + ) + func timeoutIsLiftedBeforeTheStreamsStart() async throws { + let connectionQueue = DispatchQueue(label: "test.compare.connection") + let driver = MockDatabaseDriver() + driver.applyQueryTimeoutGate = { + await withCheckedContinuation { continuation in + connectionQueue.async { continuation.resume() } + } + } + let batches = PluginRowStream.boundedBufferBatches * 3 + + let rowsRead = try await CompareRowService.readingBothSides(liftingTimeoutOn: [driver]) { + PluginRowStream.makeBounded { continuation, _ in + connectionQueue.async { + continuation.yield(.header(PluginStreamHeader(columns: ["id"], columnTypeNames: ["integer"]))) + for index in 0..(initialState: []) + let source = QueryResultExportDataSource( + tableRows: tableRows, + databaseType: .mysql, + driver: nil, + fillUnloaded: { rows in + let pull = pulls.withLock { $0 } + pullsAtEachFill.withLock { $0.append(pull) } + return Dictionary(uniqueKeysWithValues: rows.map { ($0.id, [1: PluginCellValue.text("x")]) }) + } + ) + let stream = source.streamExportRows(table: "customers", databaseName: "") + + // Off the main actor, so a fill that ran ahead of the writer would not wait behind it. + try await Task.detached { + var iterator = stream.makeAsyncIterator() + while true { + pulls.withLock { $0 += 1 } + guard try await iterator.next() != nil else { break } + } + }.value + + // Pull 1 is the header; the three chunks of 50, 50 and 20 rows are filled during pulls 2, 3 and 4. + #expect(pullsAtEachFill.withLock { $0 } == [2, 3, 4]) + } } diff --git a/apple/changelog.d/4227.fixed.md b/apple/changelog.d/4227.fixed.md new file mode 100644 index 0000000000..d6509a0a02 --- /dev/null +++ b/apple/changelog.d/4227.fixed.md @@ -0,0 +1 @@ +Copying, comparing or exporting a large SQLite, Oracle, Teradata, Cassandra or cloud table held all of it in memory. diff --git a/apple/scripts/probes/check-stream-backpressure.sh b/apple/scripts/probes/check-stream-backpressure.sh new file mode 100755 index 0000000000..c14dba646c --- /dev/null +++ b/apple/scripts/probes/check-stream-backpressure.sh @@ -0,0 +1,262 @@ +#!/usr/bin/env bash +# +# Measure how much memory a driver's row stream holds while its consumer is slower than the source. +# +# Builds a harness from the driver's own sources, fills a table on a real server, then reads it through +# `streamRows` with a consumer that waits per batch, the way a copy waits on its target's inserts. It prints the +# process footprint before the read and its peak during it. A stream with backpressure peaks at a few batches; one +# without it holds every row the consumer has not taken yet. +# +# Usage: +# scripts/probes/check-stream-backpressure.sh [--rows N] [--delay-ms N] [--root DIR] +# +# engine: postgresql, mysql, sqlite (a scratch file, no server) +# --rows rows in the probe table, default 1000000 +# --delay-ms the consumer's wait per batch, default 20 +# --root the apple/ folder whose sources to build, default this one; Libs comes from it too +# +# TP_STREAM_HOST, TP_STREAM_PORT, TP_STREAM_USER, TP_STREAM_PASSWORD and TP_STREAM_DATABASE name the server, by +# default a local one on the engine's port as the current user. TP_STREAM_BUILD_DIR keeps the harness between runs. +# Exits 1 when the read fails, 3 when the probe cannot run. + +set -uo pipefail + +ENGINE="${1:-}" +shift || true +ROWS=1000000 +DELAY_MS=20 +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +while [ $# -gt 0 ]; do + case "$1" in + --rows) ROWS="${2:?}"; shift 2 ;; + --delay-ms) DELAY_MS="${2:?}"; shift 2 ;; + --root) ROOT="$(cd "${2:?}" && pwd)"; shift 2 ;; + *) echo "unknown argument $1" >&2; exit 3 ;; + esac +done +PLUGINS="$ROOT/Plugins" + +case "$ENGINE" in + postgresql) + FOLDER=PostgreSQLDriverPlugin CLASS=PostgreSQLPlugin C_MODULE=CLibPQ + PRODUCTS="TableProCoreTypes TableProGeometry TableProKerberos TableProLogRedaction TableProPostgreSQLCore TableProSQLQuoting TableProSQLVocabulary TableProTLSClientIdentity" + LINK="$ROOT/Libs/libpq.a $ROOT/Libs/libpgcommon.a $ROOT/Libs/libpgport.a -lssl.3 -lcrypto.3 -lz -framework GSS -lcom_err" + PORT_DEFAULT=5432 DATABASE_DEFAULT=postgres + CREATE="CREATE TABLE tp_stream_probe AS SELECT g AS id, repeat(md5(g::text), 6) AS payload FROM generate_series(1, $ROWS) AS g" + ;; + mysql) + FOLDER=MySQLDriverPlugin CLASS=MySQLPlugin C_MODULE=CMariaDB + PRODUCTS="TableProCoreTypes TableProKerberos TableProLogRedaction TableProMySQLCore TableProSQLQuoting TableProSQLVocabulary" + LINK="$ROOT/Libs/libmariadb.a -lssl.3 -lcrypto.3 -liconv -lz" + PORT_DEFAULT=3306 DATABASE_DEFAULT=test + digits="(SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9)" + CREATE="CREATE TABLE tp_stream_probe AS SELECT id, REPEAT(MD5(id), 6) AS payload FROM (SELECT d0.n + d1.n * 10 + d2.n * 100 + d3.n * 1000 + d4.n * 10000 + d5.n * 100000 + d6.n * 1000000 + 1 AS id FROM $digits d0, $digits d1, $digits d2, $digits d3, $digits d4, $digits d5, $digits d6) AS numbers WHERE id <= $ROWS" + ;; + sqlite) + FOLDER=SQLiteDriverPlugin CLASS=SQLitePlugin C_MODULE="" + PRODUCTS="CSQLite TableProSQLiteCore TableProSQLQuoting TableProSQLVocabulary" + LINK="$ROOT/Libs/libsqlite3_vendored.a" + CREATE="CREATE TABLE tp_stream_probe AS WITH RECURSIVE g(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM g WHERE x < $ROWS) SELECT x AS id, hex(randomblob(96)) AS payload FROM g" + ;; + *) + echo "usage: $0 [--rows N] [--delay-ms N] [--root DIR]" >&2 + exit 3 + ;; +esac + +command -v swift > /dev/null || { echo "swift not found" >&2; exit 3; } +for library in $LINK; do + case "$library" in + /*) [ -f "$library" ] || { echo "not found: $library (run scripts/download-libs.sh)" >&2; exit 3; } ;; + esac +done + +HOST="${TP_STREAM_HOST:-127.0.0.1}" +PORT="${TP_STREAM_PORT:-${PORT_DEFAULT:-0}}" +USER_NAME="${TP_STREAM_USER:-$USER}" +PASSWORD="${TP_STREAM_PASSWORD:-}" +DATABASE="${TP_STREAM_DATABASE:-${DATABASE_DEFAULT:-}}" +if [ -n "$C_MODULE" ]; then + nc -z "$HOST" "$PORT" > /dev/null 2>&1 || { echo "no server at $HOST:$PORT" >&2; exit 3; } +fi + +WORK="${TP_STREAM_BUILD_DIR:-}" +if [ -z "$WORK" ]; then + WORK="$(mktemp -d)" + trap 'rm -rf "$WORK"' EXIT +fi +SOURCES="$WORK/Sources/Probe" +rm -rf "$SOURCES" "$WORK/CModule" +mkdir -p "$SOURCES" +if [ "$ENGINE" = sqlite ] && [ -z "$DATABASE" ]; then + DATABASE="$WORK/probe.sqlite" + rm -f "$DATABASE" "$DATABASE-wal" "$DATABASE-shm" +fi + +while IFS= read -r source; do + ln -s "$source" "$SOURCES/driver_$(basename "$source")" +done < <(find "$PLUGINS/$FOLDER" -name '*.swift' -not -path '*/Tests/*') +for shared in PluginConnectTimeout PluginQueryTimeout PluginURLSessionTaskBox; do + [ -f "$PLUGINS/Shared/$shared.swift" ] && ln -s "$PLUGINS/Shared/$shared.swift" "$SOURCES/shared_$shared.swift" +done +dependencies='.product(name: "TableProPluginKit", package: "TableProCore"), + .product(name: "TableProSQLGrammar", package: "TableProCore"), + .product(name: "TableProProcess", package: "TableProCore"),' +system_target="" +c_flags="" +if [ -n "$C_MODULE" ]; then + ln -s "$PLUGINS/$FOLDER/$C_MODULE" "$WORK/CModule" + dependencies="$dependencies + \"$C_MODULE\"," + system_target=".systemLibrary(name: \"$C_MODULE\", path: \"CModule\")," + c_flags=".unsafeFlags([\"-Xcc\", \"-I$PLUGINS/$FOLDER/$C_MODULE/include\"])," +fi +for product in $PRODUCTS; do + dependencies="$dependencies + .product(name: \"$product\", package: \"TableProCore\")," +done +flags="\"-L$ROOT/Libs\", \"-L$ROOT/Libs/dylibs\", \"-Xlinker\", \"-rpath\", \"-Xlinker\", \"$ROOT/Libs/dylibs\"" +for flag in $LINK; do + flags="$flags, \"$flag\"" +done + +cat > "$WORK/Package.swift" << MANIFEST +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "StreamBackpressureProbe", + platforms: [.macOS(.v14)], + dependencies: [ + .package(path: "$ROOT/Packages/TableProCore"), + ], + targets: [ + $system_target + .executableTarget( + name: "Probe", + dependencies: [ + $dependencies + ], + path: "Sources/Probe", + swiftSettings: [ + .swiftLanguageMode(.v6), + .enableUpcomingFeature("MemberImportVisibility"), + $c_flags + ], + linkerSettings: [.unsafeFlags([$flags])] + ), + ] +) +MANIFEST + +cat > "$SOURCES/probe_Engine.swift" << SWIFT +typealias EnginePlugin = $CLASS +SWIFT + +cat > "$SOURCES/probe_Probe.swift" << 'SWIFT' +import Darwin +import Foundation +import TableProPluginKit + +@main +enum Probe { + static func main() async { + let arguments = CommandLine.arguments + guard arguments.count >= 8, let delay = Int(arguments[7]) else { + print("usage: Probe ") + exit(3) + } + let driver = EnginePlugin().createDriver(config: DriverConnectionConfig( + host: arguments[1], port: Int(arguments[2]) ?? 0, username: arguments[3], password: arguments[4], + database: arguments[5], additionalFields: [:] + )) + do { + try await driver.connect() + try await driver.applyQueryTimeout(0) + _ = try await driver.execute(query: "DROP TABLE IF EXISTS tp_stream_probe") + _ = try await driver.execute(query: arguments[6]) + } catch { + print("cannot prepare the table: \(error)") + exit(3) + } + + let sampler = FootprintSampler() + let baseline = sampler.start() + let started = Date() + var rows = 0 + var batches = 0 + var failure: Error? + do { + for try await element in driver.streamRows(query: "SELECT id, payload FROM tp_stream_probe ORDER BY id") { + guard case .rows(let batch) = element else { continue } + rows += batch.count + batches += 1 + try await Task.sleep(for: .milliseconds(delay)) + } + } catch { + failure = error + } + let peak = sampler.stop() + _ = try? await driver.execute(query: "DROP TABLE tp_stream_probe") + driver.disconnect() + + let megabytes = { (bytes: UInt64) in String(format: "%.1f MB", Double(bytes) / 1_048_576) } + print("rows read: \(rows) in \(batches) batches, \(String(format: "%.1f", Date().timeIntervalSince(started))) s") + print("footprint before: \(megabytes(baseline))") + print("footprint peak: \(megabytes(peak))") + print("peak over baseline: \(megabytes(peak > baseline ? peak - baseline : 0))") + if let failure { + print("read failed: \(failure)") + exit(1) + } + } +} + +final class FootprintSampler: @unchecked Sendable { + private let lock = NSLock() + private var peak: UInt64 = 0 + private var running = false + + func start() -> UInt64 { + let baseline = Self.footprint() + lock.withLock { + peak = baseline + running = true + } + Thread.detachNewThread { [self] in + while lock.withLock({ running }) { + let now = Self.footprint() + lock.withLock { peak = max(peak, now) } + usleep(10_000) + } + } + return baseline + } + + func stop() -> UInt64 { + let now = Self.footprint() + return lock.withLock { + running = false + return max(peak, now) + } + } + + static func footprint() -> UInt64 { + var info = task_vm_info_data_t() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let result = withUnsafeMutablePointer(to: &info) { pointer in + pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) + } + } + return result == KERN_SUCCESS ? info.phys_footprint : 0 + } +} +SWIFT + +echo "building the $ENGINE harness from $ROOT (the first build takes a few minutes)" +(cd "$WORK" && swift build -c release > "$WORK/build.log" 2>&1) || { + grep -E "error:" "$WORK/build.log" | head -30 >&2 + exit 3 +} +"$WORK/.build/release/Probe" "$HOST" "$PORT" "$USER_NAME" "$PASSWORD" "$DATABASE" "$CREATE" "$DELAY_MS"