diff --git a/apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdRangeScan.swift b/apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdRangeScan.swift new file mode 100644 index 0000000000..b7fa51787f --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdRangeScan.swift @@ -0,0 +1,157 @@ +import Foundation + +public extension EtcdKeyRange { + init(key: Data, rangeEnd: Data?) { + self.key = key + self.rangeEnd = rangeEnd + } +} + +public struct EtcdRangeBatch { + public let items: [Item] + public let more: Bool + public let revision: Int64? + + public init(items: [Item], more: Bool, revision: Int64?) { + self.items = items + self.more = more + self.revision = revision + } +} + +public struct EtcdPageStart: Equatable, Sendable { + public let range: EtcdKeyRange + public let revision: Int64? + + public init(range: EtcdKeyRange, revision: Int64?) { + self.range = range + self.revision = revision + } +} + +// Reads a key range a bounded batch per request instead of in one response. A batch sends no sort order, so etcd +// applies the limit as it reads, in ascending key order; a descending sort makes it read the whole range first. Every +// request after the first reads at the first one's revision, so the batches are one snapshot, as a single request is. +public enum EtcdRangeScan { + public static func keyAfter(_ key: Data) -> Data { + key + Data([0]) + } + + // The rest of `range` after `key`, in ascending order. + public static func range(after key: Data, in range: EtcdKeyRange) -> EtcdKeyRange { + EtcdKeyRange(key: keyAfter(key), rangeEnd: range.rangeEnd) + } + + // The part of `range` below `key`, which is where a descending read goes on. + public static func range(before key: Data, in range: EtcdKeyRange) -> EtcdKeyRange { + EtcdKeyRange(key: range.key, rangeEnd: key) + } + + // Hands each batch to `body` until it returns false, the range ends or `maxItems` have been read. + public static func scan( + _ range: EtcdKeyRange, + batchSize: Int, + maxItems: Int, + key: (Item) -> Data, + fetch: (_ range: EtcdKeyRange, _ limit: Int64, _ revision: Int64?) async throws -> EtcdRangeBatch, + body: ([Item]) throws -> Bool + ) async throws { + var current = range + var revision: Int64? + var read = 0 + while read < maxItems { + try Task.checkCancellation() + let batch = try await fetch(current, Int64(min(max(1, batchSize), maxItems - read)), revision) + revision = revision ?? batch.revision + read += batch.items.count + guard try body(batch.items), batch.more, let last = batch.items.last else { return } + current = Self.range(after: key(last), in: range) + } + } + + // Where the page at `offset` begins, found by reading keys alone and never past the offset. Nil when the range + // has no key at that offset. + public static func pageStart( + of range: EtcdKeyRange, + skipping offset: Int, + batchSize: Int, + fetchKeys: (_ range: EtcdKeyRange, _ limit: Int64, _ revision: Int64?) async throws -> EtcdRangeBatch + ) async throws -> EtcdPageStart? { + guard offset > 0 else { return EtcdPageStart(range: range, revision: nil) } + var current = range + var revision: Int64? + var remaining = offset + while remaining > 0 { + try Task.checkCancellation() + let batch = try await fetchKeys(current, Int64(min(remaining, max(1, batchSize))), revision) + revision = revision ?? batch.revision + guard batch.more, let last = batch.items.last else { return nil } + remaining -= batch.items.count + current = Self.range(after: last, in: range) + } + return EtcdPageStart(range: current, revision: revision) + } +} + +public enum EtcdCompaction { + // etcd's ErrCompacted, sent as gRPC OutOfRange. + public static func isCompacted(grpcCode: Int?, message: String) -> Bool { + grpcCode == 11 && message.contains("required revision has been compacted") + } + + // A read pinned to its first batch's revision fails once the store compacts past it. It starts over once at the + // current revision, which is what a single request would have read. + public static func restartingOnce( + when isCompacted: (Error) -> Bool, + _ read: () async throws -> Value + ) async throws -> Value { + do { + return try await read() + } catch let error where isCompacted(error) { + return try await read() + } + } +} + +// The matching rows of one page, gathered from rows read in ascending key order. A descending page is the tail of the +// ascending matches, so it keeps only the last `offset + limit` of them rather than every row in the range. +public struct EtcdPageCollector { + private let offset: Int + private let limit: Int + private let ascending: Bool + private var skipped = 0 + private var kept: [Item] = [] + + public init(offset: Int, limit: Int, ascending: Bool) { + self.offset = max(0, offset) + self.limit = max(0, limit) + self.ascending = ascending + } + + // False once an ascending page is complete, so the read can stop. + public mutating func add(_ item: Item) -> Bool { + if ascending { + guard kept.count < limit else { return false } + if skipped < offset { + skipped += 1 + } else { + kept.append(item) + } + return kept.count < limit + } + let capacity = offset + limit + guard capacity > 0 else { return true } + kept.append(item) + if kept.count >= 2 * capacity { + kept.removeFirst(kept.count - capacity) + } + return true + } + + public var page: [Item] { + guard !ascending else { return kept } + return Array(kept.suffix(offset + limit).dropLast(offset).reversed()) + } + + var heldCount: Int { kept.count } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementCursorIndex.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementCursorIndex.swift new file mode 100644 index 0000000000..af948724c7 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementCursorIndex.swift @@ -0,0 +1,74 @@ +import Foundation + +// Answers `SQLStatementScanner.locatedStatementAtCursor` for a document that changes a little at a time: the scan +// resumes from the last recorded statement boundary before the caret instead of from the top. +public struct SQLStatementCursorIndex: Sendable { + public static let defaultCheckpointSpacing = 4_096 + + public let grammar: SQLLexicalGrammar + public private(set) var unitsScanned = 0 + + private let checkpointSpacing: Int + // In offset order, and so in `decidedThrough` order too. + private var checkpoints: [SQLStatementScanner.Checkpoint] = [] + + public init(grammar: SQLLexicalGrammar, checkpointSpacing: Int = SQLStatementCursorIndex.defaultCheckpointSpacing) { + self.grammar = grammar + self.checkpointSpacing = max(1, checkpointSpacing) + } + + public var checkpointCount: Int { + checkpoints.count + } + + // Call before the next lookup whenever the text changed at or after `location`. + public mutating func textDidChange(at location: Int) { + while let last = checkpoints.last, last.decidedThrough >= location { + checkpoints.removeLast() + } + } + + public mutating func statementAtCursor(_ cursorPosition: Int, in text: String) -> SQLStatementScanner.LocatedStatement { + let length = (text as NSString).length + let cursor = min(max(0, cursorPosition), length) + let resume = lastCheckpoint(before: cursor) + let start = resume?.offset ?? 0 + let spacing = checkpointSpacing + + var lastRecorded = checkpoints.last?.offset ?? 0 + var recorded: [SQLStatementScanner.Checkpoint] = [] + var result = SQLStatementScanner.LocatedStatement(sql: "", offset: 0, hasContent: false) + SQLStatementScanner.scan( + sql: text, + cursorPosition: cursor, + grammar: grammar, + resumingAt: resume, + onCheckpoint: { checkpoint in + guard checkpoint.offset - lastRecorded >= spacing else { return } + recorded.append(checkpoint) + lastRecorded = checkpoint.offset + }, + onStatement: { statement in + result = statement + return false + } + ) + checkpoints.append(contentsOf: recorded) + unitsScanned += max(cursor, NSMaxRange(result.range)) - start + return result + } + + private func lastCheckpoint(before offset: Int) -> SQLStatementScanner.Checkpoint? { + var low = 0 + var high = checkpoints.count + while low < high { + let middle = (low + high) / 2 + if checkpoints[middle].offset < offset { + low = middle + 1 + } else { + high = middle + } + } + return low > 0 ? checkpoints[low - 1] : nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift index 062c6802fb..7e70ec3f98 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift @@ -274,20 +274,45 @@ public enum SQLStatementScanner { return result } - // MARK: - Private + // MARK: - Resuming + + // The span of a statement that had content, kept without its text so a checkpoint stays small. + struct EndedStatement: Sendable, Equatable { + let range: NSRange + let terminator: SQLStatementTerminator + let acceptsBindParameters: Bool + + func located(in text: NSString) -> LocatedStatement { + LocatedStatement( + sql: text.substring(with: range), + offset: range.location, + hasContent: true, + terminator: terminator, + acceptsBindParameters: acceptsBindParameters + ) + } + } - /// Walks the document once and hands each segment to `onStatement`, which returns false to stop. - /// - /// Segments tile the document: a statement runs from the end of the previous one to its terminator, so it carries - /// the whitespace before it. With a `cursorPosition`, only the segment holding it is reported, or the last one - /// when the cursor sits past every terminator. Where a statement ends is the tracker's decision; this loop only - /// lexes, so a `;` inside a string, a comment or a quoted body never reaches it. A `GO` line is the exception: it - /// ends the batch, so it ends whatever statement is open whatever the tracker thinks, and is a segment of its own - /// that holds nothing to run. - private static func scan( + // A point where the scan holds no state but the last statement with content, so it can start again there. + struct Checkpoint: Sendable, Equatable { + let offset: Int + // The last offset whose text decided this boundary: an edit there or earlier moves it. + let decidedThrough: Int + let endedStatement: EndedStatement? + } + + // MARK: - Scanning + + // Segments tile the document: each runs from the end of the previous one to its terminator, whitespace included. + // With a cursor only the segment holding it is reported, or the last one when it sits past every terminator. + // Where a statement ends is the tracker's decision, except at a `GO` line, which ends the batch whatever the + // tracker thinks and is a segment of its own that runs nothing. + static func scan( sql: String, cursorPosition: Int?, grammar: SQLLexicalGrammar, + resumingAt resume: Checkpoint? = nil, + onCheckpoint: (Checkpoint) -> Void = { _ in }, onBatchSeparator: (SQLBatchSeparator) -> Void = { _ in }, onFinish: (_ holdsOpenBlock: Bool) -> Void = { _ in }, onStatement: (LocatedStatement) -> Bool @@ -299,34 +324,42 @@ public enum SQLStatementScanner { let safePosition = cursorPosition.map { min(max(0, $0), length) } var tracker = SQLStatementBoundaries.makeTracker(for: grammar) - var currentStart = 0 + var currentStart = min(max(0, resume?.offset ?? 0), length) var hasStatementContent = false - var i = 0 + var i = currentStart - var lastStatementWithContent: LocatedStatement? + var lastEndedStatement = resume?.endedStatement - /// Reports the segment ending at `end` and starts the next one there. Returns false when the scan is done. - /// - /// A caret on a SQL*Plus `/` line stands for the statement the slash ends, which is where a reader who has - /// just typed the slash expects `Cmd+Enter` to act. A caret on a `GO` line stands for the last statement of the - /// batch it ends, and for nothing when that batch is empty: the batch before is not the one the line ends. + // A caret on a SQL*Plus `/` line stands for the statement the slash ends, where `Cmd+Enter` is expected to act. + // A caret on a `GO` line stands for the last statement of its batch, and for nothing when the batch is empty. func finishSegment(at end: Int, hasContent: Bool, standsForEndedStatement: Bool = false) -> Bool { - let statement = LocatedStatement( - sql: nsQuery.substring(with: NSRange(location: currentStart, length: end - currentStart)), - offset: currentStart, - hasContent: hasContent, - terminator: tracker.terminator, - acceptsBindParameters: tracker.acceptsBindParameters - ) + let range = NSRange(location: currentStart, length: end - currentStart) if hasContent { - lastStatementWithContent = statement + lastEndedStatement = EndedStatement( + range: range, + terminator: tracker.terminator, + acceptsBindParameters: tracker.acceptsBindParameters + ) } if let cursor = safePosition { guard cursor >= currentStart, cursor <= end else { currentStart = end return true } - _ = onStatement(standsForEndedStatement ? lastStatementWithContent ?? statement : statement) + if standsForEndedStatement, let ended = lastEndedStatement { + _ = onStatement(ended.located(in: nsQuery)) + return false + } + } + let statement = LocatedStatement( + sql: nsQuery.substring(with: range), + offset: currentStart, + hasContent: hasContent, + terminator: tracker.terminator, + acceptsBindParameters: tracker.acceptsBindParameters + ) + if safePosition != nil { + _ = onStatement(statement) return false } currentStart = end @@ -361,9 +394,10 @@ public enum SQLStatementScanner { } let separatorEnd = NSMaxRange(separator.range) guard finishSegment(at: separatorEnd, hasContent: false, standsForEndedStatement: true) else { return } - lastStatementWithContent = nil + lastEndedStatement = nil tracker.reset() hasStatementContent = false + onCheckpoint(Checkpoint(offset: separatorEnd, decidedThrough: separatorEnd, endedStatement: nil)) i = separatorEnd continue } @@ -392,6 +426,7 @@ public enum SQLStatementScanner { guard finishSegment(at: i + 1, hasContent: hasStatementContent) else { return } tracker.reset() hasStatementContent = false + onCheckpoint(Checkpoint(offset: i + 1, decidedThrough: i, endedStatement: lastEndedStatement)) i += 1 continue } @@ -403,6 +438,11 @@ public enum SQLStatementScanner { guard finishSegment(at: i + 1, hasContent: false, standsForEndedStatement: true) else { return } tracker.reset() hasStatementContent = false + onCheckpoint(Checkpoint( + offset: i + 1, + decidedThrough: endOfSlashLine(nsQuery, at: i, length: length), + endedStatement: lastEndedStatement + )) i += 1 continue } @@ -439,11 +479,17 @@ public enum SQLStatementScanner { before -= 1 } guard before < 0 || text.character(at: before) == SqlLexer.newline else { return false } + let after = endOfSlashLine(text, at: offset, length: length) + return after == length || text.character(at: after) == SqlLexer.newline + } + + // The last offset `isSlashLine` reads after the slash: its line break, or the end of the text. + private static func endOfSlashLine(_ text: NSString, at offset: Int, length: Int) -> Int { var after = offset + 1 while after < length, isLineBlank(text.character(at: after)) { after += 1 } - return after == length || text.character(at: after) == SqlLexer.newline + return after } private static func isLineBlank(_ character: UInt16) -> Bool { diff --git a/apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdRangeScanTests.swift b/apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdRangeScanTests.swift new file mode 100644 index 0000000000..17241c41c8 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdRangeScanTests.swift @@ -0,0 +1,276 @@ +import Foundation +import Testing + +@testable import TableProEtcdCore + +// Answers range requests the way etcd does for a request with no sort order: ascending keys, the limit applied as it +// reads, and `more` set when the limit cut the range short. +private final class RecordingKeyspace { + struct Request: Equatable { + let range: EtcdKeyRange + let limit: Int64 + let revision: Int64? + } + + static let revision: Int64 = 7 + static let everyKey = EtcdKeyRange(key: Data([0]), rangeEnd: Data([0])) + + let keys: [Data] + private(set) var requests: [Request] = [] + + init(count: Int) { + keys = (0.. Data { + Data(String(format: "/k/%06d", index).utf8) + } + + var keysRead: Int64 { + requests.reduce(0) { $0 + $1.limit } + } + + func fetch(_ range: EtcdKeyRange, _ limit: Int64, _ revision: Int64?) -> EtcdRangeBatch { + requests.append(Request(range: range, limit: limit, revision: revision)) + let inRange = keys.filter { key in + guard !key.lexicographicallyPrecedes(range.key) else { return false } + guard let end = range.rangeEnd else { return key == range.key } + return end == Data([0]) || key.lexicographicallyPrecedes(end) + } + let taken = Array(inRange.prefix(Int(limit))) + return EtcdRangeBatch(items: taken, more: inRange.count > taken.count, revision: Self.revision) + } +} + +private struct Compacted: Error {} +private struct Refused: Error {} + +struct EtcdRangeScanTests { + @Test("A page deep in the range is found by reading keys up to its offset, never past it") + func pageStartReadsKeysUpToTheOffset() async throws { + let keyspace = RecordingKeyspace(count: 10_000) + + let start = try await EtcdRangeScan.pageStart( + of: RecordingKeyspace.everyKey, skipping: 2_500, batchSize: 1_000 + ) { keyspace.fetch($0, $1, $2) } + + let limits = keyspace.requests.map(\.limit) + #expect(limits == [1_000, 1_000, 500]) + #expect(keyspace.keysRead == 2_500) + #expect(start?.range == EtcdKeyRange( + key: RecordingKeyspace.key(2_499) + Data([0]), rangeEnd: Data([0]) + )) + #expect(start?.revision == RecordingKeyspace.revision) + } + + @Test("Every skip request after the first reads at the first one's revision") + func pageStartPinsTheRevision() async throws { + let keyspace = RecordingKeyspace(count: 5_000) + + _ = try await EtcdRangeScan.pageStart( + of: RecordingKeyspace.everyKey, skipping: 3_000, batchSize: 1_000 + ) { keyspace.fetch($0, $1, $2) } + + let revisions = keyspace.requests.map(\.revision) + #expect(revisions == [nil, RecordingKeyspace.revision, RecordingKeyspace.revision]) + } + + @Test("The first page needs no skip request") + func firstPageSendsNothing() async throws { + let keyspace = RecordingKeyspace(count: 10) + + let start = try await EtcdRangeScan.pageStart( + of: RecordingKeyspace.everyKey, skipping: 0, batchSize: 1_000 + ) { keyspace.fetch($0, $1, $2) } + + #expect(keyspace.requests.isEmpty) + #expect(start == EtcdPageStart(range: RecordingKeyspace.everyKey, revision: nil)) + } + + @Test("A page at or past the last key has no start", arguments: [10, 11, 50]) + func pagePastTheEndHasNoStart(offset: Int) async throws { + let keyspace = RecordingKeyspace(count: 10) + + let start = try await EtcdRangeScan.pageStart( + of: RecordingKeyspace.everyKey, skipping: offset, batchSize: 4 + ) { keyspace.fetch($0, $1, $2) } + + #expect(start == nil) + } + + @Test("A scan asks for one batch at a time and stops when the reader has enough") + func scanStopsWhenTheBodyDoes() async throws { + let keyspace = RecordingKeyspace(count: 10_000) + var batches: [[Data]] = [] + + try await EtcdRangeScan.scan( + RecordingKeyspace.everyKey, + batchSize: 1_000, + maxItems: .max, + key: { (key: Data) in key }, + fetch: { keyspace.fetch($0, $1, $2) }, + body: { (batch: [Data]) in + batches.append(batch) + return batches.count < 2 + } + ) + + let limits = keyspace.requests.map(\.limit) + let delivered = batches.flatMap { $0 } + let expected = (0..<2_000).map(RecordingKeyspace.key) + #expect(limits == [1_000, 1_000]) + #expect(delivered == expected) + } + + @Test("A scan reads the whole range in order, each key once") + func scanReadsEveryKeyOnce() async throws { + let keyspace = RecordingKeyspace(count: 2_345) + var seen: [Data] = [] + + try await EtcdRangeScan.scan( + RecordingKeyspace.everyKey, + batchSize: 1_000, + maxItems: .max, + key: { (key: Data) in key }, + fetch: { keyspace.fetch($0, $1, $2) }, + body: { (batch: [Data]) in + seen += batch + return true + } + ) + + #expect(seen == keyspace.keys) + #expect(keyspace.requests.count == 3) + } + + @Test("A scan stops at its row ceiling") + func scanStopsAtTheCeiling() async throws { + let keyspace = RecordingKeyspace(count: 5_000) + var seen = 0 + + try await EtcdRangeScan.scan( + RecordingKeyspace.everyKey, + batchSize: 1_000, + maxItems: 2_500, + key: { (key: Data) in key }, + fetch: { keyspace.fetch($0, $1, $2) }, + body: { (batch: [Data]) in + seen += batch.count + return true + } + ) + + let limits = keyspace.requests.map(\.limit) + #expect(seen == 2_500) + #expect(limits == [1_000, 1_000, 500]) + } + + @Test("A descending read goes on below the last key it returned") + func descendingContinuationEndsAtTheLastKey() { + let prefix = EtcdKeyRange(key: Data("/k/".utf8), prefix: true) + + #expect(EtcdRangeScan.range(before: Data("/k/5".utf8), in: prefix) + == EtcdKeyRange(key: Data("/k/".utf8), rangeEnd: Data("/k/5".utf8))) + } + + @Test("An ascending page skips its offset in matches and stops reading once full") + func ascendingCollectorStopsWhenFull() { + var collector = EtcdPageCollector(offset: 3, limit: 2, ascending: true) + var accepted: [Bool] = [] + for item in 0..<10 { + let goesOn = collector.add(item) + accepted.append(goesOn) + if !goesOn { break } + } + + #expect(collector.page == [3, 4]) + #expect(accepted == [true, true, true, true, false]) + } + + @Test("A descending page is the tail of the ascending matches, newest first") + func descendingCollectorTakesTheTail() { + var collector = EtcdPageCollector(offset: 2, limit: 3, ascending: false) + for item in 0..<10 { + _ = collector.add(item) + } + + #expect(collector.page == [7, 6, 5]) + } + + @Test("A descending page holds at most twice its offset and limit, however long the range") + func descendingCollectorStaysBounded() { + var collector = EtcdPageCollector(offset: 5, limit: 5, ascending: false) + var largestHeld = 0 + for item in 0..<100_000 { + _ = collector.add(item) + largestHeld = max(largestHeld, collector.heldCount) + } + + #expect(largestHeld <= 20) + #expect(collector.page == [99_994, 99_993, 99_992, 99_991, 99_990]) + } + + @Test("Only etcd's compacted-revision answer counts as compaction") + func compactionIsRecognized() { + #expect(EtcdCompaction.isCompacted(grpcCode: 11, message: "etcdserver: mvcc: required revision has been compacted")) + #expect(!EtcdCompaction.isCompacted(grpcCode: 11, message: "etcdserver: mvcc: required revision is a future revision")) + #expect(!EtcdCompaction.isCompacted(grpcCode: 3, message: "required revision has been compacted")) + } + + @Test("A compacted read starts over once, and a second compaction is the caller's") + func compactedReadStartsOverOnce() async { + var attempts = 0 + var failure: Error? + do { + _ = try await EtcdCompaction.restartingOnce(when: { $0 is Compacted }) { () async throws -> Int in + attempts += 1 + throw Compacted() + } + } catch { + failure = error + } + + #expect(failure is Compacted) + #expect(attempts == 2) + } + + @Test("A read that recovers on its second attempt returns that attempt's answer") + func compactedReadReturnsTheRetry() async throws { + var attempts = 0 + let answer = try await EtcdCompaction.restartingOnce(when: { $0 is Compacted }) { () async throws -> Int in + attempts += 1 + if attempts == 1 { throw Compacted() } + return 42 + } + + #expect(answer == 42) + #expect(attempts == 2) + } + + @Test("Any other failure is not retried") + func otherFailuresAreNotRetried() async { + var attempts = 0 + var failure: Error? + do { + _ = try await EtcdCompaction.restartingOnce(when: { $0 is Compacted }) { () async throws -> Int in + attempts += 1 + throw Refused() + } + } catch { + failure = error + } + + #expect(failure is Refused) + #expect(attempts == 1) + } + + @Test("A descending page past the matches is empty") + func descendingCollectorPastTheEnd() { + var collector = EtcdPageCollector(offset: 8, limit: 5, ascending: false) + for item in 0..<5 { + _ = collector.add(item) + } + + #expect(collector.page.isEmpty) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginDefaultStreamRowLimitTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginDefaultStreamRowLimitTests.swift new file mode 100644 index 0000000000..2547487c4c --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginDefaultStreamRowLimitTests.swift @@ -0,0 +1,70 @@ +import Foundation +import TableProPluginKit +import Testing + +// The default stream dropped `isTruncated`, so a result the driver cut reached the export as a whole one. +struct PluginDefaultStreamRowLimitTests { + private func header(of driver: some PluginDatabaseDriver) async throws -> PluginStreamHeader? { + for try await element in driver.streamRows(query: "SELECT id FROM t") { + if case .header(let header) = element { return header } + } + return nil + } + + @Test("A result the driver cut declares the row count it stopped at") + func cutResultDeclaresItsLimit() async throws { + let header = try await header(of: CutResultDriver(rowCount: 3, isCut: true)) + #expect(header?.rowLimit == 3) + } + + @Test("A whole result declares no limit") + func wholeResultDeclaresNoLimit() async throws { + let header = try await header(of: CutResultDriver(rowCount: 3, isCut: false)) + #expect(header != nil) + #expect(header?.rowLimit == nil) + } +} + +private final class CutResultDriver: PluginDatabaseDriver, @unchecked Sendable { + let supportsSchemas = false + let supportsTransactions = false + let currentSchema: String? = nil + let serverVersion: String? = nil + + private let rowCount: Int + private let isCut: Bool + + init(rowCount: Int, isCut: Bool) { + self.rowCount = rowCount + self.isCut = isCut + } + + func connect() async throws {} + func disconnect() {} + func ping() async throws {} + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult( + columns: ["id"], + columnTypeNames: ["INTEGER"], + rows: (0.. [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginDriverDefaultCancellationTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginDriverDefaultCancellationTests.swift new file mode 100644 index 0000000000..bb0ef0397e --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginDriverDefaultCancellationTests.swift @@ -0,0 +1,193 @@ +import Foundation +import TableProPluginKit +import Testing + +// The default row stream ran its query in an unstructured task nothing cancelled, so an export +// stopped on a driver without its own stream kept the query and the connection busy to the end. +struct PluginDriverDefaultCancellationTests { + @Test("The default row stream carries the header and every row the query returned") + func defaultStreamCarriesTheResult() async throws { + let driver = HeldQueryDriver(holdsQuery: false) + var columns: [String] = [] + var rows: [PluginRow] = [] + for try await element in driver.streamRows(query: "SELECT v FROM t") { + switch element { + case .header(let header): columns = header.columns + case .rows(let batch): rows += batch + } + } + #expect(columns == ["v"]) + #expect(rows == [["1"], ["2"]]) + #expect(driver.executeCount == 1) + } + + @Test("Cancelling the consumer of the default row stream cancels the query it runs", .timeLimit(.minutes(1))) + func consumerCancelReachesTheQuery() async { + let driver = HeldQueryDriver(holdsQuery: true) + let consumer = Task { + do { + for try await _ in driver.streamRows(query: "SELECT v FROM t") {} + } catch {} + } + await driver.waitUntilQueryStarts() + consumer.cancel() + await driver.waitUntilQueryIsCancelled() + await consumer.value + #expect(driver.executeCount == 1) + } + + @Test("Dropping the default row stream cancels the query it runs", .timeLimit(.minutes(1))) + func droppedStreamCancelsTheQuery() async { + let driver = HeldQueryDriver(holdsQuery: true) + var stream: AsyncThrowingStream? = driver.streamRows(query: "SELECT v FROM t") + await driver.waitUntilQueryStarts() + #expect(stream != nil) + stream = nil + await driver.waitUntilQueryIsCancelled() + } + + @Test("The per-table foreign key read stops at the next table once its caller is cancelled") + func foreignKeyLoopStopsOnCancel() async { + let driver = CancellingCatalogDriver(tableCount: 50) + let read = Task { try await driver.fetchAllForeignKeys(schema: nil) } + await #expect(throws: CancellationError.self) { + _ = try await read.value + } + #expect(driver.perTableReads == 1) + } + + @Test("The per-table metadata read, which skips unreadable tables, still stops once its caller is cancelled") + func tableMetadataLoopStopsOnCancel() async { + let driver = CancellingCatalogDriver(tableCount: 50) + let read = Task { try await driver.fetchAllTableMetadata(schema: nil) } + await #expect(throws: CancellationError.self) { + _ = try await read.value + } + #expect(driver.perTableReads == 1) + } +} + +private final class HeldQueryDriver: PluginDatabaseDriver, @unchecked Sendable { + let supportsSchemas = false + let supportsTransactions = false + let currentSchema: String? = nil + let serverVersion: String? = nil + + private let holdsQuery: Bool + private let lock = NSLock() + private var executions = 0 + private let started: AsyncStream + private let startedSignal: AsyncStream.Continuation + private let cancelled: AsyncStream + private let cancelledSignal: AsyncStream.Continuation + // Never finished, so only a cancel of the task reading it ends the read. + private let wire: AsyncStream + private let wireWriter: AsyncStream.Continuation + + init(holdsQuery: Bool) { + self.holdsQuery = holdsQuery + (started, startedSignal) = AsyncStream.makeStream() + (cancelled, cancelledSignal) = AsyncStream.makeStream() + (wire, wireWriter) = AsyncStream.makeStream() + } + + var executeCount: Int { lock.withLock { executions } } + + func waitUntilQueryStarts() async { + for await _ in started { return } + } + + func waitUntilQueryIsCancelled() async { + for await _ in cancelled { return } + } + + func connect() async throws {} + func disconnect() {} + func ping() async throws {} + + func execute(query: String) async throws -> PluginQueryResult { + lock.withLock { executions += 1 } + guard holdsQuery else { + return PluginQueryResult( + columns: ["v"], columnTypeNames: ["INT"], rows: [["1"], ["2"]], rowsAffected: 0, executionTime: 0 + ) + } + startedSignal.yield() + for await _ in wire {} + if Task.isCancelled { cancelledSignal.yield() } + throw CancellationError() + } + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} + +// Each per-table read cancels the task that made it and still answers, the way a driver that never +// checks for cancellation finishes its round trip. +private final class CancellingCatalogDriver: PluginDatabaseDriver, @unchecked Sendable { + let supportsSchemas = false + let supportsTransactions = false + let currentSchema: String? = nil + let serverVersion: String? = nil + + private let tableCount: Int + private let lock = NSLock() + private var reads = 0 + + init(tableCount: Int) { + self.tableCount = tableCount + } + + var perTableReads: Int { lock.withLock { reads } } + + private func cancellingRead() { + lock.withLock { reads += 1 } + withUnsafeCurrentTask { $0?.cancel() } + } + + func connect() async throws {} + func disconnect() {} + func ping() async throws {} + func execute(query: String) async throws -> PluginQueryResult { .empty } + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { + (0.. [PluginForeignKeyInfo] { + cancellingRead() + return [ + PluginForeignKeyInfo( + name: "fk_\(table)", column: "parent_id", referencedTable: "parent", referencedColumn: "id", + referencedDatabase: nil + ) + ] + } + + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + cancellingRead() + return PluginTableMetadata(tableName: table) + } + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift new file mode 100644 index 0000000000..c63a3d7bbe --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift @@ -0,0 +1,267 @@ +import Foundation +import TableProSQLGrammar +import Testing + +struct CursorIndexFixture: Sendable, CustomStringConvertible { + let typeId: String + let text: String + + var description: String { + typeId + } +} + +private let cursorIndexFixtures = [ + CursorIndexFixture(typeId: "PostgreSQL", text: """ + SELECT 1; + + -- a note; with a semicolon + SELECT 'a;b'; + CREATE FUNCTION f() RETURNS int AS $$ BEGIN RETURN 1; END; $$ LANGUAGE plpgsql; + /* c; */ SELECT E'x' + 'y';; + SELECT 2 + """), + CursorIndexFixture(typeId: "MySQL", text: """ + CREATE PROCEDURE p() + BEGIN + SELECT 1; + SELECT 2; + END; + SELECT 3;; + # hash; comment + SELECT `a;b` FROM t; + SELECT /*! 1; */ 2 + """), + CursorIndexFixture(typeId: "SQL Server", text: """ + SELECT 1 + GO + SELECT 2; + GO 3 + MERGE t USING s ON 1 = 1 WHEN MATCHED THEN DELETE; + GO + + GO + SELECT [a;b] + GO + """), + CursorIndexFixture(typeId: "Oracle", text: """ + BEGIN + NULL; + END; + / + SELECT 1 FROM dual + / + SELECT q'[a;b]' FROM dual; + / + CREATE OR REPLACE PROCEDURE p IS + BEGIN + NULL; + END; + / + """), + CursorIndexFixture(typeId: "Cassandra", text: """ + BEGIN BATCH + INSERT INTO t (a) VALUES (1); + INSERT INTO t (a) VALUES (2); + APPLY BATCH; + SELECT * FROM t; + """), +] + +private let cursorIndexInsertions = [ + ";", "'", "\nGO\n", "\n/\n", " ", "x", "$$", "BEGIN ", "END;", "--", "/*", "*/", "\n", "GO", "/", +] + +private let longScriptRow = "SELECT id, name FROM users WHERE active = true;\n" + +struct SQLStatementCursorIndexTests { + private static func grammar(_ typeId: String) -> SQLLexicalGrammar { + SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil).execution + } + + private struct Resolved: Equatable, CustomStringConvertible { + let offset: Int + let sql: String + let hasContent: Bool + let terminator: SQLStatementTerminator + let acceptsBindParameters: Bool + let contentRange: NSRange + + init(_ statement: SQLStatementScanner.LocatedStatement) { + offset = statement.offset + sql = statement.sql + hasContent = statement.hasContent + terminator = statement.terminator + acceptsBindParameters = statement.acceptsBindParameters + contentRange = statement.contentRange + } + + var description: String { + "\(offset) \(hasContent) \(sql.debugDescription)" + } + } + + private static func fresh(_ text: String, _ cursor: Int, _ grammar: SQLLexicalGrammar) -> Resolved { + Resolved(SQLStatementScanner.locatedStatementAtCursor(in: text, cursorPosition: cursor, grammar: grammar)) + } + + @Test("Every caret resolves to what a scan from the top resolves to", arguments: cursorIndexFixtures) + func everyCaretMatchesAFreshScan(_ fixture: CursorIndexFixture) { + let grammar = Self.grammar(fixture.typeId) + let text = fixture.text + let length = (text as NSString).length + var index = SQLStatementCursorIndex(grammar: grammar, checkpointSpacing: 1) + var generator = SplitMix64(seed: 7) + let shuffled = Array(0...length).shuffled(using: &generator) + + for cursor in Array(0...length) + Array((0...length).reversed()) + shuffled { + #expect(Resolved(index.statementAtCursor(cursor, in: text)) == Self.fresh(text, cursor, grammar), "caret \(cursor)") + } + #expect(index.checkpointCount > 0) + } + + @Test("After any edit every caret still resolves to what a fresh scan resolves to", arguments: cursorIndexFixtures) + func editsKeepEveryAnswer(_ fixture: CursorIndexFixture) { + let grammar = Self.grammar(fixture.typeId) + var text = fixture.text + var index = SQLStatementCursorIndex(grammar: grammar, checkpointSpacing: 1) + var generator = SplitMix64(seed: 42) + + for step in 0..<80 { + let length = (text as NSString).length + for cursor in Array(0...length).shuffled(using: &generator).prefix(24) { + _ = index.statementAtCursor(cursor, in: text) + } + + let mutable = NSMutableString(string: text) + if length > 0, Bool.random(using: &generator) { + let start = Int.random(in: 0..= length - 2 * rowLength) + + let moves = 1_000 + for step in 0.. UInt64 { + state &+= 0x9E37_79B9_7F4A_7C15 + var value = state + value = (value ^ (value >> 30)) &* 0xBF58_476D_1CE4_E5B9 + value = (value ^ (value >> 27)) &* 0x94D0_49BB_1331_11EB + return value ^ (value >> 31) + } +} diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift index fa25fee925..97fdc15781 100644 --- a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift @@ -989,6 +989,7 @@ public final class OracleCoreConnection: @unchecked Sendable { public func streamQuery( _ query: String, + batching: OracleRowBatcher, continuation: AsyncThrowingStream.Continuation ) async throws { let role = OracleTransactionRole(of: query) @@ -1000,7 +1001,14 @@ public final class OracleCoreConnection: @unchecked Sendable { let admitted = try admit(role) try await withQueryDeadline { [self] in let zones = await sessionZones(on: connection) - try await streamRows(query, options: admitted.options, zones: zones, on: connection, continuation: continuation) + try await streamRows( + query, + options: admitted.options, + zones: zones, + batching: batching, + on: connection, + continuation: continuation + ) } await recordSuccess(of: role, on: admitted.session) forgetSessionZonesIfChanged(by: query) @@ -1019,6 +1027,7 @@ public final class OracleCoreConnection: @unchecked Sendable { _ query: String, options: StatementOptions, zones: OracleSessionTimeZones, + batching: OracleRowBatcher, on connection: OracleNIO.OracleConnection, continuation: AsyncThrowingStream.Continuation ) async throws { @@ -1029,10 +1038,20 @@ public final class OracleCoreConnection: @unchecked Sendable { var decoder = CellDecoder(columns: columns, zones: zones) continuation.yield(.header(columns: columns.map(\.descriptor))) - for try await row in stream { - try Task.checkCancellation() - continuation.yield(.rows([decoder.decode(row)])) + var batcher = batching + do { + for try await row in stream { + try Task.checkCancellation() + if let batch = batcher.append(decoder.decode(row)) { + continuation.yield(.rows(batch)) + } + } + } catch { + // The rows read before a failure still reach the reader ahead of the error, as they did one by one. + if let batch = batcher.remainder() { continuation.yield(.rows(batch)) } + throw error } + if let batch = batcher.remainder() { continuation.yield(.rows(batch)) } await warnOnce(about: decoder.unsupportedTypeNames) guard columns.isEmpty else { return } diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleRowBatcher.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleRowBatcher.swift new file mode 100644 index 0000000000..ebd111018a --- /dev/null +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleRowBatcher.swift @@ -0,0 +1,43 @@ +import Foundation + +// Groups decoded rows into stream elements, so a long read pays its stream hops once per batch rather than once per row. +public struct OracleRowBatcher: Sendable { + public static let batchRows = 5_000 + + private let rowsPerBatch: Int + // The rows a capped reader takes before it stops. The batch that completes them goes out at once, so the reader + // never waits on rows past its cap. + private let rowBudget: Int? + private var delivered = 0 + private var pending: [[OracleRawCell]] = [] + + public init(rowsPerBatch: Int, rowBudget: Int? = nil) { + self.rowsPerBatch = max(1, rowsPerBatch) + self.rowBudget = rowBudget.map { max(1, $0) } + } + + public mutating func append(_ row: [OracleRawCell]) -> [[OracleRawCell]]? { + if pending.isEmpty, delivered > 0 { pending.reserveCapacity(nextBatchCapacity) } + pending.append(row) + // The first row goes out alone, since a reader times it as the moment the server began to answer. + let isFirstRow = delivered == 0 + guard isFirstRow || pending.count >= rowsPerBatch || rowBudget == delivered + pending.count else { return nil } + return take() + } + + public mutating func remainder() -> [[OracleRawCell]]? { + pending.isEmpty ? nil : take() + } + + private var nextBatchCapacity: Int { + guard let rowBudget, rowBudget > delivered else { return rowsPerBatch } + return min(rowsPerBatch, rowBudget - delivered) + } + + private mutating func take() -> [[OracleRawCell]] { + let batch = pending + delivered += batch.count + pending = [] + return batch + } +} diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleRowBatcherTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleRowBatcherTests.swift new file mode 100644 index 0000000000..55b3a32e20 --- /dev/null +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleRowBatcherTests.swift @@ -0,0 +1,93 @@ +import Foundation +import TableProOracleCore +import Testing + +struct OracleRowBatcherTests { + private static func row(_ index: Int) -> [OracleRawCell] { + [.string(String(index))] + } + + // The batch sizes the stream carries for `count` rows, in order, and the rows they carry. + private static func drive(_ batcher: OracleRowBatcher, rows count: Int) -> (sizes: [Int], rows: [[OracleRawCell]]) { + var batcher = batcher + var sizes: [Int] = [] + var rows: [[OracleRawCell]] = [] + for index in 0.. [Table] { - var keyCounts: [String: Int] = [:] + func count(_ keys: [Data], into keyCounts: inout [String: Int]) { for key in keys { keyCounts[tableName(forKey: key), default: 0] += 1 } + } + + func tables(fromKeyCounts counts: [String: Int]) -> [Table] { + var keyCounts = counts let rootKeyCount = keyCounts.removeValue(forKey: Self.rootTableName) let prefixTables = keyCounts .sorted { $0.key < $1.key } diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift index 60e0518d0d..e8d3c50f61 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift @@ -17,6 +17,13 @@ private extension Array where Element == String { var asCells: [PluginCellValue] { map(PluginCellValue.text) } } +private extension Error { + var isEtcdRevisionCompacted: Bool { + guard let fault = (self as? EtcdError)?.serverFault else { return false } + return EtcdCompaction.isCompacted(grpcCode: fault.grpcCode, message: fault.message) + } +} + final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private let config: DriverConnectionConfig private var _httpClient: EtcdHttpClient? @@ -30,7 +37,9 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "EtcdPluginDriver") private static let maxKeys = PluginRowLimits.emergencyMax - + // Values make a response as large as the data, so a batch of them is kept far smaller than a batch of keys. + private static let valueBatchSize = 1_000 + private static let keyBatchSize = 10_000 private static let columns = EtcdColumn.allCases.map(\.rawValue) private static let columnTypeNames = ["String", "String", "Int64", "Int64", "Int64", "String"] @@ -240,24 +249,34 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { ))) let range = Self.prefixRange(prefix) - var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: Int64(Self.maxKeys)) - req.sortOrder = sortAscending ? "ASCEND" : "DESCEND" - req.sortTarget = "KEY" - - let response = try await client.rangeRequest(req) - var rows: [PluginRow] = [] - - for kv in response.kvs ?? [] { - try Task.checkCancellation() - let row = Self.filterRow(kv) - guard matcher.matches(row) else { continue } - rows.append(row.cells) + guard sortAscending else { + // etcd reads only in ascending key order and sorts a descending range whole, so the range is one response. + var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: Int64(Self.maxKeys)) + req.sortOrder = "DESCEND" + req.sortTarget = "KEY" + let response = try await client.rangeRequest(req) + let rows = (response.kvs ?? []).map(Self.filterRow).filter(matcher.matches).map(\.cells) + if !rows.isEmpty { + continuation.yield(.rows(rows)) + } + continuation.finish() + return } - if !rows.isEmpty { - continuation.yield(.rows(rows)) + do { + try await scanKeyValues(in: range, client: client) { kvs in + let rows = kvs.map(Self.filterRow).filter(matcher.matches).map(\.cells) + if !rows.isEmpty { + continuation.yield(.rows(rows)) + } + return true + } + } catch let error where error.isEtcdRevisionCompacted { + // Rows already went out, so the read cannot start over at a newer revision without repeating them. + throw EtcdError.serverError(String( + localized: "etcd compacted the revision this export started from, so it stopped part way. Run it again." + )) } - continuation.finish() } @@ -278,17 +297,24 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { throw EtcdError.notConnected } - let range = Self.prefixRange(keyspace.root) - - let response = try await client.rangeRequest(EtcdRangeRequest( - key: range.key, - rangeEnd: range.rangeEnd, - limit: Int64(Self.maxKeys), - keysOnly: true - )) - - let keys = (response.kvs ?? []).map(\.key) - return keyspace.tables(forKeys: keys).map { + let keyCounts: [String: Int] = try await restartingOnCompaction { + var keyCounts: [String: Int] = [:] + try await EtcdRangeScan.scan( + Self.prefixRange(keyspace.root), + batchSize: Self.keyBatchSize, + maxItems: Self.maxKeys, + key: { $0 }, + fetch: { range, limit, revision in + try await Self.keyBatch(in: range, limit: limit, revision: revision, client: client) + }, + body: { (keys: [Data]) in + keyspace.count(keys, into: &keyCounts) + return true + } + ) + return keyCounts + } + return keyspace.tables(fromKeyCounts: keyCounts).map { PluginTableInfo(name: $0.name, type: "PREFIX", rowCount: $0.keyCount) } } @@ -841,15 +867,17 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } if let parsed = EtcdQueryBuilder.parseRangeQuery(query) { - return try await fetchKeysPage( - prefix: parsed.prefix, - offset: parsed.offset, - limit: parsed.limit, - sortAscending: parsed.sortAscending, - filter: parsed.filter, - client: client, - startTime: startTime - ) + return try await restartingOnCompaction { + try await fetchKeysPage( + prefix: parsed.prefix, + offset: parsed.offset, + limit: parsed.limit, + sortAscending: parsed.sortAscending, + filter: parsed.filter, + client: client, + startTime: startTime + ) + } } if let parsed = EtcdQueryBuilder.parseCountQuery(query) { @@ -879,20 +907,127 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { ) async throws -> PluginQueryResult { let matcher = try filter.matcher() let range = Self.prefixRange(prefix) - let fetchLimit = filter.isUnfiltered ? Int64(min(offset + limit, Self.maxKeys)) : Int64(Self.maxKeys) + let pageLimit = min(limit, max(0, Self.maxKeys - offset)) + guard pageLimit > 0 else { return emptyResult(startTime: startTime) } + + guard filter.isUnfiltered else { + // Past the row ceiling an ascending scan stops at the smallest keys, while a descending page reads from + // the largest, so that page still comes from one sorted request. + if !sortAscending, try await countKeys(prefix: prefix, client: client) > Self.maxKeys { + return try await sortedDescendingPage( + of: range, offset: offset, limit: limit, matcher: matcher, client: client, startTime: startTime + ) + } + var collector = EtcdPageCollector(offset: offset, limit: limit, ascending: sortAscending) + try await scanKeyValues(in: range, client: client) { kvs in + for kv in kvs { + let row = Self.filterRow(kv) + guard matcher.matches(row) else { continue } + guard collector.add(row) else { return false } + } + return true + } + return result(of: collector.page, startTime: startTime) + } + + let kvs = sortAscending + ? try await ascendingPage(of: range, offset: offset, limit: pageLimit, client: client) + : try await descendingPage(of: range, offset: offset, limit: pageLimit, client: client) + return result(of: kvs.map(Self.filterRow), startTime: startTime) + } - var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: fetchLimit) - req.sortOrder = sortAscending ? "ASCEND" : "DESCEND" + private func sortedDescendingPage( + of range: EtcdKeyRange, + offset: Int, + limit: Int, + matcher: EtcdRowMatcher, + client: EtcdHttpClient, + startTime: Date + ) async throws -> PluginQueryResult { + var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: Int64(Self.maxKeys)) + req.sortOrder = "DESCEND" req.sortTarget = "KEY" + let rows = (try await client.rangeRequest(req).kvs ?? []).map(Self.filterRow).filter(matcher.matches) + guard offset < rows.count else { return emptyResult(startTime: startTime) } + return result(of: Array(rows[offset ..< min(offset + limit, rows.count)]), startTime: startTime) + } - let response = try await client.rangeRequest(req) - let rows = (response.kvs ?? []).map(Self.filterRow).filter(matcher.matches) + private func restartingOnCompaction(_ read: () async throws -> Value) async throws -> Value { + try await EtcdCompaction.restartingOnce(when: { $0.isEtcdRevisionCompacted }, read) + } + + // Keys alone carry the read up to the page, then one request reads the page's values from where it starts. + private func ascendingPage( + of range: EtcdKeyRange, offset: Int, limit: Int, client: EtcdHttpClient + ) async throws -> [EtcdKeyValue] { + let start = try await EtcdRangeScan.pageStart(of: range, skipping: offset, batchSize: Self.keyBatchSize) { + try await Self.keyBatch(in: $0, limit: $1, revision: $2, client: client) + } + guard let start else { return [] } + var req = EtcdRangeRequest(key: start.range.key, rangeEnd: start.range.rangeEnd, limit: Int64(limit)) + req.revision = start.revision + return try await client.rangeRequest(req).kvs ?? [] + } - guard offset < rows.count else { - return emptyResult(startTime: startTime) + // A descending read sorts the whole range on the server however it is split, so the skip stays one request, + // of keys alone. + private func descendingPage( + of range: EtcdKeyRange, offset: Int, limit: Int, client: EtcdHttpClient + ) async throws -> [EtcdKeyValue] { + var pageRange = range + var revision: Int64? + if offset > 0 { + var skip = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: Int64(offset), keysOnly: true) + skip.sortOrder = "DESCEND" + skip.sortTarget = "KEY" + let skipped = try await client.rangeRequest(skip) + guard skipped.more == true, let last = skipped.kvs?.last else { return [] } + pageRange = EtcdRangeScan.range(before: last.key, in: range) + revision = Self.revision(of: skipped) } - let pageEnd = min(offset + limit, rows.count) - return result(of: Array(rows[offset ..< pageEnd]), startTime: startTime) + var req = EtcdRangeRequest(key: pageRange.key, rangeEnd: pageRange.rangeEnd, limit: Int64(limit)) + req.sortOrder = "DESCEND" + req.sortTarget = "KEY" + req.revision = revision + return try await client.rangeRequest(req).kvs ?? [] + } + + // Ascending key order, a bounded batch per response, all at one revision. + private func scanKeyValues( + in range: EtcdKeyRange, + client: EtcdHttpClient, + body: ([EtcdKeyValue]) throws -> Bool + ) async throws { + try await EtcdRangeScan.scan( + range, + batchSize: Self.valueBatchSize, + maxItems: Self.maxKeys, + key: { $0.key }, + fetch: { range, limit, revision in + var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: limit) + req.revision = revision + let response = try await client.rangeRequest(req) + return EtcdRangeBatch( + items: response.kvs ?? [], more: response.more ?? false, revision: Self.revision(of: response) + ) + }, + body: body + ) + } + + private static func keyBatch( + in range: EtcdKeyRange, limit: Int64, revision: Int64?, client: EtcdHttpClient + ) async throws -> EtcdRangeBatch { + var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: limit, keysOnly: true) + req.revision = revision + let response = try await client.rangeRequest(req) + return EtcdRangeBatch( + items: (response.kvs ?? []).map(\.key), more: response.more ?? false, revision: Self.revision(of: response) + ) + } + + private static func revision(of response: EtcdRangeResponse) -> Int64? { + response.header?.revision.flatMap { Int64($0) } } private func countKeys(prefix: String, client: EtcdHttpClient) async throws -> Int { diff --git a/apple/Plugins/FormatPluginTests/JSONDocumentStreamingTests.swift b/apple/Plugins/FormatPluginTests/JSONDocumentStreamingTests.swift new file mode 100644 index 0000000000..42d318e2c1 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/JSONDocumentStreamingTests.swift @@ -0,0 +1,326 @@ +// +// JSONDocumentStreamingTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +private func writeJSON(_ data: Data) throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("json-document-\(UUID().uuidString).json") + try data.write(to: url) + return url +} + +private func writeJSON(_ text: String) throws -> URL { + try writeJSON(Data(text.utf8)) +} + +private func drain(_ elements: any JSONElementSequence) throws -> [Any] { + var elements = elements + defer { elements.close() } + var values: [Any] = [] + while let element = try elements.next() { + values.append(element) + } + return values +} + +struct JSONDocumentStreamingTests { + private static let trickyRows = [ + #"{"id":1,"name":"Ada \"the\" {first} [one]","path":"C:\\temp\\","tags":["a","]","}"],"meta":{"k":[1,{"x":null}]}}"#, + #"{"id":9007199254740993,"score":-3.9192320754595876e-07,"ok":true,"none":null,"city":"Hà Nội 東京 \u00e9"}"#, + #"{"note":"line\nbreak \u2028 sep","empty":"","nested":[[[]]],"obj":{}}"#, + #"{}"#, + ] + + private func streamedRows(_ url: URL, targetTable: String? = nil, chunkSize: Int) throws -> [Any] { + var reader = try JSONDocumentReader(url: url, chunkSize: chunkSize) + let rows = try JSONDocumentOutline.rows(in: &reader, targetTable: targetTable) { reader, _ in + try reader.skipValue() + } + guard case .array(let offset, let count, _) = rows else { + reader.close() + Issue.record("Expected the rows of an array, got \(rows)") + return [] + } + let values = try drain(JSONStreamedElements(reader: reader, arrayAt: offset)) + #expect(values.count == count) + return values + } + + private func importedElements(_ json: String, targetTable: String? = nil) throws -> [Any] { + let url = try writeJSON(json) + defer { try? FileManager.default.removeItem(at: url) } + return try drain(JSONImportParsing.rowElements(at: url, targetTable: targetTable, checkCancellation: {}).elements) + } + + private func parsedElements(_ json: String, targetTable: String? = nil) throws -> [Any] { + let url = try writeJSON(json) + defer { try? FileManager.default.removeItem(at: url) } + return try JSONImportParsing.parseElements(at: url, targetTable: targetTable) + } + + @Test("Elements read one at a time equal a whole-file parse, whatever the chunk size", arguments: [1, 3, 64, 1 << 20]) + func streamedElementsMatchAWholeFileParse(chunkSize: Int) throws { + let array = "[\n " + Self.trickyRows.joined(separator: ",\n ") + "\n]" + let wrapped = #"{"public.other":[{"x":1}], "public.people" : "# + array + "}" + for (json, target) in [(array, nil), (wrapped, "people")] as [(String, String?)] { + let url = try writeJSON(json) + defer { try? FileManager.default.removeItem(at: url) } + let expected = try JSONImportParsing.parseElements(at: url, targetTable: target) + let streamed = try streamedRows(url, targetTable: target, chunkSize: chunkSize) + #expect((streamed as NSArray).isEqual(to: expected)) + } + } + + @Test("The reader holds one chunk and one element, not the file") + func readerHoldsOneChunkAndOneElement() throws { + let rows = (1...5_000).map { #"{"id":\#($0),"name":"value \#($0)","pad":"\#(String(repeating: "x", count: 50))"}"# } + let json = #"{"public.t":["# + rows.joined(separator: ",") + "]}" + let url = try writeJSON(json) + defer { try? FileManager.default.removeItem(at: url) } + let chunkSize = 256 + let longestElement = rows.map(\.utf8.count).max() ?? 0 + + var reader = try JSONDocumentReader(url: url, chunkSize: chunkSize) + let layout = try JSONDocumentOutline.rows(in: &reader, targetTable: "t") { reader, _ in try reader.skipValue() } + var mostHeld = reader.heldByteCount + guard case .array(let offset, let count, _) = layout else { + Issue.record("Expected the rows of an array, got \(layout)") + return + } + var elements = try JSONStreamedElements(reader: reader, arrayAt: offset) + defer { elements.close() } + var read = 0 + while try elements.next() != nil { + read += 1 + mostHeld = max(mostHeld, elements.heldByteCount) + } + + #expect(count == 5_000) + #expect(read == 5_000) + #expect(mostHeld <= chunkSize + longestElement) + #expect(json.utf8.count > 100 * (chunkSize + longestElement)) + } + + @Test("Every shape a whole-file parse accepts gives the same rows", arguments: [ + (#"[{"a":1},{"a":2}]"#, nil), + (#" [ ] "#, nil), + (#"[{"a":1}, 5, [1], "s", {"a":2}]"#, nil), + (#"{"users":[{"id":1}]}"#, nil), + (#"{"users":[{"id":1}],"orders":[{"id":1},{"id":2}]}"#, "orders"), + (#"{"public.users":[{"id":1}]}"#, "users"), + (#"{"id":1,"tags":["a"]}"#, nil), + (#"{}"#, nil), + (#"{"t":[]}"#, nil), + (#"{"t":[{"a":1}],"t":[{"a":2},{"a":3}]}"#, nil), + (#"{"t":5,"t":[{"a":1}]}"#, nil), + (#"{"x":[{"a":1}],"y":[1]}"#, nil), + (#"[{"a":1},]"#, nil), + (#"{"t":[{"a":1} , ],}"#, nil), + ] as [(String, String?)]) + func shapesMatchAWholeFileParse(json: String, targetTable: String?) throws { + let imported = try importedElements(json, targetTable: targetTable) + #expect((imported as NSArray).isEqual(to: try parsedElements(json, targetTable: targetTable))) + } + + // A whole-file parse hands the tables over as a dictionary, whose order changes from launch to launch. + @Test("Two tables that both match the target table resolve to the first in the file", arguments: [ + #"{"a.users":[{"n":1}],"b.users":[{"n":2}]}"#, + #"{"Users":[{"n":1}],"users":[{"n":2}]}"#, + ]) + func ambiguousTableResolvesToTheFirst(json: String) throws { + let elements = try importedElements(json, targetTable: "USERS") + #expect((elements as NSArray).isEqual(to: [["n": 1]])) + } + + @Test("A file whose structure breaks fails before its first row", arguments: [ + #"[{"a":1},{"a":2},{"a":"#, + #"{"t":[{"a":1}]"#, + #"[{"a":1}] x"#, + #"[{"a":1} {"a":2}]"#, + #"[{"a":1},,{"a":2}]"#, + #"[,]"#, + #"[{"a":1]"#, + #"{"a" 1}"#, + "42", + "", + ]) + func brokenStructureFailsUpFront(json: String) throws { + let url = try writeJSON(json) + defer { try? FileManager.default.removeItem(at: url) } + #expect(throws: (any Error).self) { + _ = try JSONImportParsing.rowElements(at: url, targetTable: nil, checkCancellation: {}) + } + } + + @Test("A UTF-16 file still reads, through JSONSerialization's own decoding") + func utf16FileStillReads() throws { + let url = try writeJSON(try #require(#"{"t":[{"name":"Ada"},{"name":"Grace"}]}"#.data(using: .utf16))) + defer { try? FileManager.default.removeItem(at: url) } + let document = try JSONImportParsing.rowElements(at: url, targetTable: nil, checkCancellation: {}) + #expect(document.count == 2) + #expect((try drain(document.elements) as NSArray).isEqual(to: [["name": "Ada"], ["name": "Grace"]])) + #expect(try JSONImportParsing.detectFields(at: url, targetTable: nil).map(\.name) == ["name"]) + } + + @Test("A file that starts with a UTF-8 byte order mark streams") + func byteOrderMarkIsSkipped() throws { + let url = try writeJSON(Data([0xEF, 0xBB, 0xBF]) + Data(#"[{"a":1}]"#.utf8)) + defer { try? FileManager.default.removeItem(at: url) } + #expect((try streamedRows(url, chunkSize: 2) as NSArray).isEqual(to: [["a": 1]])) + } + + @Test("Detection fails on a value that does not parse, in any table, as a whole-file parse did") + func detectionValidatesEveryTable() throws { + let url = try writeJSON(#"{"users":[{"id":1}],"orders":[{"id":tru}]}"#) + defer { try? FileManager.default.removeItem(at: url) } + #expect(throws: (any Error).self) { + _ = try JSONImportParsing.detectFields(at: url, targetTable: "users") + } + } + + @Test("Detection reports the fields of the first of two tables with one name") + func detectionUsesTheFirstOfTwoTablesWithOneName() throws { + let url = try writeJSON(#"{"t":[{"a":1}],"t":[{"b":2}]}"#) + defer { try? FileManager.default.removeItem(at: url) } + #expect(try JSONImportParsing.detectFields(at: url, targetTable: nil).map(\.name) == ["a"]) + } +} + +private final class RecordingSink: PluginImportDataSink, @unchecked Sendable { + let databaseTypeId = "mock" + let targetTable: String? + + private(set) var rows: [[String: PluginCellValue]] = [] + + init(targetTable: String?) { + self.targetTable = targetTable + } + + func execute(statement: String) async throws {} + func insertRow(_ values: [String: PluginCellValue]) async throws { rows.append(values) } + func insertRows(_ rows: [[String: PluginCellValue]]) async throws { self.rows.append(contentsOf: rows) } + func deleteAllRowsFromTargetTable() async throws {} + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} + func disableForeignKeyChecks() async throws {} + func enableForeignKeyChecks() async throws {} +} + +private final class DocumentSource: PluginImportSource, @unchecked Sendable { + private let url: URL + + init(url: URL) { self.url = url } + + func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + AsyncThrowingStream { $0.finish() } + } + + func fileURL() -> URL { url } + func fileSizeBytes() -> Int64 { 0 } +} + +// Serialized because `JSONImportPlugin.settings` persists through plugin storage. +@Suite(.serialized) +struct JSONDocumentImportTests { + private func runImport( + _ json: String, + errorHandling: ImportErrorHandling, + sink: RecordingSink, + progress: PluginImportProgress = PluginImportProgress(progress: Progress()) + ) async throws -> Result { + let url = try writeJSON(json) + defer { try? FileManager.default.removeItem(at: url) } + + // `settings` persists, so the developer's own preference is put back. + let plugin = JSONImportPlugin() + let original = plugin.settings + defer { plugin.settings = original } + plugin.settings.errorHandling = errorHandling + plugin.settings.wrapInTransaction = true + plugin.settings.deleteExistingRows = false + + do { + return .success(try await plugin.performImport(source: DocumentSource(url: url), sink: sink, progress: progress)) + } catch { + return .failure(error) + } + } + + @Test("An array element that is not an object is reported as a skipped row, not left out unseen") + func nonObjectElementIsReported() async throws { + let sink = RecordingSink(targetTable: "people") + let outcome = try await runImport( + #"[{"name":"Ada"}, 5, ["x"], {"name":"Grace"}]"#, errorHandling: .skipAndContinue, sink: sink + ) + guard case .success(let result) = outcome else { + Issue.record("Skip and Continue must not abort the import: \(outcome)") + return + } + #expect(result.executedStatements == 2) + #expect(result.skippedStatements == 2) + #expect(result.errors.map { $0.line } == [2, 3]) + #expect(sink.rows.compactMap { $0["name"] } == [.text("Ada"), .text("Grace")]) + } + + @Test("A stop mode fails on an array element that is not an object") + func nonObjectElementStopsAStopMode() async throws { + let outcome = try await runImport( + #"[{"name":"Ada"}, [1, "Grace"]]"#, errorHandling: .stopAndRollback, sink: RecordingSink(targetTable: "people") + ) + guard case .failure = outcome else { + Issue.record("Stop and Rollback must not import a file with a row it cannot read") + return + } + } + + // A whole-file parse refused such a file outright, so no error mode passes over the element. + @Test( + "An element that does not parse fails the import in every error mode", + arguments: [ImportErrorHandling.skipAndContinue, .stopAndRollback, .stopAndCommit] + ) + func unreadableElementFailsTheImport(errorHandling: ImportErrorHandling) async throws { + let sink = RecordingSink(targetTable: "people") + let outcome = try await runImport( + #"[{"name":"Ada"}, {"name":tru}, {"name":"Grace"}]"#, errorHandling: errorHandling, sink: sink + ) + guard case .failure = outcome else { + Issue.record("An element that does not parse must fail the import: \(outcome)") + return + } + #expect(sink.rows.isEmpty) + } + + @Test("A truncated export imports nothing, even in Skip and Continue") + func truncatedFileImportsNothing() async throws { + let sink = RecordingSink(targetTable: "people") + let outcome = try await runImport( + #"{"public.people":[{"name":"Ada"},{"name":"Grace"},{"name":"Ka"#, errorHandling: .skipAndContinue, sink: sink + ) + guard case .failure = outcome else { + Issue.record("A truncated file must fail: \(outcome)") + return + } + #expect(sink.rows.isEmpty) + } + + @Test("A table-keyed export imports the target table's rows in file order, with an exact progress total") + func tableKeyedExportImportsTheTargetTable() async throws { + let sink = RecordingSink(targetTable: "people") + let progress = PluginImportProgress(progress: Progress()) + let json = #"{"public.other":[{"name":"x"}],"public.people":[{"name":"Ada"},{"name":"Grace"},{"name":"Hedy"}]}"# + let outcome = try await runImport(json, errorHandling: .stopAndRollback, sink: sink, progress: progress) + guard case .success(let result) = outcome else { + Issue.record("A valid file must import: \(outcome)") + return + } + #expect(result.executedStatements == 3) + #expect(progress.estimatedTotalStatements == 3) + #expect(sink.rows.compactMap { $0["name"] } == [.text("Ada"), .text("Grace"), .text("Hedy")]) + } +} diff --git a/apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift b/apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift index 3d9980f0bb..367fe36762 100644 --- a/apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift +++ b/apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift @@ -14,15 +14,15 @@ struct XLSXWriterCellTypeTests { databaseTypeId: String = "" ) async throws -> String { let writer = XLSXWriter(databaseTypeId: databaseTypeId) - writer.beginSheet( + try writer.beginSheet( name: "t", columns: columnTypeNames.indices.map { "c\($0)" }, columnTypeNames: columnTypeNames, includeHeader: false, convertNullToEmpty: true ) - writer.addRows([row], convertNullToEmpty: true) - writer.finishSheet() + try writer.addRows([row], convertNullToEmpty: true) + try writer.finishSheet() let destination = FileManager.default.temporaryDirectory .appendingPathComponent("\(UUID().uuidString).xlsx") defer { try? FileManager.default.removeItem(at: destination) } diff --git a/apple/Plugins/FormatPluginTests/XLSXWriterSheetNameTests.swift b/apple/Plugins/FormatPluginTests/XLSXWriterSheetNameTests.swift index d6445fbe1e..5d66558ff8 100644 --- a/apple/Plugins/FormatPluginTests/XLSXWriterSheetNameTests.swift +++ b/apple/Plugins/FormatPluginTests/XLSXWriterSheetNameTests.swift @@ -9,9 +9,9 @@ import Testing struct XLSXWriterSheetNameTests { private static let fortyCharacterName = "customer_subscription_billing_events_log" - private func addSheet(_ name: String, to writer: XLSXWriter) { - writer.beginSheet(name: name, columns: ["id"], includeHeader: false, convertNullToEmpty: true) - writer.finishSheet() + private func addSheet(_ name: String, to writer: XLSXWriter) throws { + try writer.beginSheet(name: name, columns: ["id"], includeHeader: false, convertNullToEmpty: true) + try writer.finishSheet() } private func workbookSheetNames(of writer: XLSXWriter) async throws -> [String] { @@ -35,16 +35,16 @@ struct XLSXWriterSheetNameTests { @Test("Two tables with the same name get two sheet names") func sameNameTwiceIsNumbered() async throws { let writer = XLSXWriter() - addSheet("users", to: writer) - addSheet("users", to: writer) + try addSheet("users", to: writer) + try addSheet("users", to: writer) #expect(try await workbookSheetNames(of: writer) == ["users", "users (2)"]) } @Test("Names that differ only in case count as the same name, as Excel compares them") func caseOnlyDifferenceIsNumbered() async throws { let writer = XLSXWriter() - addSheet("users", to: writer) - addSheet("Users", to: writer) + try addSheet("users", to: writer) + try addSheet("Users", to: writer) let names = try await workbookSheetNames(of: writer) #expect(names == ["users", "Users (2)"]) #expect(areDistinctIgnoringCase(names)) @@ -53,17 +53,17 @@ struct XLSXWriterSheetNameTests { @Test("A number already taken by another table is skipped") func takenNumberIsSkipped() async throws { let writer = XLSXWriter() - addSheet("users", to: writer) - addSheet("users (2)", to: writer) - addSheet("users", to: writer) + try addSheet("users", to: writer) + try addSheet("users (2)", to: writer) + try addSheet("users", to: writer) #expect(try await workbookSheetNames(of: writer) == ["users", "users (2)", "users (3)"]) } @Test("Two long names that share their first 31 characters get two sheet names within the limit") func longNamesSharingAPrefixStayDistinct() async throws { let writer = XLSXWriter() - addSheet(Self.fortyCharacterName, to: writer) - addSheet(Self.fortyCharacterName + "_archive", to: writer) + try addSheet(Self.fortyCharacterName, to: writer) + try addSheet(Self.fortyCharacterName + "_archive", to: writer) let names = try await workbookSheetNames(of: writer) #expect(names.count == 2) #expect(names.allSatisfy { $0.count <= 31 }) @@ -73,14 +73,14 @@ struct XLSXWriterSheetNameTests { @Test("A continuation sheet of a long table keeps its number within the 31 character limit") func continuationOfLongNameKeepsItsSuffix() async throws { let writer = XLSXWriter() - writer.beginSheet(name: Self.fortyCharacterName, columns: ["id"], includeHeader: false, convertNullToEmpty: true) - writer.continueSheet( + try writer.beginSheet(name: Self.fortyCharacterName, columns: ["id"], includeHeader: false, convertNullToEmpty: true) + try writer.continueSheet( baseName: Self.fortyCharacterName, columns: ["id"], includeHeader: false, convertNullToEmpty: true ) - writer.finishSheet() + try writer.finishSheet() let names = try await workbookSheetNames(of: writer) #expect(names.count == 2) #expect(names.allSatisfy { $0.count <= 31 }) diff --git a/apple/Plugins/FormatPluginTests/XLSXWriterStreamingTests.swift b/apple/Plugins/FormatPluginTests/XLSXWriterStreamingTests.swift new file mode 100644 index 0000000000..1853d10d42 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/XLSXWriterStreamingTests.swift @@ -0,0 +1,151 @@ +// +// XLSXWriterStreamingTests.swift +// TableProTests +// + +import CryptoKit +import Foundation +import TableProPluginKit +import Testing + +struct XLSXWriterStreamingTests { + private static let columns = ["id", "name", "score", "blob", "note"] + private static let columnTypeNames = ["int4", "text", "float8", "bytea", "varchar(8)"] + + // SHA-256 of the workbook below as the writer produced it while it built every sheet in memory. + private static let workbookDigest = "7602d732018a0d8b389690d553dd671460fcc4b3dfa53d7a9fff2dcd69a63440" + + private static let closingTags = "".utf8.count + + private func temporaryURL(_ name: String) -> URL { + FileManager.default.temporaryDirectory.appendingPathComponent("xlsx-streaming-\(UUID().uuidString)-\(name)") + } + + private func fileSize(_ url: URL) throws -> Int { + let attributes = try FileManager.default.attributesOfItem(atPath: url.path(percentEncoded: false)) + return try #require((attributes[.size] as? NSNumber)?.intValue) + } + + private func buildWorkbook(_ writer: XLSXWriter) throws { + try writer.beginSheet( + name: "users", columns: Self.columns, columnTypeNames: Self.columnTypeNames, + includeHeader: true, convertNullToEmpty: false + ) + try writer.addRows([ + [.text("1"), .text("Ada & "), .text("3.5"), .bytes(Data([0xDE, 0xAD])), .null], + [.text("2"), .text("Grace \"Hopper\"\r\n'x'"), .text("1e-7"), .bytes(Data()), .text("007")], + [.text("1234567890123456789"), .text(""), .text("NaN"), .null, .text("\u{01}bell")], + ], convertNullToEmpty: false) + try writer.continueSheet( + baseName: "users", columns: Self.columns, columnTypeNames: Self.columnTypeNames, + includeHeader: true, convertNullToEmpty: false + ) + try writer.addRows([[.text("3"), .text("Hà Nội 東京"), .text("-0.50"), .null, .text("x")]], convertNullToEmpty: false) + try writer.finishSheet() + try writer.beginSheet(name: "Users", columns: ["a", "b"], includeHeader: false, convertNullToEmpty: true) + try writer.addRows((1...3_000).map { [.text(String($0)), .text("v\($0)")] }, convertNullToEmpty: true) + try writer.addRows([[.null, .text("1E5")]], convertNullToEmpty: true) + try writer.finishSheet() + try writer.beginSheet(name: "a/b:c", columns: [], includeHeader: false, convertNullToEmpty: true) + try writer.finishSheet() + } + + @Test( + "The workbook keeps every byte it had when sheets were built in memory", + arguments: [0, 64, XLSXWriter.defaultFlushThreshold] + ) + func workbookBytesAreUnchanged(flushThreshold: Int) async throws { + let destination = temporaryURL("workbook.xlsx") + defer { try? FileManager.default.removeItem(at: destination) } + try Data("an older export".utf8).write(to: destination) + + let writer = XLSXWriter(flushThreshold: flushThreshold) + try buildWorkbook(writer) + try await writer.write(to: destination) + + let digest = SHA256.hash(data: try Data(contentsOf: destination)).map { String(format: "%02x", $0) }.joined() + #expect(digest == Self.workbookDigest) + } + + @Test("A sheet holds at most one flush of XML in memory while its rows arrive") + func sheetXMLReachesItsFileAsRowsArrive() throws { + let parts = temporaryURL("parts") + let threshold = 4_096 + let writer = XLSXWriter(partsDirectory: parts, flushThreshold: threshold) + try writer.beginSheet(name: "t", columns: ["a", "b"], includeHeader: true, convertNullToEmpty: true) + try writer.addRows((1...10_000).map { [.text(String($0)), .text("value \($0)")] }, convertNullToEmpty: true) + + let sheet = parts.appendingPathComponent("sheet1.xml") + let writtenBeforeFinishing = try fileSize(sheet) + try writer.finishSheet() + let sheetSize = try fileSize(sheet) + + #expect(sheetSize > 100 * threshold) + #expect(sheetSize - writtenBeforeFinishing < threshold + Self.closingTags) + } + + @Test("A workbook past the ZIP size limit fails while its rows are still arriving") + func oversizedWorkbookFailsEarly() throws { + let writer = XLSXWriter(flushThreshold: 0, maximumArchiveSize: 20_000) + try writer.beginSheet(name: "t", columns: ["a"], includeHeader: false, convertNullToEmpty: true) + var acceptedBatches = 0 + var failure: (any Error)? + do { + for _ in 0..<1_000 { + try writer.addRows((1...100).map { [.text("row \($0)")] }, convertNullToEmpty: true) + acceptedBatches += 1 + } + } catch { + failure = error + } + #expect(failure != nil) + #expect(acceptedBatches < 10) + } + + @Test("A writer that goes away, as a cancelled export's does, removes its sheet files") + func abandonedWriterRemovesItsSheetFiles() throws { + let parts = temporaryURL("parts") + do { + let writer = XLSXWriter(partsDirectory: parts, flushThreshold: 0) + try writer.beginSheet(name: "t", columns: ["a"], includeHeader: true, convertNullToEmpty: true) + try writer.addRows([[.text("1")]], convertNullToEmpty: true) + withExtendedLifetime(writer) { + #expect(FileManager.default.fileExists(atPath: parts.appendingPathComponent("sheet1.xml").path(percentEncoded: false))) + } + } + #expect(!FileManager.default.fileExists(atPath: parts.path(percentEncoded: false))) + } + + @Test("A new export removes the sheet folders an export that crashed left behind, and only those") + func abandonedSheetFoldersAreSwept() throws { + let parent = temporaryURL("temporary") + try FileManager.default.createDirectory(at: parent, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: parent) } + let folder = { (name: String) in parent.appendingPathComponent(name, isDirectory: true) } + let exists = { (url: URL) in FileManager.default.fileExists(atPath: url.path(percentEncoded: false)) } + + let crashed = folder(XLSXWriter.partsDirectoryPrefix + "crashed") + let recent = folder(XLSXWriter.partsDirectoryPrefix + "recent") + let unrelated = folder("unrelated") + for url in [crashed, recent, unrelated] { + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + } + try Data("".utf8).write(to: crashed.appendingPathComponent("sheet1.xml")) + let running = folder(XLSXWriter.partsDirectoryPrefix + "running") + let writer = XLSXWriter(partsDirectory: running) + try writer.beginSheet(name: "t", columns: ["a"], includeHeader: true, convertNullToEmpty: true) + + let twoDaysAgo = Date(timeIntervalSinceNow: -2 * 86_400) + for url in [crashed, unrelated, running] { + try FileManager.default.setAttributes([.modificationDate: twoDaysAgo], ofItemAtPath: url.path(percentEncoded: false)) + } + XLSXWriter.removeAbandonedPartsDirectories(in: parent, modifiedBefore: Date(timeIntervalSinceNow: -3_600)) + + withExtendedLifetime(writer) { + #expect(!exists(crashed)) + #expect(exists(running)) + #expect(exists(recent)) + #expect(exists(unrelated)) + } + } +} diff --git a/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift b/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift new file mode 100644 index 0000000000..6d562e9811 --- /dev/null +++ b/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift @@ -0,0 +1,109 @@ +// +// JSONDocumentOutline.swift +// JSONImportPlugin +// + +import Foundation +import TableProPluginKit + +enum JSONDocumentRows: Equatable { + // The rows are the elements of the array that opens at `offset`. `slot` is the one `visit` was given for them. + case array(offset: Int, elementCount: Int, slot: Int) + // A single object, which is one row, or text in UTF-16 or UTF-32: `JSONSerialization` reads the file whole. + case wholeDocument +} + +// Finds which array of a `.json` file holds the rows, by the rules `JSONImportParsing.extractElements` +// applies to a parsed file, in one pass that builds nothing but member names. +enum JSONDocumentOutline { + private struct Member { + let name: String + let slot: Int + let offset: Int + let elementCount: Int + let holdsRows: Bool + } + + // `visit` gets the reader at each array element and at each member value that is not an array, + // and moves it past that value. `slot` is 0 for the elements of a top-level array, the member's + // position for the elements of a member's array, and nil for a value that cannot become a row. + static func rows( + in reader: inout JSONDocumentReader, + targetTable: String?, + visit: (inout JSONDocumentReader, _ slot: Int?) throws -> Void + ) throws -> JSONDocumentRows { + guard try reader.startDocument() else { return .wholeDocument } + guard let first = try reader.nextSignificantByte() else { throw JSONDocumentError.endsEarly } + switch first { + case JSONDocumentReader.openBracket: + let offset = reader.offset + let array = try elements(of: &reader, slot: 0, visit: visit) + try reader.expectEnd() + return .array(offset: offset, elementCount: array.count, slot: 0) + case JSONDocumentReader.openBrace: + return try memberRows(in: &reader, targetTable: targetTable, visit: visit) + case 0x22, 0x2D, 0x30...0x39, 0x66, 0x6E, 0x74: + throw JSONImportParsing.unexpectedShapeError + default: + throw JSONDocumentError.unexpectedCharacter(byteOffset: reader.offset) + } + } + + private static func elements( + of reader: inout JSONDocumentReader, + slot: Int?, + visit: (inout JSONDocumentReader, _ slot: Int?) throws -> Void + ) throws -> (count: Int, allObjects: Bool) { + guard try reader.enterArray() else { return (0, true) } + var count = 0 + var allObjects = true + repeat { + if try reader.nextSignificantByte() != JSONDocumentReader.openBrace { + allObjects = false + } + try visit(&reader, slot) + count += 1 + } while try reader.continueArray() + return (count, allObjects) + } + + private static func memberRows( + in reader: inout JSONDocumentReader, + targetTable: String?, + visit: (inout JSONDocumentReader, _ slot: Int?) throws -> Void + ) throws -> JSONDocumentRows { + var members: [Member] = [] + var names: Set = [] + if try reader.enterObject() { + repeat { + let name = try reader.memberName() + // A parsed object keeps the first of two members that share a name. + let slot = names.insert(name).inserted ? members.count : nil + let offset = reader.offset + var array = (count: 0, allObjects: false) + if try reader.nextSignificantByte() == JSONDocumentReader.openBracket { + array = try elements(of: &reader, slot: slot, visit: visit) + } else { + try visit(&reader, nil) + } + if let slot { + members.append(Member( + name: name, slot: slot, offset: offset, + elementCount: array.count, holdsRows: array.allObjects + )) + } + } while try reader.continueObject() + } + try reader.expectEnd() + + guard !members.isEmpty, members.allSatisfy({ $0.holdsRows }) else { return .wholeDocument } + let match = targetTable.flatMap { target in + members.first { $0.name.caseInsensitiveCompare(target) == .orderedSame } + ?? members.first { $0.name.split(separator: ".").last?.caseInsensitiveCompare(target) == .orderedSame } + } + guard let table = match ?? (members.count == 1 ? members[0] : nil) else { + throw JSONImportParsing.noMatchingTableError + } + return .array(offset: table.offset, elementCount: table.elementCount, slot: table.slot) + } +} diff --git a/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift b/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift new file mode 100644 index 0000000000..ea927d568d --- /dev/null +++ b/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift @@ -0,0 +1,308 @@ +// +// JSONDocumentReader.swift +// JSONImportPlugin +// + +import Foundation + +enum JSONDocumentError: LocalizedError, Equatable { + case endsEarly + case unexpectedCharacter(byteOffset: Int) + + var errorDescription: String? { + switch self { + case .endsEarly: + return String(localized: "The JSON file ends before its last value is complete.") + case .unexpectedCharacter(let byteOffset): + return String(format: String(localized: "The JSON file is not valid at byte %@."), (byteOffset + 1).formatted()) + } + } +} + +// Walks a UTF-8 JSON file one chunk at a time. It finds where each value starts and ends without +// building it, so the file is held one chunk and one value at a time. `JSONSerialization` parses +// each value a caller asks for, so every value reads exactly as it would in a whole-file parse. +struct JSONDocumentReader { + static let defaultChunkSize = 1 << 20 + + private static let quote: UInt8 = 0x22 + private static let backslash: UInt8 = 0x5C + private static let comma: UInt8 = 0x2C + private static let colon: UInt8 = 0x3A + static let openBracket: UInt8 = 0x5B + private static let closeBracket: UInt8 = 0x5D + static let openBrace: UInt8 = 0x7B + private static let closeBrace: UInt8 = 0x7D + + private let handle: FileHandle + private let chunkSize: Int + private let checkCancellation: () throws -> Void + private var buffer: [UInt8] = [] + private var position = 0 + private var bufferStart = 0 + private var valueStart: Int? + private var reachedEnd = false + + init( + url: URL, + chunkSize: Int = Self.defaultChunkSize, + checkCancellation: @escaping () throws -> Void = { try Task.checkCancellation() } + ) throws { + handle = try FileHandle(forReadingFrom: url) + self.chunkSize = max(1, chunkSize) + self.checkCancellation = checkCancellation + } + + var offset: Int { bufferStart + position } + + var heldByteCount: Int { buffer.count } + + func close() { + try? handle.close() + } + + // Steps over a UTF-8 byte order mark. False for UTF-16 or UTF-32 text, which `JSONSerialization` + // also reads: it has a byte order mark of its own or a zero byte in its first two. + mutating func startDocument() throws -> Bool { + while buffer.count - position < 4, try refill() {} + let head = buffer[position...].prefix(4) + if head.starts(with: [0xEF, 0xBB, 0xBF]) { + position += 3 + return true + } + if head.starts(with: [0xFE, 0xFF]) || head.starts(with: [0xFF, 0xFE]) { + return false + } + return !head.prefix(2).contains(0) + } + + mutating func seek(to offset: Int) throws { + try handle.seek(toOffset: UInt64(offset)) + buffer.removeAll(keepingCapacity: true) + bufferStart = offset + position = 0 + valueStart = nil + reachedEnd = false + } + + // The next byte that is not JSON whitespace, left unread, or nil at the end of the file. + mutating func nextSignificantByte() throws -> UInt8? { + while true { + while position < buffer.count { + switch buffer[position] { + case 0x20, 0x09, 0x0A, 0x0D: + position += 1 + default: + return buffer[position] + } + } + guard try refill() else { return nil } + } + } + + // MARK: - Containers + + mutating func enterArray() throws -> Bool { + try enter(opening: Self.openBracket, closing: Self.closeBracket) + } + + mutating func enterObject() throws -> Bool { + try enter(opening: Self.openBrace, closing: Self.closeBrace) + } + + mutating func continueArray() throws -> Bool { + try continueAfterValue(closing: Self.closeBracket) + } + + mutating func continueObject() throws -> Bool { + try continueAfterValue(closing: Self.closeBrace) + } + + mutating func memberName() throws -> String { + guard let byte = try nextSignificantByte() else { throw JSONDocumentError.endsEarly } + guard byte == Self.quote else { throw JSONDocumentError.unexpectedCharacter(byteOffset: offset) } + let start = offset + let name = try readValue() + guard let decoded = try JSONSerialization.jsonObject(with: name, options: .fragmentsAllowed) as? String else { + throw JSONDocumentError.unexpectedCharacter(byteOffset: start) + } + try consume(Self.colon) + return decoded + } + + mutating func expectEnd() throws { + guard try nextSignificantByte() == nil else { + throw JSONDocumentError.unexpectedCharacter(byteOffset: offset) + } + } + + // Consumes the opening bracket and tells whether an element or member follows it. + private mutating func enter(opening: UInt8, closing: UInt8) throws -> Bool { + try consume(opening) + guard let next = try nextSignificantByte() else { throw JSONDocumentError.endsEarly } + guard next == closing else { return true } + position += 1 + return false + } + + // Consumes the comma and returns true, or consumes the closing bracket and returns false. + // `JSONSerialization` takes one comma before a closing bracket, so a file it reads reads here too. + private mutating func continueAfterValue(closing: UInt8) throws -> Bool { + guard let byte = try nextSignificantByte() else { throw JSONDocumentError.endsEarly } + guard byte == Self.comma || byte == closing else { + throw JSONDocumentError.unexpectedCharacter(byteOffset: offset) + } + position += 1 + if byte == closing { return false } + guard try nextSignificantByte() == closing else { return true } + position += 1 + return false + } + + private mutating func consume(_ expected: UInt8) throws { + guard let byte = try nextSignificantByte() else { throw JSONDocumentError.endsEarly } + guard byte == expected else { throw JSONDocumentError.unexpectedCharacter(byteOffset: offset) } + position += 1 + } + + // MARK: - Values + + // The bytes of the next value. They stay in the buffer while it is read, so a value costs its own size and no more. + mutating func readValue() throws -> Data { + guard try nextSignificantByte() != nil else { throw JSONDocumentError.endsEarly } + valueStart = position + defer { valueStart = nil } + try passValue() + guard let start = valueStart else { return Data() } + return Data(buffer[start.. Bool in + while index < bytes.count { + let byte = bytes[index] + index += 1 + if escaped { + escaped = false + } else if byte == Self.backslash { + escaped = true + } else if byte == Self.quote { + return true + } + } + return false + } + position = index + if closed { return } + guard try refill() else { throw JSONDocumentError.endsEarly } + } + } + + private mutating func passContainer() throws { + var closers: [UInt8] = [] + var inString = false + var escaped = false + while true { + var index = position + var mismatched = false + let closed = buffer.withUnsafeBufferPointer { bytes -> Bool in + while index < bytes.count { + let byte = bytes[index] + index += 1 + if inString { + if escaped { + escaped = false + } else if byte == Self.backslash { + escaped = true + } else if byte == Self.quote { + inString = false + } + continue + } + switch byte { + case Self.quote: + inString = true + case Self.openBrace: + closers.append(Self.closeBrace) + case Self.openBracket: + closers.append(Self.closeBracket) + case Self.closeBrace, Self.closeBracket: + guard closers.last == byte else { + mismatched = true + return true + } + closers.removeLast() + if closers.isEmpty { return true } + default: + break + } + } + return false + } + position = index + if mismatched { throw JSONDocumentError.unexpectedCharacter(byteOffset: offset - 1) } + if closed { return } + guard try refill() else { throw JSONDocumentError.endsEarly } + } + } + + // A number, `true`, `false` or `null` runs to the next delimiter. `JSONSerialization` checks its spelling. + private mutating func passLiteral() throws { + while true { + while position < buffer.count { + switch buffer[position] { + case 0x20, 0x09, 0x0A, 0x0D, Self.comma, Self.colon, Self.quote, + Self.openBracket, Self.closeBracket, Self.openBrace, Self.closeBrace: + return + default: + position += 1 + } + } + guard try refill() else { return } + } + } + + // MARK: - Buffer + + // Drops what was read before the value in progress, then appends the next chunk. + private mutating func refill() throws -> Bool { + guard !reachedEnd else { return false } + try checkCancellation() + let keep = valueStart ?? position + if keep > 0 { + buffer.removeSubrange(0.. Any? + func close() +} + +// The elements of one array of a UTF-8 file, each read and parsed as it is reached. +struct JSONStreamedElements: JSONElementSequence { + private var reader: JSONDocumentReader + private var hasMore: Bool + + init(reader: JSONDocumentReader, arrayAt offset: Int) throws { + self.reader = reader + try self.reader.seek(to: offset) + hasMore = try self.reader.enterArray() + } + + var heldByteCount: Int { reader.heldByteCount } + + mutating func next() throws -> Any? { + guard hasMore else { return nil } + let value = try reader.readValue() + hasMore = try reader.continueArray() + let object: Any = try autoreleasepool { try JSONSerialization.jsonObject(with: value, options: .fragmentsAllowed) } + return object + } + + func close() { + reader.close() + } +} + +// The elements of a file `JSONSerialization` has already read whole. +struct JSONParsedElements: JSONElementSequence { + private let elements: [Any] + private var cursor = 0 + + init(_ elements: [Any]) { + self.elements = elements + } + + mutating func next() -> Any? { + guard cursor < elements.count else { return nil } + defer { cursor += 1 } + return elements[cursor] + } + + func close() {} +} + +// Hands a `.json` file's rows to `RowImportRunner` a batch at a time, numbered by their place in the +// array, and keeps the elements that are not objects, which Skip and Continue passes over. +struct JSONElementBatches { + private var elements: any JSONElementSequence + private let elementsPerBatch: Int + private let skipsUnreadableElements: Bool + private let maxRecordedErrors: Int + + private(set) var elementsRead = 0 + private(set) var unreadableElements: [PluginImportResult.ImportStatementError] = [] + + // Every unreadable element, counted past the end of the capped `unreadableElements` list. + private(set) var unreadableElementCount = 0 + + init( + elements: any JSONElementSequence, + elementsPerBatch: Int, + skipsUnreadableElements: Bool, + maxRecordedErrors: Int + ) { + self.elements = elements + self.elementsPerBatch = max(1, elementsPerBatch) + self.skipsUnreadableElements = skipsUnreadableElements + self.maxRecordedErrors = maxRecordedErrors + } + + mutating func next() throws -> [RowImportRunner.Entry]? { + var batch: [RowImportRunner.Entry] = [] + var elementsInBatch = 0 + while elementsInBatch < elementsPerBatch, let element = try elements.next() { + elementsInBatch += 1 + elementsRead += 1 + let number = elementsRead + do { + let row = try autoreleasepool { try JSONImportParsing.row(from: element) } + guard !row.isEmpty else { continue } + batch.append((number, row)) + } catch { + guard skipsUnreadableElements else { throw error } + recordUnreadableElement(number, error: error) + } + } + return elementsInBatch == 0 ? nil : batch + } + + func close() { + elements.close() + } + + private mutating func recordUnreadableElement(_ number: Int, error: any Error) { + unreadableElementCount += 1 + guard unreadableElements.count < maxRecordedErrors else { return } + unreadableElements.append(.init( + statement: "row \(number)", + line: number, + errorMessage: error.localizedDescription + )) + } +} diff --git a/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift b/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift index 9389cba392..453890598d 100644 --- a/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift +++ b/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift @@ -2,10 +2,6 @@ // JSONImportParsing.swift // JSONImportPlugin // -// Pure JSON parsing, row extraction, and field inference. Kept free of the -// plugin's loadable-bundle and SwiftUI surface so it can be compiled into the -// test target directly (a loadable .tableplugin cannot be linked by tests). -// import Foundation import TableProNumberFormatting @@ -33,19 +29,69 @@ enum JSONImportParsing { line.allSatisfy { $0 == 0x20 || $0 == 0x09 || $0 == 0x0D } } + static var unexpectedShapeError: PluginImportError { + .importFailed("Expected a JSON array of objects or a table-keyed object") + } + + static var noMatchingTableError: PluginImportError { + .importFailed("The file contains multiple tables and none matches the target table") + } + + // The values of a `.json` file that become rows, and how many there are. A UTF-8 file is read + // one element at a time after a pass that finds its rows and checks its structure, so a broken + // file fails before the first row is imported. + static func rowElements( + at url: URL, + targetTable: String?, + checkCancellation: @escaping () throws -> Void + ) throws -> (elements: any JSONElementSequence, count: Int) { + var reader = try JSONDocumentReader(url: url, checkCancellation: checkCancellation) + do { + let rows = try JSONDocumentOutline.rows(in: &reader, targetTable: targetTable) { reader, _ in + try reader.skipValue() + } + if case .array(let offset, let count, _) = rows { + return (try JSONStreamedElements(reader: reader, arrayAt: offset), count) + } + } catch { + reader.close() + throw error + } + reader.close() + let elements = try parseElements(at: url, targetTable: targetTable) + return (JSONParsedElements(elements), elements.count) + } + + static func row(from element: Any) throws -> [String: PluginCellValue] { + guard let object = element as? NSDictionary else { + throw PluginImportError.importFailed(String(localized: "Each array element must be a JSON object")) + } + return convertRow(object) + } + static func parseRows(at url: URL, targetTable: String?) throws -> [NSDictionary] { + try parseElements(at: url, targetTable: targetTable).compactMap { $0 as? NSDictionary } + } + + static func parseElements(at url: URL, targetTable: String?) throws -> [Any] { let data = try Data(contentsOf: url) let object = try JSONSerialization.jsonObject(with: data) - return try extractRows(from: object, targetTable: targetTable) + return try extractElements(from: object, targetTable: targetTable) } static func extractRows(from object: Any, targetTable: String?) throws -> [NSDictionary] { + try extractElements(from: object, targetTable: targetTable).compactMap { $0 as? NSDictionary } + } + + // A top-level array's elements come back whatever they are, so the import can report each one + // that is not an object instead of leaving it out unseen. + static func extractElements(from object: Any, targetTable: String?) throws -> [Any] { if let array = object as? [Any] { - return array.compactMap { $0 as? NSDictionary } + return array } guard let dict = object as? [String: Any] else { - throw PluginImportError.importFailed("Expected a JSON array of objects or a table-keyed object") + throw unexpectedShapeError } let tables = dict.compactMapValues { value -> [Any]? in @@ -59,12 +105,12 @@ enum JSONImportParsing { } if let targetTable, let match = matchTable(in: tables, to: targetTable) { - return match.compactMap { $0 as? NSDictionary } + return match } if tables.count == 1, let only = tables.values.first { - return only.compactMap { $0 as? NSDictionary } + return only } - throw PluginImportError.importFailed("The file contains multiple tables and none matches the target table") + throw noMatchingTableError } private static func matchTable(in tables: [String: [Any]], to target: String) -> [Any]? { @@ -112,11 +158,34 @@ enum JSONImportParsing { static func detectFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { try Task.checkCancellation() guard isLineDelimited(url) else { - return try detectFields(in: try parseRows(at: url, targetTable: targetTable)) + return try detectFields(inDocumentAt: url, targetTable: targetTable) } return try detectFields(inLinesAt: url) } + // Every value is parsed once, one at a time, and each array keeps a survey of its own, so the + // target table's fields are known when the pass ends. A value that does not parse fails the + // detection, as it fails a whole-file parse. + static func detectFields(inDocumentAt url: URL, targetTable: String?) throws -> [PluginImportField] { + var reader = try JSONDocumentReader(url: url) + defer { reader.close() } + var surveys: [Int: JSONFieldSurvey] = [:] + let rows = try JSONDocumentOutline.rows(in: &reader, targetTable: targetTable) { reader, slot in + let value = try reader.readValue() + try autoreleasepool { + let object = try JSONSerialization.jsonObject(with: value, options: .fragmentsAllowed) + guard let slot, let row = object as? NSDictionary else { return } + surveys[slot, default: JSONFieldSurvey()].add(row) + } + } + switch rows { + case .array(_, _, let slot): + return surveys[slot]?.fields ?? [] + case .wholeDocument: + return try detectFields(in: try parseRows(at: url, targetTable: targetTable)) + } + } + static func detectFields(inLinesAt url: URL) throws -> [PluginImportField] { var survey = JSONFieldSurvey() var lines = try JSONLineReader(url: url) diff --git a/apple/Plugins/JSONImportPlugin/JSONImportPlugin.swift b/apple/Plugins/JSONImportPlugin/JSONImportPlugin.swift index a765985859..78a8533992 100644 --- a/apple/Plugins/JSONImportPlugin/JSONImportPlugin.swift +++ b/apple/Plugins/JSONImportPlugin/JSONImportPlugin.swift @@ -47,9 +47,9 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug } private static let batchSize = 500 - /// One budget shared with the rows the runner itself recorded, so the two lists together stay - /// inside the cap the import dialog documents. The skip *count* is deliberately uncapped: a - /// truncated list must not also under-report how much of the file was left out. + // One budget shared with the rows the runner itself recorded, so the two lists together stay + // inside the cap the import dialog documents. The skip count is uncapped: a truncated list must + // not also under-report how much of the file was left out. private static let maxRecordedErrors = 1_000 private let lineDelimitedFields = JSONFieldDetectionCache() @@ -83,22 +83,24 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug unreadableLines = batches.unreadableLines unreadableLineCount = batches.unreadableLineCount } else { - let rawRows = try JSONImportParsing.parseRows(at: url, targetTable: sink.targetTable) - progress.setEstimatedTotal(rawRows.count) - var cursor = 0 + let document = try JSONImportParsing.rowElements( + at: url, targetTable: sink.targetTable, checkCancellation: progress.checkCancellation + ) + progress.setEstimatedTotal(document.count) + var batches = JSONElementBatches( + elements: document.elements, + elementsPerBatch: Self.batchSize, + skipsUnreadableElements: settings.errorHandling == .skipAndContinue, + maxRecordedErrors: Self.maxRecordedErrors + ) + defer { batches.close() } outcome = try await RowImportRunner.run( configuration: configuration, sink: sink, progress: progress ) { - guard cursor < rawRows.count else { return nil } - let end = min(cursor + Self.batchSize, rawRows.count) - let batch = (cursor.. RowImportRunner.Entry? in - let row = JSONImportParsing.convertRow(rawRows[index]) - guard !row.isEmpty else { return nil } - return (index + 1, row) - } - cursor = end - return batch + try batches.next() } + unreadableLines = batches.unreadableElements + unreadableLineCount = batches.unreadableElementCount } return PluginImportResult( diff --git a/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift b/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift index 0b5c15591c..9500f5c6cd 100644 --- a/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift +++ b/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift @@ -444,28 +444,36 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Streaming - /// The stream header has no field for classification hints, so they are added to the collected result. + // The stream header has no field for classification hints, so they are added to the collected result. func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { - let result = try await boundedQueryFromStream(query: query, rowCap: rowCap) + // The collector needs one row past the cap to tell a full result from a truncated one. + let cap = max(rowCap, 1) + let result = try await PluginBoundedStream.collect( + rowStream(query: query, rowBudget: cap < Int.max ? cap + 1 : nil), + rowCap: cap, + startedAt: Date() + ) try await reportCompilationErrors(of: query) return result.withOracleClassificationHints() } func streamRows(query: String) -> AsyncThrowingStream { + rowStream(query: query, rowBudget: nil) + } + + private func rowStream(query: String, rowBudget: Int?) -> AsyncThrowingStream { guard let core else { return AsyncThrowingStream { $0.finish(throwing: OraclePluginError(core: .notConnected)) } } + let batching = OracleRowBatcher(rowsPerBatch: OracleRowBatcher.batchRows, rowBudget: rowBudget) return PluginRowStream.make { continuation, abort in let streamTask = Task { - /// The core read runs in its own unstructured task, which the outer one cannot - /// cancel: a plain `Task {}` inherits context but is not a child. Cancelling it - /// explicitly on abort is what stops oracle-nio, whose own loop already checks - /// cancellation once it is reachable. + // A plain `Task {}` is not a child of this one, so only the explicit cancel on abort stops oracle-nio. let coreStream = AsyncThrowingStream { coreContinuation in let coreTask = Task { do { - try await core.streamQuery(query, continuation: coreContinuation) + try await core.streamQuery(query, batching: batching, continuation: coreContinuation) } catch { coreContinuation.finish(throwing: error) } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQCappedCursorRead.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQCappedCursorRead.swift new file mode 100644 index 0000000000..865cec303d --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQCappedCursorRead.swift @@ -0,0 +1,100 @@ +// +// LibPQCappedCursorRead.swift +// PostgreSQLDriverPlugin +// + +import Foundation + +// A capped read for a session that cannot cancel: inside the user's transaction block a cancel aborts the block, and +// a single-connection server takes none. Single-row mode only changes how libpq hands rows over, so the server would +// still send every row. Through a cursor, FETCH sends the cap's rows and MOVE runs the rest without sending them, so +// every row's side effects still happen, as they did when the tail was drained. +internal enum LibPQCappedCursorRead { + enum Scope: Equatable { + // A savepoint holds the cursor, so a refused DECLARE leaves the user's block as it was. + case insideTransaction + // DECLARE needs a transaction block, so the read opens and commits one of its own. + case ownTransaction + } + + struct Plan: Equatable { + let scope: Scope + // Its own round trip: the server parses a whole query string before it runs any of it, so a syntax error in + // the user's text would otherwise leave no savepoint to roll back to. + let opening: String + let statement: String + // Undoes the attempt, so a refused cursor can run again the plain way. + let recovery: String + let fetchCount: Int + } + + static let cursorName = "tablepro_capped_read" + + private static let cursorableKeywords: Set = ["select", "with", "values", "table"] + + // DECLARE refuses INTO and a data-modifying WITH, and a rolled-back stream savepoint would drop the locks of FOR + // UPDATE and FOR SHARE. A false match costs nothing but the plain read. + private static let excludedWords: Set = ["into", "insert", "update", "delete", "merge", "share"] + + static func plan(for query: String, rowCap: Int, scope: Scope) -> Plan? { + guard rowCap > 0, rowCap < Int(Int32.max), let body = cursorBody(of: query) else { return nil } + let fetchCount = rowCap + 1 + let opening: String + let closing: String + let recovery: String + switch scope { + case .insideTransaction: + opening = "SAVEPOINT \(cursorName)" + closing = "RELEASE SAVEPOINT \(cursorName)" + recovery = "ROLLBACK TO SAVEPOINT \(cursorName); RELEASE SAVEPOINT \(cursorName)" + case .ownTransaction: + opening = "BEGIN" + closing = "COMMIT" + recovery = "ROLLBACK" + } + // The body sits on lines of its own, so a trailing `--` comment cannot swallow the FETCH. + let statement = [ + "DECLARE \(cursorName) NO SCROLL CURSOR FOR", + body, + ";", + "FETCH FORWARD \(fetchCount) FROM \(cursorName);", + "MOVE FORWARD ALL FROM \(cursorName);", + "CLOSE \(cursorName);", + closing, + ].joined(separator: "\n") + return Plan(scope: scope, opening: opening, statement: statement, recovery: recovery, fetchCount: fetchCount) + } + + // One SELECT, VALUES or TABLE statement, without its trailing semicolons. Any other semicolon, even one inside a + // literal or a comment, sends the text the plain way rather than risk a second statement inside the cursor. + static func cursorBody(of query: String) -> String? { + var body = Substring(query) + while let last = body.last, last.isWhitespace || last == ";" { + body = body.dropLast() + } + guard !body.isEmpty, !body.contains(";") else { return nil } + guard let keyword = leadingKeyword(of: body), cursorableKeywords.contains(keyword) else { return nil } + let words = body.lowercased().split { !($0.isLetter || $0.isNumber || $0 == "_" || $0 == "$") } + guard !words.contains(where: { excludedWords.contains(String($0)) }) else { return nil } + return String(body) + } + + private static func leadingKeyword(of text: Substring) -> String? { + var rest = text[...] + while let first = rest.first { + if first.isWhitespace || first == "(" { + rest = rest.dropFirst() + } else if rest.hasPrefix("--") { + guard let lineEnd = rest.firstIndex(where: \.isNewline) else { return nil } + rest = rest[lineEnd...] + } else if rest.hasPrefix("/*") { + guard let commentEnd = rest.range(of: "*/") else { return nil } + rest = rest[commentEnd.upperBound...] + } else { + break + } + } + let word = rest.prefix { $0.isLetter } + return word.isEmpty ? nil : word.lowercased() + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift index f818f69baf..20b22f439c 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift @@ -80,6 +80,9 @@ final class LibPQPluginConnection: @unchecked Sendable { /// Queue-confined. Non-nil until every schema/flavor/catalog bootstrap probe has completed. private var activeConnectDeadline: LibPQConnectDeadline? private var bootstrapTransportFailed = false + // Queue-confined. Nil until a probe, sent while no block was open, showed whether the server takes savepoints: + // inside the user's block a refused SAVEPOINT aborts the block, so none is sent there until this is true. + private var takesSavepoints: Bool? var isConnected: Bool { stateLock.lock() @@ -1021,13 +1024,9 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - /// Reads at most `rowCap` rows through libpq's single-row mode, then cancels the statement and - /// drains the connection instead of pulling the rest of the result across the socket. - /// - /// One row past the cap is read so `isTruncated` can tell "exactly `rowCap` rows" from "more - /// rows exist". The cancel is not an optimization: libpq drains an abandoned result inside the - /// next `PQexec`, charging that cost to the user's next statement, and the backend stays active - /// holding its snapshot until it can finish writing to the client. + // Reads at most `rowCap` rows, plus one to tell "exactly `rowCap`" from "more", then stops the server producing + // the rest. The stop is not an optimization: libpq drains an abandoned result inside the next `PQexec`, and the + // backend holds its snapshot until it has written the whole result to the client. private func boundedQuerySync(_ query: String, rowCap: Int) throws -> LibPQPluginQueryResult { let conn = connectionHandle @@ -1039,19 +1038,27 @@ final class LibPQPluginConnection: @unchecked Sendable { defer { cancellationGate.endQuery(generation) } defer { refreshStandardConformingStrings(from: conn) } - /// Started before the drain, so a result the previous statement abandoned is charged to the - /// time before the first row rather than appearing as this query's row transfer. + // Started before the drain, so a result the previous statement abandoned is charged to the time before the + // first row rather than to this query's row transfer. let sentAt = Date() - /// Cancelling a statement inside a transaction block puts the transaction into the aborted - /// state, and every later command fails until ROLLBACK. Reading the tail of one result costs - /// less than throwing away the transaction the user opened, so the cancel is withheld here - /// and the connection is drained instead. + // A cancel inside a transaction block aborts the block the user opened, so there the statement is read + // through a cursor, and where it cannot be the connection is drained instead. let cancelsOutput = cancelsAbandonedOutput(conn) let suppressCancel = !cancelsOutput finishAbandonedResults(conn, cancellingOutput: cancelsOutput) if let ended = sessionEndedBeforeSending(conn) { throw ended } + // A Stop during one of the driver's own round trips is spent there, so each is followed by a check before + // the user's statement goes out. + probeSavepointsWhileIdle(conn) + if cancellationGate.isCancelled(generation) { throw CancellationError() } + if let plan = cappedCursorPlan(for: query, rowCap: rowCap, conn: conn), + let read = try cappedCursorRead(plan, rowCap: rowCap, conn: conn, generation: generation, sentAt: sentAt) { + return read + } + if cancellationGate.isCancelled(generation) { throw CancellationError() } + let localQuery = String(query) let sendOk = localQuery.withCString { queryPtr in PQsendQuery(conn, queryPtr) @@ -1164,6 +1171,162 @@ final class LibPQPluginConnection: @unchecked Sendable { return PostGISTypes.rendering(bounded, types: typeNames.postgisTypes) } + private static let savepointProbe = + "BEGIN; SAVEPOINT tablepro_probe; RELEASE SAVEPOINT tablepro_probe; ROLLBACK" + + // Unpinned connections (Redshift, CockroachDB) never get a driver savepoint inside the user's block. A PG-wire + // server that refuses one is found here, outside any block, where the refusal costs one round trip. + private func probeSavepointsWhileIdle(_ conn: OpaquePointer) { + guard pinsCatalogSearchPath, takesSavepoints == nil, PQtransactionStatus(conn) == PQTRANS_IDLE, + let result = Self.savepointProbe.withCString({ PQexec(conn, $0) }) + else { return } + let accepted = PQresultStatus(result) == PGRES_COMMAND_OK + let cancelled = !accepted && getResultError(from: result).sqlState == "57014" + PQclear(result) + if PQtransactionStatus(conn) != PQTRANS_IDLE, let rollback = "ROLLBACK".withCString({ PQexec(conn, $0) }) { + PQclear(rollback) + } + recordTransactionState(of: conn) + guard PQstatus(conn) == CONNECTION_OK, !cancelled else { return } + takesSavepoints = accepted + } + + private func cappedCursorPlan(for query: String, rowCap: Int, conn: OpaquePointer) -> LibPQCappedCursorRead.Plan? { + guard pinsCatalogSearchPath else { return nil } + let scope: LibPQCappedCursorRead.Scope + switch PQtransactionStatus(conn) { + case PQTRANS_INTRANS where takesSavepoints == true: + scope = .insideTransaction + case PQTRANS_IDLE where suppressServerSideCancel: + scope = .ownTransaction + default: + return nil + } + return LibPQCappedCursorRead.plan(for: query, rowCap: rowCap, scope: scope) + } + + // Nil when the caller should run the statement the plain way: after a refused DECLARE, undone before the statement + // ran, so the plain run reports any error of its own in the user's text. A failure after DECLARE is the + // statement's own and leaves the user's block aborted, as the plain read does. + private func cappedCursorRead( + _ plan: LibPQCappedCursorRead.Plan, + rowCap: Int, + conn: OpaquePointer, + generation: Int, + sentAt: Date + ) throws -> LibPQPluginQueryResult? { + guard let opened = plan.opening.withCString({ PQexec(conn, $0) }) else { + throw lostConnection(getError(from: conn), on: conn, sent: true) + } + let openingFailure = PQresultStatus(opened) == PGRES_COMMAND_OK ? nil : getResultError(from: opened) + PQclear(opened) + recordTransactionState(of: conn) + if let openingFailure { + guard PQstatus(conn) == CONNECTION_OK else { throw lostConnection(openingFailure, on: conn, sent: true) } + if cancellationGate.isCancelled(generation) { throw CancellationError() } + takesSavepoints = false + if PQtransactionStatus(conn) == PQTRANS_INERROR { throw openingFailure } + return nil + } + if cancellationGate.isCancelled(generation) { + _ = undoCappedCursorRead(plan, conn: conn) + throw CancellationError() + } + + let sendOk = plan.statement.withCString { PQsendQuery(conn, $0) } + guard sendOk != 0 else { + throw lostConnection(getError(from: conn), on: conn, sent: false) + } + guard PQsetSingleRowMode(conn) != 0 else { + _ = finishPendingResults(conn, cancellingOutput: false) + recordTransactionState(of: conn) + throw LibPQPluginError(message: "Failed to enter single-row mode", sqlState: nil, detail: nil) + } + + var metadata: ColumnMetadata? + var rows: [[PluginCellValue]] = [] + rows.reserveCapacity(min(plan.fetchCount, 10_000)) + var firstRowTime: TimeInterval? + var declared = false + var failure: LibPQPluginError? + + while let result = PQgetResult(conn) { + let status = PQresultStatus(result) + if status == PGRES_SINGLE_TUPLE || status == PGRES_TUPLES_OK { + if firstRowTime == nil { firstRowTime = Date().timeIntervalSince(sentAt) } + let columns = metadata ?? readColumnMetadata(from: result) + metadata = columns + for rowIndex in 0.. rowCap + if truncated { rows.removeLast() } + let resolvedMetadata = metadata.map { resolvingUnknownTypes($0, conn: conn) } + let bounded = LibPQPluginQueryResult( + columns: resolvedMetadata?.columns ?? [], + columnOids: resolvedMetadata?.columnOids ?? [], + columnTypeNames: resolvedMetadata?.columnTypeNames ?? [], + rows: rows, + affectedRows: 0, + commandTag: nil, + isTruncated: truncated, + firstRowTime: firstRowTime ?? Date().timeIntervalSince(sentAt) + ) + return PostGISTypes.rendering(bounded, types: typeNames.postgisTypes) + } + + private func undoCappedCursorRead(_ plan: LibPQCappedCursorRead.Plan, conn: OpaquePointer) -> Bool { + guard let result = plan.recovery.withCString({ PQexec(conn, $0) }) else { return false } + defer { PQclear(result) } + recordTransactionState(of: conn) + return PQresultStatus(result) == PGRES_COMMAND_OK + } + private func executeParameterizedQuerySync(_ query: String, parameters: [PluginCellValue]) throws -> LibPQPluginQueryResult { let conn = connectionHandle @@ -1430,7 +1593,17 @@ final class LibPQPluginConnection: @unchecked Sendable { let continuation = run.continuation let generation = run.generation let cancelsOutput = run.cancelsOutput - let suppressCancel = !cancelsOutput + // Inside the user's block a cancel aborts the block, so a savepoint holds the read: a reader that goes away + // cancels it and rolls back to the savepoint instead of draining the rest of the result. The abandoned read + // then keeps nothing it did, as a cancelled statement outside a block. + if !singleStatement { probeSavepointsWhileIdle(conn) } + if run.abort.isAborted || cancellationGate.isCancelled(generation) { throw CancellationError() } + let holdsSavepoint = !singleStatement && !cancelsOutput && openStreamSavepoint(for: run.query, conn: conn) + let suppressCancel = !cancelsOutput && !holdsSavepoint + if holdsSavepoint, run.abort.isAborted || cancellationGate.isCancelled(generation) { + if let aborted = endStreamSavepoint(conn, undoing: true) { throw aborted } + throw CancellationError() + } let sendOk = run.query.withCString { queryPtr in singleStatement @@ -1444,6 +1617,7 @@ final class LibPQPluginConnection: @unchecked Sendable { if PQsetSingleRowMode(conn) == 0 { _ = cancelAndDrain(conn, suppressCancel: suppressCancel) + if holdsSavepoint, let aborted = endStreamSavepoint(conn, undoing: true) { throw aborted } throw LibPQPluginError(message: "Failed to enter single-row mode", sqlState: nil, detail: nil) } @@ -1490,6 +1664,9 @@ final class LibPQPluginConnection: @unchecked Sendable { continuation.yield(.rows(batch)) } _ = cancelAndDrain(conn, suppressCancel: suppressCancel) + // A Stop leaves the block aborted, as it does for any statement; only a reader that left is undone. + let undoing = !cancellationGate.isCancelled(generation) + if holdsSavepoint, let aborted = endStreamSavepoint(conn, undoing: undoing) { throw aborted } throw CancellationError() } } else if status == PGRES_TUPLES_OK { @@ -1520,8 +1697,9 @@ final class LibPQPluginConnection: @unchecked Sendable { let outcome = finishPendingResults(conn, cancellingOutput: cancelsOutput) recordTransactionState(of: conn) - /// The header went out with the first row, so this stream keeps what it said; - /// the lookup is for the results that follow. + if holdsSavepoint, let aborted = endStreamSavepoint(conn, undoing: false) { throw aborted } + // The header went out with the first row, so this stream keeps what it said; the lookup is for the results + // that follow. let missing = typeNames.unresolvedOids(in: columnOids) if !missing.isEmpty { learnTypeNames(for: missing, conn: conn) @@ -1532,6 +1710,51 @@ final class LibPQPluginConnection: @unchecked Sendable { } } + private static let streamSavepoint = "SAVEPOINT tablepro_stream_read" + private static let streamSavepointRelease = "RELEASE SAVEPOINT tablepro_stream_read" + private static let streamSavepointRollback = + "ROLLBACK TO SAVEPOINT tablepro_stream_read; RELEASE SAVEPOINT tablepro_stream_read" + + // Only one read statement, since rolling back to the savepoint would undo a write, a lock or a second statement. + // Unpinned connections (Redshift, CockroachDB) never get a driver savepoint inside the user's block. + private func openStreamSavepoint(for query: String, conn: OpaquePointer) -> Bool { + guard pinsCatalogSearchPath, !suppressServerSideCancel, takesSavepoints == true, + PQtransactionStatus(conn) == PQTRANS_INTRANS, + LibPQCappedCursorRead.cursorBody(of: query) != nil, + let result = Self.streamSavepoint.withCString({ PQexec(conn, $0) }) + else { return false } + defer { PQclear(result) } + recordTransactionState(of: conn) + guard PQresultStatus(result) == PGRES_COMMAND_OK else { + takesSavepoints = false + return false + } + return true + } + + // Undoing rolls back to the savepoint whatever state the cancel left, so an abandoned read keeps nothing. Without + // it, a read that ended keeps its work and one a failure aborted stays aborted. The error is a release or rollback + // the server refused, which leaves the user's block aborted: a late cancel can land on it, and the block's state + // is reported then rather than found by the next statement. + private func endStreamSavepoint(_ conn: OpaquePointer, undoing: Bool) -> LibPQPluginError? { + let status = PQtransactionStatus(conn) + let statement: String + if undoing, status == PQTRANS_INTRANS || status == PQTRANS_INERROR { + statement = Self.streamSavepointRollback + } else if status == PQTRANS_INTRANS { + statement = Self.streamSavepointRelease + } else { + return nil + } + guard let result = statement.withCString({ PQexec(conn, $0) }) else { return getError(from: conn) } + defer { PQclear(result) } + recordTransactionState(of: conn) + guard PQresultStatus(result) != PGRES_COMMAND_OK else { return nil } + let refused = getResultError(from: result) + Self.logger.error("A stream savepoint was refused, so the transaction block is aborted") + return refused + } + // MARK: - Private Helpers var hasLostConnection: Bool { diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQCappedCursorReadTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQCappedCursorReadTests.swift new file mode 100644 index 0000000000..5f8497a740 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQCappedCursorReadTests.swift @@ -0,0 +1,92 @@ +// +// LibPQCappedCursorReadTests.swift +// PostgreSQLDriverTests +// + +import Foundation +import Testing + +struct LibPQCappedCursorReadTests { + @Test("Inside a transaction the cap plus one row are sent and the rest run on the server unsent") + func insideTransactionFetchesCapPlusOne() throws { + let plan = try #require(LibPQCappedCursorRead.plan( + for: "SELECT * FROM events", rowCap: 1_000, scope: .insideTransaction + )) + + #expect(plan.fetchCount == 1_001) + #expect(plan.opening == "SAVEPOINT tablepro_capped_read") + #expect(plan.statement == """ + DECLARE tablepro_capped_read NO SCROLL CURSOR FOR + SELECT * FROM events + ; + FETCH FORWARD 1001 FROM tablepro_capped_read; + MOVE FORWARD ALL FROM tablepro_capped_read; + CLOSE tablepro_capped_read; + RELEASE SAVEPOINT tablepro_capped_read + """) + #expect(plan.recovery == "ROLLBACK TO SAVEPOINT tablepro_capped_read; RELEASE SAVEPOINT tablepro_capped_read") + } + + @Test("Outside a transaction the cursor gets a transaction of its own") + func ownTransactionOpensAndCommits() throws { + let plan = try #require(LibPQCappedCursorRead.plan(for: "VALUES (1)", rowCap: 10, scope: .ownTransaction)) + + #expect(plan.opening == "BEGIN") + #expect(plan.statement.hasPrefix("DECLARE tablepro_capped_read NO SCROLL CURSOR FOR\nVALUES (1)\n;")) + #expect(plan.statement.hasSuffix( + "FETCH FORWARD 11 FROM tablepro_capped_read;\nMOVE FORWARD ALL FROM tablepro_capped_read;\nCLOSE tablepro_capped_read;\nCOMMIT" + )) + #expect(plan.recovery == "ROLLBACK") + } + + @Test("A trailing line comment ends before the FETCH") + func trailingLineCommentKeepsTheFetch() throws { + let plan = try #require(LibPQCappedCursorRead.plan( + for: "SELECT 1 -- tail", rowCap: 5, scope: .insideTransaction + )) + + #expect(plan.statement.contains("SELECT 1 -- tail\n;\nFETCH FORWARD 6")) + } + + @Test( + "One read statement goes through the cursor", + arguments: [ + ("SELECT * FROM t; \n", "SELECT * FROM t"), + ("select a from t;;", "select a from t"), + ("-- note\n(SELECT 1) UNION (SELECT 2)", "-- note\n(SELECT 1) UNION (SELECT 2)"), + ("/* c */ WITH x AS (SELECT 1) SELECT * FROM x", "/* c */ WITH x AS (SELECT 1) SELECT * FROM x"), + ("TABLE accounts", "TABLE accounts"), + ("SELECT updated_at, shared_by FROM t", "SELECT updated_at, shared_by FROM t"), + ] + ) + func readStatementIsCursorable(query: String, body: String) { + #expect(LibPQCappedCursorRead.cursorBody(of: query) == body) + } + + @Test( + "A statement a cursor would change or refuse runs the plain way", + arguments: [ + "SELECT 1; SELECT 2", + "SELECT ';' AS semicolon", + "SELECT * FROM t FOR UPDATE", + "SELECT * FROM t FOR NO KEY UPDATE SKIP LOCKED", + "SELECT * FROM t FOR KEY SHARE", + "SELECT * INTO copy FROM t", + "WITH gone AS (DELETE FROM t RETURNING *) SELECT * FROM gone", + "INSERT INTO t VALUES (1) RETURNING *", + "EXPLAIN SELECT 1", + "SHOW search_path", + "/* unterminated SELECT 1", + "", + " ; ", + ] + ) + func otherStatementsAreNotCursorable(query: String) { + #expect(LibPQCappedCursorRead.cursorBody(of: query) == nil) + } + + @Test("A cap FETCH cannot spell runs the plain way", arguments: [0, -1, Int(Int32.max)]) + func capOutsideFetchRangeHasNoPlan(rowCap: Int) { + #expect(LibPQCappedCursorRead.plan(for: "SELECT 1", rowCap: rowCap, scope: .insideTransaction) == nil) + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQScriptedServer.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQScriptedServer.swift new file mode 100644 index 0000000000..4b5eaf6ecd --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQScriptedServer.swift @@ -0,0 +1,170 @@ +// +// LibPQScriptedServer.swift +// PostgreSQLDriverTests +// + +import Darwin +import Foundation + +// A loopback PostgreSQL server for one plaintext session. It refuses SSL and GSS encryption, accepts any startup +// without a password, records each simple query, runs `onQuery` before it answers, and answers with an empty +// completion and the transaction status `status` picks from the query and the ones before it. +final class LibPQScriptedServer: @unchecked Sendable { + struct SetupFailure: Error {} + + static let idle = UInt8(ascii: "I") + static let inTransaction = UInt8(ascii: "T") + + private static let protocolVersion: UInt32 = 196_608 + private static let sslRequest: UInt32 = 80_877_103 + private static let gssEncryptionRequest: UInt32 = 80_877_104 + private static let pollMilliseconds: Int32 = 10_000 + private static let parameters: [(String, String)] = [ + ("server_version", "17.4"), + ("server_encoding", "UTF8"), + ("client_encoding", "UTF8"), + ("standard_conforming_strings", "on"), + ("DateStyle", "ISO, MDY"), + ("integer_datetimes", "on"), + ("TimeZone", "UTC"), + ] + + let port: Int + private let listener: Int32 + private let status: @Sendable (_ query: String, _ earlier: [String]) -> UInt8 + private let onQuery: @Sendable (String) -> Void + private let lock = NSLock() + private var received: [String] = [] + + var queries: [String] { + lock.withLock { received } + } + + init( + status: @escaping @Sendable (_ query: String, _ earlier: [String]) -> UInt8 = { _, _ in LibPQScriptedServer.idle }, + onQuery: @escaping @Sendable (String) -> Void = { _ in } + ) throws { + self.status = status + self.onQuery = onQuery + let socketFD = socket(AF_INET, SOCK_STREAM, 0) + guard socketFD >= 0 else { throw SetupFailure() } + var option: Int32 = 1 + setsockopt(socketFD, SOL_SOCKET, SO_REUSEADDR, &option, socklen_t(MemoryLayout.size)) + var address = sockaddr_in() + address.sin_family = sa_family_t(AF_INET) + address.sin_addr.s_addr = inet_addr("127.0.0.1") + let bound = withUnsafePointer(to: &address) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + bind(socketFD, $0, socklen_t(MemoryLayout.size)) + } + } + var boundAddress = sockaddr_in() + var length = socklen_t(MemoryLayout.size) + let named = withUnsafeMutablePointer(to: &boundAddress) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { getsockname(socketFD, $0, &length) } + } + guard bound == 0, Darwin.listen(socketFD, 4) == 0, named == 0 else { + Darwin.close(socketFD) + throw SetupFailure() + } + listener = socketFD + port = Int(boundAddress.sin_port.bigEndian) + } + + func start() { + DispatchQueue(label: "com.TablePro.tests.scripted-postgresql").async { self.runSession() } + } + + func close() { + Darwin.close(listener) + } + + private func runSession() { + guard let client = acceptClient() else { return } + defer { Darwin.close(client) } + guard startUp(client) else { return } + + while let type = receive(client, 1)?.first, type == UInt8(ascii: "Q"), let body = receiveBody(client) { + let query = String(decoding: body.prefix { $0 != 0 }, as: UTF8.self) + let earlier = lock.withLock { () -> [String] in + defer { received.append(query) } + return received + } + onQuery(query) + let reply = Self.message("C", Array("SELECT 0".utf8) + [0]) + Self.message("Z", [status(query, earlier)]) + guard send(client, reply) else { return } + } + } + + private func startUp(_ client: Int32) -> Bool { + while let packet = receiveBody(client), packet.count >= 4 { + let code = Self.uint32(packet.prefix(4)) + if code == Self.sslRequest || code == Self.gssEncryptionRequest { + guard send(client, [UInt8(ascii: "N")]) else { return false } + continue + } + guard code == Self.protocolVersion else { return false } + var reply = Self.message("R", Self.bytes(UInt32(0))) + for (name, value) in Self.parameters { + reply += Self.message("S", Array(name.utf8) + [0] + Array(value.utf8) + [0]) + } + reply += Self.message("K", Self.bytes(UInt32(4_242)) + Self.bytes(UInt32(7_777))) + reply += Self.message("Z", [Self.idle]) + return send(client, reply) + } + return false + } + + private static func message(_ type: Character, _ body: [UInt8]) -> [UInt8] { + [type.asciiValue ?? 0] + bytes(UInt32(body.count + 4)) + body + } + + private static func bytes(_ value: UInt32) -> [UInt8] { + [UInt8((value >> 24) & 0xFF), UInt8((value >> 16) & 0xFF), UInt8((value >> 8) & 0xFF), UInt8(value & 0xFF)] + } + + private static func uint32(_ bytes: C) -> UInt32 where C.Element == UInt8 { + bytes.reduce(0) { $0 << 8 | UInt32($1) } + } + + // A length that counts its own four bytes, then that many minus four. + private func receiveBody(_ client: Int32) -> [UInt8]? { + guard let header = receive(client, 4) else { return nil } + let length = Int(Self.uint32(header)) + guard length >= 4 else { return nil } + return length == 4 ? [] : receive(client, length - 4) + } + + private func acceptClient() -> Int32? { + var pollFD = pollfd(fd: listener, events: Int16(POLLIN), revents: 0) + guard poll(&pollFD, 1, Self.pollMilliseconds) > 0 else { return nil } + let client = accept(listener, nil, nil) + guard client >= 0 else { return nil } + var noSignal: Int32 = 1 + setsockopt(client, SOL_SOCKET, SO_NOSIGPIPE, &noSignal, socklen_t(MemoryLayout.size)) + return client + } + + private func receive(_ client: Int32, _ count: Int) -> [UInt8]? { + var bytes: [UInt8] = [] + while bytes.count < count { + var pollFD = pollfd(fd: client, events: Int16(POLLIN), revents: 0) + guard poll(&pollFD, 1, Self.pollMilliseconds) > 0 else { return nil } + var chunk = [UInt8](repeating: 0, count: count - bytes.count) + let read = recv(client, &chunk, chunk.count, 0) + guard read > 0 else { return nil } + bytes += chunk.prefix(read) + } + return bytes + } + + private func send(_ client: Int32, _ bytes: [UInt8]) -> Bool { + var offset = 0 + while offset < bytes.count { + let sent = bytes[offset...].withUnsafeBytes { Darwin.send(client, $0.baseAddress, $0.count, 0) } + guard sent > 0 else { return false } + offset += sent + } + return true + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQStopBeforeStatementTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQStopBeforeStatementTests.swift new file mode 100644 index 0000000000..8124485cb3 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQStopBeforeStatementTests.swift @@ -0,0 +1,122 @@ +// +// LibPQStopBeforeStatementTests.swift +// PostgreSQLDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +// A Stop that arrives while the driver runs a round trip of its own (the savepoint probe, the cursor read's savepoint) +// is spent on that round trip, so the user's statement must not go out after it. +@Suite(.timeLimit(.minutes(1))) +struct LibPQStopBeforeStatementTests { + private final class ConnectionBox: @unchecked Sendable { + private let lock = NSLock() + private var held: LibPQPluginConnection? + + var connection: LibPQPluginConnection? { + get { lock.withLock { held } } + set { lock.withLock { held = newValue } } + } + + func stop() { + _ = connection?.cancellationGate.cancel() + } + } + + private static func connected(to server: LibPQScriptedServer, box: ConnectionBox) async throws -> LibPQPluginConnection { + let connection = LibPQPluginConnection( + host: "127.0.0.1", port: server.port, user: "postgres", password: nil, database: "postgres" + ) + box.connection = connection + try await connection.connect() + try await connection.finishConnecting() + return connection + } + + @Test("A Stop during the savepoint probe sends none of the user's statement and reports cancelled") + func stopDuringProbeSendsNothing() async throws { + let box = ConnectionBox() + let server = try LibPQScriptedServer(onQuery: { query in + if query.contains("tablepro_probe") { box.stop() } + }) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server, box: box) + defer { connection.disconnect() } + + var failure: Error? + do { + _ = try await connection.boundedQuery("DELETE FROM big WHERE flagged", rowCap: 1_000) + } catch { + failure = error + } + + let sent = server.queries + let probed = sent.contains { $0.contains("tablepro_probe") } + let deleteSent = sent.contains { $0.contains("DELETE FROM big") } + #expect(failure is CancellationError) + #expect(probed) + #expect(!deleteSent) + } + + @Test("A Stop during the cursor read's savepoint sends no cursor and reports cancelled") + func stopDuringCursorSavepointSendsNothing() async throws { + let box = ConnectionBox() + let server = try LibPQScriptedServer( + status: { query, earlier in + (earlier + [query]).contains("BEGIN") ? LibPQScriptedServer.inTransaction : LibPQScriptedServer.idle + }, + onQuery: { query in + if query.hasPrefix("SAVEPOINT tablepro_capped_read") { box.stop() } + } + ) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server, box: box) + defer { connection.disconnect() } + + _ = try await connection.boundedQuery("SELECT 1", rowCap: 10) + _ = try await connection.executeQuery("BEGIN") + var failure: Error? + do { + _ = try await connection.boundedQuery("SELECT * FROM events", rowCap: 1_000) + } catch { + failure = error + } + + let sent = server.queries + let savepointSent = sent.contains { $0.hasPrefix("SAVEPOINT tablepro_capped_read") } + let readSent = sent.contains { $0.contains("DECLARE") || $0.contains("FROM events") } + #expect(failure is CancellationError) + #expect(savepointSent) + #expect(!readSent) + } + + @Test("A Stop during a stream's savepoint probe sends none of the user's statement and ends cancelled") + func stopDuringStreamProbeSendsNothing() async throws { + let box = ConnectionBox() + let server = try LibPQScriptedServer(onQuery: { query in + if query.contains("tablepro_probe") { box.stop() } + }) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server, box: box) + defer { connection.disconnect() } + + var failure: Error? + do { + for try await _ in connection.streamQuery("SELECT * FROM events") {} + } catch { + failure = error + } + + let sent = server.queries + let probed = sent.contains { $0.contains("tablepro_probe") } + let readSent = sent.contains { $0.contains("FROM events") } + #expect(failure is CancellationError) + #expect(probed) + #expect(!readSent) + } +} diff --git a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index c173fc6838..dc8a667c2c 100644 --- a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -1027,6 +1027,7 @@ public extension PluginDatabaseDriver { let tables = try await fetchTables(schema: schema) var result: [String: [PluginColumnInfo]] = [:] for table in tables { + try Task.checkCancellation() result[table.name] = try await fetchColumns(table: table.name, schema: schema) } return result @@ -1061,6 +1062,7 @@ public extension PluginDatabaseDriver { let tables = try await fetchTables(schema: schema) var result: [String: [PluginForeignKeyInfo]] = [:] for table in tables { + try Task.checkCancellation() let fks = try await fetchForeignKeys(table: table.name, schema: schema) if !fks.isEmpty { result[table.name] = fks } } @@ -1090,6 +1092,7 @@ public extension PluginDatabaseDriver { let tables = try await fetchTables(schema: schema) var result: [String: [PluginIndexInfo]] = [:] for table in tables { + try Task.checkCancellation() let indexes = try await fetchIndexes(table: table.name, schema: schema) if !indexes.isEmpty { result[table.name] = indexes } } @@ -1108,6 +1111,7 @@ public extension PluginDatabaseDriver { let tables = try await fetchTables(schema: schema) var result: [String: PluginTableMetadata] = [:] for table in tables { + try Task.checkCancellation() guard let metadata = try? await fetchTableMetadata(table: table.name, schema: schema) else { continue } result[table.name] = metadata } @@ -1118,6 +1122,7 @@ public extension PluginDatabaseDriver { let dbs = try await fetchDatabases() var result: [PluginDatabaseMetadata] = [] for db in dbs { + try Task.checkCancellation() do { result.append(try await fetchDatabaseMetadata(db)) } catch { @@ -1500,13 +1505,15 @@ public extension PluginDatabaseDriver { func streamRows(query: String) -> AsyncThrowingStream { AsyncThrowingStream { continuation in - Task { + let producer = Task { do { + try Task.checkCancellation() let result = try await self.execute(query: query) - let header = PluginStreamHeader( + var header = PluginStreamHeader( columns: result.columns, columnTypeNames: result.columnTypeNames ) + if result.isTruncated { header.rowLimit = result.rows.count } continuation.yield(.header(header)) if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) @@ -1516,6 +1523,8 @@ public extension PluginDatabaseDriver { continuation.finish(throwing: error) } } + // An unstructured task is no child of the consumer, so its cancel reaches execute only through here. + continuation.onTermination = { @Sendable _ in producer.cancel() } } } diff --git a/apple/Plugins/TableProPluginKit/PluginStreamTypes.swift b/apple/Plugins/TableProPluginKit/PluginStreamTypes.swift index 12cec34a01..914e3fc847 100644 --- a/apple/Plugins/TableProPluginKit/PluginStreamTypes.swift +++ b/apple/Plugins/TableProPluginKit/PluginStreamTypes.swift @@ -8,6 +8,8 @@ public struct PluginStreamHeader: Sendable { public let estimatedRowCount: Int? // See `PluginQueryResult.columnOrigins`. public var columnOrigins: [PluginColumnOrigin?]? + // The most rows the driver lets this statement return. A stream that ends at it was cut there; nil when nothing cuts it. + public var rowLimit: Int? public init(columns: [String], columnTypeNames: [String], estimatedRowCount: Int? = nil) { self.columns = columns diff --git a/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift b/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift index 31f57cd3ac..63a467ef35 100644 --- a/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift +++ b/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift @@ -66,7 +66,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug case .header(let header): columns = header.columns columnTypeNames = header.columnTypeNames - writer.beginSheet( + try writer.beginSheet( name: table.name, columns: columns, columnTypeNames: columnTypeNames, @@ -82,20 +82,20 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug if rowBatch.count >= 5_000 { let remaining = Self.maxRowsPerSheet - currentSheetRowCount if rowBatch.count <= remaining { - autoreleasepool { - writer.addRows(rowBatch, convertNullToEmpty: settings.convertNullToEmpty) + try autoreleasepool { + try writer.addRows(rowBatch, convertNullToEmpty: settings.convertNullToEmpty) } currentSheetRowCount += rowBatch.count } else { let fitting = Array(rowBatch.prefix(remaining)) let overflow = Array(rowBatch.dropFirst(remaining)) if !fitting.isEmpty { - autoreleasepool { - writer.addRows(fitting, convertNullToEmpty: settings.convertNullToEmpty) + try autoreleasepool { + try writer.addRows(fitting, convertNullToEmpty: settings.convertNullToEmpty) } currentSheetRowCount += fitting.count } - writer.continueSheet( + try writer.continueSheet( baseName: table.name, columns: columns, columnTypeNames: columnTypeNames, @@ -105,8 +105,8 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug didSplitSheets = true currentSheetRowCount = headerRowCount if !overflow.isEmpty { - autoreleasepool { - writer.addRows(overflow, convertNullToEmpty: settings.convertNullToEmpty) + try autoreleasepool { + try writer.addRows(overflow, convertNullToEmpty: settings.convertNullToEmpty) } currentSheetRowCount += overflow.count } @@ -123,19 +123,19 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug if !rowBatch.isEmpty { let remaining = Self.maxRowsPerSheet - currentSheetRowCount if rowBatch.count <= remaining { - autoreleasepool { - writer.addRows(rowBatch, convertNullToEmpty: settings.convertNullToEmpty) + try autoreleasepool { + try writer.addRows(rowBatch, convertNullToEmpty: settings.convertNullToEmpty) } currentSheetRowCount += rowBatch.count } else { let fitting = Array(rowBatch.prefix(remaining)) let overflow = Array(rowBatch.dropFirst(remaining)) if !fitting.isEmpty { - autoreleasepool { - writer.addRows(fitting, convertNullToEmpty: settings.convertNullToEmpty) + try autoreleasepool { + try writer.addRows(fitting, convertNullToEmpty: settings.convertNullToEmpty) } } - writer.continueSheet( + try writer.continueSheet( baseName: table.name, columns: columns, columnTypeNames: columnTypeNames, @@ -145,8 +145,8 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug didSplitSheets = true currentSheetRowCount = headerRowCount if !overflow.isEmpty { - autoreleasepool { - writer.addRows(overflow, convertNullToEmpty: settings.convertNullToEmpty) + try autoreleasepool { + try writer.addRows(overflow, convertNullToEmpty: settings.convertNullToEmpty) } currentSheetRowCount += overflow.count } @@ -157,15 +157,15 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug } if !isFirstBatch { - writer.finishSheet() + try writer.finishSheet() } else { - writer.beginSheet( + try writer.beginSheet( name: table.name, columns: [], includeHeader: false, convertNullToEmpty: settings.convertNullToEmpty ) - writer.finishSheet() + try writer.finishSheet() } } diff --git a/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift b/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift index a4bfb3cf5b..c404199eac 100644 --- a/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift +++ b/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift @@ -2,43 +2,44 @@ // XLSXWriter.swift // TablePro // -// Lightweight XLSX writer that creates Excel files without external dependencies. -// XLSX format = ZIP archive containing XML files (Office Open XML). -// -// Performance: Uses inline strings (no shared string table), Data buffers -// (not String concatenation), and batch row processing to handle 100K+ row -// exports with bounded memory usage. -// import Foundation import os import TableProPluginKit import zlib -/// Writes data to XLSX format using raw ZIP file construction. -/// -/// Uses inline strings (`t="inlineStr"`) instead of a shared string table -/// to avoid unbounded memory growth from caching every unique string value. -/// Rows are processed in batches and appended directly to per-sheet XML Data, -/// so raw row arrays can be released after each batch. +// Each sheet's XML goes to its own file as rows arrive, and `write(to:)` copies those files into a +// stored ZIP, so no sheet is ever held in memory whole. Strings are inline (`t="inlineStr"`), so +// there is no shared string table to grow with the data either. final class XLSXWriter { + static let defaultFlushThreshold = 1 << 20 + static let partsDirectoryPrefix = "TablePro-XLSX-" + private static let logger = Logger(subsystem: "com.TablePro", category: "XLSXWriter") - /// Per-sheet metadata and accumulated XML data - private var sheets: [(name: String, data: Data)] = [] + private static let processStart: Date = { + var info = kinfo_proc() + var size = MemoryLayout.stride + var name: [Int32] = [CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid()] + guard sysctl(&name, u_int(name.count), &info, &size, nil, 0) == 0 else { return Date() } + let start = info.kp_proc.p_un.__p_starttime + return Date(timeIntervalSince1970: TimeInterval(start.tv_sec) + TimeInterval(start.tv_usec) / 1_000_000) + }() + + private var sheets: [XLSXSheetPart] = [] + private var sheetBytesInArchive = 0 + private var partsDirectory: URL? + private var partsLock: Int32? + private let flushThreshold: Int + private let maximumArchiveSize: Int private static let maximumSheetNameLength = 31 private var takenSheetNames: Set = [] - /// Pre-cached column letter lookups private var columnLetterCache: [String] = [] - /// Tracks the current row number for the active sheet being built private var currentRowNumber: Int = 0 - /// Whether the current sheet has a header row (used for bold styling) - private var currentSheetHasHeader: Bool = false - enum CellValue: Equatable { case string(String) case number(String) @@ -113,24 +114,49 @@ final class XLSXWriter { private let bitsAreStrings: Bool - init(databaseTypeId: String = "") { + // A sheet holds at most `flushThreshold` bytes plus one row in memory before they go to its file. + // `maximumArchiveSize` is the ZIP's 4 GB offset limit, checked as the sheets grow. + init( + databaseTypeId: String = "", + partsDirectory: URL? = nil, + flushThreshold: Int = XLSXWriter.defaultFlushThreshold, + maximumArchiveSize: Int = Int(UInt32.max) + ) { bitsAreStrings = ColumnNumbers.bitsAreStrings(databaseTypeId: databaseTypeId) + self.partsDirectory = partsDirectory + self.flushThreshold = max(0, flushThreshold) + self.maximumArchiveSize = min(maximumArchiveSize, Int(UInt32.max)) + } + + deinit { + sheets.forEach { $0.closeWithoutWriting() } + defer { + if let partsLock { close(partsLock) } + } + guard let partsDirectory else { return } + do { + try FileManager.default.removeItem(at: partsDirectory) + } catch { + guard (error as? CocoaError)?.code != .fileNoSuchFile else { return } + Self.logger.warning("Could not remove the workbook's sheet files: \(error.localizedDescription, privacy: .private)") + } } // MARK: - Sheet Building API - /// Begin a new worksheet. Must be followed by `addRows` calls and then `finishSheet`. func beginSheet( name: String, columns: [String], columnTypeNames: [String] = [], includeHeader: Bool, convertNullToEmpty: Bool - ) { + ) throws { + if let previous = sheets.last, previous.isOpen { + try closeSheet(previous) + } let sheetName = unusedSheetName(for: name) currentColumnNumbers = columnTypeNames.map { ColumnNumbers(typeName: $0, bitsAreStrings: bitsAreStrings) } currentRowNumber = 0 - currentSheetHasHeader = includeHeader let maxCols = max(columns.count, columnLetterCache.count) if maxCols > columnLetterCache.count { @@ -139,25 +165,22 @@ final class XLSXWriter { } } - var d = Data() - d.appendUTF8("\n") - d.appendUTF8("") + let part = try XLSXSheetPart(name: sheetName, fileURL: try partURL(forSheet: sheets.count + 1)) + sheets.append(part) + sheetBytesInArchive += ZipBuilder.localHeaderSize(path: Self.sheetPath(sheets.count)) + part.pending.appendUTF8("\n") + part.pending.appendUTF8("") if includeHeader { let headerCells: [CellValue] = columns.map { .string($0) } - appendRow(headerCells, isHeader: true, to: &d) + appendRow(headerCells, isHeader: true, to: &part.pending) } - - sheets.append((name: sheetName, data: d)) + try flush(part) } - /// Add a batch of raw rows to the current (last) sheet. - /// Converts `[PluginCellValue]` to `CellValue` and writes XML immediately, - /// so the caller can release the raw row data after this call returns. - func addRows(_ rows: [[PluginCellValue]], convertNullToEmpty: Bool) { - guard !sheets.isEmpty else { return } - - var sheetData = sheets[sheets.count - 1].data + // Writes each row's XML as it converts it, so the caller can release the rows once this returns. + func addRows(_ rows: [[PluginCellValue]], convertNullToEmpty: Bool) throws { + guard let part = sheets.last, part.isOpen else { return } for row in rows { autoreleasepool { @@ -174,11 +197,10 @@ final class XLSXWriter { return Self.cell(forText: val, numbers: numbers) } } - appendRow(cellRow, isHeader: false, to: &sheetData) + appendRow(cellRow, isHeader: false, to: &part.pending) } + try flush(part) } - - sheets[sheets.count - 1].data = sheetData } // A text column writes text whatever its values look like. A numeric column writes a number @@ -220,23 +242,22 @@ final class XLSXWriter { return digits[...last].count } - /// Finish the current sheet by closing the XML tags. - func finishSheet() { - guard !sheets.isEmpty else { return } - sheets[sheets.count - 1].data.appendUTF8("") + func finishSheet() throws { + guard let part = sheets.last, part.isOpen else { return } + part.pending.appendUTF8("") + try closeSheet(part) } - /// Finish the current sheet and start a continuation sheet with the same columns. - /// The new sheet is named "BaseName (N)" where N increments. + // Finishes the current sheet and starts "BaseName (N)" with the same columns. func continueSheet( baseName: String, columns: [String], columnTypeNames: [String] = [], includeHeader: Bool, convertNullToEmpty: Bool - ) { - finishSheet() - beginSheet( + ) throws { + try finishSheet() + try beginSheet( name: baseName, columns: columns, columnTypeNames: columnTypeNames, @@ -245,35 +266,14 @@ final class XLSXWriter { ) } - // MARK: - Legacy Convenience API - - /// Add a complete worksheet with all rows at once (legacy compatibility). - /// For better memory usage, prefer `beginSheet` / `addRows` / `finishSheet`. - func addSheet( - name: String, - columns: [String], - columnTypeNames: [String] = [], - rows: [[PluginCellValue]], - includeHeader: Bool, - convertNullToEmpty: Bool - ) { - beginSheet( - name: name, - columns: columns, - columnTypeNames: columnTypeNames, - includeHeader: includeHeader, - convertNullToEmpty: convertNullToEmpty - ) - addRows(rows, convertNullToEmpty: convertNullToEmpty) - finishSheet() - } - - /// Write the XLSX file to the given URL func write(to url: URL) async throws { + if let part = sheets.last, part.isOpen { + try closeSheet(part) + } let entries = archiveEntries() + let maximumSize = maximumArchiveSize try await Task.detached(priority: .userInitiated) { - let zipData = try ZipBuilder.build(entries: entries) - try zipData.write(to: url, options: .atomic) + try ZipBuilder.write(entries, to: url, maximumSize: maximumSize) }.value } @@ -288,19 +288,89 @@ final class XLSXWriter { for (index, sheet) in sheets.enumerated() { entries.append(ZipFileEntry( - path: "xl/worksheets/sheet\(index + 1).xml", - data: sheet.data + path: Self.sheetPath(index + 1), + file: sheet.fileURL, + size: sheet.byteCount, + crc: sheet.crc )) } return entries } + private static func sheetPath(_ number: Int) -> String { + "xl/worksheets/sheet\(number).xml" + } + + // MARK: - Sheet Files + + private func partURL(forSheet number: Int) throws -> URL { + try openPartsDirectory().appendingPathComponent("sheet\(number).xml") + } + + // The folder stays locked while this writer lives, which tells a later export's sweep that it is in use. + private func openPartsDirectory() throws -> URL { + if let partsDirectory, partsLock != nil { return partsDirectory } + let directory: URL + if let partsDirectory { + directory = partsDirectory + } else { + let parent = FileManager.default.temporaryDirectory + Self.removeAbandonedPartsDirectories(in: parent, modifiedBefore: Self.processStart) + directory = parent.appendingPathComponent(Self.partsDirectoryPrefix + UUID().uuidString, isDirectory: true) + } + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + partsDirectory = directory + partsLock = try Self.lock(directory) + return directory + } + + // A crash or a force quit skips the deinit that removes a writer's folder. A folder older than + // this process that no writer holds, in this process or another, was left that way. + static func removeAbandonedPartsDirectories(in parent: URL, modifiedBefore cutoff: Date) { + let keys: Set = [.isDirectoryKey, .contentModificationDateKey] + guard let entries = try? FileManager.default.contentsOfDirectory( + at: parent, includingPropertiesForKeys: Array(keys) + ) else { return } + for entry in entries where entry.lastPathComponent.hasPrefix(partsDirectoryPrefix) { + guard let values = try? entry.resourceValues(forKeys: keys), values.isDirectory == true, + let modified = values.contentModificationDate, modified < cutoff, + let descriptor = try? lock(entry) else { continue } + try? FileManager.default.removeItem(at: entry) + close(descriptor) + } + } + + // `flock` is released when the descriptor closes, also when the process dies. + private static func lock(_ directory: URL) throws -> Int32 { + let descriptor = open(directory.path(percentEncoded: false), O_RDONLY | O_CLOEXEC) + guard descriptor >= 0 else { throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) } + guard flock(descriptor, LOCK_EX | LOCK_NB) == 0 else { + let code = POSIXErrorCode(rawValue: errno) ?? .EIO + close(descriptor) + throw POSIXError(code) + } + return descriptor + } + + private func flush(_ part: XLSXSheetPart) throws { + guard part.pending.count >= flushThreshold else { return } + try account(writtenBytes: part.writePending()) + } + + private func closeSheet(_ part: XLSXSheetPart) throws { + try account(writtenBytes: part.close()) + } + + // The sheets alone already past the limit means `write(to:)` would refuse the archive, so the + // export stops now instead of after reading every remaining row. + private func account(writtenBytes: Int) throws { + sheetBytesInArchive += writtenBytes + guard sheetBytesInArchive <= maximumArchiveSize else { throw ZipBuilder.ZipError.fileTooLarge } + } + // MARK: - Row XML Generation - /// Append a single row of cells to the given Data buffer using inline strings. - /// Inline strings use `t="inlineStr"` with `text` to avoid - /// the shared string table entirely (MEM-15 fix). private func appendRow(_ cells: [CellValue], isHeader: Bool, to data: inout Data) { currentRowNumber += 1 let rowNum = currentRowNumber @@ -316,7 +386,6 @@ final class XLSXWriter { switch cell { case .string(let value): if isHeader { - // Header cells get bold style (s="1") + inline string data.appendUTF8("") } else { data.appendUTF8("") @@ -335,7 +404,7 @@ final class XLSXWriter { data.appendUTF8("") } - // MARK: - XML Generation (Data-based to avoid O(n^2) String concatenation) + // MARK: - XML Generation private func contentTypesXML() -> Data { var d = Data() @@ -452,7 +521,6 @@ final class XLSXWriter { // MARK: - Data XML Helpers private extension Data { - /// Append a UTF-8 string directly to Data (O(1) amortized, no intermediate String copies) mutating func appendUTF8(_ string: String) { string.utf8.withContiguousStorageIfAvailable { buffer in if let baseAddress = buffer.baseAddress { @@ -461,9 +529,6 @@ private extension Data { } ?? self.append(contentsOf: string.utf8) } - /// Append XML-escaped text directly to Data without creating intermediate Strings. - /// Strips XML 1.0 illegal control characters (0x00–0x08, 0x0B, 0x0C, 0x0E–0x1F) - /// that can appear in binary/hex database columns and would produce malformed XML. mutating func appendXMLEscaped(_ text: String) { for byte in text.utf8 { switch byte { @@ -490,105 +555,211 @@ private extension Data { } } +// MARK: - Sheet Part + +private final class XLSXSheetPart { + let name: String + let fileURL: URL + var pending = Data() + private var handle: FileHandle? + private(set) var byteCount = 0 + private(set) var crc: UInt32 = 0 + + var isOpen: Bool { handle != nil } + + init(name: String, fileURL: URL) throws { + self.name = name + self.fileURL = fileURL + guard FileManager.default.createFile(atPath: fileURL.path(percentEncoded: false), contents: nil) else { + throw CocoaError(.fileWriteUnknown, userInfo: [NSURLErrorKey: fileURL]) + } + handle = try FileHandle(forWritingTo: fileURL) + } + + func writePending() throws -> Int { + guard let handle, !pending.isEmpty else { return 0 } + try handle.write(contentsOf: pending) + crc = ZipBuilder.crc32(of: pending, continuing: crc) + let written = pending.count + byteCount += written + pending.removeAll(keepingCapacity: true) + return written + } + + func close() throws -> Int { + guard let handle else { return 0 } + let written = try writePending() + self.handle = nil + try handle.close() + return written + } + + func closeWithoutWriting() { + try? handle?.close() + handle = nil + } +} + // MARK: - ZIP File Builder -/// Minimal ZIP file builder (store-only, no compression) +// A stored (uncompressed) ZIP, written entry by entry. A sheet's bytes are copied from its file. private struct ZipFileEntry { + enum Contents { + case data(Data) + case file(URL) + } + let path: String - let data: Data + let contents: Contents + let size: Int + let crc: UInt32 + + init(path: String, data: Data) { + self.path = path + contents = .data(data) + size = data.count + crc = ZipBuilder.crc32(of: data) + } + + init(path: String, file: URL, size: Int, crc: UInt32) { + self.path = path + contents = .file(file) + self.size = size + self.crc = crc + } } private enum ZipBuilder { enum ZipError: LocalizedError { case fileTooLarge + case tooManySheets var errorDescription: String? { switch self { case .fileTooLarge: - return "XLSX file exceeds 4 GB ZIP limit" + return String(localized: "XLSX file exceeds 4 GB ZIP limit") + case .tooManySheets: + return String(localized: "The workbook has more sheets than an XLSX file can hold.") } } } - static func build(entries: [ZipFileEntry]) throws -> Data { - var totalSize = 22 - for entry in entries { - let pathLen = entry.path.utf8.count - totalSize += 30 + pathLen + entry.data.count - totalSize += 46 + pathLen - } + private static let copyChunkSize = 1 << 20 - var output = Data(capacity: totalSize) - var centralDirectory = Data() - var offsets: [Int] = [] + static func localHeaderSize(path: String) -> Int { + 30 + path.utf8.count + } + static func write(_ entries: [ZipFileEntry], to destination: URL, maximumSize: Int) throws { + var offsets: [Int] = [] + offsets.reserveCapacity(entries.count) + var offset = 0 for entry in entries { - let currentOffset = output.count - guard currentOffset <= UInt32.max, entry.data.count <= UInt32.max else { + guard offset <= maximumSize, entry.size <= maximumSize else { throw ZipError.fileTooLarge } - offsets.append(currentOffset) - - let pathData = Data(entry.path.utf8) - let crc = zlibCRC32(entry.data) - - output.append(contentsOf: [0x50, 0x4B, 0x03, 0x04]) - output.appendUInt16(10) - output.appendUInt16(0) - output.appendUInt16(0) - output.appendUInt16(0) - output.appendUInt16(0) - output.appendUInt32(crc) - output.appendUInt32(UInt32(entry.data.count)) - output.appendUInt32(UInt32(entry.data.count)) - output.appendUInt16(UInt16(pathData.count)) - output.appendUInt16(0) - output.append(pathData) - output.append(entry.data) - - centralDirectory.append(contentsOf: [0x50, 0x4B, 0x01, 0x02]) - centralDirectory.appendUInt16(20) - centralDirectory.appendUInt16(10) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt32(crc) - centralDirectory.appendUInt32(UInt32(entry.data.count)) - centralDirectory.appendUInt32(UInt32(entry.data.count)) - centralDirectory.appendUInt16(UInt16(pathData.count)) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt32(0) - centralDirectory.appendUInt32(UInt32(currentOffset)) - centralDirectory.append(pathData) - } - - let centralDirOffset = output.count - guard centralDirOffset <= UInt32.max else { + offsets.append(offset) + offset += localHeaderSize(path: entry.path) + entry.size + } + guard offset <= maximumSize else { throw ZipError.fileTooLarge } - output.append(centralDirectory) + guard entries.count <= Int(UInt16.max) else { + throw ZipError.tooManySheets + } - output.append(contentsOf: [0x50, 0x4B, 0x05, 0x06]) - output.appendUInt16(0) - output.appendUInt16(0) - output.appendUInt16(UInt16(entries.count)) - output.appendUInt16(UInt16(entries.count)) - output.appendUInt32(UInt32(centralDirectory.count)) - output.appendUInt32(UInt32(centralDirOffset)) - output.appendUInt16(0) + let (handle, temporaryURL) = try PluginExportUtilities.beginAtomicWrite(for: destination) + var isOpen = true + do { + var centralDirectory = Data() + for (entry, entryOffset) in zip(entries, offsets) { + let pathData = Data(entry.path.utf8) + + var header = Data() + header.append(contentsOf: [0x50, 0x4B, 0x03, 0x04]) + header.appendUInt16(10) + header.appendUInt16(0) + header.appendUInt16(0) + header.appendUInt16(0) + header.appendUInt16(0) + header.appendUInt32(entry.crc) + header.appendUInt32(UInt32(entry.size)) + header.appendUInt32(UInt32(entry.size)) + header.appendUInt16(UInt16(pathData.count)) + header.appendUInt16(0) + header.append(pathData) + try handle.write(contentsOf: header) + try writeContents(of: entry, to: handle) + + centralDirectory.append(contentsOf: [0x50, 0x4B, 0x01, 0x02]) + centralDirectory.appendUInt16(20) + centralDirectory.appendUInt16(10) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt32(entry.crc) + centralDirectory.appendUInt32(UInt32(entry.size)) + centralDirectory.appendUInt32(UInt32(entry.size)) + centralDirectory.appendUInt16(UInt16(pathData.count)) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt32(0) + centralDirectory.appendUInt32(UInt32(entryOffset)) + centralDirectory.append(pathData) + } + + var end = Data() + end.append(contentsOf: [0x50, 0x4B, 0x05, 0x06]) + end.appendUInt16(0) + end.appendUInt16(0) + end.appendUInt16(UInt16(entries.count)) + end.appendUInt16(UInt16(entries.count)) + end.appendUInt32(UInt32(centralDirectory.count)) + end.appendUInt32(UInt32(offset)) + end.appendUInt16(0) + + try handle.write(contentsOf: centralDirectory) + try handle.write(contentsOf: end) + isOpen = false + try handle.close() + try PluginExportUtilities.commitAtomicWrite(from: temporaryURL, to: destination) + } catch { + if isOpen { + try? handle.close() + } + PluginExportUtilities.rollbackAtomicWrite(at: temporaryURL) + throw error + } + } - return output + private static func writeContents(of entry: ZipFileEntry, to handle: FileHandle) throws { + switch entry.contents { + case .data(let data): + try handle.write(contentsOf: data) + case .file(let url): + let reader = try FileHandle(forReadingFrom: url) + defer { try? reader.close() } + var copied = 0 + while true { + let chunk = try autoreleasepool { try reader.read(upToCount: copyChunkSize) } ?? Data() + guard !chunk.isEmpty else { break } + try handle.write(contentsOf: chunk) + copied += chunk.count + } + // The header already promised this size and CRC. + guard copied == entry.size else { + throw CocoaError(.fileReadCorruptFile, userInfo: [NSURLErrorKey: url]) + } + } } - /// CRC-32 using system zlib (hardware-accelerated) - private static func zlibCRC32(_ data: Data) -> UInt32 { + static func crc32(of data: Data, continuing crc: UInt32 = 0) -> UInt32 { data.withUnsafeBytes { buffer in - guard let ptr = buffer.baseAddress else { return 0 } - return UInt32(zlib.crc32(0, ptr.assumingMemoryBound(to: UInt8.self), uInt(buffer.count))) + guard let ptr = buffer.baseAddress, !buffer.isEmpty else { return crc } + return UInt32(zlib.crc32(uLong(crc), ptr.assumingMemoryBound(to: UInt8.self), uInt(buffer.count))) } } } diff --git a/apple/TablePro/Core/Autocomplete/SQLSchemaProvider.swift b/apple/TablePro/Core/Autocomplete/SQLSchemaProvider.swift index c58dd2fce1..a256b5de58 100644 --- a/apple/TablePro/Core/Autocomplete/SQLSchemaProvider.swift +++ b/apple/TablePro/Core/Autocomplete/SQLSchemaProvider.swift @@ -35,7 +35,9 @@ actor SQLSchemaProvider { } private var tables: [TableInfo] = [] - private var columnCache: [ColumnCacheKey: [ColumnInfo]] = [:] + private var columnCache: [ColumnCacheKey: [ColumnInfo]] = [:] { + didSet { cachedTableColumnItems = nil } + } private var columnAccessOrder: [ColumnCacheKey] = [] private var columnTasks: [ColumnCacheKey: Task<[ColumnInfo]?, Never>] = [:] private var loadTask: Task? @@ -90,6 +92,16 @@ actor SQLSchemaProvider { // Every table-slot request reads all of them, so they are built once per table list, not per keystroke. private var unqualifiedTableItems: (defaultSchema: String?, items: [SQLCompletionItem])? + private struct TableColumnItems { + let defaultSchema: String? + let eagerLoadSchema: String? + let items: [SQLCompletionItem] + } + + // Every column request before a FROM reads all of them, so they are built once per column cache and table list. + private var cachedTableColumnItems: TableColumnItems? + private(set) var tableColumnItemsBuildCount = 0 + private weak var cachedDriver: (any DatabaseDriver)? private let metadataSource: ColumnMetadataSource? private var connectionInfo: DatabaseConnection? @@ -354,6 +366,7 @@ actor SQLSchemaProvider { private func noteTableListWritten() { tableListGeneration &+= 1 unqualifiedTableItems = nil + cachedTableColumnItems = nil switch columnPreload { case .awaitingTables: startColumnPreloadIfStale(retryingFailure: true) @@ -710,12 +723,29 @@ actor SQLSchemaProvider { } } - /// Only the bulk cache can answer a statement that names no table, so once an editor has asked, a - /// stale cache starts a read here. This call answers from what is cached and never waits. + // Only the bulk cache can answer a statement that names no table, so once an editor has asked, a stale cache + // starts a read here. This call answers from what is cached and never waits. func allColumnsFromCachedTables() async -> [SQLCompletionItem] { if hasEditorDemand { startColumnPreloadIfStale(retryingFailure: false) } guard !columnCache.isEmpty else { return [] } + let defaultSchema = getDefaultSchema() + if let cached = cachedTableColumnItems, + cached.defaultSchema == defaultSchema, + cached.eagerLoadSchema == eagerLoadSchema { + return cached.items + } + let items = buildTableColumnItems() + tableColumnItemsBuildCount += 1 + cachedTableColumnItems = TableColumnItems( + defaultSchema: defaultSchema, + eagerLoadSchema: eagerLoadSchema, + items: items + ) + return items + } + + private func buildTableColumnItems() -> [SQLCompletionItem] { let canonicalNames = Dictionary( tables.map { ($0.name.lowercased(), $0.name) }, uniquingKeysWith: { first, _ in first } diff --git a/apple/TablePro/Core/ChangeTracking/AnyChangeManager.swift b/apple/TablePro/Core/ChangeTracking/AnyChangeManager.swift index 33c13b438c..5c2e7b65bf 100644 --- a/apple/TablePro/Core/ChangeTracking/AnyChangeManager.swift +++ b/apple/TablePro/Core/ChangeTracking/AnyChangeManager.swift @@ -15,6 +15,7 @@ protocol ChangeManaging: AnyObject { var supportsFieldRemoval: Bool { get } var cellWriterSupport: CellWriterSupport { get } func isRowDeleted(_ rowID: RowID) -> Bool + func pendingChange(forRow rowID: RowID) -> RowChange? func isExpression(rowID: RowID, columnIndex: Int) -> Bool func expressionColumns(forRow rowID: RowID) -> Set func recordCellChange( @@ -47,8 +48,7 @@ protocol ChangeManaging: AnyObject { func undoRowDeletion(rowID: RowID) } -/// Only the data grid tracks server-computed columns; the structure and -/// inspector grids edit schema definitions, where the concept does not apply. +// Only the data grid tracks server-computed columns; the structure and inspector grids edit schema definitions. extension ChangeManaging { var generatedColumns: Set { [] } @@ -60,11 +60,15 @@ extension ChangeManaging { var cellWriterSupport: CellWriterSupport { .none } + func pendingChange(forRow rowID: RowID) -> RowChange? { + rowChanges.first { $0.rowID == rowID } + } + func isExpression(rowID: RowID, columnIndex: Int) -> Bool { false } func expressionColumns(forRow rowID: RowID) -> Set { [] } - /// Only the data grid can show a row without a field; every other grid records the value. + // Only the data grid can show a row without a field; every other grid records the value. func recordCellChange( rowID: RowID, columnIndex: Int, @@ -84,7 +88,7 @@ extension ChangeManaging { ) } - /// Only the data grid writes SQL expressions; `cellWriterSupport` keeps every other grid from offering one. + // Only the data grid writes SQL expressions; `cellWriterSupport` keeps every other grid from offering one. func recordCellChange( rowID: RowID, columnIndex: Int, @@ -126,6 +130,10 @@ final class AnyChangeManager: ObservableObject { wrapped.isRowDeleted(rowID) } + func pendingChange(forRow rowID: RowID) -> RowChange? { + wrapped.pendingChange(forRow: rowID) + } + func isExpression(rowID: RowID, columnIndex: Int) -> Bool { wrapped.isExpression(rowID: rowID, columnIndex: columnIndex) } diff --git a/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift b/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift index 55cb981c82..1c6d8f757c 100644 --- a/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift +++ b/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift @@ -928,6 +928,14 @@ final class DataChangeManager: ObservableObject, ChangeManaging { pending.isRowInserted(rowID) } + func pendingChange(forRow rowID: RowID) -> RowChange? { + pending.firstChange(forRow: rowID) + } + + func pendingUpdate(forRow rowID: RowID) -> RowChange? { + pending.change(forRow: rowID, type: .update) + } + func isCellModified(rowID: RowID, columnIndex: Int) -> Bool { pending.isCellModified(rowID: rowID, columnIndex: columnIndex) } diff --git a/apple/TablePro/Core/ChangeTracking/PendingChanges.swift b/apple/TablePro/Core/ChangeTracking/PendingChanges.swift index aeb641a715..c4882458bd 100644 --- a/apple/TablePro/Core/ChangeTracking/PendingChanges.swift +++ b/apple/TablePro/Core/ChangeTracking/PendingChanges.swift @@ -79,6 +79,16 @@ struct PendingChanges: Equatable { return changes[idx] } + // The row's change that comes first in `changes`, which is what a scan for its id would find. + func firstChange(forRow rowID: RowID) -> RowChange? { + var first: Int? + for type in [ChangeType.insert, .update, .delete] { + guard let idx = changeIndex[RowChangeKey(rowID: rowID, type: type)] else { continue } + first = min(first ?? idx, idx) + } + return first.map { changes[$0] } + } + /// The row's cells that hold a SQL expression now: a new row's from its insert, a saved row's from its edits. func expressionColumns(forRow rowID: RowID) -> Set { if let insert = change(forRow: rowID, type: .insert) { diff --git a/apple/TablePro/Core/Coordinators/PaginationCoordinator.swift b/apple/TablePro/Core/Coordinators/PaginationCoordinator.swift index b86898ccc0..b0361ec6db 100644 --- a/apple/TablePro/Core/Coordinators/PaginationCoordinator.swift +++ b/apple/TablePro/Core/Coordinators/PaginationCoordinator.swift @@ -347,6 +347,15 @@ final class PaginationCoordinator: ObservableObject { let fetchTime = CFAbsoluteTimeGetCurrent() - start progressLog.info("[fetchAll] rows=\(result.rows.count) fetchTime=\(String(format: "%.3f", fetchTime))s") + let prepared = await TableRows.preparing( + result.rows, + rowLocators: isTableTab ? result.rowLocators : nil, + exportDocuments: result.exportDocuments, + absentCells: result.absentCells, + unloadedCells: result.unloadedCells, + shape: parent.tabSessionRegistry.tableRows(for: tabId).rowShape + ) + guard !Task.isCancelled else { // Clears the flag like every exit: a bare return stuck the tab on Loading after a late cancel (no-op `cancelQuery()` drivers). await MainActor.run { [weak self] in @@ -368,7 +377,8 @@ final class PaginationCoordinator: ObservableObject { guard let idx = parent.tabManager.tabs.firstIndex(where: { $0.id == tabId }) else { return } parent.mutateActiveTableRows(for: tabId) { rows in - rows.replace( + if let installed = rows.install(prepared) { return installed } + return rows.replace( rows: result.rows, rowLocators: isTableTab ? result.rowLocators : nil, exportDocuments: result.exportDocuments, diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift index aec7e5162d..12236dcb7f 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift @@ -67,6 +67,12 @@ private struct PrintedBatchAnswer: CarriesServerOutput { var serverOutput: PluginServerOutput = .none } +// Which query each batch's result sets belong to (nil: they are numbered), and their rows built off the main actor. +private struct BatchResultPlan { + let queries: [[SQLStatementScanner.ExecutableStatement]?] + let built: StatementRowsBuild +} + private struct BatchRun { let outcome: BatchStatementOutcome let plan: BatchTransactionPlan @@ -146,20 +152,23 @@ extension QueryExecutionCoordinator { guard let self else { return } let run = await runBatches(prepared, scope: scope, mode: mode, claim: claim, lease: lease) postRanStatements(of: prepared, run: run, scope: scope, connection: conn) + let plan = await batchResultPlan(prepared, outputs: Self.outputs(of: run.outcome), tabId: tabId) let sessionNotice = Self.runNotice(outcome: run.outcome, sessionState: run.sessionState) switch run.outcome { case .cancelled(let outputs): guard parent.tabExecution.settle(claim) else { return } parent.retireQueryTask(.claim(claim)) - keepStoppedBatches(prepared, outputs: outputs, parameters: parameters, tabId: tabId, sessionNotice: sessionNotice) + keepStoppedBatches( + prepared, outputs: outputs, parameters: parameters, tabId: tabId, sessionNotice: sessionNotice, plan: plan + ) parent.reportEndedExecutions([ EndedExecution(tabId: claim.tabId, startedAt: claim.startedAt, reason: .cancelledByUser) ]) case .completed(let outputs): applyCompletedBatches( prepared, outputs: outputs, parameters: parameters, - tabId: tabId, claim: claim, sessionNotice: sessionNotice + tabId: tabId, claim: claim, sessionNotice: sessionNotice, plan: plan ) case .failed(let outputs, let failure, let errorDescription): handleBatchFailure( @@ -177,7 +186,8 @@ extension QueryExecutionCoordinator { parameters: parameters, tabId: tabId, claim: claim, - failureOutput: run.failureOutput + failureOutput: run.failureOutput, + plan: plan ) } } @@ -367,7 +377,8 @@ extension QueryExecutionCoordinator { parameters: [QueryParameter], tabId: UUID, claim: TabExecutionClaim, - sessionNotice: String? + sessionNotice: String?, + plan: BatchResultPlan ) { guard parent.tabExecution.settle(claim) else { return } parent.retireQueryTask(.claim(claim)) @@ -383,7 +394,7 @@ extension QueryExecutionCoordinator { tabId: tabId, timing: PluginQueryTiming(total: outputs.reduce(0) { $0 + $1.executionTime }), totalRowsAffected: totalRowsAffected, - newResultSets: batchResultSets(prepared, outputs: outputs, tabId: tabId), + newResultSets: batchResultSets(prepared, outputs: outputs, tabId: tabId, plan: plan), sessionNotice: sessionNotice ) } @@ -393,7 +404,8 @@ extension QueryExecutionCoordinator { outputs: [BatchOutput], parameters: [QueryParameter], tabId: UUID, - sessionNotice: String? + sessionNotice: String?, + plan: BatchResultPlan ) { guard !outputs.isEmpty else { return } recordBatchHistory(prepared, outputs: outputs, parameters: parameters, tabId: tabId, failedIndex: nil) @@ -401,7 +413,7 @@ extension QueryExecutionCoordinator { tabId: tabId, timing: PluginQueryTiming(total: outputs.reduce(0) { $0 + $1.executionTime }), totalRowsAffected: outputs.reduce(0) { $0 + $1.rowsAffected }, - newResultSets: batchResultSets(prepared, outputs: outputs, tabId: tabId), + newResultSets: batchResultSets(prepared, outputs: outputs, tabId: tabId, plan: plan), sessionNotice: sessionNotice ) } @@ -413,7 +425,8 @@ extension QueryExecutionCoordinator { parameters: [QueryParameter], tabId: UUID, claim: TabExecutionClaim, - failureOutput: PluginServerOutput + failureOutput: PluginServerOutput, + plan: BatchResultPlan ) { let report = context.report() let printed: PluginServerOutput @@ -451,7 +464,7 @@ extension QueryExecutionCoordinator { tab.execution.executionTime = timing.total tab.execution.lastExecutedAt = Date() tab.display.replaceUnpinnedResults( - with: batchResultSets(prepared, outputs: outputs, tabId: tabId) + [errorResult] + with: batchResultSets(prepared, outputs: outputs, tabId: tabId, plan: plan) + [errorResult] ) if tab.display.isResultsCollapsed { tab.display.isResultsCollapsed = false @@ -503,25 +516,22 @@ extension QueryExecutionCoordinator { private func batchResultSets( _ prepared: [PreparedBatch], outputs: [BatchOutput], - tabId: UUID + tabId: UUID, + plan: BatchResultPlan ) -> [ResultSet] { - let databaseType = parent.connection.type let grammar = parent.lexicalGrammar let firstNumber = parent.tabManager.tabs.first { $0.id == tabId }?.display.pinnedResultCount ?? 0 + let reading = parent.tableNameReading + var builtIndex = 0 + // Taken in the order `batchResultPlan` built them: batch by batch, each batch's result sets in order. + func nextBuilt() -> StatementRows? { + defer { builtIndex += 1 } + return plan.built.statement(at: builtIndex, readBy: reading) + } var resultSets: [ResultSet] = [] - for (entry, output) in zip(prepared, outputs) { - let isPlainQuery = { (sql: String) in - QueryExecutor.qualifiesForRowCap(sql: sql, tabType: .query, databaseType: databaseType) - } + for ((entry, output), queries) in zip(zip(prepared, outputs), plan.queries) { let produced: [ResultSet] - if BatchResultMapping.mapsToStatements( - entry.batch, - resultSetCount: output.resultSets.count, - hasErrors: !output.errors.isEmpty, - grammar: grammar, - isPlainQuery: isPlainQuery - ) { - let queries = entry.batch.statements.filter { isPlainQuery($0.sql) } + if let queries { produced = zip(queries, output.resultSets).map { statement, result in let replayable = entry.parameterValues == nil && !BatchResultMapping.referencesLocalVariable(statement.sql, grammar: grammar) @@ -531,7 +541,8 @@ extension QueryExecutionCoordinator { index: firstNumber + resultSets.count, baseQuery: replayable ? statement.sql : nil, tabId: tabId, - anchor: StatementAnchor(statement) + anchor: StatementAnchor(statement), + built: nextBuilt() ) } } else if output.resultSets.isEmpty { @@ -541,7 +552,11 @@ extension QueryExecutionCoordinator { } else { produced = output.resultSets.enumerated().map { offset, result in numberedResultSet( - result, anchor: entry.batch.anchor, index: firstNumber + resultSets.count + offset, tabId: tabId + result, + anchor: entry.batch.anchor, + index: firstNumber + resultSets.count + offset, + tabId: tabId, + built: nextBuilt() ) } } @@ -551,36 +566,67 @@ extension QueryExecutionCoordinator { return resultSets } + // Decided once, before the settle: the mapping costs a classification per statement and the rows a pass each. + private func batchResultPlan(_ prepared: [PreparedBatch], outputs: [BatchOutput], tabId: UUID) async -> BatchResultPlan { + let databaseType = parent.connection.type + let grammar = parent.lexicalGrammar + let queries = zip(prepared, outputs).map { entry, output in + Self.mappedQueries(of: entry, output: output, databaseType: databaseType, grammar: grammar) + } + let sources = zip(queries, outputs).flatMap { mapped, output -> [(sql: String?, result: QueryResult)] in + guard let mapped else { return output.resultSets.map { (sql: nil, result: $0) } } + return zip(mapped, output.resultSets).map { (sql: $0.sql as String?, result: $1) } + } + return BatchResultPlan(queries: queries, built: await buildStatementRows(sources, tabId: tabId)) + } + + // The query each of a batch's result sets belongs to, or nil when the batch's shape does not pin them. + private static func mappedQueries( + of entry: PreparedBatch, + output: BatchOutput, + databaseType: DatabaseType, + grammar: SQLLexicalGrammar + ) -> [SQLStatementScanner.ExecutableStatement]? { + let isPlainQuery = { (sql: String) in + QueryExecutor.qualifiesForRowCap(sql: sql, tabType: .query, databaseType: databaseType) + } + guard BatchResultMapping.mapsToStatements( + entry.batch, + resultSetCount: output.resultSets.count, + hasErrors: !output.errors.isEmpty, + grammar: grammar, + isPlainQuery: isPlainQuery + ) else { return nil } + return entry.batch.statements.filter { isPlainQuery($0.sql) } + } + + private static func outputs(of outcome: BatchStatementOutcome) -> [BatchOutput] { + switch outcome { + case .completed(let results), .cancelled(let results), .failed(let results, _, _): + return results + } + } + private static func runNotice( outcome: BatchStatementOutcome, sessionState: PluginSessionTransactionState ) -> String? { - let outputs: [BatchOutput] - switch outcome { - case .completed(let results), .cancelled(let results), .failed(let results, _, _): - outputs = results - } - return BatchRunNotice.text( - discardedResultSetCount: outputs.reduce(0) { $0 + $1.discardedResultSetCount }, + BatchRunNotice.text( + discardedResultSetCount: outputs(of: outcome).reduce(0) { $0 + $1.discardedResultSetCount }, sessionState: sessionState ) } - private func numberedResultSet(_ result: QueryResult, anchor: StatementAnchor?, index: Int, tabId: UUID) -> ResultSet { - let columns = result.columns.map { String($0) } + private func numberedResultSet( + _ result: QueryResult, + anchor: StatementAnchor?, + index: Int, + tabId: UUID, + built: StatementRows? + ) -> ResultSet { let resultSet = ResultSet( label: ResultSet.label(tableName: nil, anchor: nil, index: index), - tableRows: TableRows.from( - queryRows: result.rows, - columns: columns, - columnTypes: result.columnTypes, - exportDocuments: result.exportDocuments, - absentCells: result.absentCells, - masks: parent.columnMasks(forTabId: tabId, table: nil, columns: columns, columnTypes: result.columnTypes), - exportPolicy: parent.exportMaskingPolicy(forTabId: tabId), - exportTable: nil, - columnOrigins: result.columnOrigins - ) + tableRows: statementTableRows(result, tableName: nil, tabId: tabId, built: built?.prepared) ) resultSet.statementAnchor = anchor resultSet.executionTime = result.executionTime diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift index 4289d16183..ead6c8e0e9 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift @@ -183,6 +183,19 @@ extension QueryExecutionCoordinator { } } + // Read before and inside `applyPhase1Result`, so rows built off the main actor install as built. A Reveal stays only on its table. + func phase1RowShape(tabId: UUID, tableName: String?, columns: [String], columnTypes: [ColumnType]) -> TableRows.RowShape { + let tab = parent.tabManager.tabs.first { $0.id == tabId } + let keepsReveal = tab?.revealsMaskedValues == true && tab?.tableContext.tableName == tableName + let policy = keepsReveal ? MaskingPolicy.none : parent.maskingPolicy + return TableRows.RowShape( + columns: columns, + masks: policy.columnMasks(table: tableName, columns: columns, columnTypes: columnTypes), + exportPolicy: policy, + exportTable: tableName + ) + } + func applyPhase1Result( // swiftlint:disable:this function_parameter_count tabId: UUID, columns: [String], @@ -210,7 +223,8 @@ extension QueryExecutionCoordinator { exportDocuments: [PluginExportDocument]? = nil, absentCells: [Int: Set] = [:], unloadedCells: [Int: [Int: Int]] = [:], - columnOrigins: [PluginColumnOrigin?]? = nil + columnOrigins: [PluginColumnOrigin?]? = nil, + prepared: TableRows.PreparedRows? = nil ) { guard let idx = parent.tabManager.tabs.firstIndex(where: { $0.id == tabId }) else { return } @@ -250,11 +264,18 @@ extension QueryExecutionCoordinator { if parent.tabManager.tabs[idx].revealsMaskedValues, parent.tabManager.tabs[idx].tableContext.tableName != tableName { parent.tabManager.mutate(at: idx) { $0.revealsMaskedValues = false } } - let masks = parent.columnMasks(forTabId: tabId, table: tableName, columns: columns, columnTypes: columnTypes) + let shape = phase1RowShape(tabId: tabId, tableName: tableName, columns: columns, columnTypes: columnTypes) + let builtRows = prepared.flatMap { $0.fits(shape) ? $0 : nil } ?? TableRows.prepare( + rows, + rowLocators: rowLocators, + exportDocuments: exportDocuments, + absentCells: absentCells, + unloadedCells: unloadedCells, + shape: shape + ) let newTableRows = TableRows.from( - queryRows: rows, - columns: columns, + prepared: builtRows, columnTypes: columnTypes, columnDefaults: resolved.columnDefaults, columnForeignKeys: resolved.columnForeignKeys, @@ -267,13 +288,6 @@ extension QueryExecutionCoordinator { rowMatchPolicy: resolved.rowMatchPolicy, hasAuthoritativeSchema: resolved.hasAuthoritativeSchema, foreignKeysFetched: resolved.foreignKeysFetched, - rowLocators: rowLocators, - exportDocuments: exportDocuments, - absentCells: absentCells, - unloadedCells: unloadedCells, - masks: masks, - exportPolicy: parent.exportMaskingPolicy(forTabId: tabId), - exportTable: tableName, columnOrigins: columnOrigins ) let previousTableName = parent.tabManager.tabs[idx].tableContext.tableName diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift index 08d9ddbe9a..9c9c7f27ba 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift @@ -7,6 +7,23 @@ import Foundation import TableProPluginKit import TableProSQLGrammar +// A statement's table name and rows, built away from the main actor for the result it becomes. +struct StatementRows: Sendable { + let tableName: String? + let prepared: TableRows.PreparedRows +} + +// The names hold only while the session still reads them the same way, so the build keeps the reading it used. +struct StatementRowsBuild: Sendable { + let reading: TableNameReading + let statements: [StatementRows] + + func statement(at index: Int, readBy current: TableNameReading) -> StatementRows? { + guard current == reading, statements.indices.contains(index) else { return nil } + return statements[index] + } +} + extension QueryExecutionCoordinator { func executeMultipleStatements( _ statements: [SQLStatementScanner.ExecutableStatement], @@ -67,24 +84,13 @@ extension QueryExecutionCoordinator { baseQueryParameterValues: [String?]? = nil, namedParameterStatement: NamedParameterStatement? = nil, tabId: UUID, - anchor: StatementAnchor? = nil + anchor: StatementAnchor? = nil, + built: StatementRows? = nil ) -> ResultSet { - let tableName = parent.extractTableName(from: sql) - let columns = result.columns.map { String($0) } - let rows = TableRows.from( - queryRows: result.rows, - columns: columns, - columnTypes: result.columnTypes, - exportDocuments: result.exportDocuments, - absentCells: result.absentCells, - masks: parent.columnMasks(forTabId: tabId, table: tableName, columns: columns, columnTypes: result.columnTypes), - exportPolicy: parent.exportMaskingPolicy(forTabId: tabId), - exportTable: tableName, - columnOrigins: result.columnOrigins - ) + let tableName = if let built { built.tableName } else { parent.extractTableName(from: sql) } let resultSet = ResultSet( label: ResultSet.label(tableName: tableName, anchor: anchor, index: index), - tableRows: rows + tableRows: statementTableRows(result, tableName: tableName, tabId: tabId, built: built?.prepared) ) resultSet.statementAnchor = anchor resultSet.executionTime = result.executionTime @@ -101,6 +107,59 @@ extension QueryExecutionCoordinator { return resultSet } + // The rows built off the main actor when the masks they were built with still apply, else built here. + func statementTableRows(_ result: QueryResult, tableName: String?, tabId: UUID, built: TableRows.PreparedRows?) -> TableRows { + let columns = result.columns.map { String($0) } + let shape = TableRows.RowShape( + columns: columns, + masks: parent.columnMasks(forTabId: tabId, table: tableName, columns: columns, columnTypes: result.columnTypes), + exportPolicy: parent.exportMaskingPolicy(forTabId: tabId), + exportTable: tableName + ) + let prepared = built.flatMap { $0.fits(shape) && $0.rows.count == result.rows.count ? $0 : nil } ?? TableRows.prepare( + result.rows, + exportDocuments: result.exportDocuments, + absentCells: result.absentCells, + shape: shape + ) + return TableRows.from(prepared: prepared, columnTypes: result.columnTypes, columnOrigins: result.columnOrigins) + } + + // A nil statement is a result no statement names, which reads no table. + func buildStatementRows(_ statements: [(sql: String?, result: QueryResult)], tabId: UUID) async -> StatementRowsBuild { + let reveals = parent.tabManager.tabs.first { $0.id == tabId }?.revealsMaskedValues == true + return await Self.statementRows( + statements, + reading: parent.tableNameReading, + policy: reveals ? .none : parent.maskingPolicy + ) + } + + // A run of 300 SELECTs holds 300 capped results, too many rows to build on the main actor. A Stop keeps them, so + // this build runs to the end. + @concurrent + nonisolated static func statementRows( + _ statements: [(sql: String?, result: QueryResult)], + reading: TableNameReading, + policy: MaskingPolicy + ) async -> StatementRowsBuild { + StatementRowsBuild(reading: reading, statements: statements.map { sql, result in + let tableName = sql.flatMap(reading.tableName(in:)) + let shape = TableRows.RowShape( + columns: result.columns, + masks: policy.columnMasks(table: tableName, columns: result.columns, columnTypes: result.columnTypes), + exportPolicy: policy, + exportTable: tableName + ) + return StatementRows( + tableName: tableName, + prepared: TableRows.prepare( + result.rows, exportDocuments: result.exportDocuments, absentCells: result.absentCells, shape: shape + ) + ) + }) + } + /// Each statement in a multi-statement run targets its own table, so each result carries its /// own identity. Nothing fetches key columns for these statements, so the origin says so and /// the result is read-only until something does. Inheriting the previous run's keys made an diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift index b3492b2e3b..c0bcf998c7 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift @@ -428,6 +428,13 @@ extension QueryExecutionCoordinator { ) ))) + let results: [QueryResult] + switch outcome { + case .completed(let ran), .cancelled(let ran), .failed(let ran, _, _): + results = ran + } + let built = await buildStatementRows(zip(sent, results).map { (sql: $0.originalSQL as String?, result: $1) }, tabId: tabId) + switch outcome { case .cancelled(let results): guard parent.tabExecution.settle(claim) else { return } @@ -437,7 +444,8 @@ extension QueryExecutionCoordinator { prepared: sent, results: results, parameters: parameters, connection: conn ), tabId: tabId, - sessionNotice: sessionNotice + sessionNotice: sessionNotice, + built: built ) parent.reportEndedExecutions([ EndedExecution(tabId: claim.tabId, startedAt: claim.startedAt, reason: .cancelledByUser) @@ -449,7 +457,8 @@ extension QueryExecutionCoordinator { ), tabId: tabId, claim: claim, - sessionNotice: sessionNotice + sessionNotice: sessionNotice, + built: built ) case .failed(let results, let failure, let errorDescription): handleMultiStatementError( @@ -468,7 +477,8 @@ extension QueryExecutionCoordinator { claim: claim, anchors: sent.map(\.anchor), timing: PluginQueryTiming.batch(of: results), - failureOutput: run.failureOutput + failureOutput: run.failureOutput, + built: built ) } } @@ -616,7 +626,8 @@ extension QueryExecutionCoordinator { private func keepStoppedStatements( history: ExecutedStatementHistory, tabId: UUID, - sessionNotice: String? + sessionNotice: String?, + built: StatementRowsBuild ) { guard !history.results.isEmpty else { return } recordExecutedStatements(history, tabId: tabId, commitOutcomeIsUnknown: false) @@ -624,7 +635,7 @@ extension QueryExecutionCoordinator { tabId: tabId, timing: PluginQueryTiming.batch(of: history.results), totalRowsAffected: history.results.reduce(0) { $0 + $1.rowsAffected }, - newResultSets: statementResultSets(history, tabId: tabId), + newResultSets: statementResultSets(history, tabId: tabId, built: built), sessionNotice: sessionNotice ) } @@ -636,7 +647,8 @@ extension QueryExecutionCoordinator { history: ExecutedStatementHistory, tabId: UUID, claim: TabExecutionClaim, - sessionNotice: String? + sessionNotice: String?, + built: StatementRowsBuild ) { guard parent.tabExecution.settle(claim) else { return } parent.retireQueryTask(.claim(claim)) @@ -654,13 +666,14 @@ extension QueryExecutionCoordinator { tabId: tabId, timing: PluginQueryTiming.batch(of: history.results), totalRowsAffected: totalRowsAffected, - newResultSets: statementResultSets(history, tabId: tabId), + newResultSets: statementResultSets(history, tabId: tabId, built: built), sessionNotice: sessionNotice ) } - private func statementResultSets(_ history: ExecutedStatementHistory, tabId: UUID) -> [ResultSet] { + private func statementResultSets(_ history: ExecutedStatementHistory, tabId: UUID, built: StatementRowsBuild) -> [ResultSet] { let firstNumber = parent.tabManager.tabs.first { $0.id == tabId }?.display.pinnedResultCount ?? 0 + let reading = parent.tableNameReading return zip(history.prepared, history.results).enumerated().map { index, pair in let (statement, result) = pair return makeStatementResultSet( @@ -673,7 +686,8 @@ extension QueryExecutionCoordinator { ? nil : NamedParameterStatement(sql: statement.originalSQL, parameters: history.parameters), tabId: tabId, - anchor: statement.anchor + anchor: statement.anchor, + built: built.statement(at: index, readBy: reading) ) } } @@ -716,7 +730,15 @@ extension QueryExecutionCoordinator { anchor: StatementAnchor? = nil, install: ResultInstall = .newResult ) async -> Bool { - await MainActor.run { [weak self] in + let prepared = await TableRows.preparing( + fetchResult.rows, + exportDocuments: fetchResult.exportDocuments, + absentCells: fetchResult.absentCells, + shape: phase1RowShape( + tabId: tabId, tableName: tableName, columns: fetchResult.columns, columnTypes: fetchResult.columnTypes + ) + ) + return await MainActor.run { [weak self] in guard let self else { return false } guard parent.tabExecution.settle(claim) else { return false } parent.retireQueryTask(.claim(claim)) @@ -764,7 +786,8 @@ extension QueryExecutionCoordinator { serverOutput: fetchResult.serverOutput, exportDocuments: fetchResult.exportDocuments, absentCells: fetchResult.absentCells, - columnOrigins: fetchResult.columnOrigins + columnOrigins: fetchResult.columnOrigins, + prepared: prepared ) let parameterValues = nativeParameters.map { $0 as? String } @@ -786,7 +809,8 @@ extension QueryExecutionCoordinator { claim: TabExecutionClaim, anchors: [StatementAnchor?], timing: PluginQueryTiming, - failureOutput: PluginServerOutput + failureOutput: PluginServerOutput, + built: StatementRowsBuild ) { let cumulativeTime = timing.total let errorDescription = context.errorDescription @@ -821,7 +845,7 @@ extension QueryExecutionCoordinator { tab.execution.executionTime = cumulativeTime tab.execution.lastExecutedAt = Date() - tab.display.replaceUnpinnedResults(with: statementResultSets(history, tabId: tabId) + [errorRS]) + tab.display.replaceUnpinnedResults(with: statementResultSets(history, tabId: tabId, built: built) + [errorRS]) if tab.display.isResultsCollapsed { tab.display.isResultsCollapsed = false } diff --git a/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift b/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift index a3baa8c780..ec3657620b 100644 --- a/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift +++ b/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift @@ -393,11 +393,24 @@ final class RowEditingCoordinator: ObservableObject { } parent.tabManager.mutate(at: tabIndex) { $0.hasUserInteraction = true } - model?.repaintAllRows() + // Undoing a command over many cells runs one step per cell, so a step repaints only its own rows. + model?.repaintRows(Self.rowIDs(touchedBy: result.action)) // The inspector reads its fields from the rows this rewrote, and nothing else it watches moves on an undo. parent.gridDisplayRevision &+= 1 } + // A step moves the pending marks of the rows it names and of no other row. + static func rowIDs(touchedBy action: UndoAction) -> [RowID] { + switch action { + case .cellEdit(let rowID, _, _, _, _, _, _, _), .rowInsertion(let rowID, _), .rowDeletion(let rowID, _, _): + return [rowID] + case .batchRowDeletion(let rows, _): + return rows.map { $0.rowID } + case .batchRowInsertion(let rows, _, _, _): + return rows.map { $0.rowID } + } + } + func copyRowsToClipboard(_ rowIDs: [RowID], includeHeaders: Bool = false) { guard let (tab, _) = parent.tabManager.selectedTabAndIndex, !rowIDs.isEmpty else { return } guard parent.unloadedValuesAreLoaded(rowIDs: rowIDs, retry: { [weak self] in diff --git a/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift b/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift index 8ff206ca42..d718cdff74 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift @@ -101,12 +101,21 @@ extension DataGridModel { atOrAfter: selection.activeCell, in: GridFindEngine.positions(of: result.matches, in: source) ) - find.setResults(result.matches, current: current, problem: result.problem) + find.setResults(result.matches, current: current, problem: result.problem, basis: findBasis(of: source)) revealCurrentFindMatch() } // The rows changed under the bar: keep the current match when it survives and leave the selection alone. func refreshFind() { + refreshFind(rereading: nil) + } + + // Only these cells (by data index) were written and these rows added, and others may have left. + func refreshFind(writtenCells: [RowID: IndexSet], addedRows: [RowID] = []) { + refreshFind(rereading: (writtenCells: writtenCells, addedRows: addedRows)) + } + + private func refreshFind(rereading changes: (writtenCells: [RowID: IndexSet], addedRows: [RowID])?) { guard find.isVisible else { return } if datasetFinder != nil { // Another page changes the grid's rows, not the owner's; only another column set searches again. @@ -115,8 +124,22 @@ extension DataGridModel { return } let source = findSource + let basis = findBasis(of: source) let previous = find.currentMatch - let result = GridFindEngine.search(find.term, options: find.options, in: source) + let result: GridFindEngine.Result + if let changes, find.loadedRowsBasis == basis, + let refreshed = GridFindEngine.refreshed( + find.matches, + writtenCells: changes.writtenCells, + addedRows: changes.addedRows, + term: find.term, + options: find.options, + in: source + ) { + result = refreshed + } else { + result = GridFindEngine.search(find.term, options: find.options, in: source) + } let current: Int? if let previous, let kept = result.matches.firstIndex(of: previous) { current = kept @@ -126,7 +149,7 @@ extension DataGridModel { in: GridFindEngine.positions(of: result.matches, in: source) ) } - find.setResults(result.matches, current: current, problem: result.problem) + find.setResults(result.matches, current: current, problem: result.problem, basis: basis) } // The owner's rows changed (an edit, an undo, a filter): search them again, keeping the current match. @@ -212,7 +235,7 @@ extension DataGridModel { } let replaced = GridFindEngine.position(of: match, in: source) guard commit(replacement.edits, actionName: String(localized: "Replace")) else { return } - refreshFind() + refreshFind(writtenCells: writtenCells(of: replacement.edits)) guard !find.matches.isEmpty else { return } let positions = GridFindEngine.positions(of: find.matches, in: findSource) find.setCurrentIndex(replaced.flatMap { GridFindEngine.index(after: $0, in: positions) } ?? 0) @@ -241,7 +264,7 @@ extension DataGridModel { ) guard !replacement.edits.isEmpty else { return replacement } guard commit(replacement.edits, actionName: String(localized: "Replace All")) else { return nil } - refreshFind() + refreshFind(writtenCells: writtenCells(of: replacement.edits)) return replacement } @@ -258,6 +281,27 @@ extension DataGridModel { ) } + // A replacement writes only cells the bar found, all in presented columns. + private func writtenCells(of edits: [DataGridCellEdit]) -> [RowID: IndexSet] { + var written: [RowID: IndexSet] = [:] + for edit in edits { + guard let column = columns.column(id: edit.columnID) else { continue } + written[edit.rowID, default: []].insert(column.dataIndex) + } + return written + } + + private func findBasis(of source: GridFindEngine.Source) -> GridFindEngine.Basis { + GridFindEngine.Basis( + term: find.term, + options: find.options, + source: source, + valueFilter: store.valueFilter, + gridSettings: AppSettingsManager.shared.dataGrid, + timeZoneGeneration: DateFormattingService.shared.systemTimeZoneGeneration + ) + } + private func stepFind(forward: Bool) { guard find.isVisible else { return } if let finder = datasetFinder { diff --git a/apple/TablePro/Core/DataGrid/DataGridModel.swift b/apple/TablePro/Core/DataGrid/DataGridModel.swift index 608496a96f..fa3d21b8a6 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel.swift @@ -144,7 +144,7 @@ final class DataGridModel: ObservableObject { restoreSelection(in: current) displayRevision &+= 1 mountedView?.apply(change) - refreshFind() + refreshFind(after: delta, in: current, previousOrder: previous) return } let change = DataGridChange.from(delta, order: order, rows: current, columns: columns) @@ -158,7 +158,37 @@ final class DataGridModel: ObservableObject { } // An edited row stays where the filter put it until the filter or the row set changes (#2667). mountedView?.apply(change) - refreshFind() + refreshFind(after: delta, in: current) + } + + // An open find bar reads again only the cells the delta wrote and the rows that now show and did not before; + // one that left drops its matches. + private func refreshFind(after delta: Delta, in current: TableRows, previousOrder: DataGridRowOrder? = nil) { + guard find.isVisible else { return } + var written: [RowID: IndexSet] = [:] + var added: [RowID] = [] + switch delta { + case .cellChanged(let row, let column): + if current.rows.indices.contains(row) { + written[current.rows[row].id] = IndexSet(integer: column) + } + case .cellsChanged(let positions): + for position in positions where current.rows.indices.contains(position.row) { + written[current.rows[position.row].id, default: []].insert(position.column) + } + case .rowsInserted(let inserted): + added = inserted.compactMap { current.rows.indices.contains($0) ? current.rows[$0].id : nil } + case .rowsRemoved: + break + case .columnsReplaced, .fullReplace: + refreshFind() + return + } + // A filtered order resolved again can bring back a row hidden since an edit, which no refresh read then. + if let previousOrder, let shown = order.filteredIDs { + added += shown.filter { previousOrder.displayIndex(of: $0, in: current) == nil } + } + refreshFind(writtenCells: written, addedRows: added) } private func resolveOrder(in rows: TableRows) -> DataGridRowOrder { diff --git a/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift b/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift index b9fbfbed40..730159e873 100644 --- a/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift +++ b/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift @@ -134,6 +134,38 @@ enum GridFindEngine { var problem: String? } + // Everything a loaded-rows search read besides the rows themselves, the settings the cell text follows included. + struct Basis: Equatable { + let term: String + let options: GridFindState.Options + let columns: DataGridColumnSet + let displayFormats: [ValueDisplayFormat?] + let databaseType: DatabaseType? + let revealsInvisibleCharacters: Bool + let valueFilter: GridValueFilterState + let gridSettings: DataGridSettings + let timeZoneGeneration: Int + + init( + term: String, + options: GridFindState.Options, + source: Source, + valueFilter: GridValueFilterState, + gridSettings: DataGridSettings, + timeZoneGeneration: Int + ) { + self.term = term + self.options = options + columns = source.columns + displayFormats = source.displayFormats + databaseType = source.databaseType + revealsInvisibleCharacters = source.revealsInvisibleCharacters + self.valueFilter = valueFilter + self.gridSettings = gridSettings + self.timeZoneGeneration = timeZoneGeneration + } + } + struct Replacement: Equatable { var edits: [DataGridCellEdit] = [] var replacements = 0 @@ -178,6 +210,68 @@ enum GridFindEngine { return Result(matches: found) } + // What `search` finds after only these cells (by data index) were written and these rows added, others having + // left, on the same basis. Nil when the term cannot match, which `search` answers as cheaply. + static func refreshed( + _ matches: [GridFindState.Match], + writtenCells: [RowID: IndexSet], + addedRows: [RowID], + term: String, + options: GridFindState.Options, + in source: Source + ) -> Result? { + guard let matcher = try? GridFindMatcher.make(term: term, options: options) else { return nil } + let searched = searchedColumns(options: options, in: source) + guard !searched.isEmpty else { return nil } + + let added = Set(addedRows) + let reread = added.union(writtenCells.keys) + var keptByRow: [RowID: [GridFindState.Match]] = [:] + for match in matches where writtenCells[match.rowID] != nil && !added.contains(match.rowID) { + keptByRow[match.rowID, default: []].append(match) + } + + var rereadMatches: [(displayRow: Int, matches: [GridFindState.Match])] = [] + for rowID in reread { + guard let displayRow = source.order.displayIndex(of: rowID, in: source.rows), + let row = source.rows.row(withID: rowID) else { continue } + let written = added.contains(rowID) ? nil : writtenCells[rowID] + let kept = keptByRow[rowID] ?? [] + var found: [GridFindState.Match] = [] + for column in searched { + if let written, !written.contains(column.dataIndex) { + if kept.contains(where: { $0.columnID == column.id }) { + found.append(GridFindState.Match(rowID: rowID, columnID: column.id)) + } + continue + } + guard let text = searchableText(in: row, column: column, source: source), + matcher.matches(text) else { continue } + found.append(GridFindState.Match(rowID: rowID, columnID: column.id)) + } + if !found.isEmpty { + rereadMatches.append((displayRow: displayRow, matches: found)) + } + } + rereadMatches.sort { $0.displayRow < $1.displayRow } + + var merged: [GridFindState.Match] = [] + merged.reserveCapacity(matches.count + rereadMatches.count) + var next = rereadMatches.startIndex + for match in matches where !reread.contains(match.rowID) { + guard let displayRow = source.order.displayIndex(of: match.rowID, in: source.rows) else { continue } + while next < rereadMatches.endIndex, rereadMatches[next].displayRow < displayRow { + merged.append(contentsOf: rereadMatches[next].matches) + next += 1 + } + merged.append(match) + } + for entry in rereadMatches[next...] { + merged.append(contentsOf: entry.matches) + } + return Result(matches: merged) + } + // A scope naming a column that is no longer presented searches every presented column. static func searchedColumns(options: GridFindState.Options, in source: Source) -> [DataGridColumn] { let candidates: [DataGridColumn] diff --git a/apple/TablePro/Core/DataGrid/GridFindState.swift b/apple/TablePro/Core/DataGrid/GridFindState.swift index 955e70e629..ba2c2f68b0 100644 --- a/apple/TablePro/Core/DataGrid/GridFindState.swift +++ b/apple/TablePro/Core/DataGrid/GridFindState.swift @@ -31,16 +31,19 @@ final class GridFindState: ObservableObject { private(set) var datasetQuery: GridDatasetFindQuery? private(set) var datasetSearch: Task? private var datasetGeneration = 0 + // What the matches were searched on, while they came from the loaded rows; nil once anything else set them. + private(set) var loadedRowsBasis: GridFindEngine.Basis? var currentMatch: Match? { guard let currentIndex, matches.indices.contains(currentIndex) else { return nil } return matches[currentIndex] } - func setResults(_ matches: [Match], current: Int?, problem: String?) { + func setResults(_ matches: [Match], current: Int?, problem: String?, basis: GridFindEngine.Basis? = nil) { self.matches = matches self.currentIndex = current.flatMap { matches.indices.contains($0) ? $0 : nil } self.problem = problem + loadedRowsBasis = basis } func setCurrentIndex(_ index: Int?) { @@ -51,6 +54,7 @@ final class GridFindState: ObservableObject { func removeMatch(at index: Int) { guard matches.indices.contains(index) else { return } matches.remove(at: index) + loadedRowsBasis = nil currentIndex = matches.isEmpty ? nil : (index < matches.count ? index : 0) } @@ -61,6 +65,7 @@ final class GridFindState: ObservableObject { term = "" replacement = "" matches = [] + loadedRowsBasis = nil currentIndex = nil problem = nil } @@ -73,6 +78,7 @@ final class GridFindState: ObservableObject { datasetQuery = query searchStopped = false problem = nil + loadedRowsBasis = nil if clearsResults { matches = [] currentIndex = nil diff --git a/apple/TablePro/Core/Database/DatabaseDriver.swift b/apple/TablePro/Core/Database/DatabaseDriver.swift index f2dae7e00a..5bae9e93a9 100644 --- a/apple/TablePro/Core/Database/DatabaseDriver.swift +++ b/apple/TablePro/Core/Database/DatabaseDriver.swift @@ -253,8 +253,6 @@ protocol DatabaseDriver: AnyObject, Sendable { /// per table, which is too expensive to run ahead of the user. var providesBulkForeignKeyFetch: Bool { get } - /// Fetch foreign keys for a specific set of tables. - /// Default implementation calls fetchAllForeignKeys and filters, or falls back to per-table. func fetchForeignKeys(forTables tableNames: [String]) async throws -> [String: [ForeignKeyInfo]] /// Fetch an approximate row count using fast database-specific metadata. @@ -816,16 +814,16 @@ extension DatabaseDriver { ) } - /// Default fetchAllDatabaseMetadata: falls back to per-database calls (N+1). - /// Drivers should override with a single bulk query where possible. func fetchAllDatabaseMetadata() async throws -> [DatabaseMetadata] { let dbNames = try await fetchDatabases() var results: [DatabaseMetadata] = [] for dbName in dbNames { + try Task.checkCancellation() do { let metadata = try await fetchDatabaseMetadata(dbName) results.append(metadata) } catch { + try Self.rethrowIfCancelled(error) results.append(DatabaseMetadata.minimal(name: dbName)) } } @@ -842,72 +840,67 @@ extension DatabaseDriver { func fetchAllForeignKeys() async throws -> [String: [ForeignKeyInfo]] { let allTables = try await fetchTables() - var result: [String: [ForeignKeyInfo]] = [:] - for table in allTables { - do { - let fks = try await fetchForeignKeys(table: table.name) - if !fks.isEmpty { result[table.name] = fks } - } catch { - Logger(subsystem: "com.TablePro", category: "DatabaseDriver") - .debug("Failed to fetch foreign keys for \(table.name): \(error.localizedDescription)") - } + return try await readEachTable(allTables.map(\.name), skippingFailures: true, reading: "foreign keys") { name in + let keys = try await fetchForeignKeys(table: name) + return keys.isEmpty ? nil : keys } - return result } func fetchForeignKeys(forTables tableNames: [String]) async throws -> [String: [ForeignKeyInfo]] { - // For small subsets, per-table fetch avoids scanning the entire schema - if tableNames.count <= 5 { - var result: [String: [ForeignKeyInfo]] = [:] - for tableName in tableNames { - do { - let fks = try await fetchForeignKeys(table: tableName) - if !fks.isEmpty { result[tableName] = fks } - } catch { - Logger(subsystem: "com.TablePro", category: "DatabaseDriver") - .debug("Failed to fetch foreign keys for \(tableName): \(error.localizedDescription)") - } - } - return result + if providesBulkForeignKeyFetch, tableNames.count > 5 { + let all = try await fetchAllForeignKeys() + let nameSet = Set(tableNames) + return all.filter { nameSet.contains($0.key) } + } + // Without a bulk query the schema-wide read is a round trip per schema table; past five names an error still fails it. + return try await readEachTable(tableNames, skippingFailures: tableNames.count <= 5, reading: "foreign keys") { name in + let keys = try await fetchForeignKeys(table: name) + return keys.isEmpty ? nil : keys } - let all = try await fetchAllForeignKeys() - let nameSet = Set(tableNames) - return all.filter { nameSet.contains($0.key) } } func fetchIndexes(forTables tableNames: [String]) async throws -> [String: [IndexInfo]] { - var result: [String: [IndexInfo]] = [:] - for tableName in tableNames { + try await readEachTable(tableNames, skippingFailures: true, reading: "indexes") { name in + let indexes = try await fetchIndexes(table: name) + return indexes.isEmpty ? nil : indexes + } + } + + private func readEachTable( + _ tableNames: [String], + skippingFailures: Bool, + reading what: String, + _ read: (String) async throws -> Value? + ) async throws -> [String: Value] { + var result: [String: Value] = [:] + for name in tableNames { + try Task.checkCancellation() do { - let indexes = try await fetchIndexes(table: tableName) - if !indexes.isEmpty { result[tableName] = indexes } + if let value = try await read(name) { result[name] = value } } catch { + try Self.rethrowIfCancelled(error) + guard skippingFailures else { throw error } Logger(subsystem: "com.TablePro", category: "DatabaseDriver") - .debug("Failed to fetch indexes for \(tableName): \(error.localizedDescription)") + .debug("Skipping \(what) for \(name): \(error.localizedDescription)") } } return result } + private static func rethrowIfCancelled(_ error: any Error) throws { + try Task.checkCancellation() + if error is CancellationError { throw error } + } + func sampleFieldPaths(table: String, limit: Int) async throws -> [PluginFieldPath] { [] } - /// Default fetchAllColumns: falls back to per-table fetchColumns (N+1). - /// Drivers should override with a single bulk query where possible. func fetchAllColumns() async throws -> [String: [ColumnInfo]] { let allTables = try await fetchTables() - var result: [String: [ColumnInfo]] = [:] - for table in allTables { - do { - let columns = try await fetchColumns(table: table.name) - result[table.name] = columns - } catch { - Logger(subsystem: "com.TablePro", category: "DatabaseDriver") - .debug("Skipping columns for table '\(table.name)': \(error.localizedDescription)") - } + return try await readEachTable(allTables.map(\.name), skippingFailures: true, reading: "columns") { name in + try await fetchColumns(table: name) } - return result } /// Default: no dependent types (MySQL/SQLite don't have standalone enum types) diff --git a/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift b/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift index ee30d84cc6..4c67711e74 100644 --- a/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift +++ b/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift @@ -26,7 +26,7 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable private let pagination: PaginationCapability private let appliesRowScope: Bool private let commentableKinds: Set - private let cappedTables = OSAllocatedUnfairLock<[String]>(initialState: []) + private let cappedTables = OSAllocatedUnfairLock<[(table: String, limit: Int)]>(initialState: []) init(driver: DatabaseDriver, databaseType: DatabaseType) { let snapshot = PluginMetadataRegistry.shared.snapshot(for: databaseType) @@ -59,15 +59,13 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable (driver as? PluginDriverAdapter)?.schemaPluginDriver } - /// One line per table that stopped at the engine's row ceiling, so a partial copy is never - /// reported as the whole table. + // One line per table whose rows stopped at a ceiling, so a partial copy is never reported as the whole table. var cappedTableWarnings: [String] { - guard let maximum = pagination.maximumRows else { return [] } - return cappedTables.withLock { $0 }.map { table in + cappedTables.withLock { $0 }.map { capped in String( format: String(localized: "%1$@: only the first %2$lld rows were read, the most this database returns from one query."), - table, - maximum + capped.table, + capped.limit ) } } @@ -88,7 +86,12 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable // A table export only reads, whatever the statement the driver chose classifies as. var context = PluginStatementContext() context.readOnly = true - return pluginDriver.streamExportRows(query: customQuery, context: context) + // The driver's own export query is not the app's capped SELECT, so only a limit it declares applies. + return reportingCut( + of: pluginDriver.streamExportRows(query: customQuery, context: context), + table: table, + engineCeiling: nil + ) } let query = leadingRowsQuery( columns: "*", @@ -132,26 +135,17 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable guard let adapter = driver as? PluginDriverAdapter else { return AsyncThrowingStream { $0.finish(throwing: PluginExportError.exportFailed("No plugin driver available")) } } - let stream = adapter.streamExportRows(query: query) - guard let maximum = pagination.maximumRows else { return stream } + return reportingCut(of: adapter.streamExportRows(query: query), table: table, engineCeiling: pagination.maximumRows) + } + + private func reportingCut( + of stream: AsyncThrowingStream, + table: String, + engineCeiling: Int? + ) -> AsyncThrowingStream { let cappedTables = cappedTables - return AsyncThrowingStream { continuation in - let task = Task { - var rowCount = 0 - do { - for try await element in stream { - if case .rows(let rows, _) = element { rowCount += rows.count } - continuation.yield(element) - } - if rowCount >= maximum { - cappedTables.withLock { $0.append(table) } - } - continuation.finish() - } catch { - continuation.finish(throwing: error) - } - } - continuation.onTermination = { _ in task.cancel() } + return ExportRowCut.reporting(stream, engineCeiling: engineCeiling) { limit in + cappedTables.withLock { $0.append((table, limit)) } } } diff --git a/apple/TablePro/Core/Plugins/ExportRowCut.swift b/apple/TablePro/Core/Plugins/ExportRowCut.swift new file mode 100644 index 0000000000..04efb07dc6 --- /dev/null +++ b/apple/TablePro/Core/Plugins/ExportRowCut.swift @@ -0,0 +1,88 @@ +// +// ExportRowCut.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +// A stream that ends at the engine's ceiling or the driver's declared `rowLimit` wrote a partial file. +// One consumer reads an export stream, so one call at a time reaches the iterator. +final class ExportRowCut: @unchecked Sendable { + private var iterator: AsyncThrowingStream.AsyncIterator + private let engineCeiling: Int? + private let reading: @Sendable (Element) -> (header: PluginStreamHeader?, rows: Int) + private var declaredLimit: Int? + private var rowCount = 0 + private var isSettled = false + + private init( + upstream: AsyncThrowingStream, + engineCeiling: Int?, + reading: @escaping @Sendable (Element) -> (header: PluginStreamHeader?, rows: Int) + ) { + self.iterator = upstream.makeAsyncIterator() + self.engineCeiling = engineCeiling + self.reading = reading + } + + private func next() async throws -> Element? { + guard let element = try await iterator.next() else { return nil } + let read = reading(element) + if let header = read.header { declaredLimit = header.rowLimit } + rowCount += read.rows + return element + } + + private func settle() -> Int? { + guard !isSettled else { return nil } + isSettled = true + return [engineCeiling, declaredLimit].compactMap { $0 }.filter { rowCount >= $0 }.min() + } + + // Pulled rather than pumped by a task of its own, which kept every row a slow writer had not taken yet. + static func reporting( + _ upstream: AsyncThrowingStream, + engineCeiling: Int?, + reading: @escaping @Sendable (Element) -> (header: PluginStreamHeader?, rows: Int), + onCut: @escaping @Sendable (Int) -> Void + ) -> AsyncThrowingStream { + let cut = ExportRowCut(upstream: upstream, engineCeiling: engineCeiling, reading: reading) + return AsyncThrowingStream(unfolding: { + guard let element = try await cut.next() else { + if let limit = cut.settle() { onCut(limit) } + return nil + } + return element + }) + } +} + +extension ExportRowCut where Element == PluginStreamElement { + static func reporting( + _ upstream: AsyncThrowingStream, + engineCeiling: Int?, + onCut: @escaping @Sendable (Int) -> Void + ) -> AsyncThrowingStream { + reporting(upstream, engineCeiling: engineCeiling, reading: { element in + switch element { + case .header(let header): return (header, 0) + case .rows(let rows): return (nil, rows.count) + } + }, onCut: onCut) + } +} + +extension ExportRowCut where Element == PluginExportStreamElement { + static func reporting( + _ upstream: AsyncThrowingStream, + engineCeiling: Int?, + onCut: @escaping @Sendable (Int) -> Void + ) -> AsyncThrowingStream { + reporting(upstream, engineCeiling: engineCeiling, reading: { element in + if case .header(let header) = element { return (header, 0) } + if case .rows(let rows, _) = element { return (nil, rows.count) } + return (nil, 0) + }, onCut: onCut) + } +} diff --git a/apple/TablePro/Core/Plugins/StreamingQueryExportDataSource.swift b/apple/TablePro/Core/Plugins/StreamingQueryExportDataSource.swift index 35ec58638b..eb5520437a 100644 --- a/apple/TablePro/Core/Plugins/StreamingQueryExportDataSource.swift +++ b/apple/TablePro/Core/Plugins/StreamingQueryExportDataSource.swift @@ -18,9 +18,15 @@ final class StreamingQueryExportDataSource: PluginExportDataSource, @unchecked S private let query: String private let driver: DatabaseDriver private let dbType: DatabaseType + private let rowCut = OSAllocatedUnfairLock(initialState: nil) private static let logger = Logger(subsystem: "com.TablePro", category: "StreamingQueryExport") + // Where the driver stopped the stream at a limit it declared, so the file is partial. + var driverRowCut: Int? { + rowCut.withLock { $0 } + } + init(query: String, driver: DatabaseDriver, databaseType: DatabaseType) { self.query = query self.driver = driver @@ -33,7 +39,10 @@ final class StreamingQueryExportDataSource: PluginExportDataSource, @unchecked S guard let adapter = driver as? PluginDriverAdapter else { return AsyncThrowingStream { $0.finish(throwing: PluginExportError.exportFailed("No plugin driver available")) } } - return adapter.streamRows(query: query) + let rowCut = rowCut + return ExportRowCut.reporting(adapter.streamRows(query: query), engineCeiling: nil) { limit in + rowCut.withLock { $0 = limit } + } } func streamExportRows(for object: PluginExportTable) -> AsyncThrowingStream { @@ -44,7 +53,10 @@ final class StreamingQueryExportDataSource: PluginExportDataSource, @unchecked S guard let adapter = driver as? PluginDriverAdapter else { return AsyncThrowingStream { $0.finish(throwing: PluginExportError.exportFailed("No plugin driver available")) } } - return adapter.streamExportRows(query: query) + let rowCut = rowCut + return ExportRowCut.reporting(adapter.streamExportRows(query: query), engineCeiling: nil) { limit in + rowCut.withLock { $0 = limit } + } } func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { diff --git a/apple/TablePro/Core/Services/Export/ExportService.swift b/apple/TablePro/Core/Services/Export/ExportService.swift index b63587ffd7..5276597a82 100644 --- a/apple/TablePro/Core/Services/Export/ExportService.swift +++ b/apple/TablePro/Core/Services/Export/ExportService.swift @@ -475,16 +475,21 @@ final class ExportService: ObservableObject { let capWarning = Self.leadingRowsCapWarning( exportedRows: progress.processedRows, - pagination: PaginationCapability.of(databaseType) + pagination: PaginationCapability.of(databaseType), + driverCut: dataSource.driverRowCut ) state.warnings = result.warnings + [capWarning].compactMap { $0 } state.notes = result.notes } - /// A query result exported from an engine that returns only its leading rows stops at the - /// engine's ceiling, so a file that reached it is named as partial rather than passing as whole. - static func leadingRowsCapWarning(exportedRows: Int, pagination: PaginationCapability) -> String? { - guard let maximum = pagination.maximumRows, exportedRows >= maximum else { return nil } + // A file that reached the engine's ceiling, or the limit the driver cut its stream at, is named as partial. + static func leadingRowsCapWarning( + exportedRows: Int, + pagination: PaginationCapability, + driverCut: Int? = nil + ) -> String? { + let engineCut = pagination.maximumRows.flatMap { exportedRows >= $0 ? $0 : nil } + guard let maximum = engineCut ?? driverCut else { return nil } return String( format: String(localized: "Only the first %lld rows were exported, the most this database returns from one query."), maximum diff --git a/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift b/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift index 1b51ce9465..2d85197c26 100644 --- a/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift +++ b/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift @@ -6,6 +6,22 @@ import Foundation import TableProPluginKit +// The lease a task's work runs under. That work keeps the lease until a lease it asks for returns, so such a +// request never waits for room; a task started inside keeps the exemption only while the lease is open. +@MainActor +private final class PooledLease { + let connectionId: UUID + var isOpen = true + + init(connectionId: UUID) { + self.connectionId = connectionId + } +} + +private enum PooledLeaseContext { + @TaskLocal static var open: [PooledLease] = [] +} + @MainActor final class MetadataConnectionPool { static let shared = MetadataConnectionPool() @@ -39,6 +55,7 @@ final class MetadataConnectionPool { /// cancelling the caller has to be forwarded explicitly or a stopped query /// would keep running with nobody waiting on it. func runSerially( + connectionId: UUID, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T ) async throws -> T { let previous = tail @@ -46,7 +63,11 @@ final class MetadataConnectionPool { let work = Task { @MainActor () async throws -> T in await previous.value try Task.checkCancellation() - return try await body(driver) + let lease = PooledLease(connectionId: connectionId) + defer { lease.isOpen = false } + return try await PooledLeaseContext.$open.withValue(PooledLeaseContext.open + [lease]) { + try await body(driver) + } } tail = Task { @MainActor in _ = try? await work.value } return try await withTaskCancellationHandler( @@ -87,10 +108,22 @@ final class MetadataConnectionPool { let continuation: CheckedContinuation } + @MainActor + private final class RoomTicket { + let key: Key + var continuation: CheckedContinuation? + var isWithdrawn = false + + init(key: Key) { + self.key = key + } + } + private var entries: [Key: Entry] = [:] private var pending: [Key: PendingOpen] = [:] private var transportReplacements: [UUID: Int] = [:] private var transportWaiters: [UUID: [TransportWaiter]] = [:] + private var roomQueues: [UUID: [RoomTicket]] = [:] private let openDriver: DriverOpener static let maxPerConnection = 6 private var sweeper: Task? @@ -127,7 +160,7 @@ final class MetadataConnectionPool { entry.inFlightCount += 1 entry.lastUsed = Date() defer { releaseEntry(entry, connectionId: scope.connectionId) } - return try await entry.runSerially(body) + return try await entry.runSerially(connectionId: scope.connectionId, body) } /// Closes only the leases attached to one database, which is what a rename of that database @@ -219,6 +252,10 @@ final class MetadataConnectionPool { internal func transportWaiterCount(for connectionId: UUID) -> Int { transportWaiters[connectionId]?.count ?? 0 } + + internal func roomWaiterCount(for connectionId: UUID) -> Int { + roomQueues[connectionId]?.filter { $0.continuation != nil }.count ?? 0 + } #endif /// Whether the connection's transport is being rebuilt, which ends every connection dialed @@ -235,6 +272,7 @@ final class MetadataConnectionPool { return } trimIdleEntries(for: connectionId) + handRoom(on: connectionId) } private func closeOrDeferEntry(forKey key: Key) { @@ -244,23 +282,30 @@ final class MetadataConnectionPool { } else { entry.closeWhenIdle = true } + handRoom(on: key.scope.connectionId) } private func closeEntries( withdrawingOpensAs withdrawal: Withdrawal, where matches: (DatabaseScope) -> Bool ) { + var withdrawnFrom: Set = [] for key in pending.keys where matches(key.scope) { guard let open = pending.removeValue(forKey: key) else { continue } open.withdrawal = withdrawal open.task.cancel() + withdrawnFrom.insert(key.scope.connectionId) } if withdrawal == .closed { failTransportWaiters(where: matches) + withdrawRoomTickets(where: matches) } for key in entries.keys where matches(key.scope) { closeOrDeferEntry(forKey: key) } + for connectionId in withdrawnFrom { + handRoom(on: connectionId) + } stopSweeperIfEmpty() } @@ -286,6 +331,13 @@ final class MetadataConnectionPool { if let entry = reusableEntry(forKey: key) { return entry } + if pending[key] == nil { + try await waitForRoom(forKey: key) + // A wait can outlast its reason: the key opened meanwhile, or the transport is being rebuilt. + if pending[key] != nil || entries[key] != nil || isReplacingTransport(for: scope.connectionId) { + continue + } + } let open = try pending[key] ?? startOpen(forKey: key) let failure = await completion(of: open, forKey: key) /// A withdrawn open can end either way: a driver whose connect honours cancellation @@ -313,6 +365,7 @@ final class MetadataConnectionPool { private func forgetSpentOpen(_ open: PendingOpen, forKey key: Key) { guard pending[key] === open else { return } pending.removeValue(forKey: key) + handRoom(on: key.scope.connectionId) } /// A cached entry is only worth reusing while it is both connected and recent. The @@ -356,6 +409,8 @@ final class MetadataConnectionPool { open.waiterCount -= 1 if open.waiterCount == 0, pending[key] === open { pending.removeValue(forKey: key) + // A failed open frees its place, and a taken one leaves an entry that is now evictable once idle. + handRoom(on: key.scope.connectionId) } } do { @@ -408,6 +463,88 @@ final class MetadataConnectionPool { waiter.continuation.resume(throwing: CancellationError()) } + // First come first served, interactive ahead of bulk; a caller arriving while others wait goes behind them. + private func waitForRoom(forKey key: Key) async throws { + let connectionId = key.scope.connectionId + guard !PooledLeaseContext.open.contains(where: { $0.isOpen && $0.connectionId == connectionId }) else { return } + let ticket = RoomTicket(key: key) + roomQueues[connectionId, default: []].append(ticket) + defer { leaveRoomQueue(ticket) } + while !mayOpen(ticket) { + try await parkForRoom(ticket) + try Task.checkCancellation() + if ticket.isWithdrawn { throw CancellationError() } + } + } + + private func mayOpen(_ ticket: RoomTicket) -> Bool { + let key = ticket.key + let connectionId = key.scope.connectionId + if pending[key] != nil || entries[key] != nil || isReplacingTransport(for: connectionId) { return true } + return nextInLine(on: connectionId) === ticket && hasRoom(for: key.workload, on: connectionId) + } + + private func nextInLine(on connectionId: UUID) -> RoomTicket? { + let line = roomQueues[connectionId] ?? [] + return line.first { $0.key.workload == .interactive } ?? line.first + } + + // Wakes only the caller next in line, so a freed place goes to it rather than to whoever resumes first. + private func handRoom(on connectionId: UUID) { + guard let next = nextInLine(on: connectionId), + let continuation = next.continuation, + mayOpen(next) + else { return } + next.continuation = nil + continuation.resume() + } + + private func leaveRoomQueue(_ ticket: RoomTicket) { + let connectionId = ticket.key.scope.connectionId + roomQueues[connectionId]?.removeAll { $0 === ticket } + if roomQueues[connectionId]?.isEmpty == true { + roomQueues.removeValue(forKey: connectionId) + } + handRoom(on: connectionId) + } + + private func parkForRoom(_ ticket: RoomTicket) async throws { + try await withTaskCancellationHandler( + operation: { try await parkUntilHanded(ticket) }, + onCancel: { [weak self] in + Task { @MainActor in + self?.failRoomTicket(ticket) + } + } + ) + } + + private func parkUntilHanded(_ ticket: RoomTicket) async throws { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + guard !Task.isCancelled, !ticket.isWithdrawn else { + continuation.resume(throwing: CancellationError()) + return + } + ticket.continuation = continuation + } + } + + private func failRoomTicket(_ ticket: RoomTicket) { + guard let continuation = ticket.continuation else { return } + ticket.continuation = nil + continuation.resume(throwing: CancellationError()) + } + + // Same reason as `failTransportWaiters`: a parked caller would otherwise open what was just closed. + private func withdrawRoomTickets(where matches: (DatabaseScope) -> Bool) { + for line in roomQueues.values { + for ticket in line where matches(ticket.key.scope) { + ticket.isWithdrawn = true + failRoomTicket(ticket) + } + } + } + /// Opens a second connection to the session's server, on `scope`'s database, through the /// transport the session already holds. Every tunnel manager closes its tunnel before building a /// new one, so dialing the session's effective endpoint is what keeps the other connections up. @@ -607,21 +744,40 @@ final class MetadataConnectionPool { /// left alone whatever its age: `lastUsed` is stamped when the work starts, so a long export /// would otherwise have its own connection closed underneath it. internal func sweepIdleEntries(now: Date = Date()) { + var sweptFrom: Set = [] for (key, entry) in entries where entry.inFlightCount == 0 && Self.isStale(entry.lastUsed, now: now) { entries.removeValue(forKey: key) entry.driver.disconnect() + sweptFrom.insert(key.scope.connectionId) + } + for connectionId in sweptFrom { + handRoom(on: connectionId) } stopSweeperIfEmpty() } - /// Makes room for one more before an open. The caller never waits for room, so a burst of opens - /// can still run past the limit while its work is in flight; `trimIdleEntries` hands the extra - /// back as that work ends. + // A caller with no room waits in `waitForRoom`, so only a lease taken inside another one's work opens past the limit. private func evictIdleIfNeeded(for connectionId: UUID) { guard heldCount(for: connectionId) >= Self.maxPerConnection else { return } evictLeastRecentlyUsedIdleEntry(for: connectionId) } + // Bulk never takes the last place, so a completion read or a sidebar expand never waits behind a burst of bulk work. + private func hasRoom(for workload: Workload, on connectionId: UUID) -> Bool { + let placeIsFree = heldCount(for: connectionId) < Self.maxPerConnection + || leastRecentlyUsedIdleKey(for: connectionId) != nil + guard workload == .bulk else { return placeIsFree } + return placeIsFree && busyCount(for: connectionId) < Self.maxPerConnection - 1 + } + + private func busyCount(for connectionId: UUID) -> Int { + var keys = Set(pending.keys.filter { $0.scope.connectionId == connectionId }) + for (key, entry) in entries where key.scope.connectionId == connectionId && entry.inFlightCount > 0 { + keys.insert(key) + } + return keys.count + } + /// Keeps a connection at the limit once its work ends. Closing only one idle entry per open, as /// the pool used to, left a burst's high-water mark standing until the idle sweep: a tree Refresh /// across many databases held every connection it opened for ten minutes (#3103). @@ -640,16 +796,21 @@ final class MetadataConnectionPool { return keys.count } - /// An entry whose open is still listed has callers that have not taken it yet, so it is not idle - /// however it looks. @discardableResult private func evictLeastRecentlyUsedIdleEntry(for connectionId: UUID) -> Bool { + guard let key = leastRecentlyUsedIdleKey(for: connectionId), + let entry = entries.removeValue(forKey: key) + else { return false } + entry.driver.disconnect() + return true + } + + /// An entry whose open is still listed has callers that have not taken it yet, so it is not idle + /// however it looks. + private func leastRecentlyUsedIdleKey(for connectionId: UUID) -> Key? { let idle = entries.filter { key, entry in key.scope.connectionId == connectionId && entry.inFlightCount == 0 && pending[key] == nil } - guard let oldest = idle.min(by: { $0.value.lastUsed < $1.value.lastUsed }) else { return false } - entries.removeValue(forKey: oldest.key) - oldest.value.driver.disconnect() - return true + return idle.min(by: { $0.value.lastUsed < $1.value.lastUsed })?.key } } diff --git a/apple/TablePro/Core/Services/Query/QuerySqlParser.swift b/apple/TablePro/Core/Services/Query/QuerySqlParser.swift index 13b77e1805..76a0d66045 100644 --- a/apple/TablePro/Core/Services/Query/QuerySqlParser.swift +++ b/apple/TablePro/Core/Services/Query/QuerySqlParser.swift @@ -121,3 +121,14 @@ enum QuerySqlParser { return EnumValueParser.parseMySQLEnumOrSet(from: "ENUM(\(valuesString))") } } + +// What `extractTableName` reads from the connection and its session, held as a value so a parse can run off the main actor. +struct TableNameReading: Equatable, Sendable { + let dialect: SqlDialect + let readings: SQLLexicalReadings + let browseSchema: String? + + func tableName(in sql: String) -> String? { + QuerySqlParser.extractTableName(from: sql, dialect: dialect, readings: readings, browseSchema: browseSchema) + } +} diff --git a/apple/TablePro/Core/Storage/ConnectionStorage.swift b/apple/TablePro/Core/Storage/ConnectionStorage.swift index f1f9cfbd49..c80bb3bb79 100644 --- a/apple/TablePro/Core/Storage/ConnectionStorage.swift +++ b/apple/TablePro/Core/Storage/ConnectionStorage.swift @@ -29,6 +29,8 @@ final class ConnectionStorage { private var cachedConnections: [DatabaseConnection]? private(set) var lastLoadFailed = false + private(set) var fileReadCount = 0 + private(set) var fileWriteCount = 0 /// Whether the file on disk is the one TablePro last wrote. False once it has been edited by /// something else, which is the signal to refuse to run a connection's password source. @@ -91,6 +93,7 @@ final class ConnectionStorage { func loadConnections() -> [DatabaseConnection] { if let cached = cachedConnections { return cached } + fileReadCount += 1 guard let storedConnections = file.load() else { lastLoadFailed = true return [] @@ -139,6 +142,7 @@ final class ConnectionStorage { @discardableResult private func persist(_ connections: [DatabaseConnection], byUser: Bool = true) -> Bool { guard file.save(connections.map { StoredConnection(from: $0) }, byUser: byUser) else { return false } + fileWriteCount += 1 cachedConnections = nil return true } diff --git a/apple/TablePro/Core/Sync/SyncCoordinator.swift b/apple/TablePro/Core/Sync/SyncCoordinator.swift index 3455061e16..3cb725ec6b 100644 --- a/apple/TablePro/Core/Sync/SyncCoordinator.swift +++ b/apple/TablePro/Core/Sync/SyncCoordinator.swift @@ -861,6 +861,17 @@ final class SyncCoordinator: ObservableObject { var tableFavorites: [FavoriteTablesStorage.FavoriteEntry] = [] let databaseFavoriteTombstoneIds = Set(metadataStorage.tombstones(for: .favoriteDatabase).map(\.id)) var tableFolderRecords: [CKRecord] = [] + var connectionBatch: RemoteConnectionBatch? + + func persistConnectionBatch() { + guard let batch = connectionBatch else { return } + connectionBatch = nil + switch persistRemoteConnections(batch) { + case .applied: actualConnectionChanges = true + case .failed: persistenceFailed = true + case .skipped: break + } + } for record in changedRecords { if let echoGuard, echoGuard.withholds(record.recordID, tracker: changeTracker) { @@ -868,13 +879,13 @@ final class SyncCoordinator: ObservableObject { continue } guard let type = SyncRecordType(rawValue: record.recordType), settings.syncs(type) else { continue } + // A record whose apply can read or write the library sees every connection record before it. + if !Self.leavesConnectionLibraryAlone(type) { + persistConnectionBatch() + } switch type { case .connection: - switch applyRemoteConnection(record, tombstoneIds: connectionTombstoneIds) { - case .applied: actualConnectionChanges = true - case .failed: persistenceFailed = true - case .skipped: break - } + stageRemoteConnection(record, tombstoneIds: connectionTombstoneIds, into: &connectionBatch) case .group: switch applyRemoteGroup(record, tombstoneIds: groupTombstoneIds) { case .applied: groupsOrTagsChanged = true @@ -911,6 +922,7 @@ final class SyncCoordinator: ObservableObject { break } } + persistConnectionBatch() applyRemoteTableFolderRecords(tableFolderRecords) var effects = applyRemoteDeletions( @@ -982,51 +994,81 @@ final class SyncCoordinator: ObservableObject { } } - private func applyRemoteConnection(_ record: CKRecord, tombstoneIds: Set) -> RemoteApplyOutcome { + // Groups and tags write only their own store, and the rest are collected and applied after the loop. + private static func leavesConnectionLibraryAlone(_ type: SyncRecordType) -> Bool { + switch type { + case .connection, .group, .tag, .tableFavorite, .tableFolder, .tableFolderItem, .favorite, .favoriteFolder: + return true + case .settings, .sshProfile, .credentialProfile, .favoriteDatabase: + return false + } + } + + // Applied to one list and written once, where each record used to read, verify, decode, encode and write the + // whole library. + private func stageRemoteConnection( + _ record: CKRecord, + tombstoneIds: Set, + into pending: inout RemoteConnectionBatch? + ) { let remoteConnection: DatabaseConnection do { remoteConnection = try SyncRecordMapper.toConnection(record) } catch { Self.logger.error("Skipping remote connection \(record.recordID.recordName, privacy: .public): \(error.publicLogShape, privacy: .public)") - return .skipped + return } if tombstoneIds.contains(remoteConnection.id.uuidString) { - return .skipped + return + } + + let batch: RemoteConnectionBatch + if let existing = pending { + batch = existing + } else { + batch = RemoteConnectionBatch(connections: services.connectionStorage.loadConnections()) + pending = batch } - var connections = services.connectionStorage.loadConnections() var incoming = remoteConnection - let index = connections.firstIndex(where: { $0.id == remoteConnection.id }) + let index = batch.index(of: remoteConnection.id) if let index { - guard !connections[index].localOnly else { return .skipped } + guard !batch.connections[index].localOnly else { return } if changeTracker.dirtyRecords(for: .connection).contains(remoteConnection.id.uuidString) { guard let reconciled = mergeLocalEdits( into: record, - localConnection: connections[index] + localConnection: batch.connections[index] ) else { - return .skipped + return } incoming = reconciled } } // A record an older build pushed can still carry a connection string's secret in plain text. let kept = services.connectionStorage.keepingMongoURISecretsInKeychain(incoming) - let movedSecrets = kept.additionalFields != incoming.additionalFields + if kept.additionalFields != incoming.additionalFields { + batch.cleanedIds.insert(remoteConnection.id.uuidString) + } if let index { - connections[index] = kept.adoptingDeviceLocalState(from: connections[index]) + batch.replace(at: index, with: kept.adoptingDeviceLocalState(from: batch.connections[index])) } else { var inserted = kept inserted.origin = .synced - connections.append(inserted) + batch.append(inserted) } - guard services.connectionStorage.saveConnections(connections, byUser: false) else { - Self.logger.error("Failed to apply remote connection update: persistence error for \(remoteConnection.id, privacy: .public)") + batch.stagedCount += 1 + } + + private func persistRemoteConnections(_ batch: RemoteConnectionBatch) -> RemoteApplyOutcome { + guard batch.stagedCount > 0 else { return .skipped } + guard services.connectionStorage.saveConnections(batch.connections, byUser: false) else { + Self.logger.error( + "Failed to apply \(batch.stagedCount, privacy: .public) remote connection update(s): persistence error" + ) return .failed } - if movedSecrets { - connectionsCleanedOnPull.insert(remoteConnection.id.uuidString) - } + connectionsCleanedOnPull.formUnion(batch.cleanedIds) return .applied } @@ -1266,6 +1308,35 @@ final class SyncCoordinator: ObservableObject { } } +// A reference, so staging a record changes the list in place instead of copying it. +private final class RemoteConnectionBatch { + private(set) var connections: [DatabaseConnection] + var cleanedIds: Set = [] + var stagedCount = 0 + private var indexById: [UUID: Int] + + init(connections: [DatabaseConnection]) { + self.connections = connections + indexById = Dictionary( + connections.enumerated().map { ($0.element.id, $0.offset) }, + uniquingKeysWith: { first, _ in first } + ) + } + + func index(of id: UUID) -> Int? { + indexById[id] + } + + func replace(at index: Int, with connection: DatabaseConnection) { + connections[index] = connection + } + + func append(_ connection: DatabaseConnection) { + indexById[connection.id] = connections.count + connections.append(connection) + } +} + /// What starts a run while sync is on. Releasing it detaches every observer; the network monitor /// stops in its own deinit. private final class SyncTriggers { diff --git a/apple/TablePro/Models/Query/ResultSetMenuModel.swift b/apple/TablePro/Models/Query/ResultSetMenuModel.swift index 65399f008e..fbe9fe7442 100644 --- a/apple/TablePro/Models/Query/ResultSetMenuModel.swift +++ b/apple/TablePro/Models/Query/ResultSetMenuModel.swift @@ -21,6 +21,21 @@ struct ResultSetMenuModel: Equatable { let entries: [ResultSetMenuEntry] let activeOrdinal: Int let total: Int + // Counted once here: the pull-down names every entry, and counting per entry made a 10,000-statement run quadratic. + private let positions: [UUID: Int] + private let occurrences: [Int?] + + init(entries: [ResultSetMenuEntry], activeOrdinal: Int, total: Int) { + self.entries = entries + self.activeOrdinal = activeOrdinal + self.total = total + positions = Dictionary(entries.enumerated().map { ($1.id, $0) }) { first, _ in first } + occurrences = Self.occurrenceNumbers(in: entries.map(\.label)) + } + + static func == (lhs: ResultSetMenuModel, rhs: ResultSetMenuModel) -> Bool { + lhs.entries == rhs.entries && lhs.activeOrdinal == rhs.activeOrdinal && lhs.total == rhs.total + } // HIG, Tab views: "Avoid providing more than six tabs". Past that the pull-down alone lists them. static let maximumTabCount = 6 @@ -48,28 +63,39 @@ struct ResultSetMenuModel: Equatable { } func displayLabel(of entry: ResultSetMenuEntry) -> String { - guard let index = entries.firstIndex(where: { $0.id == entry.id }) else { return entry.label } - return Self.displayLabel(at: index, in: entries.map(\.label)) + guard let index = positions[entry.id] else { return entry.label } + return Self.counted(entries[index].label, occurrence: occurrences[index]) } // The name is cut before the occurrence is added, so "(2)" always survives. func tabTitle(of entry: ResultSetMenuEntry) -> String { - guard let index = entries.firstIndex(where: { $0.id == entry.id }) else { return entry.label } - let labels = entries.map(\.label) - let name = Self.truncated(entry.label, limit: Self.tabTitleCharacterLimit) - guard let occurrence = Self.occurrence(at: index, in: labels) else { return name } - return String(format: String(localized: "%1$@ (%2$d)"), name, occurrence) + guard let index = positions[entry.id] else { return entry.label } + return Self.counted(Self.truncated(entry.label, limit: Self.tabTitleCharacterLimit), occurrence: occurrences[index]) } // Two results read from one table share a name; the later ones carry their occurrence so two never look alike. static func displayLabel(at index: Int, in labels: [String]) -> String { - guard let occurrence = occurrence(at: index, in: labels) else { return labels[index] } - return String(format: String(localized: "%1$@ (%2$d)"), labels[index], occurrence) + let earlier = labels[.. 0 ? earlier + 1 : nil) } - private static func occurrence(at index: Int, in labels: [String]) -> Int? { - let earlier = labels[.. 0 ? earlier + 1 : nil + // Every name at once, for a caller naming each result of a tab. + static func displayLabels(for labels: [String]) -> [String] { + zip(labels, occurrenceNumbers(in: labels)).map { counted($0, occurrence: $1) } + } + + private static func occurrenceNumbers(in labels: [String]) -> [Int?] { + var seen: [String: Int] = [:] + return labels.map { label in + let earlier = seen[label, default: 0] + seen[label] = earlier + 1 + return earlier > 0 ? earlier + 1 : nil + } + } + + private static func counted(_ name: String, occurrence: Int?) -> String { + guard let occurrence else { return name } + return String(format: String(localized: "%1$@ (%2$d)"), name, occurrence) } // A pinned result is pinned precisely so that nothing takes it away. diff --git a/apple/TablePro/Models/Query/TabSessionRegistry.swift b/apple/TablePro/Models/Query/TabSessionRegistry.swift index 6b2429bd89..377a09e64b 100644 --- a/apple/TablePro/Models/Query/TabSessionRegistry.swift +++ b/apple/TablePro/Models/Query/TabSessionRegistry.swift @@ -45,10 +45,13 @@ final class TabSessionRegistry { session.rowSetRevision &+= 1 } + // `mutate` must not read this tab's rows back from the registry: they are out of the session while it runs. @discardableResult func updateTableRows(for tabId: UUID, _ mutate: (inout TableRows) -> Delta) -> Delta { let session = ensureSession(for: tabId) var rows = session.tableRows + // A buffer the session still held would be copied whole by the first write, once per edit. + session.tableRows = TableRows() let delta = mutate(&rows) session.tableRows = rows session.dataRevision &+= 1 diff --git a/apple/TablePro/Models/Query/TableRows.swift b/apple/TablePro/Models/Query/TableRows.swift index c175fc3c73..09592ca312 100644 --- a/apple/TablePro/Models/Query/TableRows.swift +++ b/apple/TablePro/Models/Query/TableRows.swift @@ -395,29 +395,15 @@ struct TableRows: Sendable { absentCells: [Int: Set] = [:], unloadedCells: [Int: [Int: Int]] = [:] ) -> Delta { - var rebuilt = ContiguousArray() - rebuilt.reserveCapacity(replacementRows.count) - var rebuiltIndex = [RowID: Int]() - rebuiltIndex.reserveCapacity(replacementRows.count) - for (idx, values) in replacementRows.enumerated() { - let normalized = Self.normalize(values: masks.row(values), toCount: columns.count) - let row = Row( - id: .existing(offset + idx), - values: normalized, - absentColumns: Self.clamp(absentCells[idx] ?? [], toCount: columns.count), - unloadedColumns: Self.clamp(unloadedCells[idx] ?? [:], toCount: columns.count), - exportSnapshot: Self.exportSnapshot( - documents: exportDocuments, index: idx, count: replacementRows.count, - columns: columns, values: normalized, rawValues: values, absent: absentCells[idx] ?? [], masks: masks, - policy: exportPolicy, table: exportTable - ) - ) - rebuilt.append(row) - rebuiltIndex[row.id] = idx - } - rows = rebuilt - indexByID = rebuiltIndex - rowLocators = Self.locators(replacementLocators, rowCount: replacementRows.count, offset: offset) + adopt(Self.prepare( + replacementRows, + offset: offset, + rowLocators: replacementLocators, + exportDocuments: exportDocuments, + absentCells: absentCells, + unloadedCells: unloadedCells, + shape: rowShape + )) return .fullReplace } @@ -510,25 +496,50 @@ struct TableRows: Sendable { exportTable: String? = nil, columnOrigins: [PluginColumnOrigin?]? = nil ) -> TableRows { - var rows = ContiguousArray() - rows.reserveCapacity(queryRows.count) - for (index, values) in queryRows.enumerated() { - let normalized = normalize(values: masks.row(values), toCount: columns.count) - rows.append(Row( - id: .existing(index), - values: normalized, - absentColumns: clamp(absentCells[index] ?? [], toCount: columns.count), - unloadedColumns: clamp(unloadedCells[index] ?? [:], toCount: columns.count), - exportSnapshot: exportSnapshot( - documents: exportDocuments, index: index, count: queryRows.count, - columns: columns, values: normalized, rawValues: values, absent: absentCells[index] ?? [], masks: masks, - policy: exportPolicy, table: exportTable - ) - )) - } + from( + prepared: prepare( + queryRows, + rowLocators: rowLocators, + exportDocuments: exportDocuments, + absentCells: absentCells, + unloadedCells: unloadedCells, + shape: RowShape(columns: columns, masks: masks, exportPolicy: exportPolicy, exportTable: exportTable) + ), + columnTypes: columnTypes, + columnDefaults: columnDefaults, + columnForeignKeys: columnForeignKeys, + columnEnumValues: columnEnumValues, + columnNullable: columnNullable, + columnComments: columnComments, + columnIdentity: columnIdentity, + columnDeclaredTypes: columnDeclaredTypes, + generatedColumns: generatedColumns, + rowMatchPolicy: rowMatchPolicy, + hasAuthoritativeSchema: hasAuthoritativeSchema, + foreignKeysFetched: foreignKeysFetched, + columnOrigins: columnOrigins + ) + } + + static func from( + prepared: PreparedRows, + columnTypes: [ColumnType], + columnDefaults: [String: String?] = [:], + columnForeignKeys: [String: ForeignKeyInfo] = [:], + columnEnumValues: [String: [String]] = [:], + columnNullable: [String: Bool] = [:], + columnComments: [String: String] = [:], + columnIdentity: [String: IdentityKind] = [:], + columnDeclaredTypes: [String: String] = [:], + generatedColumns: Set = [], + rowMatchPolicy: RowMatchPolicy = .none, + hasAuthoritativeSchema: Bool = false, + foreignKeysFetched: Bool = false, + columnOrigins: [PluginColumnOrigin?]? = nil + ) -> TableRows { + let shape = prepared.shape var tableRows = TableRows( - rows: rows, - columns: columns, + columns: shape.columns, columnTypes: columnTypes, columnDefaults: columnDefaults, columnForeignKeys: columnForeignKeys, @@ -542,14 +553,125 @@ struct TableRows: Sendable { hasAuthoritativeSchema: hasAuthoritativeSchema, foreignKeysFetched: foreignKeysFetched ) - tableRows.rowLocators = locators(rowLocators, rowCount: queryRows.count, offset: 0) - tableRows.masks = masks - tableRows.exportPolicy = exportPolicy - tableRows.exportTable = exportTable - tableRows.reportedOrigins = columnOrigins.map { (columns: columns, origins: $0) } + tableRows.adopt(prepared) + tableRows.masks = shape.masks + tableRows.exportPolicy = shape.exportPolicy + tableRows.exportTable = shape.exportTable + tableRows.reportedOrigins = columnOrigins.map { (columns: shape.columns, origins: $0) } return tableRows } + var rowShape: RowShape { + RowShape(columns: columns, masks: masks, exportPolicy: exportPolicy, exportTable: exportTable) + } + + // Nil when the rows were built for other columns or masks, as after a masking rule saved while they were built. + mutating func install(_ prepared: PreparedRows) -> Delta? { + guard prepared.fits(rowShape) else { return nil } + adopt(prepared) + return .fullReplace + } + + private mutating func adopt(_ prepared: PreparedRows) { + rows = prepared.rows + indexByID = prepared.indexByID + rowLocators = prepared.rowLocators + } + + static let cancellationCheckInterval = 4_096 + + // A Fetch All or a first page past the row cap is millions of rows, too many to build on the main actor. + // A Stop ends the build within one interval of rows; the result is then incomplete and never installs. + @concurrent + static func preparing( + _ queryRows: Source, + offset: Int = 0, + rowLocators: [String?]? = nil, + exportDocuments: [PluginExportDocument]? = nil, + absentCells: [Int: Set] = [:], + unloadedCells: [Int: [Int: Int]] = [:], + shape: RowShape + ) async -> PreparedRows where Source.Element == [PluginCellValue] { + build( + queryRows, + offset: offset, + rowLocators: rowLocators, + exportDocuments: exportDocuments, + absentCells: absentCells, + unloadedCells: unloadedCells, + shape: shape, + stopsOnCancel: true + ) + } + + // Never stops early: it is also the fallback that has to land every row, inside a task a Stop may have cancelled. + static func prepare( + _ queryRows: Source, + offset: Int = 0, + rowLocators: [String?]? = nil, + exportDocuments: [PluginExportDocument]? = nil, + absentCells: [Int: Set] = [:], + unloadedCells: [Int: [Int: Int]] = [:], + shape: RowShape + ) -> PreparedRows where Source.Element == [PluginCellValue] { + build( + queryRows, + offset: offset, + rowLocators: rowLocators, + exportDocuments: exportDocuments, + absentCells: absentCells, + unloadedCells: unloadedCells, + shape: shape, + stopsOnCancel: false + ) + } + + private static func build( + _ queryRows: Source, + offset: Int, + rowLocators: [String?]?, + exportDocuments: [PluginExportDocument]?, + absentCells: [Int: Set], + unloadedCells: [Int: [Int: Int]], + shape: RowShape, + stopsOnCancel: Bool + ) -> PreparedRows where Source.Element == [PluginCellValue] { + let count = queryRows.count + let columnCount = shape.columns.count + var rows = ContiguousArray() + rows.reserveCapacity(count) + var index = [RowID: Int]() + index.reserveCapacity(count) + for (position, rowIndex) in zip(0..., queryRows.indices) { + if stopsOnCancel, position.isMultiple(of: cancellationCheckInterval), Task.isCancelled { + return PreparedRows(shape: shape, rows: [], indexByID: [:], rowLocators: [:], isComplete: false) + } + let values = queryRows[rowIndex] + let normalized = normalize(values: shape.masks.row(values), toCount: columnCount) + let absent = absentCells[position] ?? [] + let row = Row( + id: .existing(offset + position), + values: normalized, + absentColumns: clamp(absent, toCount: columnCount), + unloadedColumns: clamp(unloadedCells[position] ?? [:], toCount: columnCount), + exportSnapshot: exportSnapshot( + documents: exportDocuments, index: position, count: count, + columns: shape.columns, values: normalized, rawValues: values, absent: absent, masks: shape.masks, + policy: shape.exportPolicy, table: shape.exportTable + ) + ) + rows.append(row) + index[row.id] = position + } + return PreparedRows( + shape: shape, + rows: rows, + indexByID: index, + rowLocators: locators(rowLocators, rowCount: count, offset: offset), + isComplete: true + ) + } + private static func exportSnapshot( documents: [PluginExportDocument]?, index: Int, count: Int, columns: [String], values: ContiguousArray, rawValues: [PluginCellValue], @@ -646,3 +768,27 @@ extension TableRows { return columnEnumValues.filter { !setColumns.contains($0.key) } } } + +extension TableRows { + // What building a row reads from the rows it joins: the masks, and the column names export documents match. + struct RowShape: Equatable, Sendable { + let columns: [String] + let masks: ColumnMasks + let exportPolicy: MaskingPolicy + let exportTable: String? + } + + // Rows built from one result with their index, which install by assignment while the shape still holds. + struct PreparedRows: Sendable { + let shape: RowShape + let rows: ContiguousArray + fileprivate let indexByID: [RowID: Int] + fileprivate let rowLocators: [RowID: String] + // False when a Stop ended the build early: the rows are not the result. + let isComplete: Bool + + func fits(_ target: RowShape) -> Bool { + isComplete && shape == target + } + } +} diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 01982c4328..1c647f72e3 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -69660,6 +69660,10 @@ } } }, + "Each array element must be a JSON object" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Each Browse filter needs an Attribute and an Operator" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -74486,6 +74490,10 @@ } } }, + "etcd compacted the revision this export started from, so it stopped part way. Run it again." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "etcd has no %@ column." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -189445,6 +189453,14 @@ }, "The JSON array must contain only objects." : { + }, + "The JSON file ends before its last value is complete." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The JSON file is not valid at byte %@." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "The JSON is nested more deeply than DynamoDB allows" : { "extractionState" : "manual", @@ -196074,6 +196090,10 @@ }, "The wait/io/table/sql/handler instrument is disabled, so the server records no index reads." : { + }, + "The workbook has more sheets than an XLSX file can hold." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "The workbook has no worksheet to read." : { "extractionState" : "manual", @@ -218197,6 +218217,10 @@ } } }, + "XLSX file exceeds 4 GB ZIP limit" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "XXXXX-XXXXX-XXXXX-XXXXX-XXXXX" : { "extractionState" : "stale", "localizations" : { diff --git a/apple/TablePro/Views/DataGrid/Cells/RowVisualIndex.swift b/apple/TablePro/Views/DataGrid/Cells/RowVisualIndex.swift index 140f135bcc..dc5c2339ca 100644 --- a/apple/TablePro/Views/DataGrid/Cells/RowVisualIndex.swift +++ b/apple/TablePro/Views/DataGrid/Cells/RowVisualIndex.swift @@ -43,7 +43,7 @@ final class RowVisualIndex { } let isInserted = changeManager.insertedRowIDs.contains(rowID) - if let rowChange = changeManager.rowChanges.first(where: { $0.rowID == rowID }) { + if let rowChange = changeManager.pendingChange(forRow: rowID) { states[rowID] = Self.makeState(for: rowChange, inserted: isInserted) return } diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift index b4124734ad..e4a9b39fb1 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift @@ -7,22 +7,48 @@ import AppKit import SwiftUI import TableProPluginKit +// Each loaded value as the cell shows it, counted, in the order first seen. +struct ColumnValueTally: Sendable { + var counts: [String: Int] = [:] + var order: [String] = [] + var nullCount = 0 + + func sorted() -> [ColumnDistinctValue] { + var values = order + .sorted { $0.localizedStandardCompare($1) == .orderedAscending } + .map { ColumnDistinctValue(display: $0, isNull: false, count: counts[$0] ?? 0) } + if nullCount > 0 { + values.insert(ColumnDistinctValue(display: "", isNull: true, count: nullCount), at: 0) + } + return values + } + + // A column of unique values makes n log n locale-aware comparisons, too slow for the main actor. + @concurrent + func sortedOffTheMainActor() async -> [ColumnDistinctValue] { + sorted() + } +} + extension DataGridCommands { // Every loaded value as the cell shows it, which is what a value filter stores and matches; NULL first. func distinctValues(for column: DataGridColumn) -> [ColumnDistinctValue] { + valueTally(for: column).sorted() + } + + // Formatting reads main-actor state, so the tally is taken here and only the sort leaves the main actor. + func valueTally(for column: DataGridColumn) -> ColumnValueTally { let rows = model.rows let dataIndex = column.dataIndex - guard dataIndex >= 0, dataIndex < rows.columns.count else { return [] } + guard dataIndex >= 0, dataIndex < rows.columns.count else { return ColumnValueTally() } let format = model.displayFormat(for: column) let databaseType = configuration.traits.databaseType - var counts: [String: Int] = [:] - var order: [String] = [] - var nullCount = 0 + var tally = ColumnValueTally() for row in rows.rows { guard dataIndex < row.values.count else { continue } let raw = row.values[dataIndex] if case .null = raw { - nullCount += 1 + tally.nullCount += 1 continue } let display = GridDisplayOrderResolver.displayValue( @@ -31,18 +57,12 @@ extension DataGridCommands { displayFormat: format, databaseType: databaseType ) - if counts[display] == nil { - order.append(display) + if tally.counts[display] == nil { + tally.order.append(display) } - counts[display, default: 0] += 1 + tally.counts[display, default: 0] += 1 } - var values = order - .sorted { $0.localizedStandardCompare($1) == .orderedAscending } - .map { ColumnDistinctValue(display: $0, isNull: false, count: counts[$0] ?? 0) } - if nullCount > 0 { - values.insert(ColumnDistinctValue(display: "", isNull: true, count: nullCount), at: 0) - } - return values + return tally } } @@ -58,11 +78,42 @@ extension DataGridViewController { guard let self, let current = self.model.columns.column(id: columnID) else { return } self.presentValueFilter(for: current) }) else { return } - guard let anchor = model.anchor(for: column) else { return } + guard model.anchor(for: column) != nil else { return } - let values = gridCommands.distinctValues(for: column) + let tally = gridCommands.valueTally(for: column) let initialFilter = model.store.valueFilter.filter(forColumn: dataIndex) + let loadedRowCount = rows.count + let listedModel = model dismissValueFilterPopover() + // A reload, insert or removal while the list sorts cancels it, as it would close the open popover. + menuState.valueFilterListing = Task { [weak self] in + let values = await tally.sortedOffTheMainActor() + guard !Task.isCancelled, let self else { return } + self.menuState.valueFilterListing = nil + // A grid that took another tab's model meanwhile shows other rows. + guard self.model === listedModel, + let current = self.model.columns.column(id: columnID), + let anchor = self.model.anchor(for: current) else { return } + // A connection switch unparents the grid without dismantling it, and NSPopover raises on a view with no window. + guard let window = anchor.view.window, window.isVisible else { return } + self.showValueFilterPopover( + for: current, + values: values, + loadedRowCount: loadedRowCount, + initialFilter: initialFilter, + anchor: anchor + ) + } + } + + private func showValueFilterPopover( + for column: DataGridColumn, + values: [ColumnDistinctValue], + loadedRowCount: Int, + initialFilter: ColumnValueFilter?, + anchor: (view: NSView, rect: NSRect) + ) { + let columnID = column.id menuState.valueFilterPopover = PopoverPresenter.show( relativeTo: anchor.rect, of: anchor.view, @@ -71,7 +122,7 @@ extension DataGridViewController { ColumnValueFilterPopover( columnName: column.name, values: values, - loadedRowCount: rows.count, + loadedRowCount: loadedRowCount, initialFilter: initialFilter, // A change closes the popover once applied, in followRowChangesWithPopovers; declining the alert keeps the picks. onApply: { filter in @@ -89,6 +140,8 @@ extension DataGridViewController { } func dismissValueFilterPopover() { + menuState.valueFilterListing?.cancel() + menuState.valueFilterListing = nil guard let popover = menuState.valueFilterPopover else { return } menuState.valueFilterPopover = nil popover.close() diff --git a/apple/TablePro/Views/DataGrid/DataGridViewController+Menus.swift b/apple/TablePro/Views/DataGrid/DataGridViewController+Menus.swift index 022f1e79bd..0aba998017 100644 --- a/apple/TablePro/Views/DataGrid/DataGridViewController+Menus.swift +++ b/apple/TablePro/Views/DataGrid/DataGridViewController+Menus.swift @@ -9,6 +9,8 @@ import AppKit final class DataGridMenuState { let loader = DataGridValueLoader() var valueFilterPopover: NSPopover? + // The value list sorting before its popover opens. + var valueFilterListing: Task? } extension DataGridViewController { diff --git a/apple/TablePro/Views/Editor/StatementRunController.swift b/apple/TablePro/Views/Editor/StatementRunController.swift index 0ad9853f8e..7909dcd81a 100644 --- a/apple/TablePro/Views/Editor/StatementRunController.swift +++ b/apple/TablePro/Views/Editor/StatementRunController.swift @@ -15,17 +15,10 @@ enum StatementNavigationDirection { case next } -/// Keeps the editor's per-statement decorations in step with the document. -/// -/// Two things are drawn from one scanner: the run control the gutter puts beside each statement, and the band behind -/// the statement the caret is in. Both come from ``SQLStatementScanner``, which is also what decides the SQL that -/// actually runs, so a control cannot offer to run something different from what it is drawn beside. -/// -/// The two are refreshed differently because they cost differently. The band resolves one statement through -/// ``SQLStatementScanner/locatedStatementAtCursor(in:cursorPosition:grammar:)``, which stops at the caret, so it is -/// the same call the execution path makes and it can run on every keystroke. The controls need every statement in the -/// document, which is a full pass: measured at 31ms on a document at the size limit, well past a frame, so that one is -/// debounced. Nothing is lost by the delay, since the controls only show while the pointer is over the gutter. +// The gutter's run controls and the caret's statement band both come from the scanner execution uses, so a control +// never offers to run something other than what it is drawn beside. The band runs on every caret move, so a SQL +// document resolves it through `SQLStatementCursorIndex`, which rescans only from the last boundary before the caret. +// The controls need every statement, a full pass of 31ms at the size limit, so that refresh is debounced. @MainActor final class StatementRunController { /// The largest document decorations are computed for, in UTF-16 units. @@ -44,11 +37,24 @@ final class StatementRunController { static let synchronousRefreshLimit = 64_000 var sizeLimit = StatementRunController.defaultSizeLimit - var grammar: SQLLexicalGrammar = .ansi + var grammar: SQLLexicalGrammar = .ansi { + didSet { if grammar != oldValue { caretIndex = nil } } + } /// How the document divides into statements. JavaScript engines do not end a statement at every /// semicolon, so the gutter controls and the caret band have to ask the same splitter execution does. - var statementModel: QueryStatementModel = .sql + var statementModel: QueryStatementModel = .sql { + didSet { if statementModel != oldValue { caretIndex = nil } } + } + + // The band asks on every caret move, so a SQL document keeps its statement boundaries between edits. + private var caretIndex: SQLStatementCursorIndex? + private weak var caretIndexStorage: NSTextStorage? + private var caretIndexEdits: StorageEditObservation? + + var caretIndexUnitsScanned: Int { + caretIndex?.unitsScanned ?? 0 + } /// Called with the SQL the pressed control stands for, read from the document at the moment of the press, and /// where that SQL starts in the document. @@ -206,6 +212,8 @@ final class StatementRunController { func clear(in controller: TextViewController?) { pendingRefresh?.cancel() pendingRefresh = nil + caretIndex = nil + caretIndexEdits = nil controller?.runnableStatements = [] controller?.highlightedStatementRange = nil controller?.onRunStatement = nil @@ -246,11 +254,7 @@ final class StatementRunController { ) } - /// The span the band covers, or `nil` when there is nothing to mark. - /// - /// A non-empty selection runs verbatim rather than being widened to its statement, so the band steps aside and - /// lets the selection speak for itself. Multiple carets resolve through the first one, which is the caret the - /// execution path reads too. + // A selection runs verbatim, so the band steps aside for it. Several carets resolve through the first, as execution does. private func highlightRange(in controller: TextViewController) -> NSRange? { guard isHighlightEnabled, let textView = controller.textView, @@ -262,13 +266,65 @@ final class StatementRunController { let text = textView.string guard (text as NSString).length <= sizeLimit else { return nil } - let statement = QueryStatementScanner.locatedStatementAtCursor( - in: text, - cursorPosition: selection.range.location, - model: statementModel, - grammar: grammar - ) + let statement: SQLStatementScanner.LocatedStatement + if statementModel == .sql, let storage = textView.textStorage { + statement = indexedStatement(at: selection.range.location, in: text, storage: storage) + } else { + statement = QueryStatementScanner.locatedStatementAtCursor( + in: text, + cursorPosition: selection.range.location, + model: statementModel, + grammar: grammar + ) + } guard statement.hasContent, statement.contentRange.length > 0 else { return nil } return statement.contentRange } + + private func indexedStatement( + at cursor: Int, + in text: String, + storage: NSTextStorage + ) -> SQLStatementScanner.LocatedStatement { + if caretIndex == nil || caretIndexStorage !== storage { + startCaretIndex(on: storage) + } + // Inside an editing batch the text has changed but the storage has not posted its notification yet. + if storage.editedMask.contains(.editedCharacters) { + caretIndex?.textDidChange(at: storage.editedRange.location) + } + return caretIndex?.statementAtCursor(cursor, in: text) + ?? SQLStatementScanner.locatedStatementAtCursor(in: text, cursorPosition: cursor, grammar: grammar) + } + + private func startCaretIndex(on storage: NSTextStorage) { + caretIndex = SQLStatementCursorIndex(grammar: grammar) + caretIndexStorage = storage + // Every way the text changes passes through the storage, including the ones that skip the editor's delegate. + let token = NotificationCenter.default.addObserver( + forName: NSTextStorage.didProcessEditingNotification, + object: storage, + queue: nil + ) { [weak self] _ in + MainActor.assumeIsolated { self?.caretIndexStorageDidProcessEditing() } + } + caretIndexEdits = StorageEditObservation(token) + } + + private func caretIndexStorageDidProcessEditing() { + guard let storage = caretIndexStorage, storage.editedMask.contains(.editedCharacters) else { return } + caretIndex?.textDidChange(at: storage.editedRange.location) + } +} + +private final class StorageEditObservation { + private let token: any NSObjectProtocol + + init(_ token: any NSObjectProtocol) { + self.token = token + } + + deinit { + NotificationCenter.default.removeObserver(token) + } } diff --git a/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift b/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift index 86e0de6639..f9f0e6a53f 100644 --- a/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift +++ b/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift @@ -142,7 +142,7 @@ final class DataTabSQLExporter: DataGridSQLExporting { private func sourceRow(_ row: Row, target: Target) -> SQLRowToStatementConverter.SourceRow { var stored = Array(row.values) - let pendingUpdate = target.changeManager.rowChanges.first { $0.rowID == row.id && $0.type == .update } + let pendingUpdate = target.changeManager.pendingUpdate(forRow: row.id) for cellChange in pendingUpdate?.cellChanges ?? [] where stored.indices.contains(cellChange.columnIndex) { stored[cellChange.columnIndex] = cellChange.oldValue } @@ -161,7 +161,7 @@ final class DataTabSQLExporter: DataGridSQLExporting { if target.changeManager.insertedRowIDs.contains(row.id) { return Set(row.values.indices.filter { row.values[$0] == marker }) } - let pendingUpdate = target.changeManager.rowChanges.first { $0.rowID == row.id && $0.type == .update } + let pendingUpdate = target.changeManager.pendingUpdate(forRow: row.id) return Set(pendingUpdate?.cellChanges.filter { $0.newValue == marker }.map(\.columnIndex) ?? []) } diff --git a/apple/TablePro/Views/Main/Child/MainEditorContentView.swift b/apple/TablePro/Views/Main/Child/MainEditorContentView.swift index ad00263ed6..81b48e98af 100644 --- a/apple/TablePro/Views/Main/Child/MainEditorContentView.swift +++ b/apple/TablePro/Views/Main/Child/MainEditorContentView.swift @@ -1021,7 +1021,7 @@ struct MainEditorContentView: View { isPinned: resultSet.isPinned, isActive: resultSet.id == activeId, ordinal: index + 1, - isComparable: coordinator.isComparable(resultSet, in: tab) + isComparable: coordinator.isComparable(resultSet, in: tab, activeResultSetId: activeId) ) } return ResultSetMenuModel( diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift index 581bd55786..48861aa5be 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift @@ -136,7 +136,8 @@ extension MainContentCoordinator { exportDocuments: [PluginExportDocument]? = nil, absentCells: [Int: Set] = [:], unloadedCells: [Int: [Int: Int]] = [:], - columnOrigins: [PluginColumnOrigin?]? = nil + columnOrigins: [PluginColumnOrigin?]? = nil, + prepared: TableRows.PreparedRows? = nil ) { queryExecutionCoordinator.applyPhase1Result( tabId: tabId, @@ -164,7 +165,42 @@ extension MainContentCoordinator { exportDocuments: exportDocuments, absentCells: absentCells, unloadedCells: unloadedCells, - columnOrigins: columnOrigins + columnOrigins: columnOrigins, + prepared: prepared + ) + } + + func phase1RowShape(tabId: UUID, tableName: String?, columns: [String], columnTypes: [ColumnType]) -> TableRows.RowShape { + queryExecutionCoordinator.phase1RowShape(tabId: tabId, tableName: tableName, columns: columns, columnTypes: columnTypes) + } + + func preparedPhase1Rows( + _ fetchResult: QueryFetchResult, + tabId: UUID, + tableName: String?, + isTableTab: Bool + ) async -> TableRows.PreparedRows { + await TableRows.preparing( + fetchResult.rows, + rowLocators: isTableTab ? fetchResult.rowLocators : nil, + exportDocuments: fetchResult.exportDocuments, + absentCells: fetchResult.absentCells, + unloadedCells: fetchResult.unloadedCells, + shape: phase1RowShape( + tabId: tabId, tableName: tableName, columns: fetchResult.columns, columnTypes: fetchResult.columnTypes + ) + ) + } + + func extractTableName(from sql: String) -> String? { + tableNameReading.tableName(in: sql) + } + + var tableNameReading: TableNameReading { + TableNameReading( + dialect: sqlDialect, + readings: connection.type.lexicalReadings, + browseSchema: services.databaseManager.session(for: connectionId)?.browseSchema ) } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ResultComparison.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ResultComparison.swift index 0a683dbc8a..77d770a22a 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ResultComparison.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ResultComparison.swift @@ -24,9 +24,10 @@ extension MainContentCoordinator { let left = comparisonSide(of: resultSet, in: tab) else { return } let leftId = resultSet.id let tabId = tab.id + let activeId = tab.display.activeResultSet?.id let partner = Self.defaultComparisonPartner( of: leftId, - among: tab.display.resultSets.filter { isComparable($0, in: tab) }.map(\.id) + among: tab.display.resultSets.filter { isComparable($0, in: tab, activeResultSetId: activeId) }.map(\.id) ) let model = ResultComparisonModel(left: left, preferredRightId: partner) { Self.comparisonCandidates(excluding: leftId, homeTabId: tabId) @@ -55,13 +56,15 @@ extension MainContentCoordinator { for tab in coordinator.tabManager.tabs where tab.tabType == .query { let isHome = tab.id == homeTabId let tabId = tab.id - for resultSet in tab.display.resultSets + let activeId = tab.display.activeResultSet?.id + let titles = ResultSetMenuModel.displayLabels(for: tab.display.resultSets.map(\.label)) + for (index, resultSet) in tab.display.resultSets.enumerated() where resultSet.id != leftId && (isHome || resultSet.isPinned) - && coordinator.isComparable(resultSet, in: tab) { + && coordinator.isComparable(resultSet, in: tab, activeResultSetId: activeId) { let resultId = resultSet.id let candidate = ResultComparisonCandidate( id: resultId, - title: comparisonTitle(of: resultSet, in: tab.display), + title: titles[index], group: isHome ? String(localized: "This Tab") : coordinator.comparisonDetail(for: tab), isPinned: resultSet.isPinned, read: { [weak coordinator] in @@ -87,7 +90,7 @@ extension MainContentCoordinator { func comparisonSide(of resultSet: ResultSet, in tab: QueryTab) -> ResultComparisonSide? { guard !resultSet.isExplainResult else { return nil } - let rows = comparisonRows(of: resultSet, in: tab) + let rows = comparisonRows(of: resultSet, in: tab, activeResultSetId: tab.display.activeResultSet?.id) guard !rows.columns.isEmpty else { return nil } let primaryKey: [String] if let origin = resultSet.origin, origin.keysResolved, origin.tableName?.isEmpty == false { @@ -111,12 +114,18 @@ extension MainContentCoordinator { } func isComparable(_ resultSet: ResultSet, in tab: QueryTab) -> Bool { - !resultSet.isExplainResult && !comparisonRows(of: resultSet, in: tab).columns.isEmpty + isComparable(resultSet, in: tab, activeResultSetId: tab.display.activeResultSet?.id) + } + + // A pass over every result reads the active id once: finding it per result made the pass quadratic. + func isComparable(_ resultSet: ResultSet, in tab: QueryTab, activeResultSetId: UUID?) -> Bool { + !resultSet.isExplainResult + && !comparisonRows(of: resultSet, in: tab, activeResultSetId: activeResultSetId).columns.isEmpty } // The active result's rows live in the tab's shared buffer, which page loads and Fetch All write to. - private func comparisonRows(of resultSet: ResultSet, in tab: QueryTab) -> TableRows { - guard tab.display.activeResultSet?.id == resultSet.id else { return resultSet.tableRows } + private func comparisonRows(of resultSet: ResultSet, in tab: QueryTab, activeResultSetId: UUID?) -> TableRows { + guard activeResultSetId == resultSet.id else { return resultSet.tableRows } return tabSessionRegistry.tableRows(for: tab.id) } } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+UnloadedValues.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+UnloadedValues.swift index f3bd273ecc..90b35b5799 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+UnloadedValues.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+UnloadedValues.swift @@ -208,7 +208,7 @@ extension MainContentCoordinator { // The server holds the row under its key as last read, not as a pending edit has it. private func storedValue(of row: Row, column: Int) -> PluginCellValue { - let pendingUpdate = changeManager.rowChanges.first { $0.rowID == row.id && $0.type == .update } + let pendingUpdate = changeManager.pendingUpdate(forRow: row.id) if let change = pendingUpdate?.cellChanges.first(where: { $0.columnIndex == column }) { return change.oldValue } diff --git a/apple/TablePro/Views/Main/MainContentCoordinator.swift b/apple/TablePro/Views/Main/MainContentCoordinator.swift index 8ec9847488..b338583e42 100644 --- a/apple/TablePro/Views/Main/MainContentCoordinator.swift +++ b/apple/TablePro/Views/Main/MainContentCoordinator.swift @@ -1315,6 +1315,7 @@ final class MainContentCoordinator: ObservableObject { : nil // Rows and definition commit together after a definition change, so new columns never sit under old keys or defaults. let definition = needsDefinition ? try? await schemaTask?.value : nil + let prepared = await preparedPhase1Rows(fetchResult, tabId: tabId, tableName: tableName, isTableTab: isTableTab) let applied = await MainActor.run { [weak self] () -> Bool in guard let self else { return false } @@ -1360,7 +1361,8 @@ final class MainContentCoordinator: ObservableObject { exportDocuments: fetchResult.exportDocuments, absentCells: fetchResult.absentCells, unloadedCells: fetchResult.unloadedCells, - columnOrigins: fetchResult.columnOrigins + columnOrigins: fetchResult.columnOrigins, + prepared: prepared ) scheduleTraceCompletion(traceToken, outcome: .completed) @@ -1458,17 +1460,6 @@ final class MainContentCoordinator: ObservableObject { return result } - // MARK: - SQL Parsing - - func extractTableName(from sql: String) -> String? { - QuerySqlParser.extractTableName( - from: sql, - dialect: sqlDialect, - readings: connection.type.lexicalReadings, - browseSchema: services.databaseManager.session(for: connectionId)?.browseSchema - ) - } - // MARK: - Sorting func handleSortStateChanged(_ newState: SortState) { diff --git a/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift index 9884d8d6b2..ffc62eab00 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift @@ -123,7 +123,10 @@ nonisolated final class OracleDriver: DatabaseDriver, @unchecked Sendable { let coreStream = AsyncThrowingStream { coreContinuation in let coreTask = Task { do { - try await core.streamQuery(query, continuation: coreContinuation) + // The result buffer shows rows as they arrive, so none waits on a batch to fill. + try await core.streamQuery( + query, batching: OracleRowBatcher(rowsPerBatch: 1), continuation: coreContinuation + ) } catch { coreContinuation.finish(throwing: error) } diff --git a/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderFallbackTests.swift b/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderFallbackTests.swift index 414aacc08c..ec3b9f7aab 100644 --- a/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderFallbackTests.swift +++ b/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderFallbackTests.swift @@ -536,4 +536,62 @@ struct SQLSchemaProviderFallbackTests { // Documentation should reference the canonical table name "Users" #expect(items[0].documentation?.contains("Users") == true) } + + // MARK: - Built once per cache + + // Every keystroke before a FROM asks again; at 300 tables of 40 columns each request rebuilt 12,000 items. + @Test("Completions before a FROM reuse the column items while the cache and table list stand") + func columnItemsAreBuiltOncePerCache() async { + let driver = MockFallbackDriver() + var tables: [TableInfo] = [] + for table in 0..<30 { + let name = "table_\(table)" + tables.append(TestFixtures.makeTableInfo(name: name)) + driver.columnsPerTable[name] = (0..<10).map { + TestFixtures.makeColumnInfo(name: "col_\(table)_\($0)", isPrimaryKey: false) + } + } + driver.tablesToReturn = tables + + let provider = SQLSchemaProvider() + await provider.loadSchema(using: driver, connection: TestFixtures.makeConnection()) + await populateCache(provider: provider, driver: driver) + + let first = await provider.allColumnsFromCachedTables().map(\.label) + for _ in 0..<50 { + let again = await provider.allColumnsFromCachedTables().map(\.label) + #expect(again == first) + } + #expect(first.count == 300) + #expect(await provider.tableColumnItemsBuildCount == 1) + } + + @Test("A newly cached table, a new table list and a cleared cache each reach the column items") + func columnItemsFollowTheCacheAndTableList() async { + let driver = MockFallbackDriver() + driver.tablesToReturn = [TestFixtures.makeTableInfo(name: "users"), TestFixtures.makeTableInfo(name: "orders")] + driver.columnsPerTable = [ + "users": [TestFixtures.makeColumnInfo(name: "id")], + "orders": [TestFixtures.makeColumnInfo(name: "total", dataType: "DECIMAL", isPrimaryKey: false)] + ] + let provider = SQLSchemaProvider() + await provider.loadSchema(using: driver, connection: TestFixtures.makeConnection()) + _ = await provider.getColumns(for: "users") + let usersOnly = await provider.allColumnsFromCachedTables().map(\.label) + #expect(usersOnly == ["id"]) + + _ = await provider.getColumns(for: "orders") + let both = Set(await provider.allColumnsFromCachedTables().map(\.label)) + #expect(both == ["id", "total"]) + + await provider.updateTables([TestFixtures.makeTableInfo(name: "Users"), TestFixtures.makeTableInfo(name: "orders")]) + let renamed = await provider.allColumnsFromCachedTables() + let idDocumentation = renamed.first { $0.label == "id" }?.documentation + #expect(idDocumentation?.contains("Users") == true) + + await provider.clearColumnCache() + let cleared = await provider.allColumnsFromCachedTables() + #expect(cleared.isEmpty) + #expect(await provider.tableColumnItemsBuildCount == 3) + } } diff --git a/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift b/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift index 6c3ed73cdd..35cc1d7d7f 100644 --- a/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift +++ b/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift @@ -169,6 +169,97 @@ struct GridFindEngineTests { #expect(search("a b", in: source(rows, revealsInvisibleCharacters: true)).matches.isEmpty) } + // MARK: - Refreshing + + @Test("Reading only the written cells and the added rows again finds what a full search finds") + func refreshMatchesAFullSearch() throws { + var rows = Self.people + let before = search("act", in: source(rows)).matches + rows.edit(row: 1, column: 2, value: .text("acting")) + rows.edit(row: 2, column: 0, value: .text("done")) + rows.appendInsertedRow(values: [.text("react"), .text("Dee"), .null]) + let inserted = try #require(rows.rows.last?.id) + rows.remove(at: IndexSet(integer: 0)) + + let refreshed = GridFindEngine.refreshed( + before, + writtenCells: [.existing(1): IndexSet(integer: 2), .existing(2): IndexSet(integer: 0)], + addedRows: [inserted], + term: "act", + options: GridFindState.Options(), + in: source(rows) + ) + + let expected = [ + match(1, Self.status), + match(1, Self.remark), + match(2, Self.remark), + GridFindState.Match(rowID: inserted, columnID: Self.status) + ] + #expect(refreshed?.matches == expected) + #expect(search("act", in: source(rows)).matches == expected) + } + + @Test("Under a value filter a refresh keeps the filter's order and leaves out a hidden row it rereads") + func refreshFollowsTheFilteredOrder() { + let order = DataGridRowOrder(filteredIDs: [.existing(2), .existing(0)], revision: 1) + var rows = Self.people + let before = search("act", in: source(rows, order: order)).matches + rows.edit(row: 0, column: 2, value: .text("react")) + rows.edit(row: 1, column: 1, value: .text("Actor")) + + let refreshed = GridFindEngine.refreshed( + before, + writtenCells: [.existing(0): IndexSet(integer: 2), .existing(1): IndexSet(integer: 1)], + addedRows: [], + term: "act", + options: GridFindState.Options(), + in: source(rows, order: order) + ) + + #expect(refreshed?.matches == search("act", in: source(rows, order: order)).matches) + #expect(refreshed?.matches == [match(2, Self.status), match(2, Self.remark), match(0, Self.status), match(0, Self.remark)]) + } + + @Test("A term that cannot match leaves the refresh to a full search") + func refreshWithoutATerm() { + let blank = GridFindEngine.refreshed( + [], writtenCells: [:], addedRows: [.existing(0)], term: " ", options: GridFindState.Options(), in: source() + ) + let invalid = GridFindEngine.refreshed( + [], writtenCells: [:], addedRows: [], term: "(", options: options(regex: true), in: source() + ) + + #expect(blank == nil) + #expect(invalid == nil) + } + + // The cost pin: the caller names the written cells, and every other cell keeps its match without being read. + @Test("Only the written cells of a row are read again") + func refreshReadsOnlyTheWrittenCells() { + var rows = Self.people + let before = search("act", in: source(rows)).matches + rows.edit(row: 2, column: 0, value: .text("done")) + rows.edit(row: 2, column: 1, value: .text("Actor")) + + let refreshed = GridFindEngine.refreshed( + before, + writtenCells: [.existing(2): IndexSet(integer: 1)], + addedRows: [], + term: "act", + options: GridFindState.Options(), + in: source(rows) + ) + + #expect(refreshed?.matches == [ + match(0, Self.status), + match(1, Self.status), + match(2, Self.status), + match(2, Self.name), + match(2, Self.remark) + ]) + } + // MARK: - Binary columns private static let payload = TableRows( @@ -360,6 +451,11 @@ struct DataGridModelFindTests { ) private static let company = DataGridColumnID(name: "company", occurrence: 0) + private static let city = DataGridColumnID(name: "city", occurrence: 0) + + private func match(_ row: Int, _ column: DataGridColumnID) -> GridFindState.Match { + GridFindState.Match(rowID: .existing(row), columnID: column) + } private func mount( _ configure: (inout DataGridConfiguration) -> Void = { _ in } @@ -555,6 +651,92 @@ struct DataGridModelFindTests { #expect(model.find.matches.count == 1) } + private func fullSearch(_ model: DataGridModel, _ owner: FindGridOwner) -> [GridFindState.Match] { + let source = GridFindEngine.Source(rows: owner.tableRows, order: model.order, columns: model.columns) + return GridFindEngine.search(model.find.term, options: model.find.options, in: source).matches + } + + @Test("Edits under an open bar keep the matches a full search finds") + func editsKeepTheMatchesInStep() { + let (model, owner) = mount() + search(model, "acme") + + owner.tableRows.rows[2].values[1] = .text("Umbrella") + model.apply(.cellChanged(row: 2, column: 1)) + owner.tableRows.rows[0].values[2] = .text("Acme Heights") + owner.tableRows.rows[1].values[2] = .text("Falls") + model.apply(.cellsChanged([CellPosition(row: 0, column: 2), CellPosition(row: 1, column: 2)])) + + #expect(model.find.matches == [match(0, Self.city), match(3, Self.company), match(3, Self.city)]) + #expect(model.find.matches == fullSearch(model, owner)) + } + + @Test("Rows added or removed under an open bar keep the matches a full search finds") + func insertsAndRemovalsKeepTheMatchesInStep() { + let (model, owner) = mount() + search(model, "acme") + + let inserted = owner.tableRows.appendInsertedRow(values: [.text("5"), .text("Acme East"), .text("Oslo")]) + model.apply(inserted) + let insertedID = owner.tableRows.rows[4].id + #expect(model.find.matches.last == GridFindState.Match(rowID: insertedID, columnID: Self.company)) + #expect(model.find.matches == fullSearch(model, owner)) + + let removed = owner.tableRows.remove(at: IndexSet(integer: 1)) + model.apply(removed) + + #expect(model.find.matches == [ + match(2, Self.company), + match(3, Self.company), + match(3, Self.city), + GridFindState.Match(rowID: insertedID, columnID: Self.company) + ]) + #expect(model.find.matches == fullSearch(model, owner)) + } + + // An edit keeps its row on screen (#2667); an insert hides it, and once undone and inserted again it shows again. + @Test("A row a value filter shows again after an insert is searched again") + func rowBackInTheFilteredOrderIsSearched() { + let (model, owner) = mount() + var filter = GridValueFilterState() + filter.set( + ColumnValueFilter(selectedValues: ["Acme Falls", "Paris", "Acme"], includesNull: false), + columnName: "city", + forColumn: 2 + ) + model.setValueFilter(filter) + search(model, "acme") + #expect(model.find.matches == [match(1, Self.city), match(2, Self.company), match(3, Self.company), match(3, Self.city)]) + + owner.tableRows.rows[2].values[2] = .text("Lyon") + model.apply(.cellChanged(row: 2, column: 2)) + let firstRow = owner.tableRows.appendInsertedRow(values: [.text("5"), .text("Nova"), .text("Oslo")]) + model.apply(firstRow) + #expect(model.displayIndex(of: .existing(2)) == nil) + let firstRowID = owner.tableRows.rows[4].id + model.apply(owner.tableRows.remove(rowIDs: [firstRowID])) + owner.tableRows.rows[2].values[2] = .text("Paris") + model.apply(.cellChanged(row: 2, column: 2)) + let secondRow = owner.tableRows.appendInsertedRow(values: [.text("6"), .text("Vega"), .text("Oslo")]) + model.apply(secondRow) + + #expect(model.displayIndex(of: .existing(2)) != nil) + #expect(model.find.matches == [match(1, Self.city), match(2, Self.company), match(3, Self.company), match(3, Self.city)]) + #expect(model.find.matches == fullSearch(model, owner)) + } + + @Test("A term typed but not yet searched makes the next refresh search every row") + func changedTermSearchesEveryRow() { + let (model, owner) = mount() + search(model, "acme") + + model.find.term = "paris" + owner.tableRows.rows[0].values[2] = .text("Paris") + model.apply(.cellChanged(row: 3, column: 1)) + + #expect(model.find.matches == [match(0, Self.city), match(2, Self.city)]) + } + @Test("The counter names the page whenever the owner holds more rows, Search All Rows offered or not") func pageScopeFollowsTheOwnersRows() { let (model, owner) = mount() diff --git a/apple/TableProTests/Core/Database/ForeignKeysForTablesReadTests.swift b/apple/TableProTests/Core/Database/ForeignKeysForTablesReadTests.swift new file mode 100644 index 0000000000..49c838b1a0 --- /dev/null +++ b/apple/TableProTests/Core/Database/ForeignKeysForTablesReadTests.swift @@ -0,0 +1,172 @@ +// +// ForeignKeysForTablesReadTests.swift +// TableProTests +// +// Asking for six tables' foreign keys on a driver without a bulk read fell into the schema-wide +// default, one round trip per table in the schema, and a cancelled caller kept the loop going. +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class CatalogReadLog: @unchecked Sendable { + private let lock = NSLock() + private var tables: [String] = [] + private var listings = 0 + private var bulk = 0 + + var perTableReads: [String] { lock.withLock { tables } } + var tableListings: Int { lock.withLock { listings } } + var bulkReads: Int { lock.withLock { bulk } } + + func recordTable(_ name: String) { lock.withLock { tables.append(name) } } + func recordListing() { lock.withLock { listings += 1 } } + func recordBulk() { lock.withLock { bulk += 1 } } +} + +private func foreignKey(on table: String) -> PluginForeignKeyInfo { + PluginForeignKeyInfo( + name: "fk_\(table)", column: "parent_id", referencedTable: "parent", referencedColumn: "id", + referencedDatabase: nil + ) +} + +private func tableInfos(_ count: Int) -> [PluginTableInfo] { + (0.. [PluginTableInfo] { + log.recordListing() + return tableInfos(tableCount) + } + + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { + log.recordTable(table) + if cancelsCallerOnRead { withUnsafeCurrentTask { $0?.cancel() } } + if let failure = failures[table] { throw failure } + return [foreignKey(on: table)] + } +} + +private final class BulkCatalogDriver: PluginDriverStub, @unchecked Sendable { + var supportsSchemas: Bool { true } + var supportsTransactions: Bool { false } + var currentSchema: String? { "public" } + var serverVersion: String? { nil } + var providesBulkForeignKeyFetch: Bool { true } + + let log = CatalogReadLog() + private let tableCount: Int + + init(tableCount: Int) { + self.tableCount = tableCount + } + + func ping() async throws {} + + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { + log.recordTable(table) + return [foreignKey(on: table)] + } + + func fetchAllForeignKeys(schema: String?) async throws -> [String: [PluginForeignKeyInfo]] { + log.recordBulk() + return Dictionary(uniqueKeysWithValues: tableInfos(tableCount).map { ($0.name, [foreignKey(on: $0.name)]) }) + } +} + +struct ForeignKeysForTablesReadTests { + private let sixTables = (0..<6).map { "t\($0)" } + + private func adapter(_ driver: any PluginDatabaseDriver) -> PluginDriverAdapter { + PluginDriverAdapter(connection: DatabaseConnection(name: "Test", type: .postgresql), pluginDriver: driver) + } + + @Test("Six tables on a driver without a bulk read cost six reads, not one per table in the schema") + func perTableDriverReadsOnlyTheTablesAskedFor() async throws { + let driver = PerTableCatalogDriver(tableCount: 500) + + let keys = try await adapter(driver).fetchForeignKeys(forTables: sixTables) + + #expect(driver.log.perTableReads == sixTables) + #expect(driver.log.tableListings == 0) + #expect(Set(keys.keys) == Set(sixTables)) + #expect(keys["t3"]?.first?.name == "fk_t3") + } + + @Test("A driver with a bulk read still answers more than five tables from one query") + func bulkDriverKeepsItsSingleQuery() async throws { + let driver = BulkCatalogDriver(tableCount: 500) + + let keys = try await adapter(driver).fetchForeignKeys(forTables: sixTables) + + #expect(driver.log.bulkReads == 1) + #expect(driver.log.perTableReads.isEmpty) + #expect(Set(keys.keys) == Set(sixTables)) + } + + @Test("Past five tables without a bulk read, a table that cannot be read still fails the read") + func perTableDriverPastFiveStillFails() async { + let driver = PerTableCatalogDriver(tableCount: 500, failures: ["t2": DatabaseError.queryFailed("denied")]) + + await #expect(throws: DatabaseError.self) { + _ = try await adapter(driver).fetchForeignKeys(forTables: sixTables) + } + #expect(driver.log.perTableReads == ["t0", "t1", "t2"]) + } + + @Test("Up to five tables, a table that cannot be read is left out and the rest are returned") + func smallSetSkipsAnUnreadableTable() async throws { + let driver = PerTableCatalogDriver(tableCount: 500, failures: ["t1": DatabaseError.queryFailed("denied")]) + + let keys = try await adapter(driver).fetchForeignKeys(forTables: ["t0", "t1", "t2"]) + + #expect(Set(keys.keys) == ["t0", "t2"]) + } + + @Test("A cancelled caller stops the per-table read at the next table") + func cancelledCallerStopsTheLoop() async { + let driver = PerTableCatalogDriver(tableCount: 500, cancelsCallerOnRead: true) + let reader = adapter(driver) + + let read = Task { try await reader.fetchForeignKeys(forTables: ["t0", "t1", "t2"]) } + await #expect(throws: CancellationError.self) { + _ = try await read.value + } + #expect(driver.log.perTableReads == ["t0"]) + } + + @Test("A table read that reports a cancellation ends the read instead of being skipped") + func cancellationFromTheDriverIsNotSkipped() async { + let driver = PerTableCatalogDriver(tableCount: 500, failures: ["t0": CancellationError()]) + + await #expect(throws: CancellationError.self) { + _ = try await adapter(driver).fetchForeignKeys(forTables: ["t0", "t1", "t2"]) + } + #expect(driver.log.perTableReads == ["t0"]) + } +} diff --git a/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterRowLimitTests.swift b/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterRowLimitTests.swift index 0e380bac45..6704212a34 100644 --- a/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterRowLimitTests.swift +++ b/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterRowLimitTests.swift @@ -19,6 +19,99 @@ private final class StreamRecordingDriver: PluginDriverStub, @unchecked Sendable func quoteIdentifier(_ name: String) -> String { "\"\(name)\"" } } +// No stream of its own, so its rows reach the export through PluginKit's default, the way SAP HANA's do. +private final class CuttingQueryDriver: PluginDriverStub, @unchecked Sendable { + private let rowCount: Int + private let isCut: Bool + + init(rowCount: Int, isCut: Bool) { + self.rowCount = rowCount + self.isCut = isCut + } + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult( + columns: ["id"], + columnTypeNames: ["INTEGER"], + rows: (0.. String { "\"\(name)\"" } +} + +// A SAP HANA export over 5,000,000 rows wrote the first 5,000,000 and reported success: the default +// stream dropped the result's cut, so nothing named the file as partial. +@MainActor +struct ExportDriverRowCutTests { + private let orders = PluginExportTable(name: "orders", databaseName: "SALES", tableType: "table", schema: "SALES", kind: .table) + + private func exportAdapter(_ driver: CuttingQueryDriver, type: DatabaseType) -> ExportDataSourceAdapter { + ExportDataSourceAdapter( + driver: PluginDriverAdapter(connection: TestFixtures.makeConnection(type: type), pluginDriver: driver), + databaseType: type + ) + } + + private func rowCount(_ stream: AsyncThrowingStream) async throws -> Int { + var count = 0 + for try await element in stream { + if case .rows(let rows) = element { count += rows.count } + } + return count + } + + @Test("A table export the driver cut at its row limit is named as partial") + func cutTableExportIsReported() async throws { + let adapter = exportAdapter(CuttingQueryDriver(rowCount: 3, isCut: true), type: .sapHana) + + let exported = try await rowCount(adapter.streamRows(for: orders)) + + #expect(exported == 3) + #expect(adapter.cappedTableWarnings.count == 1) + #expect(adapter.cappedTableWarnings.first?.contains("orders") == true) + #expect(adapter.cappedTableWarnings.first?.contains("3") == true) + } + + @Test("A table export the driver read whole carries no warning") + func wholeTableExportIsNotReported() async throws { + let adapter = exportAdapter(CuttingQueryDriver(rowCount: 3, isCut: false), type: .sapHana) + + let exported = try await rowCount(adapter.streamRows(for: orders)) + + #expect(exported == 3) + #expect(adapter.cappedTableWarnings.isEmpty) + } + + @Test("A table export that reaches the engine's own ceiling is still named as partial") + func engineCeilingIsStillReported() async throws { + let adapter = exportAdapter(CuttingQueryDriver(rowCount: 10_000, isCut: false), type: .cloudflareR2SQL) + + _ = try await rowCount(adapter.streamRows(for: orders)) + + #expect(adapter.cappedTableWarnings.count == 1) + } + + @Test("A query export the driver cut is named as partial at the limit it declared") + func cutQueryExportIsReported() async throws { + let driver = PluginDriverAdapter( + connection: TestFixtures.makeConnection(type: .sapHana), + pluginDriver: CuttingQueryDriver(rowCount: 3, isCut: true) + ) + let source = StreamingQueryExportDataSource(query: "SELECT id FROM orders", driver: driver, databaseType: .sapHana) + + let exported = try await rowCount(source.streamRows(table: "result", databaseName: "")) + + #expect(exported == 3) + #expect(source.driverRowCut == 3) + #expect(ExportService.leadingRowsCapWarning(exportedRows: 3, pagination: .offset, driverCut: source.driverRowCut) != nil) + #expect(ExportService.leadingRowsCapWarning(exportedRows: 3, pagination: .offset, driverCut: nil) == nil) + } +} + struct ExportDataSourceAdapterRowLimitTests { private func streamedQuery( type: DatabaseType, diff --git a/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift index 983f92b642..739be16695 100644 --- a/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift +++ b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift @@ -5,7 +5,8 @@ // The pool closed at most one idle connection each time a new one was opened, so a burst of opens // on distinct keys, which a tree restore or a Refresh of every expanded database produces, left // every connection it opened standing until the idle sweep: 13 to 16 backends for one saved -// connection on a server 1.5 seconds away, against a limit of 6 (#3103). +// connection on a server 1.5 seconds away, against a limit of 6 (#3103). Trimming at the end of +// the burst still let it dial every key at once, so the limit now holds while the burst runs. // import Foundation @@ -20,6 +21,8 @@ private final class CountingOpener { private let connectDelaySeconds: Double private(set) var opened: [MockDatabaseDriver] = [] private(set) var openedByDatabase: [String: MockDatabaseDriver] = [:] + private(set) var openOrder: [String] = [] + private(set) var peakOpenCount = 0 var afterOpen: ((DatabaseScope) -> Void)? var deadOnArrival: Set = [] @@ -37,12 +40,19 @@ private final class CountingOpener { driver.hasLostConnection = deadOnArrival.remove(scope.database) != nil opened.append(driver) openedByDatabase[scope.database] = driver + openOrder.append(scope.database) + peakOpenCount = max(peakOpenCount, openCount) try await driver.connect() afterOpen?(scope) return driver } } +@MainActor +private final class StartedTaskBox { + var task: Task? +} + /// Holds a lease's body until the test lets it go, so a test can keep work in flight. @MainActor private final class LeaseGate { @@ -85,12 +95,13 @@ struct MetadataConnectionPoolCapacityTests { private func fill( _ pool: MetadataConnectionPool, _ connection: DatabaseConnection, - databases: [String] + databases: [String], + workload: MetadataConnectionPool.Workload = .interactive ) async throws -> (gates: [LeaseGate], leases: [Task]) { let gates = databases.map { _ in LeaseGate() } let leases = zip(databases, gates).map { database, gate in Task { @MainActor in - try await pool.withDriver(scope: scope(connection, database)) { _ in + try await pool.withDriver(scope: scope(connection, database), workload: workload) { _ in await gate.wait() } } @@ -126,34 +137,248 @@ struct MetadataConnectionPoolCapacityTests { } #expect(opener.opened.count == limit * 2) + #expect(opener.peakOpenCount == limit) #expect(opener.openCount == limit) #expect(pool.heldConnectionCount(for: connection.id) == limit) } - @Test("Each lease that ends past the limit closes the connection unused longest, never one still working") - func trimClosesOnlyIdleConnections() async throws { + @Test("A burst on twice the limit never has more than the limit open while it runs") + func burstNeverRunsPastTheLimit() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + let gates = (0..] = [] + for (database, workload) in arrivals { + let place = scope(connection, database) + waiting.append(Task { @MainActor in + try await pool.withDriver(scope: place, workload: workload) { _ in } + }) + let parked = waiting.count + try await waitUntil { pool.roomWaiterCount(for: connection.id) == parked } + } + + gates[0].open() + _ = try? await leases[0].value + try await waiting[1].value + try await waiting[3].value + #expect(Array(opener.openOrder.dropFirst(limit)) == ["i1", "i2"]) + #expect(pool.roomWaiterCount(for: connection.id) == 2) + + let lateBulk = scope(connection, "b3") + let late = Task { @MainActor in + try await pool.withDriver(scope: lateBulk, workload: .bulk) { _ in } + } + try await waitUntil { pool.roomWaiterCount(for: connection.id) == 3 } + try await pool.withDriver(scope: scope(connection, "i3")) { _ in } + #expect(Array(opener.openOrder.dropFirst(limit)) == ["i1", "i2", "i3"]) + + gates[1].open() + _ = try? await leases[1].value + try await waiting[0].value + try await waiting[2].value + try await late.value + #expect(Array(opener.openOrder.dropFirst(limit)) == ["i1", "i2", "i3", "b1", "b2", "b3"]) + await release(gates, leases) + } + + @Test("Bulk work never takes the last place, so an interactive lease runs while five bulk leases hold the rest") + func bulkLeavesOnePlaceForInteractive() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + let (gates, leases) = try await fill(pool, connection, databases: (0.. PullResult { + PullResult(changedRecords: records, deletedRecordIDs: [], newToken: nil) + } + + private static func remoteConnections(_ count: Int) -> [DatabaseConnection] { + (0.. (LibraryWork, [DatabaseConnection]) { + let environment = try SyncTestEnvironment(label: "sync-connection-pull-batch") + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + let readsBefore = environment.connections.fileReadCount + let writesBefore = environment.connections.fileWriteCount + + let applied = await coordinator.applyPullResult( + Self.pull(connections.map { SyncRecordMapper.toCKRecord($0, in: Self.zoneID) }), + settings: .default + ) + + #expect(applied) + let work = LibraryWork( + reads: environment.connections.fileReadCount - readsBefore, + writes: environment.connections.fileWriteCount - writesBefore + ) + return (work, environment.connections.loadConnections()) + } + + // Each record used to read, verify, decode, encode and write the whole library: 300 of each at 300 connections. + @Test("A pull reads and writes the library the same number of times for 10 connections as for 300") + func pullWorkDoesNotGrowWithTheBatch() async throws { + let few = Self.remoteConnections(10) + let many = Self.remoteConnections(300) + + let (fewWork, fewStored) = try await pullIntoEmptyLibrary(few) + let (manyWork, manyStored) = try await pullIntoEmptyLibrary(many) + + #expect(fewWork.writes == 1) + #expect(manyWork == fewWork) + let fewIds = fewStored.map(\.id) + let manyIds = manyStored.map(\.id) + #expect(fewIds == few.map(\.id)) + #expect(manyIds == many.map(\.id)) + let allSynced = manyStored.allSatisfy { $0.origin == .synced } + #expect(allSynced) + } + + @Test("A credential profile that arrives after a connection still writes its username through to it") + func profileAfterAConnectionSeesTheConnection() async throws { + let environment = try SyncTestEnvironment(label: "sync-connection-pull-order") + let profile = CredentialProfile(name: "Prod", username: "profile-user") + var local = TestFixtures.makeConnection(name: "Linked") + local.username = "local-user" + local.credentialMode = .profile(id: profile.id) + #expect(environment.connections.saveConnections([local])) + var remote = local + remote.username = "remote-user" + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + + let applied = await coordinator.applyPullResult( + Self.pull([ + SyncRecordMapper.toCKRecord(remote, in: Self.zoneID), + SyncRecordMapper.toCKRecord(profile, in: Self.zoneID), + ]), + settings: .default + ) + + #expect(applied) + #expect(environment.connections.loadConnection(id: local.id)?.username == "profile-user") + } + + @Test("A pull whose library cannot be read writes none of its connections and is not acknowledged") + func unreadableLibraryRefusesTheWholeBatch() async throws { + let environment = try SyncTestEnvironment(label: "sync-connection-pull-unreadable") + let fileURL = environment.directory.appendingPathComponent("connections.json") + let garbage = Data("not a connection list".utf8) + try garbage.write(to: fileURL) + environment.connections.invalidateCache() + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + let writesBefore = environment.connections.fileWriteCount + + let applied = await coordinator.applyPullResult( + Self.pull(Self.remoteConnections(3).map { SyncRecordMapper.toCKRecord($0, in: Self.zoneID) }), + settings: .default + ) + + #expect(!applied) + #expect(environment.connections.fileWriteCount == writesBefore) + #expect(try Data(contentsOf: fileURL) == garbage) + } +} diff --git a/apple/TableProTests/Models/Query/ResultSetMenuModelTests.swift b/apple/TableProTests/Models/Query/ResultSetMenuModelTests.swift index 105d0d2c51..b75d4a25ed 100644 --- a/apple/TableProTests/Models/Query/ResultSetMenuModelTests.swift +++ b/apple/TableProTests/Models/Query/ResultSetMenuModelTests.swift @@ -95,6 +95,20 @@ struct ResultSetMenuModelTests { #expect(model.entries.map(model.displayLabel(of:)) == ["users", "orders", "users (2)", "users (3)"]) } + // A 10,000-statement run counted each entry's earlier uses per entry, about 10^8 comparisons to open the chooser. + @Test("Every name of a long run is counted the way one name alone is") + func longRunNamesMatchSingleNameCounting() { + let labels = (0 ..< 3_000).map { index in ["users", "orders", "Result \(index)"][index % 3] } + let model = Self.make(labels: labels, activeIndex: 0) + let expected = labels.indices.map { ResultSetMenuModel.displayLabel(at: $0, in: labels) } + + #expect(model.entries.map { model.displayLabel(of: $0) } == expected) + #expect(ResultSetMenuModel.displayLabels(for: labels) == expected) + #expect(Array(expected.prefix(6)) == ["users", "orders", "Result 2", "users (2)", "orders (2)", "Result 5"]) + #expect(expected.last == "Result 2999") + #expect(model.tabTitle(of: model.entries[2_997]) == "users (1000)") + } + @Test("A long name is cut on a whole character, never inside an emoji") func truncationKeepsWholeCharacters() { let label = "Monthly revenue per region 📈 for Q3" diff --git a/apple/TableProTests/Models/Query/TabSessionRegistryTests.swift b/apple/TableProTests/Models/Query/TabSessionRegistryTests.swift index c13f0553a7..94ab532c4f 100644 --- a/apple/TableProTests/Models/Query/TabSessionRegistryTests.swift +++ b/apple/TableProTests/Models/Query/TabSessionRegistryTests.swift @@ -118,6 +118,32 @@ struct TabSessionRegistryTests { #expect(session.dataRevision > before) } + // A buffer allocated while the old one is still held cannot share its address, so a copy always moves it. + @Test("An edit writes into the session's buffer instead of copying every row") + func editWritesInPlace() { + let registry = TabSessionRegistry() + let session = TabSession() + registry.register(session) + registry.setTableRows(makeRows((0..<1_000).map { "r\($0)" }), for: session.id) + + for row in [0, 499, 999] { + let before = Self.bufferAddress(registry.tableRows(for: session.id)) + registry.updateTableRows(for: session.id) { rows in + rows.edit(row: row, column: 0, value: .text("edited")) + } + #expect(Self.bufferAddress(registry.tableRows(for: session.id)) == before) + } + + let rows = registry.tableRows(for: session.id) + #expect(rows.rows.count == 1_000) + #expect(rows.rows[499].values[0] == .text("edited")) + #expect(rows.rows[500].values[0] == .text("r500")) + } + + private static func bufferAddress(_ rows: TableRows) -> UInt { + rows.rows.withUnsafeBufferPointer { UInt(bitPattern: $0.baseAddress) } + } + // MARK: - rowSetRevision @Test("A cell edit moves dataRevision and leaves rowSetRevision alone") diff --git a/apple/TableProTests/Models/Query/TableRowsPreparedRowsTests.swift b/apple/TableProTests/Models/Query/TableRowsPreparedRowsTests.swift new file mode 100644 index 0000000000..61af8dd4a7 --- /dev/null +++ b/apple/TableProTests/Models/Query/TableRowsPreparedRowsTests.swift @@ -0,0 +1,168 @@ +// +// TableRowsPreparedRowsTests.swift +// TableProTests +// +// A Fetch All or a large first page builds one Row per fetched row. That build runs off the main actor, and the main +// actor only assigns the finished buffer, so a 2,000,000-row result no longer freezes the window while it is copied. +// + +import Foundation +import os +@testable import TablePro +import TableProPluginKit +import Testing + +@MainActor +struct TableRowsPreparedRowsTests { + private static let columns = ["id", "email"] + private static let types: [ColumnType] = [.integer(rawType: nil), .text(rawType: nil)] + private static let shape = TableRows.RowShape(columns: columns, masks: .none, exportPolicy: .none, exportTable: nil) + + private static func fetched(_ count: Int) -> [[PluginCellValue]] { + (0 ..< count).map { [.text("\($0)"), .text("user\($0)@example.com")] } + } + + private static func address(of rows: ContiguousArray) -> UnsafeRawPointer? { + rows.withUnsafeBufferPointer { UnsafeRawPointer($0.baseAddress) } + } + + @Test("Preparing a result reads every row off the main actor") + func preparingReadsOffTheMainActor() async { + let source = MainThreadReadCountingRows(base: Self.fetched(500)) + + let prepared = await TableRows.preparing(source, shape: Self.shape) + + #expect(prepared.rows.count == 500) + #expect(source.reads == MainThreadReadCountingRows.Reads(total: 500, onMain: 0)) + } + + @Test("The counting source sees reads made on the main actor") + func countingSourceSeesMainActorReads() { + let source = MainThreadReadCountingRows(base: Self.fetched(50)) + + _ = TableRows.prepare(source, shape: Self.shape) + + #expect(source.reads == MainThreadReadCountingRows.Reads(total: 50, onMain: 50)) + } + + @Test("Installing prepared rows assigns the built buffer and its index instead of building them again") + func installAdoptsTheBuiltBuffer() { + var table = TableRows.from(queryRows: [[.text("0"), .text("old@example.com")]], columns: Self.columns, columnTypes: Self.types) + let prepared = TableRows.prepare( + Self.fetched(3), offset: 10, rowLocators: ["first", nil, "third"], shape: table.rowShape + ) + + #expect(table.install(prepared) == .fullReplace) + #expect(Self.address(of: table.rows) == Self.address(of: prepared.rows)) + #expect(table.rows.map { $0.id } == [.existing(10), .existing(11), .existing(12)]) + #expect(table.index(of: .existing(12)) == 2) + #expect(table.rowLocator(for: .existing(10)) == "first") + #expect(table.rowLocator(for: .existing(11)) == nil) + #expect(table.rowLocator(for: .existing(12)) == "third") + } + + @Test("Rows prepared before a masking rule was saved are refused, and the held rows stay") + func installRefusesRowsBuiltForOtherMasks() { + var table = TableRows.from(queryRows: [[.text("0"), .text("old@example.com")]], columns: Self.columns, columnTypes: Self.types) + let prepared = TableRows.prepare(Self.fetched(3), shape: table.rowShape) + table.addMasks(ColumnMasks(styles: [1: .full])) + + #expect(table.install(prepared) == nil) + #expect(table.rows.map { $0.id } == [.existing(0)]) + #expect(table.value(at: 0, column: 1) == ColumnMasks(styles: [1: .full]).value(.text("old@example.com"), column: 1)) + } + + @Test("Prepared rows install exactly what a replace builds: masks, padding, absent and unloaded cells, documents") + func installMatchesReplace() { + let masks = ColumnMasks(styles: [1: .email]) + let base = TableRows.from( + queryRows: [[.text("0"), .text("old@example.com")]], + columns: Self.columns, + columnTypes: Self.types, + masks: masks, + exportTable: "users" + ) + let incoming: [[PluginCellValue]] = [[.text("1"), .text("jane@example.com")], [.text("2")], [.text("3"), .null, .text("extra")]] + let documents = incoming.indices.map { index in + PluginExportDocument(fields: [PluginExportField( + name: "email", canonicalJSON: "\"doc\(index)@example.com\"", typeName: "string", cellValue: .text("doc\(index)@example.com") + )]) + } + let absent: [Int: Set] = [1: [1]] + let unloaded: [Int: [Int: Int]] = [2: [1: 4_000]] + + var replaced = base + replaced.replace(rows: incoming, offset: 5, exportDocuments: documents, absentCells: absent, unloadedCells: unloaded) + var installed = base + let prepared = TableRows.prepare( + incoming, offset: 5, exportDocuments: documents, absentCells: absent, unloadedCells: unloaded, shape: base.rowShape + ) + + #expect(installed.install(prepared) == .fullReplace) + #expect(installed.rows == replaced.rows) + #expect(installed.index(of: .existing(7)) == replaced.index(of: .existing(7))) + #expect(installed.value(at: 0, column: 1) != .text("jane@example.com")) + } + + // A Stop on a 2,000,000-row Fetch All left the tab on Loading until every row was built. + @Test("A Stop during the off-main build ends it at the next interval, and what it built never installs") + func stopEndsTheBuildWithinAnInterval() async { + let interval = TableRows.cancellationCheckInterval + let source = MainThreadReadCountingRows(base: Self.fetched(interval * 3), cancelsTaskAtRow: interval) + + let prepared = await Task { await TableRows.preparing(source, shape: Self.shape) }.value + var table = TableRows.from(queryRows: [[.text("0"), .text("old@example.com")]], columns: Self.columns, columnTypes: Self.types) + + #expect(source.reads.total == interval * 2) + #expect(!prepared.isComplete) + #expect(prepared.rows.isEmpty) + #expect(table.install(prepared) == nil) + #expect(table.rows.map { $0.id } == [.existing(0)]) + } + + @Test("The build on the caller's actor lands every row even inside a stopped task") + func callerBuildIgnoresAStop() async { + let interval = TableRows.cancellationCheckInterval + let source = MainThreadReadCountingRows(base: Self.fetched(interval * 2), cancelsTaskAtRow: 0) + + let prepared = await Task { TableRows.prepare(source, shape: Self.shape) }.value + + #expect(source.reads.total == interval * 2) + #expect(prepared.isComplete) + #expect(prepared.rows.count == interval * 2) + } +} + +// Counts which thread read each row, so a test can tell where a build ran. +private struct MainThreadReadCountingRows: RandomAccessCollection, Sendable { + struct Reads: Equatable, Sendable { + var total = 0 + var onMain = 0 + } + + let base: [[PluginCellValue]] + // Cancels the task reading the row at this position, the way a Stop lands in the middle of a build. + let cancelsTaskAtRow: Int? + private let counter = OSAllocatedUnfairLock(initialState: Reads()) + + init(base: [[PluginCellValue]], cancelsTaskAtRow: Int? = nil) { + self.base = base + self.cancelsTaskAtRow = cancelsTaskAtRow + } + + var reads: Reads { counter.withLock { $0 } } + var startIndex: Int { base.startIndex } + var endIndex: Int { base.endIndex } + + subscript(position: Int) -> [PluginCellValue] { + let onMain = Thread.isMainThread + counter.withLock { counted in + counted.total += 1 + if onMain { counted.onMain += 1 } + } + if position == cancelsTaskAtRow { + withUnsafeCurrentTask { $0?.cancel() } + } + return base[position] + } +} diff --git a/apple/TableProTests/Views/DataGrid/DataGridValueFilterTests.swift b/apple/TableProTests/Views/DataGrid/DataGridValueFilterTests.swift index 4a313e942b..3d0242a892 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridValueFilterTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridValueFilterTests.swift @@ -343,6 +343,52 @@ struct DataGridValueFilterTests { #expect(grid.owner.displayOrderRequests == 2) } + // The click returns before the list is sorted, so the main actor never waits on the comparisons. + @Test("Filter Values sorts its list away from the click, and a reload while it sorts opens nothing") + func valueListSortsBeforeThePopoverOpens() async throws { + var configuration = DataGridConfiguration() + configuration.commands = [.valueFilter] + let grid = HostedDataGrid(rows: Self.statuses, configuration: configuration) + defer { grid.close() } + let status = try #require(grid.model.columns.columns.first { $0.name == "status" }) + + grid.controller.presentValueFilter(for: status) + + let listing = try #require(grid.controller.menuState.valueFilterListing) + #expect(grid.controller.menuState.valueFilterPopover == nil) + + grid.model.reloadRows() + await listing.value + + #expect(grid.controller.menuState.valueFilterListing == nil) + #expect(grid.controller.menuState.valueFilterPopover == nil) + } + + // A connection switch unparents the grid without dismantling it, and NSPopover raises on a view with no window. + @Test("A list that finishes after the grid left its window, or while the window is off screen, opens nothing") + func valueListOpensOnlyInAVisibleWindow() async throws { + var configuration = DataGridConfiguration() + configuration.commands = [.valueFilter] + let grid = HostedDataGrid(rows: Self.statuses, configuration: configuration) + defer { grid.close() } + let status = try #require(grid.model.columns.columns.first { $0.name == "status" }) + #expect(!grid.window.isVisible) + + grid.controller.presentValueFilter(for: status) + let offScreen = try #require(grid.controller.menuState.valueFilterListing) + await offScreen.value + #expect(grid.controller.menuState.valueFilterPopover == nil) + + grid.controller.presentValueFilter(for: status) + let unparented = try #require(grid.controller.menuState.valueFilterListing) + grid.controller.view.removeFromSuperview() + await unparented.value + + #expect(grid.controller.view.window == nil) + #expect(grid.controller.menuState.valueFilterListing == nil) + #expect(grid.controller.menuState.valueFilterPopover == nil) + } + @Test("Replacing the rows closes an open value filter, whose values were the old rows'") func replacingTheRowsClosesTheValueFilter() { let grid = HostedDataGrid(rows: Self.statuses) diff --git a/apple/TableProTests/Views/DataGrid/RowVisualIndexTests.swift b/apple/TableProTests/Views/DataGrid/RowVisualIndexTests.swift index 395c5de217..5d8ba4999e 100644 --- a/apple/TableProTests/Views/DataGrid/RowVisualIndexTests.swift +++ b/apple/TableProTests/Views/DataGrid/RowVisualIndexTests.swift @@ -9,6 +9,120 @@ import TableProPluginKit import TableProTheme import Testing +@MainActor +private final class CountingChangeManager: ChangeManaging { + var changes: [RowChange] = [] + private(set) var scans = 0 + private(set) var lookups = 0 + + var hasChanges: Bool { !changes.isEmpty } + var reloadVersion: Int { 0 } + var canRedo: Bool { false } + var insertedRowIDs: Set { [] } + + var rowChanges: [RowChange] { + scans += 1 + return changes + } + + func pendingChange(forRow rowID: RowID) -> RowChange? { + lookups += 1 + return changes.first { $0.rowID == rowID } + } + + func isRowDeleted(_ rowID: RowID) -> Bool { false } + + func recordCellChange( + rowID: RowID, + columnIndex: Int, + columnName: String, + oldValue: PluginCellValue, + newValue: PluginCellValue, + originalRow: [PluginCellValue]? + ) {} + + func undoRowDeletion(rowID: RowID) {} +} + +@MainActor +private final class RepaintRecordingView: DataGridMountedView { + private(set) var changes: [DataGridChange] = [] + + func apply(_ change: DataGridChange) { + changes.append(change) + } + + func selectionDidChange() {} + func configurationDidChange() {} + func reveal(row: Int, displayColumn: Int?) {} + func revealColumn(_ displayColumn: Int) {} + func sortDidChange() {} + func summaryRulesDidChange() {} + func viewportSample() -> (row: Int, offset: CGFloat)? { nil } + func beginEditing(row: Int, displayColumn: Int) -> Bool { false } + var isEditing: Bool { false } + func commitEditing() -> Bool { true } + func discardEditing() {} + func focus() -> Bool { false } + func anchor(forColumn displayColumn: Int) -> (view: NSView, rect: NSRect)? { nil } +} + +@MainActor +struct UndoStepRepaintTests { + private let fixture = CoordinatorFixture() + + // Undoing a command over every row runs one step per cell, so a step that repaints every row costs rows squared. + @Test("Undoing a step repaints the row it names and no other") + func undoRepaintsOnlyItsRow() { + let rows = TableRows.from( + queryRows: (0..<40).map { row -> [PluginCellValue] in [.text("\(row)"), .text("b")] }, + columns: ["id", "name"], + columnTypes: [.text(rawType: nil), .text(rawType: nil)], + hasAuthoritativeSchema: true + ) + let tab = fixture.addTab( + QueryTab(title: "users", query: "SELECT * FROM users", tabType: .table, tableName: "users"), + rows: rows + ) + let model = fixture.coordinator.gridModel(forTab: tab.id) + let view = RepaintRecordingView() + model.attach(view) + + fixture.coordinator.handleUndoResult(UndoResult( + action: .cellEdit( + rowID: .existing(7), columnIndex: 1, columnName: "name", + previousValue: "a", newValue: "b", originalRow: ["7", "a"] + ), + needsRowRemoval: false, + needsRowRestore: false, + restoreRow: nil + )) + + var repainted = IndexSet() + var marksRepainted = false + for change in view.changes { + guard case .rowsUpdated(let indexes, let columns) = change else { continue } + repainted.formUnion(indexes) + if columns == nil { marksRepainted = true } + } + #expect(repainted == IndexSet(integer: 7)) + #expect(marksRepainted) + #expect(fixture.coordinator.tabSessionRegistry.tableRows(for: tab.id).rows[7].values[1] == "a") + } + + @Test("A batch step names every row it holds") + func batchStepNamesItsRows() { + let deletion = UndoAction.batchRowDeletion(rows: [(rowID: .existing(1), originalRow: []), (rowID: .existing(3), originalRow: [])]) + let insertion = UndoAction.batchRowInsertion( + rows: [InsertedRowLocation(rowID: .existing(5), storageIndex: 5)], + rowValues: [[]] + ) + + #expect(RowEditingCoordinator.rowIDs(touchedBy: deletion) == [.existing(1), .existing(3)]) + #expect(RowEditingCoordinator.rowIDs(touchedBy: insertion) == [.existing(5)]) + } +} + @MainActor struct RowVisualIndexTests { private func makeManager() -> DataChangeManager { @@ -110,6 +224,55 @@ struct RowVisualIndexTests { #expect(index.visualState(for: inserted).expressionColumns == [1]) } + @Test("Updating rows looks each one's change up by its id, never scanning every pending change") + func updateRowLooksTheChangeUp() { + let manager = CountingChangeManager() + manager.changes = (0..<2_000).map { row in + RowChange( + rowID: .existing(row), + type: .update, + cellChanges: [CellChange(columnIndex: 1, columnName: "name", oldValue: "a", newValue: "b")] + ) + } + let changeManager = AnyChangeManager(manager) + let index = RowVisualIndex() + + for row in 0..<2_000 { + index.updateRow(.existing(row), from: changeManager) + } + + #expect(manager.scans == 0) + #expect(manager.lookups == 2_000) + #expect(index.visualState(for: .existing(1_999)).modifiedColumns == [1]) + } + + @Test("A row's pending change is the one a scan of every change finds, after removals reorder them") + func pendingChangeMatchesAScan() { + let manager = makeManager() + for row in 0..<6 { + manager.recordCellChange( + rowID: .existing(row), columnIndex: 1, columnName: "name", + oldValue: "a", newValue: "b", originalRow: [.text("\(row)"), "a"] + ) + } + let inserted = RowID.inserted(UUID()) + manager.recordRowInsertion(rowID: inserted, values: [.null, .null]) + manager.recordRowDeletion(rowID: .existing(2), originalRow: ["2", "b"]) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "b", newValue: "a", originalRow: ["0", "b"] + ) + manager.undoRowDeletion(rowID: .existing(2)) + manager.recordRowDeletion(rowID: .existing(4), originalRow: ["4", "b"]) + + for rowID in (0..<8).map(RowID.existing) + [inserted] { + let scanned = manager.rowChanges.first { $0.rowID == rowID } + #expect(manager.pendingChange(forRow: rowID) == scanned) + } + #expect(manager.pendingChange(forRow: .existing(4))?.type == .delete) + #expect(manager.pendingChange(forRow: .existing(0)) == nil) + } + @Test("A deleted or new row tints the whole row from its theme slot, a highlighted one with the rule's wash") func rowTintFollowsTheState() { let scope = ThemeScope.live diff --git a/apple/TableProTests/Views/Editor/StatementRunControllerTests.swift b/apple/TableProTests/Views/Editor/StatementRunControllerTests.swift index 249dff0232..70fe90613f 100644 --- a/apple/TableProTests/Views/Editor/StatementRunControllerTests.swift +++ b/apple/TableProTests/Views/Editor/StatementRunControllerTests.swift @@ -174,6 +174,81 @@ struct StatementRunControllerTests { #expect(controller.highlightedStatementRange == nil) } + // MARK: - The band across edits + + // Long enough that the band's index records boundaries, which an edit near the top has to drop. + private let indexedScript = String(repeating: "SELECT 1;\n", count: 1_000) + + private func freshBand(in controller: TextViewController, caret: Int) -> NSRange? { + let statement = SQLStatementScanner.locatedStatementAtCursor(in: controller.textView.string, cursorPosition: caret) + guard statement.hasContent, statement.contentRange.length > 0 else { return nil } + return statement.contentRange + } + + private func band(at caret: Int, in controller: TextViewController, subject: StatementRunController) -> NSRange? { + controller.setCursorPositions([CursorPosition(range: NSRange(location: caret, length: 0))]) + subject.refreshHighlight(in: controller) + return controller.highlightedStatementRange + } + + @Test("An edit typed into the editor reaches a band that already read the script") + func typedEditReachesTheBand() throws { + let controller = makeController(text: indexedScript) + let subject = makeSubject() + subject.install(on: controller) + #expect(band(at: 9_995, in: controller, subject: subject) == NSRange(location: 9_990, length: 9)) + + // An open comment, because the editor pairs a typed quote, and it swallows every boundary the band had read. + controller.textView.replaceCharacters(in: NSRange(location: 0, length: 0), with: "/*") + try #require(controller.textView.string.hasPrefix("/*SELECT")) + + #expect(freshBand(in: controller, caret: 9_995) == nil) + #expect(band(at: 9_995, in: controller, subject: subject) == nil) + } + + @Test("Text set on the text view without the editor's edit path reaches the band") + func textSetDirectlyReachesTheBand() { + let controller = makeController(text: indexedScript) + let subject = makeSubject() + subject.install(on: controller) + #expect(band(at: 9_995, in: controller, subject: subject) == NSRange(location: 9_990, length: 9)) + + controller.textView.string = "'" + indexedScript + + #expect(band(at: 9_995, in: controller, subject: subject) == freshBand(in: controller, caret: 9_995)) + } + + @Test("A replaced document gets a band read from the new text") + func replacedDocumentReachesTheBand() { + let controller = makeController(text: indexedScript) + let subject = makeSubject() + subject.install(on: controller) + #expect(band(at: 9_995, in: controller, subject: subject) == NSRange(location: 9_990, length: 9)) + + controller.setText("'" + indexedScript) + + #expect(band(at: 9_995, in: controller, subject: subject) == freshBand(in: controller, caret: 9_995)) + } + + // 5,000 statements: resolving from the top walks about 50,000 units per caret move. + @Test("Moving the caret through a long script rescans only near the caret") + func caretMovesRescanOnlyNearTheCaret() { + let controller = makeController(text: String(repeating: "SELECT 1;\n", count: 5_000)) + let subject = makeSubject() + subject.install(on: controller) + let start = 49_995 + _ = band(at: start, in: controller, subject: subject) + let warmed = subject.caretIndexUnitsScanned + + let moves = 500 + for step in 1...moves { + _ = band(at: start - step, in: controller, subject: subject) + } + + #expect(controller.highlightedStatementRange == freshBand(in: controller, caret: start - moves)) + #expect(subject.caretIndexUnitsScanned - warmed <= moves * (SQLStatementCursorIndex.defaultCheckpointSpacing + 20)) + } + // MARK: - Callback @Test("Pressing a control reports that control's own statement, not the caret's") diff --git a/apple/TableProTests/Views/Main/ResultRowsBuildOffMainTests.swift b/apple/TableProTests/Views/Main/ResultRowsBuildOffMainTests.swift new file mode 100644 index 0000000000..ce095016eb --- /dev/null +++ b/apple/TableProTests/Views/Main/ResultRowsBuildOffMainTests.swift @@ -0,0 +1,212 @@ +// +// ResultRowsBuildOffMainTests.swift +// TableProTests +// +// A query's rows are built off the main actor, under the shape the tab will install them with. The main actor then +// assigns the buffer, and only builds again when that shape changed while the rows were built. +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProSQLGrammar +import Testing + +@MainActor +struct ResultRowsBuildOffMainTests { + private static let columns = ["id", "email"] + private static let types: [ColumnType] = [.integer(rawType: "int"), .text(rawType: "varchar")] + private static let fetched: [[PluginCellValue]] = [[.text("1"), .text("jane@example.com")], [.text("2"), .text("john@example.com")]] + + private let fixture: CoordinatorFixture = { + var connection = TestFixtures.makeConnection(database: "shop") + connection.maskingRules = [MaskingRule(pattern: "*.email", style: .email)] + return CoordinatorFixture(connection: connection) + }() + + private static func address(of rows: ContiguousArray) -> UnsafeRawPointer? { + rows.withUnsafeBufferPointer { UnsafeRawPointer($0.baseAddress) } + } + + private func addUsersTab() -> QueryTab { + fixture.addTab(QueryTab(title: "users", query: "SELECT * FROM users", tabType: .table, tableName: "users")) + } + + private func load(_ prepared: TableRows.PreparedRows, into tab: QueryTab) -> TableRows { + fixture.coordinator.applyPhase1Result( + tabId: tab.id, + columns: Self.columns, + columnTypes: Self.types, + rows: Self.fetched, + executionTime: 0, + rowsAffected: 0, + statusMessage: nil, + tableName: "users", + isEditable: true, + metadata: nil, + hasSchema: false, + read: TableFreshness.Read(startedAt: .now, includesDefinition: false), + sql: "SELECT * FROM users", + connection: fixture.connection, + prepared: prepared + ) + return fixture.coordinator.tabSessionRegistry.tableRows(for: tab.id) + } + + @Test("A first page built off the main actor is installed without another pass over its rows") + func firstPageAdoptsThePreparedRows() async { + let tab = addUsersTab() + let shape = fixture.coordinator.phase1RowShape(tabId: tab.id, tableName: "users", columns: Self.columns, columnTypes: Self.types) + let prepared = await TableRows.preparing(Self.fetched, shape: shape) + + let installed = load(prepared, into: tab) + + #expect(Self.address(of: installed.rows) == Self.address(of: prepared.rows)) + #expect(installed.value(at: 0, column: 1) == .text("j•••@example.com")) + #expect(installed.masks.isMasked(1)) + } + + @Test("A first page built before a masking rule was saved is built again under the rule") + func firstPageRebuildsRowsBuiltForOtherMasks() async { + let unmasked = TableRows.RowShape(columns: Self.columns, masks: .none, exportPolicy: .none, exportTable: "users") + let prepared = await TableRows.preparing(Self.fetched, shape: unmasked) + + let installed = load(prepared, into: addUsersTab()) + + #expect(Self.address(of: installed.rows) != Self.address(of: prepared.rows)) + #expect(installed.value(at: 0, column: 1) == .text("j•••@example.com")) + #expect(installed.value(at: 1, column: 1) == .text("j•••@example.com")) + } + + @Test("A Run All statement's rows built off the main actor become its result without another pass") + func runAllStatementAdoptsBuiltRows() async throws { + let tab = fixture.addTab(QueryTab(title: "Query", query: "SELECT * FROM users", tabType: .query)) + let runner: QueryExecutionCoordinator = fixture.coordinator.queryExecutionCoordinator + let result = ScriptAnsweringDriver.resultSet(columns: Self.columns, rows: [["1", "jane@example.com"], ["2", "john@example.com"]]) + + let build = await runner.buildStatementRows([(sql: "SELECT * FROM users", result: result)], tabId: tab.id) + let built = try #require(build.statement(at: 0, readBy: fixture.coordinator.tableNameReading)) + let resultSet = runner.makeStatementResultSet( + result: result, sql: "SELECT * FROM users", index: 0, baseQuery: nil, tabId: tab.id, built: built + ) + + #expect(built.tableName == fixture.coordinator.extractTableName(from: "SELECT * FROM users")) + #expect(Self.address(of: resultSet.tableRows.rows) == Self.address(of: built.prepared.rows)) + #expect(resultSet.tableRows.value(at: 1, column: 1) == .text("j•••@example.com")) + #expect(resultSet.tableRows.masks.isMasked(1)) + } + + @Test("A Run All statement built before a masking rule applied is built again under it") + func runAllStatementRebuildsRowsBuiltForOtherMasks() async throws { + let tab = fixture.addTab(QueryTab(title: "Query", query: "SELECT * FROM users", tabType: .query)) + let runner: QueryExecutionCoordinator = fixture.coordinator.queryExecutionCoordinator + let result = ScriptAnsweringDriver.resultSet(columns: Self.columns, rows: [["1", "jane@example.com"]]) + let reading = fixture.coordinator.tableNameReading + + let build = await QueryExecutionCoordinator.statementRows( + [(sql: "SELECT * FROM users", result: result)], reading: reading, policy: .none + ) + let built = try #require(build.statement(at: 0, readBy: reading)) + let resultSet = runner.makeStatementResultSet( + result: result, sql: "SELECT * FROM users", index: 0, baseQuery: nil, tabId: tab.id, built: built + ) + + #expect(Self.address(of: resultSet.tableRows.rows) != Self.address(of: built.prepared.rows)) + #expect(resultSet.tableRows.value(at: 0, column: 1) == .text("j•••@example.com")) + } + + @Test("Table names read under another session schema are not used") + func statementNamesNeedTheSameReading() async { + let reading = TableNameReading(dialect: .generic, readings: .single(.ansi), browseSchema: "public") + let moved = TableNameReading(dialect: .generic, readings: .single(.ansi), browseSchema: "sales") + let result = ScriptAnsweringDriver.resultSet(columns: ["n"], rows: [["1"]]) + + let build = await QueryExecutionCoordinator.statementRows( + [(sql: "SELECT * FROM public.users", result: result)], reading: reading, policy: .none + ) + + #expect(build.statement(at: 0, readBy: reading)?.tableName == "users") + #expect(build.statement(at: 0, readBy: moved) == nil) + #expect(build.statement(at: 1, readBy: reading) == nil) + } + + private func runBatch(answeredWith resultSets: [QueryResult]) async throws -> [ResultSet] { + let connection = fixture.connection + let driver = ScriptAnsweringDriver(connection: connection, answer: { _ in ScriptAnsweringDriver.batch(resultSets) }) + var session = ConnectionSession(connection: connection, driver: driver) + session.status = .connected + session.browseDatabase = connection.database + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let tab = fixture.addTab(QueryTab(title: "Query", query: "SELECT * FROM users", tabType: .query)) + let runner: QueryExecutionCoordinator = fixture.coordinator.queryExecutionCoordinator + runner.executeBatches(runner.executionBatches(in: "SELECT * FROM users"), parameters: []) + let handle = try #require(fixture.coordinator.queryTasks.task(for: tab.id)) + await handle.value + return try #require(fixture.tabManager.tabs.first { $0.id == tab.id }).display.resultSets + } + + @Test("A batch's result set that maps to its query is named after its table and masked") + func batchResultMappedToItsQuery() async throws { + let users = ScriptAnsweringDriver.resultSet(columns: Self.columns, rows: [["1", "jane@example.com"], ["2", "john@example.com"]]) + + let results = try await runBatch(answeredWith: [users]) + + let result = try #require(results.first) + #expect(results.count == 1) + #expect(result.label == "users") + #expect(result.tableRows.rows.count == 2) + #expect(result.tableRows.value(at: 1, column: 1) == .text("j•••@example.com")) + } + + @Test("A batch's numbered result sets are each masked and keep their order") + func batchNumberedResultsAreMasked() async throws { + let first = ScriptAnsweringDriver.resultSet(columns: Self.columns, rows: [["1", "jane@example.com"]]) + let second = ScriptAnsweringDriver.resultSet(columns: Self.columns, rows: [["2", "john@example.com"], ["3", "ann@example.com"]]) + + let results = try await runBatch(answeredWith: [first, second]) + + #expect(results.map { $0.tableRows.rows.count } == [1, 2]) + #expect(results.map { $0.tableRows.value(at: 0, column: 0) } == [.text("1"), .text("2")]) + #expect(results.allSatisfy { $0.tableRows.value(at: 0, column: 1) != .text("jane@example.com") }) + #expect(results.last?.tableRows.value(at: 1, column: 1) == .text("a•••@example.com")) + } + + @Test("Fetch All lands every row, masked the way the page it replaces is") + func fetchAllMasksLikeTheHeldPage() async throws { + let connection = fixture.connection + let driver = ScriptAnsweringDriver(connection: connection, statementAnswer: { _ in + ScriptAnsweringDriver.resultSet( + columns: ["id", "email"], + rows: [["1", "jane@example.com"], ["2", "john@example.com"], ["3", "ann@example.com"]] + ) + }) + var session = ConnectionSession(connection: connection, driver: driver) + session.status = .connected + session.browseDatabase = connection.database + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let pageMasks = ColumnMasks(styles: [1: .full]) + let tab = fixture.addTab( + QueryTab(title: "Query", query: "SELECT * FROM users", tabType: .query), + rows: TableRows.from(queryRows: [Self.fetched[0]], columns: Self.columns, columnTypes: Self.types, masks: pageMasks) + ) + + fixture.coordinator.paginationCoordinator.performFetchAll( + tabId: tab.id, + baseQuery: "SELECT * FROM users", + scope: DatabaseScope(connectionId: connection.id, database: connection.database, schema: nil) + ) + let handle = try #require(fixture.coordinator.queryTasks.task(for: tab.id)) + await handle.value + + let rows = fixture.coordinator.tabSessionRegistry.tableRows(for: tab.id) + let hidden = pageMasks.value(.text("jane@example.com"), column: 1) + #expect(rows.rows.map { $0.id } == [.existing(0), .existing(1), .existing(2)]) + #expect(rows.rows.map { $0.values[0] } == [.text("1"), .text("2"), .text("3")]) + #expect(rows.rows.allSatisfy { $0.values[1] == hidden }) + #expect(rows.masks == pageMasks) + } +} diff --git a/apple/changelog.d/4181.fixed.1.md b/apple/changelog.d/4181.fixed.1.md new file mode 100644 index 0000000000..fe23d7f460 --- /dev/null +++ b/apple/changelog.d/4181.fixed.1.md @@ -0,0 +1 @@ +Column suggestions before FROM were slow to appear in schemas with many tables. diff --git a/apple/changelog.d/4181.fixed.10.md b/apple/changelog.d/4181.fixed.10.md new file mode 100644 index 0000000000..ec73a7630f --- /dev/null +++ b/apple/changelog.d/4181.fixed.10.md @@ -0,0 +1 @@ +Stopping an export from SAP HANA, Teradata, InfluxDB, Cloudflare R2 SQL or Memcached left its query running. diff --git a/apple/changelog.d/4181.fixed.11.md b/apple/changelog.d/4181.fixed.11.md new file mode 100644 index 0000000000..3fe26a5af6 --- /dev/null +++ b/apple/changelog.d/4181.fixed.11.md @@ -0,0 +1 @@ +AI chat could read foreign keys for every table in a large schema on Snowflake, Trino, ClickHouse and others. diff --git a/apple/changelog.d/4181.fixed.12.md b/apple/changelog.d/4181.fixed.12.md new file mode 100644 index 0000000000..d25eeefa74 --- /dev/null +++ b/apple/changelog.d/4181.fixed.12.md @@ -0,0 +1 @@ +Exporting more than 5,000,000 rows from SAP HANA wrote only the first 5,000,000 and reported success. diff --git a/apple/changelog.d/4181.fixed.13.md b/apple/changelog.d/4181.fixed.13.md new file mode 100644 index 0000000000..fc42943d59 --- /dev/null +++ b/apple/changelog.d/4181.fixed.13.md @@ -0,0 +1 @@ +Fetch All and large query results froze the window while their rows were loaded. diff --git a/apple/changelog.d/4181.fixed.14.md b/apple/changelog.d/4181.fixed.14.md new file mode 100644 index 0000000000..f6abe06d63 --- /dev/null +++ b/apple/changelog.d/4181.fixed.14.md @@ -0,0 +1 @@ +Run All on a script with thousands of statements froze the window and its result chooser. diff --git a/apple/changelog.d/4181.fixed.15.md b/apple/changelog.d/4181.fixed.15.md new file mode 100644 index 0000000000..14b9122b65 --- /dev/null +++ b/apple/changelog.d/4181.fixed.15.md @@ -0,0 +1 @@ +A capped PostgreSQL query or a cancelled export inside a transaction read the whole result. diff --git a/apple/changelog.d/4181.fixed.16.md b/apple/changelog.d/4181.fixed.16.md new file mode 100644 index 0000000000..a7045f946b --- /dev/null +++ b/apple/changelog.d/4181.fixed.16.md @@ -0,0 +1 @@ +Exporting a large Oracle table was far slower than the same export from other databases. diff --git a/apple/changelog.d/4181.fixed.17.md b/apple/changelog.d/4181.fixed.17.md new file mode 100644 index 0000000000..9444c35513 --- /dev/null +++ b/apple/changelog.d/4181.fixed.17.md @@ -0,0 +1 @@ +Browsing, filtering or exporting a large etcd prefix downloaded every key and value at once. diff --git a/apple/changelog.d/4181.fixed.2.md b/apple/changelog.d/4181.fixed.2.md new file mode 100644 index 0000000000..292e54419a --- /dev/null +++ b/apple/changelog.d/4181.fixed.2.md @@ -0,0 +1 @@ +The app froze during the first iCloud sync of a large connection library. diff --git a/apple/changelog.d/4181.fixed.3.md b/apple/changelog.d/4181.fixed.3.md new file mode 100644 index 0000000000..a38e0f31f3 --- /dev/null +++ b/apple/changelog.d/4181.fixed.3.md @@ -0,0 +1 @@ +XLSX export held the whole workbook in memory several times and hit the 4 GB limit only after reading every row. diff --git a/apple/changelog.d/4181.fixed.4.md b/apple/changelog.d/4181.fixed.4.md new file mode 100644 index 0000000000..9c83f750b0 --- /dev/null +++ b/apple/changelog.d/4181.fixed.4.md @@ -0,0 +1 @@ +JSON import used several times the file's size in memory and silently skipped array items that were not objects. diff --git a/apple/changelog.d/4181.fixed.5.md b/apple/changelog.d/4181.fixed.5.md new file mode 100644 index 0000000000..6fd33d00f9 --- /dev/null +++ b/apple/changelog.d/4181.fixed.5.md @@ -0,0 +1 @@ +Editing, adding or removing rows with Find open in a large result froze the app. diff --git a/apple/changelog.d/4181.fixed.6.md b/apple/changelog.d/4181.fixed.6.md new file mode 100644 index 0000000000..2651f531ab --- /dev/null +++ b/apple/changelog.d/4181.fixed.6.md @@ -0,0 +1 @@ +Filling a column, pasting over many rows or undoing it in a large result hung the app. diff --git a/apple/changelog.d/4181.fixed.7.md b/apple/changelog.d/4181.fixed.7.md new file mode 100644 index 0000000000..369cddca38 --- /dev/null +++ b/apple/changelog.d/4181.fixed.7.md @@ -0,0 +1 @@ +Typing in the row inspector after Fetch All on a large result lagged on every key. diff --git a/apple/changelog.d/4181.fixed.8.md b/apple/changelog.d/4181.fixed.8.md new file mode 100644 index 0000000000..46de700998 --- /dev/null +++ b/apple/changelog.d/4181.fixed.8.md @@ -0,0 +1 @@ +Opening Filter Values on a column with many distinct values froze the app while it sorted them. diff --git a/apple/changelog.d/4181.fixed.9.md b/apple/changelog.d/4181.fixed.9.md new file mode 100644 index 0000000000..3fe3b283dd --- /dev/null +++ b/apple/changelog.d/4181.fixed.9.md @@ -0,0 +1 @@ +Refreshing many databases or schemas at once could open more server connections than the server allows. diff --git a/apple/changelog.d/4181.fixed.md b/apple/changelog.d/4181.fixed.md new file mode 100644 index 0000000000..59c28971d3 --- /dev/null +++ b/apple/changelog.d/4181.fixed.md @@ -0,0 +1 @@ +Typing or moving the caret near the end of a long SQL script lagged while the current statement was highlighted.